diff --git a/package-lock.json b/package-lock.json index 73bf2177b1..fb287dc07b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { - "name": "promptbook", + "name": "promptbook-engine", "version": "0.86.10", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "promptbook", + "name": "promptbook-engine", "version": "0.86.10", "funding": [ { @@ -60,6 +60,7 @@ "xyzt": "5.4.3" }, "devDependencies": { + "@biomejs/biome": "1.9.4", "@rollup/plugin-json": "6.1.0", "@rollup/plugin-typescript": "8.3.0", "@types/crypto-js": "4.2.2", @@ -1101,6 +1102,170 @@ "dev": true, "license": "MIT" }, + "node_modules/@biomejs/biome": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.9.4.tgz", + "integrity": "sha512-1rkd7G70+o9KkTn5KLmDYXihGoTaIGO9PIIN2ZB7UJxFrWw04CZHPYiMRjYsaDvVV7hP1dYNRLxSANLaBFGpog==", + "dev": true, + "hasInstallScript": true, + "license": "MIT OR Apache-2.0", + "bin": { + "biome": "bin/biome" + }, + "engines": { + "node": ">=14.21.3" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/biome" + }, + "optionalDependencies": { + "@biomejs/cli-darwin-arm64": "1.9.4", + "@biomejs/cli-darwin-x64": "1.9.4", + "@biomejs/cli-linux-arm64": "1.9.4", + "@biomejs/cli-linux-arm64-musl": "1.9.4", + "@biomejs/cli-linux-x64": "1.9.4", + "@biomejs/cli-linux-x64-musl": "1.9.4", + "@biomejs/cli-win32-arm64": "1.9.4", + "@biomejs/cli-win32-x64": "1.9.4" + } + }, + "node_modules/@biomejs/cli-darwin-arm64": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.9.4.tgz", + "integrity": "sha512-bFBsPWrNvkdKrNCYeAp+xo2HecOGPAy9WyNyB/jKnnedgzl4W4Hb9ZMzYNbf8dMCGmUdSavlYHiR01QaYR58cw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-darwin-x64": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.9.4.tgz", + "integrity": "sha512-ngYBh/+bEedqkSevPVhLP4QfVPCpb+4BBe2p7Xs32dBgs7rh9nY2AIYUL6BgLw1JVXV8GlpKmb/hNiuIxfPfZg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-arm64": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.9.4.tgz", + "integrity": "sha512-fJIW0+LYujdjUgJJuwesP4EjIBl/N/TcOX3IvIHJQNsAqvV2CHIogsmA94BPG6jZATS4Hi+xv4SkBBQSt1N4/g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-arm64-musl": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.9.4.tgz", + "integrity": "sha512-v665Ct9WCRjGa8+kTr0CzApU0+XXtRgwmzIf1SeKSGAv+2scAlW6JR5PMFo6FzqqZ64Po79cKODKf3/AAmECqA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-x64": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.9.4.tgz", + "integrity": "sha512-lRCJv/Vi3Vlwmbd6K+oQ0KhLHMAysN8lXoCI7XeHlxaajk06u7G+UsFSO01NAs5iYuWKmVZjmiOzJ0OJmGsMwg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-x64-musl": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.9.4.tgz", + "integrity": "sha512-gEhi/jSBhZ2m6wjV530Yy8+fNqG8PAinM3oV7CyO+6c3CEh16Eizm21uHVsyVBEB6RIM8JHIl6AGYCv6Q6Q9Tg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-win32-arm64": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.9.4.tgz", + "integrity": "sha512-tlbhLk+WXZmgwoIKwHIHEBZUwxml7bRJgk0X2sPyNR3S93cdRq6XulAZRQJ17FYGGzWne0fgrXBKpl7l4M87Hg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-win32-x64": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.9.4.tgz", + "integrity": "sha512-8Y5wMhVIPaWe6jw2H+KlEm4wP/f7EW3810ZLmDlrEEy5KvBsb9ECEfu/kMWD484ijfQ8+nIi0giMgu9g1UAuuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=14.21.3" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", diff --git a/package.json b/package.json index 0f3b259dee..864c4fba56 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "postversion": "npm run update-version-in-config && ts-node ./scripts/generate-examples-jsons/generate-examples-jsons.ts --commit && npm run generate-packages --commit && npm run generate-documentation && npm run import-markdowns && git push && git push --tags && npm run use-packages" }, "devDependencies": { + "@biomejs/biome": "1.9.4", "@rollup/plugin-json": "6.1.0", "@rollup/plugin-typescript": "8.3.0", "@types/crypto-js": "4.2.2", diff --git a/src/_packages/anthropic-claude.index.ts b/src/_packages/anthropic-claude.index.ts index 30090fc5b9..4213bad648 100644 --- a/src/_packages/anthropic-claude.index.ts +++ b/src/_packages/anthropic-claude.index.ts @@ -1,20 +1,18 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/anthropic-claude` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { ANTHROPIC_CLAUDE_MODELS } from '../llm-providers/anthropic-claude/anthropic-claude-models'; -import { AnthropicClaudeExecutionTools } from '../llm-providers/anthropic-claude/AnthropicClaudeExecutionTools'; -import type { AnthropicClaudeExecutionToolsOptions } from '../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions'; -import type { AnthropicClaudeExecutionToolsDirectOptions } from '../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions'; -import type { AnthropicClaudeExecutionToolsProxiedOptions } from '../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions'; -import { createAnthropicClaudeExecutionTools } from '../llm-providers/anthropic-claude/createAnthropicClaudeExecutionTools'; -import { _AnthropicClaudeRegistration } from '../llm-providers/anthropic-claude/register-constructor'; - +import { AnthropicClaudeExecutionTools } from "../llm-providers/anthropic-claude/AnthropicClaudeExecutionTools"; +import type { AnthropicClaudeExecutionToolsOptions } from "../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions"; +import type { AnthropicClaudeExecutionToolsDirectOptions } from "../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions"; +import type { AnthropicClaudeExecutionToolsProxiedOptions } from "../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions"; +import { ANTHROPIC_CLAUDE_MODELS } from "../llm-providers/anthropic-claude/anthropic-claude-models"; +import { createAnthropicClaudeExecutionTools } from "../llm-providers/anthropic-claude/createAnthropicClaudeExecutionTools"; +import { _AnthropicClaudeRegistration } from "../llm-providers/anthropic-claude/register-constructor"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/anthropic-claude` export { ANTHROPIC_CLAUDE_MODELS }; export { AnthropicClaudeExecutionTools }; diff --git a/src/_packages/azure-openai.index.ts b/src/_packages/azure-openai.index.ts index 9f03f40e6d..49d8c23b54 100644 --- a/src/_packages/azure-openai.index.ts +++ b/src/_packages/azure-openai.index.ts @@ -1,17 +1,15 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/azure-openai` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { AzureOpenAiExecutionTools } from '../llm-providers/azure-openai/AzureOpenAiExecutionTools'; -import type { AzureOpenAiExecutionToolsOptions } from '../llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions'; -import { createAzureOpenAiExecutionTools } from '../llm-providers/azure-openai/createAzureOpenAiExecutionTools'; -import { _AzureOpenAiRegistration } from '../llm-providers/azure-openai/register-constructor'; - +import { AzureOpenAiExecutionTools } from "../llm-providers/azure-openai/AzureOpenAiExecutionTools"; +import type { AzureOpenAiExecutionToolsOptions } from "../llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions"; +import { createAzureOpenAiExecutionTools } from "../llm-providers/azure-openai/createAzureOpenAiExecutionTools"; +import { _AzureOpenAiRegistration } from "../llm-providers/azure-openai/register-constructor"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/azure-openai` export { AzureOpenAiExecutionTools }; export type { AzureOpenAiExecutionToolsOptions }; diff --git a/src/_packages/browser.index.ts b/src/_packages/browser.index.ts index c2522cbe91..9e1de9fea4 100644 --- a/src/_packages/browser.index.ts +++ b/src/_packages/browser.index.ts @@ -1,17 +1,15 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/browser` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { SimplePromptInterfaceTools } from '../dialogs/simple-prompt/SimplePromptInterfaceTools'; -import { $provideScrapersForBrowser } from '../scrapers/_common/register/$provideScrapersForBrowser'; -import { getLocalStorage } from '../storage/local-storage/getLocalStorage'; -import { getSessionStorage } from '../storage/local-storage/getSessionStorage'; - +import { SimplePromptInterfaceTools } from "../dialogs/simple-prompt/SimplePromptInterfaceTools"; +import { $provideScrapersForBrowser } from "../scrapers/_common/register/$provideScrapersForBrowser"; +import { getLocalStorage } from "../storage/local-storage/getLocalStorage"; +import { getSessionStorage } from "../storage/local-storage/getSessionStorage"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/browser` export { SimplePromptInterfaceTools }; export { $provideScrapersForBrowser }; diff --git a/src/_packages/cli.index.ts b/src/_packages/cli.index.ts index b9ec86f1c5..035d906287 100644 --- a/src/_packages/cli.index.ts +++ b/src/_packages/cli.index.ts @@ -1,40 +1,38 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/cli` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { _CLI } from '../cli/main'; -import { _AnthropicClaudeMetadataRegistration } from '../llm-providers/anthropic-claude/register-configuration'; -import { _AnthropicClaudeRegistration } from '../llm-providers/anthropic-claude/register-constructor'; -import { _AzureOpenAiMetadataRegistration } from '../llm-providers/azure-openai/register-configuration'; -import { _AzureOpenAiRegistration } from '../llm-providers/azure-openai/register-constructor'; -import { _DeepseekMetadataRegistration } from '../llm-providers/deepseek/register-configuration'; -import { _DeepseekRegistration } from '../llm-providers/deepseek/register-constructor'; -import { _GoogleMetadataRegistration } from '../llm-providers/google/register-configuration'; -import { _GoogleRegistration } from '../llm-providers/google/register-constructor'; -import { _OpenAiMetadataRegistration } from '../llm-providers/openai/register-configuration'; -import { _OpenAiAssistantMetadataRegistration } from '../llm-providers/openai/register-configuration'; -import { _OpenAiRegistration } from '../llm-providers/openai/register-constructor'; -import { _OpenAiAssistantRegistration } from '../llm-providers/openai/register-constructor'; -import { _BoilerplateScraperRegistration } from '../scrapers/_boilerplate/register-constructor'; -import { _BoilerplateScraperMetadataRegistration } from '../scrapers/_boilerplate/register-metadata'; -import { _LegacyDocumentScraperRegistration } from '../scrapers/document-legacy/register-constructor'; -import { _LegacyDocumentScraperMetadataRegistration } from '../scrapers/document-legacy/register-metadata'; -import { _DocumentScraperRegistration } from '../scrapers/document/register-constructor'; -import { _DocumentScraperMetadataRegistration } from '../scrapers/document/register-metadata'; -import { _MarkdownScraperRegistration } from '../scrapers/markdown/register-constructor'; -import { _MarkdownScraperMetadataRegistration } from '../scrapers/markdown/register-metadata'; -import { _MarkitdownScraperRegistration } from '../scrapers/markitdown/register-constructor'; -import { _MarkitdownScraperMetadataRegistration } from '../scrapers/markitdown/register-metadata'; -import { _PdfScraperRegistration } from '../scrapers/pdf/register-constructor'; -import { _PdfScraperMetadataRegistration } from '../scrapers/pdf/register-metadata'; -import { _WebsiteScraperRegistration } from '../scrapers/website/register-constructor'; -import { _WebsiteScraperMetadataRegistration } from '../scrapers/website/register-metadata'; - +import { _CLI } from "../cli/main"; +import { _AnthropicClaudeMetadataRegistration } from "../llm-providers/anthropic-claude/register-configuration"; +import { _AnthropicClaudeRegistration } from "../llm-providers/anthropic-claude/register-constructor"; +import { _AzureOpenAiMetadataRegistration } from "../llm-providers/azure-openai/register-configuration"; +import { _AzureOpenAiRegistration } from "../llm-providers/azure-openai/register-constructor"; +import { _DeepseekMetadataRegistration } from "../llm-providers/deepseek/register-configuration"; +import { _DeepseekRegistration } from "../llm-providers/deepseek/register-constructor"; +import { _GoogleMetadataRegistration } from "../llm-providers/google/register-configuration"; +import { _GoogleRegistration } from "../llm-providers/google/register-constructor"; +import { _OpenAiMetadataRegistration } from "../llm-providers/openai/register-configuration"; +import { _OpenAiAssistantMetadataRegistration } from "../llm-providers/openai/register-configuration"; +import { _OpenAiRegistration } from "../llm-providers/openai/register-constructor"; +import { _OpenAiAssistantRegistration } from "../llm-providers/openai/register-constructor"; +import { _BoilerplateScraperRegistration } from "../scrapers/_boilerplate/register-constructor"; +import { _BoilerplateScraperMetadataRegistration } from "../scrapers/_boilerplate/register-metadata"; +import { _LegacyDocumentScraperRegistration } from "../scrapers/document-legacy/register-constructor"; +import { _LegacyDocumentScraperMetadataRegistration } from "../scrapers/document-legacy/register-metadata"; +import { _DocumentScraperRegistration } from "../scrapers/document/register-constructor"; +import { _DocumentScraperMetadataRegistration } from "../scrapers/document/register-metadata"; +import { _MarkdownScraperRegistration } from "../scrapers/markdown/register-constructor"; +import { _MarkdownScraperMetadataRegistration } from "../scrapers/markdown/register-metadata"; +import { _MarkitdownScraperRegistration } from "../scrapers/markitdown/register-constructor"; +import { _MarkitdownScraperMetadataRegistration } from "../scrapers/markitdown/register-metadata"; +import { _PdfScraperRegistration } from "../scrapers/pdf/register-constructor"; +import { _PdfScraperMetadataRegistration } from "../scrapers/pdf/register-metadata"; +import { _WebsiteScraperRegistration } from "../scrapers/website/register-constructor"; +import { _WebsiteScraperMetadataRegistration } from "../scrapers/website/register-metadata"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/cli` export { _CLI }; export { _AnthropicClaudeMetadataRegistration }; diff --git a/src/_packages/core.index.ts b/src/_packages/core.index.ts index f972f379fa..a1401a8ee7 100644 --- a/src/_packages/core.index.ts +++ b/src/_packages/core.index.ts @@ -1,142 +1,140 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/core` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { collectionToJson } from '../collection/collectionToJson'; -import { createCollectionFromJson } from '../collection/constructors/createCollectionFromJson'; -import { createCollectionFromPromise } from '../collection/constructors/createCollectionFromPromise'; -import { createCollectionFromUrl } from '../collection/constructors/createCollectionFromUrl'; -import { createSubcollection } from '../collection/constructors/createSubcollection'; -import { NAME } from '../config'; -import { ADMIN_EMAIL } from '../config'; -import { ADMIN_GITHUB_NAME } from '../config'; -import { CLAIM } from '../config'; -import { LOGO_LIGHT_SRC } from '../config'; -import { LOGO_DARK_SRC } from '../config'; -import { DEFAULT_BOOK_TITLE } from '../config'; -import { DEFAULT_TASK_TITLE } from '../config'; -import { DEFAULT_PROMPT_TASK_TITLE } from '../config'; -import { DEFAULT_BOOK_OUTPUT_PARAMETER_NAME } from '../config'; -import { DEFAULT_MAX_FILE_SIZE } from '../config'; -import { MAX_FILENAME_LENGTH } from '../config'; -import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from '../config'; -import { DEFAULT_MAX_PARALLEL_COUNT } from '../config'; -import { DEFAULT_MAX_EXECUTION_ATTEMPTS } from '../config'; -import { DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH } from '../config'; -import { DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL } from '../config'; -import { DEFAULT_BOOKS_DIRNAME } from '../config'; -import { DEFAULT_DOWNLOAD_CACHE_DIRNAME } from '../config'; -import { DEFAULT_EXECUTION_CACHE_DIRNAME } from '../config'; -import { DEFAULT_SCRAPE_CACHE_DIRNAME } from '../config'; -import { DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME } from '../config'; -import { DEFAULT_REMOTE_URL } from '../config'; -import { DEFAULT_REMOTE_URL_PATH } from '../config'; -import { DEFAULT_CSV_SETTINGS } from '../config'; -import { DEFAULT_IS_VERBOSE } from '../config'; -import { SET_IS_VERBOSE } from '../config'; -import { DEFAULT_IS_AUTO_INSTALLED } from '../config'; -import { DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME } from '../config'; -import { ORDER_OF_PIPELINE_JSON } from '../constants'; -import { RESERVED_PARAMETER_NAMES } from '../constants'; -import { compilePipeline } from '../conversion/compilePipeline'; -import { parsePipeline } from '../conversion/parsePipeline'; -import { pipelineJsonToString } from '../conversion/pipelineJsonToString'; -import { prettifyPipelineString } from '../conversion/prettify/prettifyPipelineString'; -import { extractParameterNamesFromTask } from '../conversion/utils/extractParameterNamesFromTask'; -import { validatePipeline } from '../conversion/validation/validatePipeline'; -import { CallbackInterfaceTools } from '../dialogs/callback/CallbackInterfaceTools'; -import type { CallbackInterfaceToolsOptions } from '../dialogs/callback/CallbackInterfaceToolsOptions'; -import { BoilerplateError } from '../errors/0-BoilerplateError'; -import { PROMPTBOOK_ERRORS } from '../errors/0-index'; -import { AbstractFormatError } from '../errors/AbstractFormatError'; -import { CollectionError } from '../errors/CollectionError'; -import { EnvironmentMismatchError } from '../errors/EnvironmentMismatchError'; -import { ExpectError } from '../errors/ExpectError'; -import { KnowledgeScrapeError } from '../errors/KnowledgeScrapeError'; -import { LimitReachedError } from '../errors/LimitReachedError'; -import { MissingToolsError } from '../errors/MissingToolsError'; -import { NotFoundError } from '../errors/NotFoundError'; -import { NotYetImplementedError } from '../errors/NotYetImplementedError'; -import { ParseError } from '../errors/ParseError'; -import { PipelineExecutionError } from '../errors/PipelineExecutionError'; -import { PipelineLogicError } from '../errors/PipelineLogicError'; -import { PipelineUrlError } from '../errors/PipelineUrlError'; -import { UnexpectedError } from '../errors/UnexpectedError'; -import { createPipelineExecutor } from '../execution/createPipelineExecutor/00-createPipelineExecutor'; -import { embeddingVectorToString } from '../execution/embeddingVectorToString'; -import { executionReportJsonToString } from '../execution/execution-report/executionReportJsonToString'; -import type { ExecutionReportStringOptions } from '../execution/execution-report/ExecutionReportStringOptions'; -import { ExecutionReportStringOptionsDefaults } from '../execution/execution-report/ExecutionReportStringOptions'; -import { addUsage } from '../execution/utils/addUsage'; -import { isPassingExpectations } from '../execution/utils/checkExpectations'; -import { ZERO_USAGE } from '../execution/utils/usage-constants'; -import { UNCERTAIN_USAGE } from '../execution/utils/usage-constants'; -import { usageToHuman } from '../execution/utils/usageToHuman'; -import { usageToWorktime } from '../execution/utils/usageToWorktime'; -import { CsvFormatDefinition } from '../formats/csv/CsvFormatDefinition'; -import { CsvFormatError } from '../formats/csv/CsvFormatError'; -import { MANDATORY_CSV_SETTINGS } from '../formats/csv/CsvSettings'; -import { TextFormatDefinition } from '../formats/text/TextFormatDefinition'; -import { BoilerplateFormfactorDefinition } from '../formfactors/_boilerplate/BoilerplateFormfactorDefinition'; -import { ChatbotFormfactorDefinition } from '../formfactors/chatbot/ChatbotFormfactorDefinition'; -import { GeneratorFormfactorDefinition } from '../formfactors/generator/GeneratorFormfactorDefinition'; -import { GenericFormfactorDefinition } from '../formfactors/generic/GenericFormfactorDefinition'; -import { ImageGeneratorFormfactorDefinition } from '../formfactors/image-generator/ImageGeneratorFormfactorDefinition'; -import { FORMFACTOR_DEFINITIONS } from '../formfactors/index'; -import { MatcherFormfactorDefinition } from '../formfactors/matcher/MatcherFormfactorDefinition'; -import { SheetsFormfactorDefinition } from '../formfactors/sheets/SheetsFormfactorDefinition'; -import { TranslatorFormfactorDefinition } from '../formfactors/translator/TranslatorFormfactorDefinition'; -import { $llmToolsMetadataRegister } from '../llm-providers/_common/register/$llmToolsMetadataRegister'; -import { $llmToolsRegister } from '../llm-providers/_common/register/$llmToolsRegister'; -import { createLlmToolsFromConfiguration } from '../llm-providers/_common/register/createLlmToolsFromConfiguration'; -import { cacheLlmTools } from '../llm-providers/_common/utils/cache/cacheLlmTools'; -import { countTotalUsage } from '../llm-providers/_common/utils/count-total-usage/countTotalUsage'; -import { limitTotalUsage } from '../llm-providers/_common/utils/count-total-usage/limitTotalUsage'; -import { _AnthropicClaudeMetadataRegistration } from '../llm-providers/anthropic-claude/register-configuration'; -import { _AzureOpenAiMetadataRegistration } from '../llm-providers/azure-openai/register-configuration'; -import { _DeepseekMetadataRegistration } from '../llm-providers/deepseek/register-configuration'; -import { _GoogleMetadataRegistration } from '../llm-providers/google/register-configuration'; -import { joinLlmExecutionTools } from '../llm-providers/multiple/joinLlmExecutionTools'; -import { MultipleLlmExecutionTools } from '../llm-providers/multiple/MultipleLlmExecutionTools'; -import { _OpenAiMetadataRegistration } from '../llm-providers/openai/register-configuration'; -import { _OpenAiAssistantMetadataRegistration } from '../llm-providers/openai/register-configuration'; -import { preparePersona } from '../personas/preparePersona'; -import { book } from '../pipeline/book-notation'; -import { isValidPipelineString } from '../pipeline/isValidPipelineString'; -import { GENERIC_PIPELINE_INTERFACE } from '../pipeline/PipelineInterface/constants'; -import { getPipelineInterface } from '../pipeline/PipelineInterface/getPipelineInterface'; -import { isPipelineImplementingInterface } from '../pipeline/PipelineInterface/isPipelineImplementingInterface'; -import { isPipelineInterfacesEqual } from '../pipeline/PipelineInterface/isPipelineInterfacesEqual'; -import { EXPECTATION_UNITS } from '../pipeline/PipelineJson/Expectations'; -import { validatePipelineString } from '../pipeline/validatePipelineString'; -import { isPipelinePrepared } from '../prepare/isPipelinePrepared'; -import { preparePipeline } from '../prepare/preparePipeline'; -import { prepareTasks } from '../prepare/prepareTasks'; -import { unpreparePipeline } from '../prepare/unpreparePipeline'; -import { _BoilerplateScraperMetadataRegistration } from '../scrapers/_boilerplate/register-metadata'; -import { prepareKnowledgePieces } from '../scrapers/_common/prepareKnowledgePieces'; -import { $scrapersMetadataRegister } from '../scrapers/_common/register/$scrapersMetadataRegister'; -import { $scrapersRegister } from '../scrapers/_common/register/$scrapersRegister'; -import { makeKnowledgeSourceHandler } from '../scrapers/_common/utils/makeKnowledgeSourceHandler'; -import { _LegacyDocumentScraperMetadataRegistration } from '../scrapers/document-legacy/register-metadata'; -import { _DocumentScraperMetadataRegistration } from '../scrapers/document/register-metadata'; -import { _MarkdownScraperMetadataRegistration } from '../scrapers/markdown/register-metadata'; -import { _MarkitdownScraperMetadataRegistration } from '../scrapers/markitdown/register-metadata'; -import { _PdfScraperMetadataRegistration } from '../scrapers/pdf/register-metadata'; -import { _WebsiteScraperMetadataRegistration } from '../scrapers/website/register-metadata'; -import { BlackholeStorage } from '../storage/blackhole/BlackholeStorage'; -import { MemoryStorage } from '../storage/memory/MemoryStorage'; -import { PrefixStorage } from '../storage/utils/PrefixStorage'; -import { MODEL_VARIANTS } from '../types/ModelVariant'; -import { NonTaskSectionTypes } from '../types/SectionType'; -import { SectionTypes } from '../types/SectionType'; -import { TaskTypes } from '../types/TaskType'; - +import { collectionToJson } from "../collection/collectionToJson"; +import { createCollectionFromJson } from "../collection/constructors/createCollectionFromJson"; +import { createCollectionFromPromise } from "../collection/constructors/createCollectionFromPromise"; +import { createCollectionFromUrl } from "../collection/constructors/createCollectionFromUrl"; +import { createSubcollection } from "../collection/constructors/createSubcollection"; +import { NAME } from "../config"; +import { ADMIN_EMAIL } from "../config"; +import { ADMIN_GITHUB_NAME } from "../config"; +import { CLAIM } from "../config"; +import { LOGO_LIGHT_SRC } from "../config"; +import { LOGO_DARK_SRC } from "../config"; +import { DEFAULT_BOOK_TITLE } from "../config"; +import { DEFAULT_TASK_TITLE } from "../config"; +import { DEFAULT_PROMPT_TASK_TITLE } from "../config"; +import { DEFAULT_BOOK_OUTPUT_PARAMETER_NAME } from "../config"; +import { DEFAULT_MAX_FILE_SIZE } from "../config"; +import { MAX_FILENAME_LENGTH } from "../config"; +import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from "../config"; +import { DEFAULT_MAX_PARALLEL_COUNT } from "../config"; +import { DEFAULT_MAX_EXECUTION_ATTEMPTS } from "../config"; +import { DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH } from "../config"; +import { DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL } from "../config"; +import { DEFAULT_BOOKS_DIRNAME } from "../config"; +import { DEFAULT_DOWNLOAD_CACHE_DIRNAME } from "../config"; +import { DEFAULT_EXECUTION_CACHE_DIRNAME } from "../config"; +import { DEFAULT_SCRAPE_CACHE_DIRNAME } from "../config"; +import { DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME } from "../config"; +import { DEFAULT_REMOTE_URL } from "../config"; +import { DEFAULT_REMOTE_URL_PATH } from "../config"; +import { DEFAULT_CSV_SETTINGS } from "../config"; +import { DEFAULT_IS_VERBOSE } from "../config"; +import { SET_IS_VERBOSE } from "../config"; +import { DEFAULT_IS_AUTO_INSTALLED } from "../config"; +import { DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME } from "../config"; +import { ORDER_OF_PIPELINE_JSON } from "../constants"; +import { RESERVED_PARAMETER_NAMES } from "../constants"; +import { compilePipeline } from "../conversion/compilePipeline"; +import { parsePipeline } from "../conversion/parsePipeline"; +import { pipelineJsonToString } from "../conversion/pipelineJsonToString"; +import { prettifyPipelineString } from "../conversion/prettify/prettifyPipelineString"; +import { extractParameterNamesFromTask } from "../conversion/utils/extractParameterNamesFromTask"; +import { validatePipeline } from "../conversion/validation/validatePipeline"; +import { CallbackInterfaceTools } from "../dialogs/callback/CallbackInterfaceTools"; +import type { CallbackInterfaceToolsOptions } from "../dialogs/callback/CallbackInterfaceToolsOptions"; +import { BoilerplateError } from "../errors/0-BoilerplateError"; +import { PROMPTBOOK_ERRORS } from "../errors/0-index"; +import { AbstractFormatError } from "../errors/AbstractFormatError"; +import { CollectionError } from "../errors/CollectionError"; +import { EnvironmentMismatchError } from "../errors/EnvironmentMismatchError"; +import { ExpectError } from "../errors/ExpectError"; +import { KnowledgeScrapeError } from "../errors/KnowledgeScrapeError"; +import { LimitReachedError } from "../errors/LimitReachedError"; +import { MissingToolsError } from "../errors/MissingToolsError"; +import { NotFoundError } from "../errors/NotFoundError"; +import { NotYetImplementedError } from "../errors/NotYetImplementedError"; +import { ParseError } from "../errors/ParseError"; +import { PipelineExecutionError } from "../errors/PipelineExecutionError"; +import { PipelineLogicError } from "../errors/PipelineLogicError"; +import { PipelineUrlError } from "../errors/PipelineUrlError"; +import { UnexpectedError } from "../errors/UnexpectedError"; +import { createPipelineExecutor } from "../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { embeddingVectorToString } from "../execution/embeddingVectorToString"; +import type { ExecutionReportStringOptions } from "../execution/execution-report/ExecutionReportStringOptions"; +import { ExecutionReportStringOptionsDefaults } from "../execution/execution-report/ExecutionReportStringOptions"; +import { executionReportJsonToString } from "../execution/execution-report/executionReportJsonToString"; +import { addUsage } from "../execution/utils/addUsage"; +import { isPassingExpectations } from "../execution/utils/checkExpectations"; +import { ZERO_USAGE } from "../execution/utils/usage-constants"; +import { UNCERTAIN_USAGE } from "../execution/utils/usage-constants"; +import { usageToHuman } from "../execution/utils/usageToHuman"; +import { usageToWorktime } from "../execution/utils/usageToWorktime"; +import { CsvFormatDefinition } from "../formats/csv/CsvFormatDefinition"; +import { CsvFormatError } from "../formats/csv/CsvFormatError"; +import { MANDATORY_CSV_SETTINGS } from "../formats/csv/CsvSettings"; +import { TextFormatDefinition } from "../formats/text/TextFormatDefinition"; +import { BoilerplateFormfactorDefinition } from "../formfactors/_boilerplate/BoilerplateFormfactorDefinition"; +import { ChatbotFormfactorDefinition } from "../formfactors/chatbot/ChatbotFormfactorDefinition"; +import { GeneratorFormfactorDefinition } from "../formfactors/generator/GeneratorFormfactorDefinition"; +import { GenericFormfactorDefinition } from "../formfactors/generic/GenericFormfactorDefinition"; +import { ImageGeneratorFormfactorDefinition } from "../formfactors/image-generator/ImageGeneratorFormfactorDefinition"; +import { FORMFACTOR_DEFINITIONS } from "../formfactors/index"; +import { MatcherFormfactorDefinition } from "../formfactors/matcher/MatcherFormfactorDefinition"; +import { SheetsFormfactorDefinition } from "../formfactors/sheets/SheetsFormfactorDefinition"; +import { TranslatorFormfactorDefinition } from "../formfactors/translator/TranslatorFormfactorDefinition"; +import { $llmToolsMetadataRegister } from "../llm-providers/_common/register/$llmToolsMetadataRegister"; +import { $llmToolsRegister } from "../llm-providers/_common/register/$llmToolsRegister"; +import { createLlmToolsFromConfiguration } from "../llm-providers/_common/register/createLlmToolsFromConfiguration"; +import { cacheLlmTools } from "../llm-providers/_common/utils/cache/cacheLlmTools"; +import { countTotalUsage } from "../llm-providers/_common/utils/count-total-usage/countTotalUsage"; +import { limitTotalUsage } from "../llm-providers/_common/utils/count-total-usage/limitTotalUsage"; +import { _AnthropicClaudeMetadataRegistration } from "../llm-providers/anthropic-claude/register-configuration"; +import { _AzureOpenAiMetadataRegistration } from "../llm-providers/azure-openai/register-configuration"; +import { _DeepseekMetadataRegistration } from "../llm-providers/deepseek/register-configuration"; +import { _GoogleMetadataRegistration } from "../llm-providers/google/register-configuration"; +import { MultipleLlmExecutionTools } from "../llm-providers/multiple/MultipleLlmExecutionTools"; +import { joinLlmExecutionTools } from "../llm-providers/multiple/joinLlmExecutionTools"; +import { _OpenAiMetadataRegistration } from "../llm-providers/openai/register-configuration"; +import { _OpenAiAssistantMetadataRegistration } from "../llm-providers/openai/register-configuration"; +import { preparePersona } from "../personas/preparePersona"; +import { GENERIC_PIPELINE_INTERFACE } from "../pipeline/PipelineInterface/constants"; +import { getPipelineInterface } from "../pipeline/PipelineInterface/getPipelineInterface"; +import { isPipelineImplementingInterface } from "../pipeline/PipelineInterface/isPipelineImplementingInterface"; +import { isPipelineInterfacesEqual } from "../pipeline/PipelineInterface/isPipelineInterfacesEqual"; +import { EXPECTATION_UNITS } from "../pipeline/PipelineJson/Expectations"; +import { book } from "../pipeline/book-notation"; +import { isValidPipelineString } from "../pipeline/isValidPipelineString"; +import { validatePipelineString } from "../pipeline/validatePipelineString"; +import { isPipelinePrepared } from "../prepare/isPipelinePrepared"; +import { preparePipeline } from "../prepare/preparePipeline"; +import { prepareTasks } from "../prepare/prepareTasks"; +import { unpreparePipeline } from "../prepare/unpreparePipeline"; +import { _BoilerplateScraperMetadataRegistration } from "../scrapers/_boilerplate/register-metadata"; +import { prepareKnowledgePieces } from "../scrapers/_common/prepareKnowledgePieces"; +import { $scrapersMetadataRegister } from "../scrapers/_common/register/$scrapersMetadataRegister"; +import { $scrapersRegister } from "../scrapers/_common/register/$scrapersRegister"; +import { makeKnowledgeSourceHandler } from "../scrapers/_common/utils/makeKnowledgeSourceHandler"; +import { _LegacyDocumentScraperMetadataRegistration } from "../scrapers/document-legacy/register-metadata"; +import { _DocumentScraperMetadataRegistration } from "../scrapers/document/register-metadata"; +import { _MarkdownScraperMetadataRegistration } from "../scrapers/markdown/register-metadata"; +import { _MarkitdownScraperMetadataRegistration } from "../scrapers/markitdown/register-metadata"; +import { _PdfScraperMetadataRegistration } from "../scrapers/pdf/register-metadata"; +import { _WebsiteScraperMetadataRegistration } from "../scrapers/website/register-metadata"; +import { BlackholeStorage } from "../storage/blackhole/BlackholeStorage"; +import { MemoryStorage } from "../storage/memory/MemoryStorage"; +import { PrefixStorage } from "../storage/utils/PrefixStorage"; +import { MODEL_VARIANTS } from "../types/ModelVariant"; +import { NonTaskSectionTypes } from "../types/SectionType"; +import { SectionTypes } from "../types/SectionType"; +import { TaskTypes } from "../types/TaskType"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/core` export { collectionToJson }; export { createCollectionFromJson }; diff --git a/src/_packages/deepseek.index.ts b/src/_packages/deepseek.index.ts index 4eabfe965f..10ebc6b70c 100644 --- a/src/_packages/deepseek.index.ts +++ b/src/_packages/deepseek.index.ts @@ -1,16 +1,14 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/deepseek` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { createDeepseekExecutionTools } from '../llm-providers/deepseek/createDeepseekExecutionTools'; -import type { DeepseekExecutionToolsOptions } from '../llm-providers/deepseek/DeepseekExecutionToolsOptions'; -import { _DeepseekRegistration } from '../llm-providers/deepseek/register-constructor'; - +import type { DeepseekExecutionToolsOptions } from "../llm-providers/deepseek/DeepseekExecutionToolsOptions"; +import { createDeepseekExecutionTools } from "../llm-providers/deepseek/createDeepseekExecutionTools"; +import { _DeepseekRegistration } from "../llm-providers/deepseek/register-constructor"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/deepseek` export { createDeepseekExecutionTools }; export type { DeepseekExecutionToolsOptions }; diff --git a/src/_packages/documents.index.ts b/src/_packages/documents.index.ts index 0831942661..5abaa2f6d3 100644 --- a/src/_packages/documents.index.ts +++ b/src/_packages/documents.index.ts @@ -1,16 +1,14 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/documents` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { createDocumentScraper } from '../scrapers/document/createDocumentScraper'; -import { DocumentScraper } from '../scrapers/document/DocumentScraper'; -import { _DocumentScraperRegistration } from '../scrapers/document/register-constructor'; - +import { DocumentScraper } from "../scrapers/document/DocumentScraper"; +import { createDocumentScraper } from "../scrapers/document/createDocumentScraper"; +import { _DocumentScraperRegistration } from "../scrapers/document/register-constructor"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/documents` export { createDocumentScraper }; export { DocumentScraper }; diff --git a/src/_packages/editable.index.ts b/src/_packages/editable.index.ts index ea869c4ce7..d3c7e75e51 100644 --- a/src/_packages/editable.index.ts +++ b/src/_packages/editable.index.ts @@ -1,57 +1,55 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/editable` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { getParserForCommand } from '../commands/_common/getParserForCommand'; -import { parseCommand } from '../commands/_common/parseCommand'; -import type { CommandParser } from '../commands/_common/types/CommandParser'; -import type { PipelineBothCommandParser } from '../commands/_common/types/CommandParser'; -import type { PipelineHeadCommandParser } from '../commands/_common/types/CommandParser'; -import type { PipelineTaskCommandParser } from '../commands/_common/types/CommandParser'; -import type { CommandUsagePlace } from '../commands/_common/types/CommandUsagePlaces'; -import type { BookVersionCommand } from '../commands/BOOK_VERSION/BookVersionCommand'; -import { bookVersionCommandParser } from '../commands/BOOK_VERSION/bookVersionCommandParser'; -import { expectCommandParser } from '../commands/EXPECT/expectCommandParser'; -import type { ForeachCommand } from '../commands/FOREACH/ForeachCommand'; -import { foreachCommandParser } from '../commands/FOREACH/foreachCommandParser'; -import { formatCommandParser } from '../commands/FORMAT/formatCommandParser'; -import type { FormfactorCommand } from '../commands/FORMFACTOR/FormfactorCommand'; -import { formfactorCommandParser } from '../commands/FORMFACTOR/formfactorCommandParser'; -import { COMMANDS } from '../commands/index'; -import type { JokerCommand } from '../commands/JOKER/JokerCommand'; -import { jokerCommandParser } from '../commands/JOKER/jokerCommandParser'; -import type { KnowledgeCommand } from '../commands/KNOWLEDGE/KnowledgeCommand'; -import { knowledgeCommandParser } from '../commands/KNOWLEDGE/knowledgeCommandParser'; -import { knowledgeSourceContentToName } from '../commands/KNOWLEDGE/utils/knowledgeSourceContentToName'; -import type { ModelCommand } from '../commands/MODEL/ModelCommand'; -import { modelCommandParser } from '../commands/MODEL/modelCommandParser'; -import type { ParameterCommand } from '../commands/PARAMETER/ParameterCommand'; -import { parameterCommandParser } from '../commands/PARAMETER/parameterCommandParser'; -import type { PersonaCommand } from '../commands/PERSONA/PersonaCommand'; -import { personaCommandParser } from '../commands/PERSONA/personaCommandParser'; -import type { PostprocessCommand } from '../commands/POSTPROCESS/PostprocessCommand'; -import { postprocessCommandParser } from '../commands/POSTPROCESS/postprocessCommandParser'; -import type { SectionCommand } from '../commands/SECTION/SectionCommand'; -import { sectionCommandParser } from '../commands/SECTION/sectionCommandParser'; -import type { UrlCommand } from '../commands/URL/UrlCommand'; -import { urlCommandParser } from '../commands/URL/urlCommandParser'; -import type { ActionCommand } from '../commands/X_ACTION/ActionCommand'; -import { actionCommandParser } from '../commands/X_ACTION/actionCommandParser'; -import type { InstrumentCommand } from '../commands/X_INSTRUMENT/InstrumentCommand'; -import { instrumentCommandParser } from '../commands/X_INSTRUMENT/instrumentCommandParser'; -import { addPipelineCommand } from '../utils/editable/edit-pipeline-string/addPipelineCommand'; -import { deflatePipeline } from '../utils/editable/edit-pipeline-string/deflatePipeline'; -import { removePipelineCommand } from '../utils/editable/edit-pipeline-string/removePipelineCommand'; -import type { PipelineEditableSerialized } from '../utils/editable/types/PipelineEditableSerialized'; -import { isFlatPipeline } from '../utils/editable/utils/isFlatPipeline'; -import { renamePipelineParameter } from '../utils/editable/utils/renamePipelineParameter'; -import { stringifyPipelineJson } from '../utils/editable/utils/stringifyPipelineJson'; - +import type { BookVersionCommand } from "../commands/BOOK_VERSION/BookVersionCommand"; +import { bookVersionCommandParser } from "../commands/BOOK_VERSION/bookVersionCommandParser"; +import { expectCommandParser } from "../commands/EXPECT/expectCommandParser"; +import type { ForeachCommand } from "../commands/FOREACH/ForeachCommand"; +import { foreachCommandParser } from "../commands/FOREACH/foreachCommandParser"; +import { formatCommandParser } from "../commands/FORMAT/formatCommandParser"; +import type { FormfactorCommand } from "../commands/FORMFACTOR/FormfactorCommand"; +import { formfactorCommandParser } from "../commands/FORMFACTOR/formfactorCommandParser"; +import type { JokerCommand } from "../commands/JOKER/JokerCommand"; +import { jokerCommandParser } from "../commands/JOKER/jokerCommandParser"; +import type { KnowledgeCommand } from "../commands/KNOWLEDGE/KnowledgeCommand"; +import { knowledgeCommandParser } from "../commands/KNOWLEDGE/knowledgeCommandParser"; +import { knowledgeSourceContentToName } from "../commands/KNOWLEDGE/utils/knowledgeSourceContentToName"; +import type { ModelCommand } from "../commands/MODEL/ModelCommand"; +import { modelCommandParser } from "../commands/MODEL/modelCommandParser"; +import type { ParameterCommand } from "../commands/PARAMETER/ParameterCommand"; +import { parameterCommandParser } from "../commands/PARAMETER/parameterCommandParser"; +import type { PersonaCommand } from "../commands/PERSONA/PersonaCommand"; +import { personaCommandParser } from "../commands/PERSONA/personaCommandParser"; +import type { PostprocessCommand } from "../commands/POSTPROCESS/PostprocessCommand"; +import { postprocessCommandParser } from "../commands/POSTPROCESS/postprocessCommandParser"; +import type { SectionCommand } from "../commands/SECTION/SectionCommand"; +import { sectionCommandParser } from "../commands/SECTION/sectionCommandParser"; +import type { UrlCommand } from "../commands/URL/UrlCommand"; +import { urlCommandParser } from "../commands/URL/urlCommandParser"; +import type { ActionCommand } from "../commands/X_ACTION/ActionCommand"; +import { actionCommandParser } from "../commands/X_ACTION/actionCommandParser"; +import type { InstrumentCommand } from "../commands/X_INSTRUMENT/InstrumentCommand"; +import { instrumentCommandParser } from "../commands/X_INSTRUMENT/instrumentCommandParser"; +import { getParserForCommand } from "../commands/_common/getParserForCommand"; +import { parseCommand } from "../commands/_common/parseCommand"; +import type { CommandParser } from "../commands/_common/types/CommandParser"; +import type { PipelineBothCommandParser } from "../commands/_common/types/CommandParser"; +import type { PipelineHeadCommandParser } from "../commands/_common/types/CommandParser"; +import type { PipelineTaskCommandParser } from "../commands/_common/types/CommandParser"; +import type { CommandUsagePlace } from "../commands/_common/types/CommandUsagePlaces"; +import { COMMANDS } from "../commands/index"; +import { addPipelineCommand } from "../utils/editable/edit-pipeline-string/addPipelineCommand"; +import { deflatePipeline } from "../utils/editable/edit-pipeline-string/deflatePipeline"; +import { removePipelineCommand } from "../utils/editable/edit-pipeline-string/removePipelineCommand"; +import type { PipelineEditableSerialized } from "../utils/editable/types/PipelineEditableSerialized"; +import { isFlatPipeline } from "../utils/editable/utils/isFlatPipeline"; +import { renamePipelineParameter } from "../utils/editable/utils/renamePipelineParameter"; +import { stringifyPipelineJson } from "../utils/editable/utils/stringifyPipelineJson"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/editable` export { getParserForCommand }; export { parseCommand }; diff --git a/src/_packages/execute-javascript.index.ts b/src/_packages/execute-javascript.index.ts index 683b5bb773..3958bc6b6f 100644 --- a/src/_packages/execute-javascript.index.ts +++ b/src/_packages/execute-javascript.index.ts @@ -1,16 +1,14 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/execute-javascript` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { JavascriptEvalExecutionTools } from '../scripting/javascript/JavascriptEvalExecutionTools'; -import { JavascriptExecutionTools } from '../scripting/javascript/JavascriptExecutionTools'; -import { POSTPROCESSING_FUNCTIONS } from '../scripting/javascript/postprocessing-functions'; - +import { JavascriptEvalExecutionTools } from "../scripting/javascript/JavascriptEvalExecutionTools"; +import { JavascriptExecutionTools } from "../scripting/javascript/JavascriptExecutionTools"; +import { POSTPROCESSING_FUNCTIONS } from "../scripting/javascript/postprocessing-functions"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/execute-javascript` export { JavascriptEvalExecutionTools }; export { JavascriptExecutionTools }; diff --git a/src/_packages/fake-llm.index.ts b/src/_packages/fake-llm.index.ts index c3fb3cab48..fa7be20486 100644 --- a/src/_packages/fake-llm.index.ts +++ b/src/_packages/fake-llm.index.ts @@ -1,15 +1,13 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/fake-llm` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { MockedEchoLlmExecutionTools } from '../llm-providers/mocked/MockedEchoLlmExecutionTools'; -import { MockedFackedLlmExecutionTools } from '../llm-providers/mocked/MockedFackedLlmExecutionTools'; - +import { MockedEchoLlmExecutionTools } from "../llm-providers/mocked/MockedEchoLlmExecutionTools"; +import { MockedFackedLlmExecutionTools } from "../llm-providers/mocked/MockedFackedLlmExecutionTools"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/fake-llm` export { MockedEchoLlmExecutionTools }; export { MockedFackedLlmExecutionTools }; diff --git a/src/_packages/google.index.ts b/src/_packages/google.index.ts index 2bd0a611ee..8dd02dc77d 100644 --- a/src/_packages/google.index.ts +++ b/src/_packages/google.index.ts @@ -1,17 +1,15 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/google` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { createGoogleExecutionTools } from '../llm-providers/google/createGoogleExecutionTools'; -import type { GoogleExecutionToolsOptions } from '../llm-providers/google/GoogleExecutionToolsOptions'; -import { _GoogleRegistration } from '../llm-providers/google/register-constructor'; -import type { VercelExecutionToolsOptions } from '../llm-providers/vercel/VercelExecutionToolsOptions'; - +import type { GoogleExecutionToolsOptions } from "../llm-providers/google/GoogleExecutionToolsOptions"; +import { createGoogleExecutionTools } from "../llm-providers/google/createGoogleExecutionTools"; +import { _GoogleRegistration } from "../llm-providers/google/register-constructor"; +import type { VercelExecutionToolsOptions } from "../llm-providers/vercel/VercelExecutionToolsOptions"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/google` export { createGoogleExecutionTools }; export type { GoogleExecutionToolsOptions }; diff --git a/src/_packages/legacy-documents.index.ts b/src/_packages/legacy-documents.index.ts index 0356dde0e3..7deacf193b 100644 --- a/src/_packages/legacy-documents.index.ts +++ b/src/_packages/legacy-documents.index.ts @@ -1,16 +1,14 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/legacy-documents` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { createLegacyDocumentScraper } from '../scrapers/document-legacy/createLegacyDocumentScraper'; -import { LegacyDocumentScraper } from '../scrapers/document-legacy/LegacyDocumentScraper'; -import { _LegacyDocumentScraperRegistration } from '../scrapers/document-legacy/register-constructor'; - +import { LegacyDocumentScraper } from "../scrapers/document-legacy/LegacyDocumentScraper"; +import { createLegacyDocumentScraper } from "../scrapers/document-legacy/createLegacyDocumentScraper"; +import { _LegacyDocumentScraperRegistration } from "../scrapers/document-legacy/register-constructor"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/legacy-documents` export { createLegacyDocumentScraper }; export { LegacyDocumentScraper }; diff --git a/src/_packages/markdown-utils.index.ts b/src/_packages/markdown-utils.index.ts index 8575d780b1..6fac956bc2 100644 --- a/src/_packages/markdown-utils.index.ts +++ b/src/_packages/markdown-utils.index.ts @@ -1,36 +1,34 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/markdown-utils` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { extractBlock } from '../postprocessing/utils/extractBlock'; -import { extractJsonBlock } from '../postprocessing/utils/extractJsonBlock'; -import { createMarkdownScraper } from '../scrapers/markdown/createMarkdownScraper'; -import { MarkdownScraper } from '../scrapers/markdown/MarkdownScraper'; -import { _MarkdownScraperRegistration } from '../scrapers/markdown/register-constructor'; -import type { string_markdown } from '../types/typeAliases'; -import type { string_markdown_section } from '../types/typeAliases'; -import type { string_markdown_section_content } from '../types/typeAliases'; -import type { string_markdown_text } from '../types/typeAliases'; -import type { string_markdown_codeblock_language } from '../types/typeAliases'; -import { addAutoGeneratedSection } from '../utils/markdown/addAutoGeneratedSection'; -import { createMarkdownChart } from '../utils/markdown/createMarkdownChart'; -import { createMarkdownTable } from '../utils/markdown/createMarkdownTable'; -import { escapeMarkdownBlock } from '../utils/markdown/escapeMarkdownBlock'; -import { extractAllBlocksFromMarkdown } from '../utils/markdown/extractAllBlocksFromMarkdown'; -import { extractAllListItemsFromMarkdown } from '../utils/markdown/extractAllListItemsFromMarkdown'; -import { extractOneBlockFromMarkdown } from '../utils/markdown/extractOneBlockFromMarkdown'; -import { flattenMarkdown } from '../utils/markdown/flattenMarkdown'; -import type { MarkdownSection } from '../utils/markdown/parseMarkdownSection'; -import { parseMarkdownSection } from '../utils/markdown/parseMarkdownSection'; -import { removeMarkdownComments } from '../utils/markdown/removeMarkdownComments'; -import { removeMarkdownFormatting } from '../utils/markdown/removeMarkdownFormatting'; -import { splitMarkdownIntoSections } from '../utils/markdown/splitMarkdownIntoSections'; - +import { extractBlock } from "../postprocessing/utils/extractBlock"; +import { extractJsonBlock } from "../postprocessing/utils/extractJsonBlock"; +import { MarkdownScraper } from "../scrapers/markdown/MarkdownScraper"; +import { createMarkdownScraper } from "../scrapers/markdown/createMarkdownScraper"; +import { _MarkdownScraperRegistration } from "../scrapers/markdown/register-constructor"; +import type { string_markdown } from "../types/typeAliases"; +import type { string_markdown_section } from "../types/typeAliases"; +import type { string_markdown_section_content } from "../types/typeAliases"; +import type { string_markdown_text } from "../types/typeAliases"; +import type { string_markdown_codeblock_language } from "../types/typeAliases"; +import { addAutoGeneratedSection } from "../utils/markdown/addAutoGeneratedSection"; +import { createMarkdownChart } from "../utils/markdown/createMarkdownChart"; +import { createMarkdownTable } from "../utils/markdown/createMarkdownTable"; +import { escapeMarkdownBlock } from "../utils/markdown/escapeMarkdownBlock"; +import { extractAllBlocksFromMarkdown } from "../utils/markdown/extractAllBlocksFromMarkdown"; +import { extractAllListItemsFromMarkdown } from "../utils/markdown/extractAllListItemsFromMarkdown"; +import { extractOneBlockFromMarkdown } from "../utils/markdown/extractOneBlockFromMarkdown"; +import { flattenMarkdown } from "../utils/markdown/flattenMarkdown"; +import type { MarkdownSection } from "../utils/markdown/parseMarkdownSection"; +import { parseMarkdownSection } from "../utils/markdown/parseMarkdownSection"; +import { removeMarkdownComments } from "../utils/markdown/removeMarkdownComments"; +import { removeMarkdownFormatting } from "../utils/markdown/removeMarkdownFormatting"; +import { splitMarkdownIntoSections } from "../utils/markdown/splitMarkdownIntoSections"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/markdown-utils` export { extractBlock }; export { extractJsonBlock }; diff --git a/src/_packages/markitdown.index.ts b/src/_packages/markitdown.index.ts index 41fd44b261..bb5b59ea0d 100644 --- a/src/_packages/markitdown.index.ts +++ b/src/_packages/markitdown.index.ts @@ -1,16 +1,14 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/markitdown` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { createMarkitdownScraper } from '../scrapers/markitdown/createMarkitdownScraper'; -import { MarkitdownScraper } from '../scrapers/markitdown/MarkitdownScraper'; -import { _MarkitdownScraperRegistration } from '../scrapers/markitdown/register-constructor'; - +import { MarkitdownScraper } from "../scrapers/markitdown/MarkitdownScraper"; +import { createMarkitdownScraper } from "../scrapers/markitdown/createMarkitdownScraper"; +import { _MarkitdownScraperRegistration } from "../scrapers/markitdown/register-constructor"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/markitdown` export { createMarkitdownScraper }; export { MarkitdownScraper }; diff --git a/src/_packages/node.index.ts b/src/_packages/node.index.ts index 74ccbc4174..e004b1c80c 100644 --- a/src/_packages/node.index.ts +++ b/src/_packages/node.index.ts @@ -1,23 +1,21 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/node` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { createCollectionFromDirectory } from '../collection/constructors/createCollectionFromDirectory'; -import { $provideExecutablesForNode } from '../executables/$provideExecutablesForNode'; -import { $provideExecutionToolsForNode } from '../execution/utils/$provideExecutionToolsForNode'; -import { $provideLlmToolsConfigurationFromEnv } from '../llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv'; -import { $provideLlmToolsFromEnv } from '../llm-providers/_common/register/$provideLlmToolsFromEnv'; -import { $provideFilesystemForNode } from '../scrapers/_common/register/$provideFilesystemForNode'; -import { $provideScrapersForNode } from '../scrapers/_common/register/$provideScrapersForNode'; -import { FileCacheStorage } from '../storage/file-cache-storage/FileCacheStorage'; -import { $execCommand } from '../utils/execCommand/$execCommand'; -import { $execCommands } from '../utils/execCommand/$execCommands'; - +import { createCollectionFromDirectory } from "../collection/constructors/createCollectionFromDirectory"; +import { $provideExecutablesForNode } from "../executables/$provideExecutablesForNode"; +import { $provideExecutionToolsForNode } from "../execution/utils/$provideExecutionToolsForNode"; +import { $provideLlmToolsConfigurationFromEnv } from "../llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv"; +import { $provideLlmToolsFromEnv } from "../llm-providers/_common/register/$provideLlmToolsFromEnv"; +import { $provideFilesystemForNode } from "../scrapers/_common/register/$provideFilesystemForNode"; +import { $provideScrapersForNode } from "../scrapers/_common/register/$provideScrapersForNode"; +import { FileCacheStorage } from "../storage/file-cache-storage/FileCacheStorage"; +import { $execCommand } from "../utils/execCommand/$execCommand"; +import { $execCommands } from "../utils/execCommand/$execCommands"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/node` export { createCollectionFromDirectory }; export { $provideExecutablesForNode }; diff --git a/src/_packages/openai.index.ts b/src/_packages/openai.index.ts index aa78e5b13e..d666092356 100644 --- a/src/_packages/openai.index.ts +++ b/src/_packages/openai.index.ts @@ -1,22 +1,20 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/openai` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { createOpenAiAssistantExecutionTools } from '../llm-providers/openai/createOpenAiAssistantExecutionTools'; -import { createOpenAiExecutionTools } from '../llm-providers/openai/createOpenAiExecutionTools'; -import { OPENAI_MODELS } from '../llm-providers/openai/openai-models'; -import { OpenAiAssistantExecutionTools } from '../llm-providers/openai/OpenAiAssistantExecutionTools'; -import type { OpenAiAssistantExecutionToolsOptions } from '../llm-providers/openai/OpenAiAssistantExecutionToolsOptions'; -import { OpenAiExecutionTools } from '../llm-providers/openai/OpenAiExecutionTools'; -import type { OpenAiExecutionToolsOptions } from '../llm-providers/openai/OpenAiExecutionToolsOptions'; -import { _OpenAiRegistration } from '../llm-providers/openai/register-constructor'; -import { _OpenAiAssistantRegistration } from '../llm-providers/openai/register-constructor'; - +import { OpenAiAssistantExecutionTools } from "../llm-providers/openai/OpenAiAssistantExecutionTools"; +import type { OpenAiAssistantExecutionToolsOptions } from "../llm-providers/openai/OpenAiAssistantExecutionToolsOptions"; +import { OpenAiExecutionTools } from "../llm-providers/openai/OpenAiExecutionTools"; +import type { OpenAiExecutionToolsOptions } from "../llm-providers/openai/OpenAiExecutionToolsOptions"; +import { createOpenAiAssistantExecutionTools } from "../llm-providers/openai/createOpenAiAssistantExecutionTools"; +import { createOpenAiExecutionTools } from "../llm-providers/openai/createOpenAiExecutionTools"; +import { OPENAI_MODELS } from "../llm-providers/openai/openai-models"; +import { _OpenAiRegistration } from "../llm-providers/openai/register-constructor"; +import { _OpenAiAssistantRegistration } from "../llm-providers/openai/register-constructor"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/openai` export { createOpenAiAssistantExecutionTools }; export { createOpenAiExecutionTools }; diff --git a/src/_packages/pdf.index.ts b/src/_packages/pdf.index.ts index c55594b96b..f3ad0b8412 100644 --- a/src/_packages/pdf.index.ts +++ b/src/_packages/pdf.index.ts @@ -1,19 +1,17 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/pdf` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { createMarkitdownScraper } from '../scrapers/markitdown/createMarkitdownScraper'; -import { MarkitdownScraper } from '../scrapers/markitdown/MarkitdownScraper'; -import { _MarkitdownScraperRegistration } from '../scrapers/markitdown/register-constructor'; -import { createPdfScraper } from '../scrapers/pdf/createPdfScraper'; -import { PdfScraper } from '../scrapers/pdf/PdfScraper'; -import { _PdfScraperRegistration } from '../scrapers/pdf/register-constructor'; - +import { MarkitdownScraper } from "../scrapers/markitdown/MarkitdownScraper"; +import { createMarkitdownScraper } from "../scrapers/markitdown/createMarkitdownScraper"; +import { _MarkitdownScraperRegistration } from "../scrapers/markitdown/register-constructor"; +import { PdfScraper } from "../scrapers/pdf/PdfScraper"; +import { createPdfScraper } from "../scrapers/pdf/createPdfScraper"; +import { _PdfScraperRegistration } from "../scrapers/pdf/register-constructor"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/pdf` export { createMarkitdownScraper }; export { MarkitdownScraper }; diff --git a/src/_packages/remote-client.index.ts b/src/_packages/remote-client.index.ts index d8b4cd6d36..7b4db1d6c6 100644 --- a/src/_packages/remote-client.index.ts +++ b/src/_packages/remote-client.index.ts @@ -1,21 +1,19 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/remote-client` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { compilePipelineOnRemoteServer } from '../conversion/compilePipelineOnRemoteServer'; -import { RemoteLlmExecutionTools } from '../llm-providers/remote/RemoteLlmExecutionTools'; -import { preparePipelineOnRemoteServer } from '../prepare/preparePipelineOnRemoteServer'; -import type { PromptbookServer_Identification } from '../remote-server/socket-types/_subtypes/PromptbookServer_Identification'; -import type { PromptbookServer_ApplicationIdentification } from '../remote-server/socket-types/_subtypes/PromptbookServer_Identification'; -import type { PromptbookServer_AnonymousIdentification } from '../remote-server/socket-types/_subtypes/PromptbookServer_Identification'; -import type { RemoteClientOptions } from '../remote-server/types/RemoteClientOptions'; -import type { RemoteServerOptions } from '../remote-server/types/RemoteServerOptions'; - +import { compilePipelineOnRemoteServer } from "../conversion/compilePipelineOnRemoteServer"; +import { RemoteLlmExecutionTools } from "../llm-providers/remote/RemoteLlmExecutionTools"; +import { preparePipelineOnRemoteServer } from "../prepare/preparePipelineOnRemoteServer"; +import type { PromptbookServer_Identification } from "../remote-server/socket-types/_subtypes/PromptbookServer_Identification"; +import type { PromptbookServer_ApplicationIdentification } from "../remote-server/socket-types/_subtypes/PromptbookServer_Identification"; +import type { PromptbookServer_AnonymousIdentification } from "../remote-server/socket-types/_subtypes/PromptbookServer_Identification"; +import type { RemoteClientOptions } from "../remote-server/types/RemoteClientOptions"; +import type { RemoteServerOptions } from "../remote-server/types/RemoteServerOptions"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/remote-client` export { compilePipelineOnRemoteServer }; export { RemoteLlmExecutionTools }; diff --git a/src/_packages/remote-server.index.ts b/src/_packages/remote-server.index.ts index c9480eeb03..27b09d8770 100644 --- a/src/_packages/remote-server.index.ts +++ b/src/_packages/remote-server.index.ts @@ -1,18 +1,16 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/remote-server` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import type { PromptbookServer_Identification } from '../remote-server/socket-types/_subtypes/PromptbookServer_Identification'; -import type { PromptbookServer_ApplicationIdentification } from '../remote-server/socket-types/_subtypes/PromptbookServer_Identification'; -import type { PromptbookServer_AnonymousIdentification } from '../remote-server/socket-types/_subtypes/PromptbookServer_Identification'; -import { startRemoteServer } from '../remote-server/startRemoteServer'; -import type { RemoteServerOptions } from '../remote-server/types/RemoteServerOptions'; - +import type { PromptbookServer_Identification } from "../remote-server/socket-types/_subtypes/PromptbookServer_Identification"; +import type { PromptbookServer_ApplicationIdentification } from "../remote-server/socket-types/_subtypes/PromptbookServer_Identification"; +import type { PromptbookServer_AnonymousIdentification } from "../remote-server/socket-types/_subtypes/PromptbookServer_Identification"; +import { startRemoteServer } from "../remote-server/startRemoteServer"; +import type { RemoteServerOptions } from "../remote-server/types/RemoteServerOptions"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/remote-server` export type { PromptbookServer_Identification }; export type { PromptbookServer_ApplicationIdentification }; diff --git a/src/_packages/templates.index.ts b/src/_packages/templates.index.ts index 39ba8b282e..1a7945ee08 100644 --- a/src/_packages/templates.index.ts +++ b/src/_packages/templates.index.ts @@ -1,13 +1,11 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/templates` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { getBookTemplates } from '../other/templates/getBookTemplates'; - +import { getBookTemplates } from "../other/templates/getBookTemplates"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/templates` export { getBookTemplates }; diff --git a/src/_packages/types.index.ts b/src/_packages/types.index.ts index d6837efc93..e527f210e2 100644 --- a/src/_packages/types.index.ts +++ b/src/_packages/types.index.ts @@ -1,287 +1,287 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/types` -import type { PipelineCollection } from '../collection/PipelineCollection'; -import type { Command } from '../commands/_common/types/Command'; -import type { CommandParser } from '../commands/_common/types/CommandParser'; -import type { PipelineBothCommandParser } from '../commands/_common/types/CommandParser'; -import type { PipelineHeadCommandParser } from '../commands/_common/types/CommandParser'; -import type { PipelineTaskCommandParser } from '../commands/_common/types/CommandParser'; -import type { CommandParserInput } from '../commands/_common/types/CommandParser'; -import type { CommandType } from '../commands/_common/types/CommandType'; -import type { CommandUsagePlace } from '../commands/_common/types/CommandUsagePlaces'; -import type { BookVersionCommand } from '../commands/BOOK_VERSION/BookVersionCommand'; -import type { ExpectCommand } from '../commands/EXPECT/ExpectCommand'; -import type { ForeachCommand } from '../commands/FOREACH/ForeachCommand'; -import type { ForeachJson } from '../commands/FOREACH/ForeachJson'; -import type { FormatCommand } from '../commands/FORMAT/FormatCommand'; -import type { FormfactorCommand } from '../commands/FORMFACTOR/FormfactorCommand'; -import type { JokerCommand } from '../commands/JOKER/JokerCommand'; -import type { KnowledgeCommand } from '../commands/KNOWLEDGE/KnowledgeCommand'; -import type { ModelCommand } from '../commands/MODEL/ModelCommand'; -import type { ParameterCommand } from '../commands/PARAMETER/ParameterCommand'; -import type { PersonaCommand } from '../commands/PERSONA/PersonaCommand'; -import type { PostprocessCommand } from '../commands/POSTPROCESS/PostprocessCommand'; -import type { SectionCommand } from '../commands/SECTION/SectionCommand'; -import type { UrlCommand } from '../commands/URL/UrlCommand'; -import type { ActionCommand } from '../commands/X_ACTION/ActionCommand'; -import type { InstrumentCommand } from '../commands/X_INSTRUMENT/InstrumentCommand'; -import type { PrettifyOptions } from '../conversion/prettify/PrettifyOptions'; -import type { renderPipelineMermaidOptions } from '../conversion/prettify/renderPipelineMermaidOptions'; -import type { CallbackInterfaceToolsOptions } from '../dialogs/callback/CallbackInterfaceToolsOptions'; -import type { ErrorJson } from '../errors/utils/ErrorJson'; -import type { LocateAppOptions } from '../executables/locateApp'; -import type { AbstractTaskResult } from '../execution/AbstractTaskResult'; -import type { AvailableModel } from '../execution/AvailableModel'; -import type { CommonToolsOptions } from '../execution/CommonToolsOptions'; -import type { CreatePipelineExecutorOptions } from '../execution/createPipelineExecutor/00-CreatePipelineExecutorOptions'; -import type { EmbeddingVector } from '../execution/EmbeddingVector'; -import type { Executables } from '../execution/Executables'; -import type { ExecutionPromptReportJson } from '../execution/execution-report/ExecutionPromptReportJson'; -import type { ExecutionReportJson } from '../execution/execution-report/ExecutionReportJson'; -import type { ExecutionReportString } from '../execution/execution-report/ExecutionReportString'; -import type { ExecutionReportStringOptions } from '../execution/execution-report/ExecutionReportStringOptions'; -import type { ExecutionTask } from '../execution/ExecutionTask'; -import type { PreparationTask } from '../execution/ExecutionTask'; -import type { AbstractTask } from '../execution/ExecutionTask'; -import type { Task } from '../execution/ExecutionTask'; -import type { ExecutionTools } from '../execution/ExecutionTools'; -import type { FilesystemTools } from '../execution/FilesystemTools'; -import type { LlmExecutionTools } from '../execution/LlmExecutionTools'; -import type { LlmExecutionToolsConstructor } from '../execution/LlmExecutionToolsConstructor'; -import type { PipelineExecutor } from '../execution/PipelineExecutor'; -import type { PipelineExecutorResult } from '../execution/PipelineExecutorResult'; -import type { PromptbookFetch } from '../execution/PromptbookFetch'; -import type { PromptResult } from '../execution/PromptResult'; -import type { CompletionPromptResult } from '../execution/PromptResult'; -import type { ChatPromptResult } from '../execution/PromptResult'; -import type { EmbeddingPromptResult } from '../execution/PromptResult'; -import type { PromptResultUsage } from '../execution/PromptResultUsage'; -import type { PromptResultUsageCounts } from '../execution/PromptResultUsage'; -import type { ScriptExecutionTools } from '../execution/ScriptExecutionTools'; -import type { ScriptExecutionToolsExecuteOptions } from '../execution/ScriptExecutionTools'; -import type { UncertainNumber } from '../execution/UncertainNumber'; -import type { UserInterfaceTools } from '../execution/UserInterfaceTools'; -import type { UserInterfaceToolsPromptDialogOptions } from '../execution/UserInterfaceTools'; -import type { FormatSubvalueDefinition } from '../formats/_common/FormatSubvalueDefinition'; -import type { CsvSettings } from '../formats/csv/CsvSettings'; -import type { AbstractFormfactorDefinition } from '../formfactors/_common/AbstractFormfactorDefinition'; -import type { FormfactorDefinition } from '../formfactors/_common/FormfactorDefinition'; -import type { string_formfactor_name } from '../formfactors/_common/string_formfactor_name'; -import type { LlmToolsConfiguration } from '../llm-providers/_common/register/LlmToolsConfiguration'; -import type { LlmToolsMetadata } from '../llm-providers/_common/register/LlmToolsMetadata'; -import type { LlmToolsOptions } from '../llm-providers/_common/register/LlmToolsOptions'; -import type { CacheItem } from '../llm-providers/_common/utils/cache/CacheItem'; -import type { CacheLlmToolsOptions } from '../llm-providers/_common/utils/cache/CacheLlmToolsOptions'; -import type { LlmExecutionToolsWithTotalUsage } from '../llm-providers/_common/utils/count-total-usage/LlmExecutionToolsWithTotalUsage'; -import type { AnthropicClaudeExecutionToolsOptions } from '../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions'; -import type { AnthropicClaudeExecutionToolsDirectOptions } from '../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions'; -import type { AnthropicClaudeExecutionToolsProxiedOptions } from '../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions'; -import type { AzureOpenAiExecutionToolsOptions } from '../llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions'; -import type { DeepseekExecutionToolsOptions } from '../llm-providers/deepseek/DeepseekExecutionToolsOptions'; -import type { GoogleExecutionToolsOptions } from '../llm-providers/google/GoogleExecutionToolsOptions'; -import type { OpenAiAssistantExecutionToolsOptions } from '../llm-providers/openai/OpenAiAssistantExecutionToolsOptions'; -import type { OpenAiExecutionToolsOptions } from '../llm-providers/openai/OpenAiExecutionToolsOptions'; -import type { VercelExecutionToolsOptions } from '../llm-providers/vercel/VercelExecutionToolsOptions'; -import type { VercelProvider } from '../llm-providers/vercel/VercelProvider'; -import type { IsPipelineImplementingInterfaceOptions } from '../pipeline/PipelineInterface/isPipelineImplementingInterface'; -import type { PipelineInterface } from '../pipeline/PipelineInterface/PipelineInterface'; -import type { CommonTaskJson } from '../pipeline/PipelineJson/CommonTaskJson'; -import type { DialogTaskJson } from '../pipeline/PipelineJson/DialogTaskJson'; -import type { Expectations } from '../pipeline/PipelineJson/Expectations'; -import type { ExpectationUnit } from '../pipeline/PipelineJson/Expectations'; -import type { ExpectationAmount } from '../pipeline/PipelineJson/Expectations'; -import type { KnowledgePiecePreparedJson } from '../pipeline/PipelineJson/KnowledgePieceJson'; -import type { KnowledgeSourceJson } from '../pipeline/PipelineJson/KnowledgeSourceJson'; -import type { KnowledgeSourcePreparedJson } from '../pipeline/PipelineJson/KnowledgeSourceJson'; -import type { ParameterJson } from '../pipeline/PipelineJson/ParameterJson'; -import type { InputParameterJson } from '../pipeline/PipelineJson/ParameterJson'; -import type { IntermediateParameterJson } from '../pipeline/PipelineJson/ParameterJson'; -import type { OutputParameterJson } from '../pipeline/PipelineJson/ParameterJson'; -import type { CommonParameterJson } from '../pipeline/PipelineJson/ParameterJson'; -import type { PersonaJson } from '../pipeline/PipelineJson/PersonaJson'; -import type { PersonaPreparedJson } from '../pipeline/PipelineJson/PersonaJson'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import type { PreparationJson } from '../pipeline/PipelineJson/PreparationJson'; -import type { PromptTaskJson } from '../pipeline/PipelineJson/PromptTaskJson'; -import type { ScriptTaskJson } from '../pipeline/PipelineJson/ScriptTaskJson'; -import type { SimpleTaskJson } from '../pipeline/PipelineJson/SimpleTaskJson'; -import type { TaskJson } from '../pipeline/PipelineJson/TaskJson'; -import type { PipelineString } from '../pipeline/PipelineString'; -import type { PrepareAndScrapeOptions } from '../prepare/PrepareAndScrapeOptions'; -import type { PromptbookServer_Identification } from '../remote-server/socket-types/_subtypes/PromptbookServer_Identification'; -import type { PromptbookServer_ApplicationIdentification } from '../remote-server/socket-types/_subtypes/PromptbookServer_Identification'; -import type { PromptbookServer_AnonymousIdentification } from '../remote-server/socket-types/_subtypes/PromptbookServer_Identification'; -import type { RemoteClientOptions } from '../remote-server/types/RemoteClientOptions'; -import type { RemoteServerOptions } from '../remote-server/types/RemoteServerOptions'; -import type { AnonymousRemoteServerOptions } from '../remote-server/types/RemoteServerOptions'; -import type { ApplicationRemoteServerOptions } from '../remote-server/types/RemoteServerOptions'; -import type { ApplicationRemoteServerClientOptions } from '../remote-server/types/RemoteServerOptions'; -import type { Converter } from '../scrapers/_common/Converter'; -import type { ScraperAndConverterMetadata } from '../scrapers/_common/register/ScraperAndConverterMetadata'; -import type { ScraperConstructor } from '../scrapers/_common/register/ScraperConstructor'; -import type { Scraper } from '../scrapers/_common/Scraper'; -import type { ScraperSourceHandler } from '../scrapers/_common/Scraper'; -import type { ScraperIntermediateSource } from '../scrapers/_common/ScraperIntermediateSource'; -import type { JavascriptExecutionToolsOptions } from '../scripting/javascript/JavascriptExecutionToolsOptions'; -import type { PostprocessingFunction } from '../scripting/javascript/JavascriptExecutionToolsOptions'; -import type { PromptbookStorage } from '../storage/_common/PromptbookStorage'; -import type { FileCacheStorageOptions } from '../storage/file-cache-storage/FileCacheStorageOptions'; -import type { IntermediateFilesStrategy } from '../types/IntermediateFilesStrategy'; -import type { ModelRequirements } from '../types/ModelRequirements'; -import type { CompletionModelRequirements } from '../types/ModelRequirements'; -import type { ChatModelRequirements } from '../types/ModelRequirements'; -import type { EmbeddingModelRequirements } from '../types/ModelRequirements'; -import type { ModelVariant } from '../types/ModelVariant'; -import type { NonEmptyArray } from '../types/NonEmptyArray'; -import type { NonEmptyReadonlyArray } from '../types/NonEmptyArray'; -import type { Prompt } from '../types/Prompt'; -import type { CompletionPrompt } from '../types/Prompt'; -import type { ChatPrompt } from '../types/Prompt'; -import type { EmbeddingPrompt } from '../types/Prompt'; -import type { ScriptLanguage } from '../types/ScriptLanguage'; -import type { SectionType } from '../types/SectionType'; -import type { TaskType } from '../types/TaskType'; -import type { string_char_emoji } from '../types/typeAliasEmoji'; -import type { string_business_category_name } from '../types/typeAliases'; -import type { string_model_name } from '../types/typeAliases'; -import type { string_prompt } from '../types/typeAliases'; -import type { string_template } from '../types/typeAliases'; -import type { string_text_prompt } from '../types/typeAliases'; -import type { string_chat_prompt } from '../types/typeAliases'; -import type { string_system_message } from '../types/typeAliases'; -import type { string_completion_prompt } from '../types/typeAliases'; -import type { string_page } from '../types/typeAliases'; -import type { string_mime_type } from '../types/typeAliases'; -import type { string_mime_type_with_wildcard } from '../types/typeAliases'; -import type { string_char } from '../types/typeAliases'; -import type { string_name } from '../types/typeAliases'; -import type { string_parameter_name } from '../types/typeAliases'; -import type { string_parameter_value } from '../types/typeAliases'; -import type { Parameters } from '../types/typeAliases'; -import type { InputParameters } from '../types/typeAliases'; -import type { string_reserved_parameter_name } from '../types/typeAliases'; -import type { ReservedParameters } from '../types/typeAliases'; -import type { string_title } from '../types/typeAliases'; -import type { string_persona_description } from '../types/typeAliases'; -import type { string_model_description } from '../types/typeAliases'; -import type { string_knowledge_source_content } from '../types/typeAliases'; -import type { string_knowledge_source_link } from '../types/typeAliases'; -import type { string_html } from '../types/typeAliases'; -import type { string_xml } from '../types/typeAliases'; -import type { string_markdown } from '../types/typeAliases'; -import type { string_markdown_section } from '../types/typeAliases'; -import type { string_markdown_section_content } from '../types/typeAliases'; -import type { string_markdown_text } from '../types/typeAliases'; -import type { string_markdown_codeblock_language } from '../types/typeAliases'; -import type { string_promptbook_documentation_url } from '../types/typeAliases'; -import type { string_domain } from '../types/typeAliases'; -import type { string_tdl } from '../types/typeAliases'; -import type { string_css } from '../types/typeAliases'; -import type { string_svg } from '../types/typeAliases'; -import type { string_script } from '../types/typeAliases'; -import type { string_javascript } from '../types/typeAliases'; -import type { string_json } from '../types/typeAliases'; -import type { string_css_class } from '../types/typeAliases'; -import type { string_css_property } from '../types/typeAliases'; -import type { string_css_value } from '../types/typeAliases'; -import type { string_css_selector } from '../types/typeAliases'; -import type { string_url } from '../types/typeAliases'; -import type { string_base_url } from '../types/typeAliases'; -import type { string_pipeline_root_url } from '../types/typeAliases'; -import type { string_pipeline_url } from '../types/typeAliases'; -import type { string_pipeline_url_with_task_hash } from '../types/typeAliases'; -import type { string_data_url } from '../types/typeAliases'; -import type { string_base64 } from '../types/typeAliases'; -import type { string_href } from '../types/typeAliases'; -import type { string_url_image } from '../types/typeAliases'; -import type { string_executable_path } from '../types/typeAliases'; -import type { string_uri } from '../types/typeAliases'; -import type { string_uri_part } from '../types/typeAliases'; -import type { string_hostname } from '../types/typeAliases'; -import type { string_host } from '../types/typeAliases'; -import type { string_protocol } from '../types/typeAliases'; -import type { string_email } from '../types/typeAliases'; -import type { string_emails } from '../types/typeAliases'; -import type { string_uuid } from '../types/typeAliases'; -import type { string_app_id } from '../types/typeAliases'; -import type { string_user_id } from '../types/typeAliases'; -import type { string_sha256 } from '../types/typeAliases'; -import type { string_semantic_version } from '../types/typeAliases'; -import type { string_version_dependency } from '../types/typeAliases'; -import type { string_file_extension } from '../types/typeAliases'; -import type { string_absolute_filename } from '../types/typeAliases'; -import type { string_relative_filename } from '../types/typeAliases'; -import type { string_filename } from '../types/typeAliases'; -import type { string_absolute_dirname } from '../types/typeAliases'; -import type { string_relative_dirname } from '../types/typeAliases'; -import type { string_dirname } from '../types/typeAliases'; -import type { string_person_fullname } from '../types/typeAliases'; -import type { string_person_profile } from '../types/typeAliases'; -import type { string_license } from '../types/typeAliases'; -import type { string_attribute } from '../types/typeAliases'; -import type { string_attribute_value_scope } from '../types/typeAliases'; -import type { string_color } from '../types/typeAliases'; -import type { string_translate_name } from '../types/typeAliases'; -import type { string_translate_name_not_normalized } from '../types/typeAliases'; -import type { string_translate_language } from '../types/typeAliases'; -import type { string_javascript_name } from '../types/typeAliases'; -import type { string_postprocessing_function_name } from '../types/typeAliases'; -import type { id } from '../types/typeAliases'; -import type { task_id } from '../types/typeAliases'; -import type { string_token } from '../types/typeAliases'; -import type { string_license_token } from '../types/typeAliases'; -import type { string_password } from '../types/typeAliases'; -import type { string_ssh_key } from '../types/typeAliases'; -import type { string_pgp_key } from '../types/typeAliases'; -import type { string_date_iso8601 } from '../types/typeAliases'; -import type { number_usd } from '../types/typeAliases'; -import type { number_id } from '../types/typeAliases'; -import type { number_linecol_number } from '../types/typeAliases'; -import type { number_tokens } from '../types/typeAliases'; -import type { number_positive } from '../types/typeAliases'; -import type { number_negative } from '../types/typeAliases'; -import type { number_integer } from '../types/typeAliases'; -import type { number_port } from '../types/typeAliases'; -import type { number_percent } from '../types/typeAliases'; -import type { number_model_temperature } from '../types/typeAliases'; -import type { number_seed } from '../types/typeAliases'; -import type { number_likeness } from '../types/typeAliases'; -import type { number_miliseconds } from '../types/typeAliases'; -import type { number_seconds } from '../types/typeAliases'; -import type { number_minutes } from '../types/typeAliases'; -import type { number_hours } from '../types/typeAliases'; -import type { number_days } from '../types/typeAliases'; -import type { number_weeks } from '../types/typeAliases'; -import type { number_months } from '../types/typeAliases'; -import type { number_years } from '../types/typeAliases'; -import type { number_bytes } from '../types/typeAliases'; -import type { number_kilobytes } from '../types/typeAliases'; -import type { number_megabytes } from '../types/typeAliases'; -import type { number_gigabytes } from '../types/typeAliases'; -import type { number_terabytes } from '../types/typeAliases'; -import type { Registered } from '../utils/$Register'; -import type { Registration } from '../utils/$Register'; -import type { PipelineEditableSerialized } from '../utils/editable/types/PipelineEditableSerialized'; -import type { ExecCommandOptions } from '../utils/execCommand/ExecCommandOptions'; -import type { ExecCommandOptionsAdvanced } from '../utils/execCommand/ExecCommandOptions'; -import type { FromtoItems } from '../utils/FromtoItems'; -import type { CodeBlock } from '../utils/markdown/extractAllBlocksFromMarkdown'; -import type { MarkdownSection } from '../utils/markdown/parseMarkdownSection'; -import type { string_keyword } from '../utils/normalization/IKeywords'; -import type { Keywords } from '../utils/normalization/IKeywords'; -import type { string_kebab_case } from '../utils/normalization/normalize-to-kebab-case'; -import type { string_camelCase } from '../utils/normalization/normalizeTo_camelCase'; -import type { string_PascalCase } from '../utils/normalization/normalizeTo_PascalCase'; -import type { string_SCREAMING_CASE } from '../utils/normalization/normalizeTo_SCREAMING_CASE'; -import type { string_snake_case } from '../utils/normalization/normalizeTo_snake_case'; -import type { OrderJsonOptions } from '../utils/normalization/orderJson'; -import type { empty_object } from '../utils/organization/empty_object'; -import type { really_any } from '../utils/organization/really_any'; -import type { TODO_any } from '../utils/organization/TODO_any'; -import type { CheckSerializableAsJsonOptions } from '../utils/serialization/checkSerializableAsJson'; -import type { ExportJsonOptions } from '../utils/serialization/exportJson'; -import type { string_promptbook_version } from '../version'; +import type { PipelineCollection } from "../collection/PipelineCollection"; +import type { BookVersionCommand } from "../commands/BOOK_VERSION/BookVersionCommand"; +import type { ExpectCommand } from "../commands/EXPECT/ExpectCommand"; +import type { ForeachCommand } from "../commands/FOREACH/ForeachCommand"; +import type { ForeachJson } from "../commands/FOREACH/ForeachJson"; +import type { FormatCommand } from "../commands/FORMAT/FormatCommand"; +import type { FormfactorCommand } from "../commands/FORMFACTOR/FormfactorCommand"; +import type { JokerCommand } from "../commands/JOKER/JokerCommand"; +import type { KnowledgeCommand } from "../commands/KNOWLEDGE/KnowledgeCommand"; +import type { ModelCommand } from "../commands/MODEL/ModelCommand"; +import type { ParameterCommand } from "../commands/PARAMETER/ParameterCommand"; +import type { PersonaCommand } from "../commands/PERSONA/PersonaCommand"; +import type { PostprocessCommand } from "../commands/POSTPROCESS/PostprocessCommand"; +import type { SectionCommand } from "../commands/SECTION/SectionCommand"; +import type { UrlCommand } from "../commands/URL/UrlCommand"; +import type { ActionCommand } from "../commands/X_ACTION/ActionCommand"; +import type { InstrumentCommand } from "../commands/X_INSTRUMENT/InstrumentCommand"; +import type { Command } from "../commands/_common/types/Command"; +import type { CommandParser } from "../commands/_common/types/CommandParser"; +import type { PipelineBothCommandParser } from "../commands/_common/types/CommandParser"; +import type { PipelineHeadCommandParser } from "../commands/_common/types/CommandParser"; +import type { PipelineTaskCommandParser } from "../commands/_common/types/CommandParser"; +import type { CommandParserInput } from "../commands/_common/types/CommandParser"; +import type { CommandType } from "../commands/_common/types/CommandType"; +import type { CommandUsagePlace } from "../commands/_common/types/CommandUsagePlaces"; +import type { PrettifyOptions } from "../conversion/prettify/PrettifyOptions"; +import type { renderPipelineMermaidOptions } from "../conversion/prettify/renderPipelineMermaidOptions"; +import type { CallbackInterfaceToolsOptions } from "../dialogs/callback/CallbackInterfaceToolsOptions"; +import type { ErrorJson } from "../errors/utils/ErrorJson"; +import type { LocateAppOptions } from "../executables/locateApp"; +import type { AbstractTaskResult } from "../execution/AbstractTaskResult"; +import type { AvailableModel } from "../execution/AvailableModel"; +import type { CommonToolsOptions } from "../execution/CommonToolsOptions"; +import type { EmbeddingVector } from "../execution/EmbeddingVector"; +import type { Executables } from "../execution/Executables"; +import type { ExecutionTask } from "../execution/ExecutionTask"; +import type { PreparationTask } from "../execution/ExecutionTask"; +import type { AbstractTask } from "../execution/ExecutionTask"; +import type { Task } from "../execution/ExecutionTask"; +import type { ExecutionTools } from "../execution/ExecutionTools"; +import type { FilesystemTools } from "../execution/FilesystemTools"; +import type { LlmExecutionTools } from "../execution/LlmExecutionTools"; +import type { LlmExecutionToolsConstructor } from "../execution/LlmExecutionToolsConstructor"; +import type { PipelineExecutor } from "../execution/PipelineExecutor"; +import type { PipelineExecutorResult } from "../execution/PipelineExecutorResult"; +import type { PromptResult } from "../execution/PromptResult"; +import type { CompletionPromptResult } from "../execution/PromptResult"; +import type { ChatPromptResult } from "../execution/PromptResult"; +import type { EmbeddingPromptResult } from "../execution/PromptResult"; +import type { PromptResultUsage } from "../execution/PromptResultUsage"; +import type { PromptResultUsageCounts } from "../execution/PromptResultUsage"; +import type { PromptbookFetch } from "../execution/PromptbookFetch"; +import type { ScriptExecutionTools } from "../execution/ScriptExecutionTools"; +import type { ScriptExecutionToolsExecuteOptions } from "../execution/ScriptExecutionTools"; +import type { UncertainNumber } from "../execution/UncertainNumber"; +import type { UserInterfaceTools } from "../execution/UserInterfaceTools"; +import type { UserInterfaceToolsPromptDialogOptions } from "../execution/UserInterfaceTools"; +import type { CreatePipelineExecutorOptions } from "../execution/createPipelineExecutor/00-CreatePipelineExecutorOptions"; +import type { ExecutionPromptReportJson } from "../execution/execution-report/ExecutionPromptReportJson"; +import type { ExecutionReportJson } from "../execution/execution-report/ExecutionReportJson"; +import type { ExecutionReportString } from "../execution/execution-report/ExecutionReportString"; +import type { ExecutionReportStringOptions } from "../execution/execution-report/ExecutionReportStringOptions"; +import type { FormatSubvalueDefinition } from "../formats/_common/FormatSubvalueDefinition"; +import type { CsvSettings } from "../formats/csv/CsvSettings"; +import type { AbstractFormfactorDefinition } from "../formfactors/_common/AbstractFormfactorDefinition"; +import type { FormfactorDefinition } from "../formfactors/_common/FormfactorDefinition"; +import type { string_formfactor_name } from "../formfactors/_common/string_formfactor_name"; +import type { LlmToolsConfiguration } from "../llm-providers/_common/register/LlmToolsConfiguration"; +import type { LlmToolsMetadata } from "../llm-providers/_common/register/LlmToolsMetadata"; +import type { LlmToolsOptions } from "../llm-providers/_common/register/LlmToolsOptions"; +import type { CacheItem } from "../llm-providers/_common/utils/cache/CacheItem"; +import type { CacheLlmToolsOptions } from "../llm-providers/_common/utils/cache/CacheLlmToolsOptions"; +import type { LlmExecutionToolsWithTotalUsage } from "../llm-providers/_common/utils/count-total-usage/LlmExecutionToolsWithTotalUsage"; +import type { AnthropicClaudeExecutionToolsOptions } from "../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions"; +import type { AnthropicClaudeExecutionToolsDirectOptions } from "../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions"; +import type { AnthropicClaudeExecutionToolsProxiedOptions } from "../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions"; +import type { AzureOpenAiExecutionToolsOptions } from "../llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions"; +import type { DeepseekExecutionToolsOptions } from "../llm-providers/deepseek/DeepseekExecutionToolsOptions"; +import type { GoogleExecutionToolsOptions } from "../llm-providers/google/GoogleExecutionToolsOptions"; +import type { OpenAiAssistantExecutionToolsOptions } from "../llm-providers/openai/OpenAiAssistantExecutionToolsOptions"; +import type { OpenAiExecutionToolsOptions } from "../llm-providers/openai/OpenAiExecutionToolsOptions"; +import type { VercelExecutionToolsOptions } from "../llm-providers/vercel/VercelExecutionToolsOptions"; +import type { VercelProvider } from "../llm-providers/vercel/VercelProvider"; +import type { PipelineInterface } from "../pipeline/PipelineInterface/PipelineInterface"; +import type { IsPipelineImplementingInterfaceOptions } from "../pipeline/PipelineInterface/isPipelineImplementingInterface"; +import type { CommonTaskJson } from "../pipeline/PipelineJson/CommonTaskJson"; +import type { DialogTaskJson } from "../pipeline/PipelineJson/DialogTaskJson"; +import type { Expectations } from "../pipeline/PipelineJson/Expectations"; +import type { ExpectationUnit } from "../pipeline/PipelineJson/Expectations"; +import type { ExpectationAmount } from "../pipeline/PipelineJson/Expectations"; +import type { KnowledgePiecePreparedJson } from "../pipeline/PipelineJson/KnowledgePieceJson"; +import type { KnowledgeSourceJson } from "../pipeline/PipelineJson/KnowledgeSourceJson"; +import type { KnowledgeSourcePreparedJson } from "../pipeline/PipelineJson/KnowledgeSourceJson"; +import type { ParameterJson } from "../pipeline/PipelineJson/ParameterJson"; +import type { InputParameterJson } from "../pipeline/PipelineJson/ParameterJson"; +import type { IntermediateParameterJson } from "../pipeline/PipelineJson/ParameterJson"; +import type { OutputParameterJson } from "../pipeline/PipelineJson/ParameterJson"; +import type { CommonParameterJson } from "../pipeline/PipelineJson/ParameterJson"; +import type { PersonaJson } from "../pipeline/PipelineJson/PersonaJson"; +import type { PersonaPreparedJson } from "../pipeline/PipelineJson/PersonaJson"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import type { PreparationJson } from "../pipeline/PipelineJson/PreparationJson"; +import type { PromptTaskJson } from "../pipeline/PipelineJson/PromptTaskJson"; +import type { ScriptTaskJson } from "../pipeline/PipelineJson/ScriptTaskJson"; +import type { SimpleTaskJson } from "../pipeline/PipelineJson/SimpleTaskJson"; +import type { TaskJson } from "../pipeline/PipelineJson/TaskJson"; +import type { PipelineString } from "../pipeline/PipelineString"; +import type { PrepareAndScrapeOptions } from "../prepare/PrepareAndScrapeOptions"; +import type { PromptbookServer_Identification } from "../remote-server/socket-types/_subtypes/PromptbookServer_Identification"; +import type { PromptbookServer_ApplicationIdentification } from "../remote-server/socket-types/_subtypes/PromptbookServer_Identification"; +import type { PromptbookServer_AnonymousIdentification } from "../remote-server/socket-types/_subtypes/PromptbookServer_Identification"; +import type { RemoteClientOptions } from "../remote-server/types/RemoteClientOptions"; +import type { RemoteServerOptions } from "../remote-server/types/RemoteServerOptions"; +import type { AnonymousRemoteServerOptions } from "../remote-server/types/RemoteServerOptions"; +import type { ApplicationRemoteServerOptions } from "../remote-server/types/RemoteServerOptions"; +import type { ApplicationRemoteServerClientOptions } from "../remote-server/types/RemoteServerOptions"; +import type { Converter } from "../scrapers/_common/Converter"; +import type { Scraper } from "../scrapers/_common/Scraper"; +import type { ScraperSourceHandler } from "../scrapers/_common/Scraper"; +import type { ScraperIntermediateSource } from "../scrapers/_common/ScraperIntermediateSource"; +import type { ScraperAndConverterMetadata } from "../scrapers/_common/register/ScraperAndConverterMetadata"; +import type { ScraperConstructor } from "../scrapers/_common/register/ScraperConstructor"; +import type { JavascriptExecutionToolsOptions } from "../scripting/javascript/JavascriptExecutionToolsOptions"; +import type { PostprocessingFunction } from "../scripting/javascript/JavascriptExecutionToolsOptions"; +import type { PromptbookStorage } from "../storage/_common/PromptbookStorage"; +import type { FileCacheStorageOptions } from "../storage/file-cache-storage/FileCacheStorageOptions"; +import type { IntermediateFilesStrategy } from "../types/IntermediateFilesStrategy"; +import type { ModelRequirements } from "../types/ModelRequirements"; +import type { CompletionModelRequirements } from "../types/ModelRequirements"; +import type { ChatModelRequirements } from "../types/ModelRequirements"; +import type { EmbeddingModelRequirements } from "../types/ModelRequirements"; +import type { ModelVariant } from "../types/ModelVariant"; +import type { NonEmptyArray } from "../types/NonEmptyArray"; +import type { NonEmptyReadonlyArray } from "../types/NonEmptyArray"; +import type { Prompt } from "../types/Prompt"; +import type { CompletionPrompt } from "../types/Prompt"; +import type { ChatPrompt } from "../types/Prompt"; +import type { EmbeddingPrompt } from "../types/Prompt"; +import type { ScriptLanguage } from "../types/ScriptLanguage"; +import type { SectionType } from "../types/SectionType"; +import type { TaskType } from "../types/TaskType"; +import type { string_char_emoji } from "../types/typeAliasEmoji"; +import type { string_business_category_name } from "../types/typeAliases"; +import type { string_model_name } from "../types/typeAliases"; +import type { string_prompt } from "../types/typeAliases"; +import type { string_template } from "../types/typeAliases"; +import type { string_text_prompt } from "../types/typeAliases"; +import type { string_chat_prompt } from "../types/typeAliases"; +import type { string_system_message } from "../types/typeAliases"; +import type { string_completion_prompt } from "../types/typeAliases"; +import type { string_page } from "../types/typeAliases"; +import type { string_mime_type } from "../types/typeAliases"; +import type { string_mime_type_with_wildcard } from "../types/typeAliases"; +import type { string_char } from "../types/typeAliases"; +import type { string_name } from "../types/typeAliases"; +import type { string_parameter_name } from "../types/typeAliases"; +import type { string_parameter_value } from "../types/typeAliases"; +import type { Parameters } from "../types/typeAliases"; +import type { InputParameters } from "../types/typeAliases"; +import type { string_reserved_parameter_name } from "../types/typeAliases"; +import type { ReservedParameters } from "../types/typeAliases"; +import type { string_title } from "../types/typeAliases"; +import type { string_persona_description } from "../types/typeAliases"; +import type { string_model_description } from "../types/typeAliases"; +import type { string_knowledge_source_content } from "../types/typeAliases"; +import type { string_knowledge_source_link } from "../types/typeAliases"; +import type { string_html } from "../types/typeAliases"; +import type { string_xml } from "../types/typeAliases"; +import type { string_markdown } from "../types/typeAliases"; +import type { string_markdown_section } from "../types/typeAliases"; +import type { string_markdown_section_content } from "../types/typeAliases"; +import type { string_markdown_text } from "../types/typeAliases"; +import type { string_markdown_codeblock_language } from "../types/typeAliases"; +import type { string_promptbook_documentation_url } from "../types/typeAliases"; +import type { string_domain } from "../types/typeAliases"; +import type { string_tdl } from "../types/typeAliases"; +import type { string_css } from "../types/typeAliases"; +import type { string_svg } from "../types/typeAliases"; +import type { string_script } from "../types/typeAliases"; +import type { string_javascript } from "../types/typeAliases"; +import type { string_json } from "../types/typeAliases"; +import type { string_css_class } from "../types/typeAliases"; +import type { string_css_property } from "../types/typeAliases"; +import type { string_css_value } from "../types/typeAliases"; +import type { string_css_selector } from "../types/typeAliases"; +import type { string_url } from "../types/typeAliases"; +import type { string_base_url } from "../types/typeAliases"; +import type { string_pipeline_root_url } from "../types/typeAliases"; +import type { string_pipeline_url } from "../types/typeAliases"; +import type { string_pipeline_url_with_task_hash } from "../types/typeAliases"; +import type { string_data_url } from "../types/typeAliases"; +import type { string_base64 } from "../types/typeAliases"; +import type { string_href } from "../types/typeAliases"; +import type { string_url_image } from "../types/typeAliases"; +import type { string_executable_path } from "../types/typeAliases"; +import type { string_uri } from "../types/typeAliases"; +import type { string_uri_part } from "../types/typeAliases"; +import type { string_hostname } from "../types/typeAliases"; +import type { string_host } from "../types/typeAliases"; +import type { string_protocol } from "../types/typeAliases"; +import type { string_email } from "../types/typeAliases"; +import type { string_emails } from "../types/typeAliases"; +import type { string_uuid } from "../types/typeAliases"; +import type { string_app_id } from "../types/typeAliases"; +import type { string_user_id } from "../types/typeAliases"; +import type { string_sha256 } from "../types/typeAliases"; +import type { string_semantic_version } from "../types/typeAliases"; +import type { string_version_dependency } from "../types/typeAliases"; +import type { string_file_extension } from "../types/typeAliases"; +import type { string_absolute_filename } from "../types/typeAliases"; +import type { string_relative_filename } from "../types/typeAliases"; +import type { string_filename } from "../types/typeAliases"; +import type { string_absolute_dirname } from "../types/typeAliases"; +import type { string_relative_dirname } from "../types/typeAliases"; +import type { string_dirname } from "../types/typeAliases"; +import type { string_person_fullname } from "../types/typeAliases"; +import type { string_person_profile } from "../types/typeAliases"; +import type { string_license } from "../types/typeAliases"; +import type { string_attribute } from "../types/typeAliases"; +import type { string_attribute_value_scope } from "../types/typeAliases"; +import type { string_color } from "../types/typeAliases"; +import type { string_translate_name } from "../types/typeAliases"; +import type { string_translate_name_not_normalized } from "../types/typeAliases"; +import type { string_translate_language } from "../types/typeAliases"; +import type { string_javascript_name } from "../types/typeAliases"; +import type { string_postprocessing_function_name } from "../types/typeAliases"; +import type { id } from "../types/typeAliases"; +import type { task_id } from "../types/typeAliases"; +import type { string_token } from "../types/typeAliases"; +import type { string_license_token } from "../types/typeAliases"; +import type { string_password } from "../types/typeAliases"; +import type { string_ssh_key } from "../types/typeAliases"; +import type { string_pgp_key } from "../types/typeAliases"; +import type { string_date_iso8601 } from "../types/typeAliases"; +import type { number_usd } from "../types/typeAliases"; +import type { number_id } from "../types/typeAliases"; +import type { number_linecol_number } from "../types/typeAliases"; +import type { number_tokens } from "../types/typeAliases"; +import type { number_positive } from "../types/typeAliases"; +import type { number_negative } from "../types/typeAliases"; +import type { number_integer } from "../types/typeAliases"; +import type { number_port } from "../types/typeAliases"; +import type { number_percent } from "../types/typeAliases"; +import type { number_model_temperature } from "../types/typeAliases"; +import type { number_seed } from "../types/typeAliases"; +import type { number_likeness } from "../types/typeAliases"; +import type { number_miliseconds } from "../types/typeAliases"; +import type { number_seconds } from "../types/typeAliases"; +import type { number_minutes } from "../types/typeAliases"; +import type { number_hours } from "../types/typeAliases"; +import type { number_days } from "../types/typeAliases"; +import type { number_weeks } from "../types/typeAliases"; +import type { number_months } from "../types/typeAliases"; +import type { number_years } from "../types/typeAliases"; +import type { number_bytes } from "../types/typeAliases"; +import type { number_kilobytes } from "../types/typeAliases"; +import type { number_megabytes } from "../types/typeAliases"; +import type { number_gigabytes } from "../types/typeAliases"; +import type { number_terabytes } from "../types/typeAliases"; +import type { Registered } from "../utils/$Register"; +import type { Registration } from "../utils/$Register"; +import type { FromtoItems } from "../utils/FromtoItems"; +import type { PipelineEditableSerialized } from "../utils/editable/types/PipelineEditableSerialized"; +import type { ExecCommandOptions } from "../utils/execCommand/ExecCommandOptions"; +import type { ExecCommandOptionsAdvanced } from "../utils/execCommand/ExecCommandOptions"; +import type { CodeBlock } from "../utils/markdown/extractAllBlocksFromMarkdown"; +import type { MarkdownSection } from "../utils/markdown/parseMarkdownSection"; +import type { string_keyword } from "../utils/normalization/IKeywords"; +import type { Keywords } from "../utils/normalization/IKeywords"; +import type { string_kebab_case } from "../utils/normalization/normalize-to-kebab-case"; +import type { string_PascalCase } from "../utils/normalization/normalizeTo_PascalCase"; +import type { string_SCREAMING_CASE } from "../utils/normalization/normalizeTo_SCREAMING_CASE"; +import type { string_camelCase } from "../utils/normalization/normalizeTo_camelCase"; +import type { string_snake_case } from "../utils/normalization/normalizeTo_snake_case"; +import type { OrderJsonOptions } from "../utils/normalization/orderJson"; +import type { TODO_any } from "../utils/organization/TODO_any"; +import type { empty_object } from "../utils/organization/empty_object"; +import type { really_any } from "../utils/organization/really_any"; +import type { CheckSerializableAsJsonOptions } from "../utils/serialization/checkSerializableAsJson"; +import type { ExportJsonOptions } from "../utils/serialization/exportJson"; +import type { string_promptbook_version } from "../version"; // Note: Entities of the `@promptbook/types` export type { PipelineCollection }; diff --git a/src/_packages/utils.index.ts b/src/_packages/utils.index.ts index 4b3dbac061..2aa3c1d3a1 100644 --- a/src/_packages/utils.index.ts +++ b/src/_packages/utils.index.ts @@ -1,94 +1,92 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/utils` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { VALUE_STRINGS } from '../config'; -import { SMALL_NUMBER } from '../config'; -import { renderPromptbookMermaid } from '../conversion/prettify/renderPipelineMermaidOptions'; -import { extractVariablesFromScript } from '../conversion/utils/extractVariablesFromScript'; -import { deserializeError } from '../errors/utils/deserializeError'; -import { serializeError } from '../errors/utils/serializeError'; -import { forEachAsync } from '../execution/utils/forEachAsync'; -import { isValidJsonString } from '../formats/json/utils/isValidJsonString'; -import { prompt } from '../pipeline/prompt-notation'; -import { promptTemplate } from '../pipeline/prompt-notation'; -import { $getCurrentDate } from '../utils/$getCurrentDate'; -import { $isRunningInBrowser } from '../utils/environment/$isRunningInBrowser'; -import { $isRunningInJest } from '../utils/environment/$isRunningInJest'; -import { $isRunningInNode } from '../utils/environment/$isRunningInNode'; -import { $isRunningInWebWorker } from '../utils/environment/$isRunningInWebWorker'; -import { CHARACTERS_PER_STANDARD_LINE } from '../utils/expectation-counters/constants'; -import { LINES_PER_STANDARD_PAGE } from '../utils/expectation-counters/constants'; -import { countCharacters } from '../utils/expectation-counters/countCharacters'; -import { countLines } from '../utils/expectation-counters/countLines'; -import { countPages } from '../utils/expectation-counters/countPages'; -import { countParagraphs } from '../utils/expectation-counters/countParagraphs'; -import { splitIntoSentences } from '../utils/expectation-counters/countSentences'; -import { countSentences } from '../utils/expectation-counters/countSentences'; -import { countWords } from '../utils/expectation-counters/countWords'; -import { CountUtils } from '../utils/expectation-counters/index'; -import { capitalize } from '../utils/normalization/capitalize'; -import { decapitalize } from '../utils/normalization/decapitalize'; -import { DIACRITIC_VARIANTS_LETTERS } from '../utils/normalization/DIACRITIC_VARIANTS_LETTERS'; -import type { string_keyword } from '../utils/normalization/IKeywords'; -import type { Keywords } from '../utils/normalization/IKeywords'; -import { isValidKeyword } from '../utils/normalization/isValidKeyword'; -import { nameToUriPart } from '../utils/normalization/nameToUriPart'; -import { nameToUriParts } from '../utils/normalization/nameToUriParts'; -import type { string_kebab_case } from '../utils/normalization/normalize-to-kebab-case'; -import { normalizeToKebabCase } from '../utils/normalization/normalize-to-kebab-case'; -import type { string_camelCase } from '../utils/normalization/normalizeTo_camelCase'; -import { normalizeTo_camelCase } from '../utils/normalization/normalizeTo_camelCase'; -import type { string_PascalCase } from '../utils/normalization/normalizeTo_PascalCase'; -import { normalizeTo_PascalCase } from '../utils/normalization/normalizeTo_PascalCase'; -import type { string_SCREAMING_CASE } from '../utils/normalization/normalizeTo_SCREAMING_CASE'; -import { normalizeTo_SCREAMING_CASE } from '../utils/normalization/normalizeTo_SCREAMING_CASE'; -import { normalizeTo_snake_case } from '../utils/normalization/normalizeTo_snake_case'; -import { normalizeWhitespaces } from '../utils/normalization/normalizeWhitespaces'; -import { orderJson } from '../utils/normalization/orderJson'; -import { parseKeywords } from '../utils/normalization/parseKeywords'; -import { parseKeywordsFromString } from '../utils/normalization/parseKeywordsFromString'; -import { removeDiacritics } from '../utils/normalization/removeDiacritics'; -import { searchKeywords } from '../utils/normalization/searchKeywords'; -import { suffixUrl } from '../utils/normalization/suffixUrl'; -import { titleToName } from '../utils/normalization/titleToName'; -import { spaceTrim } from '../utils/organization/spaceTrim'; -import { extractParameterNames } from '../utils/parameters/extractParameterNames'; -import { numberToString } from '../utils/parameters/numberToString'; -import { templateParameters } from '../utils/parameters/templateParameters'; -import { valueToString } from '../utils/parameters/valueToString'; -import { parseNumber } from '../utils/parseNumber'; -import { removeEmojis } from '../utils/removeEmojis'; -import { removeQuotes } from '../utils/removeQuotes'; -import { $deepFreeze } from '../utils/serialization/$deepFreeze'; -import { checkSerializableAsJson } from '../utils/serialization/checkSerializableAsJson'; -import { clonePipeline } from '../utils/serialization/clonePipeline'; -import { deepClone } from '../utils/serialization/deepClone'; -import { exportJson } from '../utils/serialization/exportJson'; -import { isSerializableAsJson } from '../utils/serialization/isSerializableAsJson'; -import { difference } from '../utils/sets/difference'; -import { intersection } from '../utils/sets/intersection'; -import { union } from '../utils/sets/union'; -import { trimCodeBlock } from '../utils/trimCodeBlock'; -import { trimEndOfCodeBlock } from '../utils/trimEndOfCodeBlock'; -import { unwrapResult } from '../utils/unwrapResult'; -import { isValidEmail } from '../utils/validators/email/isValidEmail'; -import { isRootPath } from '../utils/validators/filePath/isRootPath'; -import { isValidFilePath } from '../utils/validators/filePath/isValidFilePath'; -import { isValidJavascriptName } from '../utils/validators/javascriptName/isValidJavascriptName'; -import { isValidPromptbookVersion } from '../utils/validators/semanticVersion/isValidPromptbookVersion'; -import { isValidSemanticVersion } from '../utils/validators/semanticVersion/isValidSemanticVersion'; -import { isHostnameOnPrivateNetwork } from '../utils/validators/url/isHostnameOnPrivateNetwork'; -import { isUrlOnPrivateNetwork } from '../utils/validators/url/isUrlOnPrivateNetwork'; -import { isValidPipelineUrl } from '../utils/validators/url/isValidPipelineUrl'; -import { isValidUrl } from '../utils/validators/url/isValidUrl'; -import { isValidUuid } from '../utils/validators/uuid/isValidUuid'; - +import { VALUE_STRINGS } from "../config"; +import { SMALL_NUMBER } from "../config"; +import { renderPromptbookMermaid } from "../conversion/prettify/renderPipelineMermaidOptions"; +import { extractVariablesFromScript } from "../conversion/utils/extractVariablesFromScript"; +import { deserializeError } from "../errors/utils/deserializeError"; +import { serializeError } from "../errors/utils/serializeError"; +import { forEachAsync } from "../execution/utils/forEachAsync"; +import { isValidJsonString } from "../formats/json/utils/isValidJsonString"; +import { prompt } from "../pipeline/prompt-notation"; +import { promptTemplate } from "../pipeline/prompt-notation"; +import { $getCurrentDate } from "../utils/$getCurrentDate"; +import { $isRunningInBrowser } from "../utils/environment/$isRunningInBrowser"; +import { $isRunningInJest } from "../utils/environment/$isRunningInJest"; +import { $isRunningInNode } from "../utils/environment/$isRunningInNode"; +import { $isRunningInWebWorker } from "../utils/environment/$isRunningInWebWorker"; +import { CHARACTERS_PER_STANDARD_LINE } from "../utils/expectation-counters/constants"; +import { LINES_PER_STANDARD_PAGE } from "../utils/expectation-counters/constants"; +import { countCharacters } from "../utils/expectation-counters/countCharacters"; +import { countLines } from "../utils/expectation-counters/countLines"; +import { countPages } from "../utils/expectation-counters/countPages"; +import { countParagraphs } from "../utils/expectation-counters/countParagraphs"; +import { splitIntoSentences } from "../utils/expectation-counters/countSentences"; +import { countSentences } from "../utils/expectation-counters/countSentences"; +import { countWords } from "../utils/expectation-counters/countWords"; +import { CountUtils } from "../utils/expectation-counters/index"; +import { DIACRITIC_VARIANTS_LETTERS } from "../utils/normalization/DIACRITIC_VARIANTS_LETTERS"; +import type { string_keyword } from "../utils/normalization/IKeywords"; +import type { Keywords } from "../utils/normalization/IKeywords"; +import { capitalize } from "../utils/normalization/capitalize"; +import { decapitalize } from "../utils/normalization/decapitalize"; +import { isValidKeyword } from "../utils/normalization/isValidKeyword"; +import { nameToUriPart } from "../utils/normalization/nameToUriPart"; +import { nameToUriParts } from "../utils/normalization/nameToUriParts"; +import type { string_kebab_case } from "../utils/normalization/normalize-to-kebab-case"; +import { normalizeToKebabCase } from "../utils/normalization/normalize-to-kebab-case"; +import type { string_PascalCase } from "../utils/normalization/normalizeTo_PascalCase"; +import { normalizeTo_PascalCase } from "../utils/normalization/normalizeTo_PascalCase"; +import type { string_SCREAMING_CASE } from "../utils/normalization/normalizeTo_SCREAMING_CASE"; +import { normalizeTo_SCREAMING_CASE } from "../utils/normalization/normalizeTo_SCREAMING_CASE"; +import type { string_camelCase } from "../utils/normalization/normalizeTo_camelCase"; +import { normalizeTo_camelCase } from "../utils/normalization/normalizeTo_camelCase"; +import { normalizeTo_snake_case } from "../utils/normalization/normalizeTo_snake_case"; +import { normalizeWhitespaces } from "../utils/normalization/normalizeWhitespaces"; +import { orderJson } from "../utils/normalization/orderJson"; +import { parseKeywords } from "../utils/normalization/parseKeywords"; +import { parseKeywordsFromString } from "../utils/normalization/parseKeywordsFromString"; +import { removeDiacritics } from "../utils/normalization/removeDiacritics"; +import { searchKeywords } from "../utils/normalization/searchKeywords"; +import { suffixUrl } from "../utils/normalization/suffixUrl"; +import { titleToName } from "../utils/normalization/titleToName"; +import { spaceTrim } from "../utils/organization/spaceTrim"; +import { extractParameterNames } from "../utils/parameters/extractParameterNames"; +import { numberToString } from "../utils/parameters/numberToString"; +import { templateParameters } from "../utils/parameters/templateParameters"; +import { valueToString } from "../utils/parameters/valueToString"; +import { parseNumber } from "../utils/parseNumber"; +import { removeEmojis } from "../utils/removeEmojis"; +import { removeQuotes } from "../utils/removeQuotes"; +import { $deepFreeze } from "../utils/serialization/$deepFreeze"; +import { checkSerializableAsJson } from "../utils/serialization/checkSerializableAsJson"; +import { clonePipeline } from "../utils/serialization/clonePipeline"; +import { deepClone } from "../utils/serialization/deepClone"; +import { exportJson } from "../utils/serialization/exportJson"; +import { isSerializableAsJson } from "../utils/serialization/isSerializableAsJson"; +import { difference } from "../utils/sets/difference"; +import { intersection } from "../utils/sets/intersection"; +import { union } from "../utils/sets/union"; +import { trimCodeBlock } from "../utils/trimCodeBlock"; +import { trimEndOfCodeBlock } from "../utils/trimEndOfCodeBlock"; +import { unwrapResult } from "../utils/unwrapResult"; +import { isValidEmail } from "../utils/validators/email/isValidEmail"; +import { isRootPath } from "../utils/validators/filePath/isRootPath"; +import { isValidFilePath } from "../utils/validators/filePath/isValidFilePath"; +import { isValidJavascriptName } from "../utils/validators/javascriptName/isValidJavascriptName"; +import { isValidPromptbookVersion } from "../utils/validators/semanticVersion/isValidPromptbookVersion"; +import { isValidSemanticVersion } from "../utils/validators/semanticVersion/isValidSemanticVersion"; +import { isHostnameOnPrivateNetwork } from "../utils/validators/url/isHostnameOnPrivateNetwork"; +import { isUrlOnPrivateNetwork } from "../utils/validators/url/isUrlOnPrivateNetwork"; +import { isValidPipelineUrl } from "../utils/validators/url/isValidPipelineUrl"; +import { isValidUrl } from "../utils/validators/url/isValidUrl"; +import { isValidUuid } from "../utils/validators/uuid/isValidUuid"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/utils` export { VALUE_STRINGS }; export { SMALL_NUMBER }; diff --git a/src/_packages/vercel.index.ts b/src/_packages/vercel.index.ts index 0fdedbd863..f3317af1e4 100644 --- a/src/_packages/vercel.index.ts +++ b/src/_packages/vercel.index.ts @@ -1,15 +1,13 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/vercel` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { createExecutionToolsFromVercelProvider } from '../llm-providers/vercel/createExecutionToolsFromVercelProvider'; -import type { VercelProvider } from '../llm-providers/vercel/VercelProvider'; - +import type { VercelProvider } from "../llm-providers/vercel/VercelProvider"; +import { createExecutionToolsFromVercelProvider } from "../llm-providers/vercel/createExecutionToolsFromVercelProvider"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/vercel` export { createExecutionToolsFromVercelProvider }; export type { VercelProvider }; diff --git a/src/_packages/website-crawler.index.ts b/src/_packages/website-crawler.index.ts index 097639e47f..1171c00bd1 100644 --- a/src/_packages/website-crawler.index.ts +++ b/src/_packages/website-crawler.index.ts @@ -1,16 +1,14 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/website-crawler` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { createWebsiteScraper } from '../scrapers/website/createWebsiteScraper'; -import { _WebsiteScraperRegistration } from '../scrapers/website/register-constructor'; -import { WebsiteScraper } from '../scrapers/website/WebsiteScraper'; - +import { WebsiteScraper } from "../scrapers/website/WebsiteScraper"; +import { createWebsiteScraper } from "../scrapers/website/createWebsiteScraper"; +import { _WebsiteScraperRegistration } from "../scrapers/website/register-constructor"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/website-crawler` export { createWebsiteScraper }; export { _WebsiteScraperRegistration }; diff --git a/src/_packages/wizzard.index.ts b/src/_packages/wizzard.index.ts index 73459fc94e..1702724df3 100644 --- a/src/_packages/wizzard.index.ts +++ b/src/_packages/wizzard.index.ts @@ -1,40 +1,38 @@ // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten // `@promptbook/wizzard` -import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { _AnthropicClaudeMetadataRegistration } from '../llm-providers/anthropic-claude/register-configuration'; -import { _AnthropicClaudeRegistration } from '../llm-providers/anthropic-claude/register-constructor'; -import { _AzureOpenAiMetadataRegistration } from '../llm-providers/azure-openai/register-configuration'; -import { _AzureOpenAiRegistration } from '../llm-providers/azure-openai/register-constructor'; -import { _DeepseekMetadataRegistration } from '../llm-providers/deepseek/register-configuration'; -import { _DeepseekRegistration } from '../llm-providers/deepseek/register-constructor'; -import { _GoogleMetadataRegistration } from '../llm-providers/google/register-configuration'; -import { _GoogleRegistration } from '../llm-providers/google/register-constructor'; -import { _OpenAiMetadataRegistration } from '../llm-providers/openai/register-configuration'; -import { _OpenAiAssistantMetadataRegistration } from '../llm-providers/openai/register-configuration'; -import { _OpenAiRegistration } from '../llm-providers/openai/register-constructor'; -import { _OpenAiAssistantRegistration } from '../llm-providers/openai/register-constructor'; -import { _BoilerplateScraperRegistration } from '../scrapers/_boilerplate/register-constructor'; -import { _BoilerplateScraperMetadataRegistration } from '../scrapers/_boilerplate/register-metadata'; -import { _LegacyDocumentScraperRegistration } from '../scrapers/document-legacy/register-constructor'; -import { _LegacyDocumentScraperMetadataRegistration } from '../scrapers/document-legacy/register-metadata'; -import { _DocumentScraperRegistration } from '../scrapers/document/register-constructor'; -import { _DocumentScraperMetadataRegistration } from '../scrapers/document/register-metadata'; -import { _MarkdownScraperRegistration } from '../scrapers/markdown/register-constructor'; -import { _MarkdownScraperMetadataRegistration } from '../scrapers/markdown/register-metadata'; -import { _MarkitdownScraperRegistration } from '../scrapers/markitdown/register-constructor'; -import { _MarkitdownScraperMetadataRegistration } from '../scrapers/markitdown/register-metadata'; -import { _PdfScraperRegistration } from '../scrapers/pdf/register-constructor'; -import { _PdfScraperMetadataRegistration } from '../scrapers/pdf/register-metadata'; -import { _WebsiteScraperRegistration } from '../scrapers/website/register-constructor'; -import { _WebsiteScraperMetadataRegistration } from '../scrapers/website/register-metadata'; -import { wizzard } from '../wizzard/wizzard'; - +import { _AnthropicClaudeMetadataRegistration } from "../llm-providers/anthropic-claude/register-configuration"; +import { _AnthropicClaudeRegistration } from "../llm-providers/anthropic-claude/register-constructor"; +import { _AzureOpenAiMetadataRegistration } from "../llm-providers/azure-openai/register-configuration"; +import { _AzureOpenAiRegistration } from "../llm-providers/azure-openai/register-constructor"; +import { _DeepseekMetadataRegistration } from "../llm-providers/deepseek/register-configuration"; +import { _DeepseekRegistration } from "../llm-providers/deepseek/register-constructor"; +import { _GoogleMetadataRegistration } from "../llm-providers/google/register-configuration"; +import { _GoogleRegistration } from "../llm-providers/google/register-constructor"; +import { _OpenAiMetadataRegistration } from "../llm-providers/openai/register-configuration"; +import { _OpenAiAssistantMetadataRegistration } from "../llm-providers/openai/register-configuration"; +import { _OpenAiRegistration } from "../llm-providers/openai/register-constructor"; +import { _OpenAiAssistantRegistration } from "../llm-providers/openai/register-constructor"; +import { _BoilerplateScraperRegistration } from "../scrapers/_boilerplate/register-constructor"; +import { _BoilerplateScraperMetadataRegistration } from "../scrapers/_boilerplate/register-metadata"; +import { _LegacyDocumentScraperRegistration } from "../scrapers/document-legacy/register-constructor"; +import { _LegacyDocumentScraperMetadataRegistration } from "../scrapers/document-legacy/register-metadata"; +import { _DocumentScraperRegistration } from "../scrapers/document/register-constructor"; +import { _DocumentScraperMetadataRegistration } from "../scrapers/document/register-metadata"; +import { _MarkdownScraperRegistration } from "../scrapers/markdown/register-constructor"; +import { _MarkdownScraperMetadataRegistration } from "../scrapers/markdown/register-metadata"; +import { _MarkitdownScraperRegistration } from "../scrapers/markitdown/register-constructor"; +import { _MarkitdownScraperMetadataRegistration } from "../scrapers/markitdown/register-metadata"; +import { _PdfScraperRegistration } from "../scrapers/pdf/register-constructor"; +import { _PdfScraperMetadataRegistration } from "../scrapers/pdf/register-metadata"; +import { _WebsiteScraperRegistration } from "../scrapers/website/register-constructor"; +import { _WebsiteScraperMetadataRegistration } from "../scrapers/website/register-metadata"; +import { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION } from "../version"; +import { wizzard } from "../wizzard/wizzard"; // Note: Exporting version from each package export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION }; - // Note: Entities of the `@promptbook/wizzard` export { _AnthropicClaudeMetadataRegistration }; export { _AnthropicClaudeRegistration }; diff --git a/src/cli/cli-commands/_boilerplate.ts b/src/cli/cli-commands/_boilerplate.ts index 4458d0e584..bab9ce47e1 100644 --- a/src/cli/cli-commands/_boilerplate.ts +++ b/src/cli/cli-commands/_boilerplate.ts @@ -1,8 +1,8 @@ -import colors from 'colors'; +import colors from "colors"; import type { - Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, -} from 'commander'; -import spaceTrim from 'spacetrim'; + Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, +} from "commander"; +import spaceTrim from "spacetrim"; /** * Initializes `boilerplate` command for Promptbook CLI utilities @@ -12,21 +12,20 @@ import spaceTrim from 'spacetrim'; * @private internal function of `promptbookCli` */ export function $initializeBoilerplateCommand(program: Program) { - const boilerplateCommand = program.command('boilerplate'); - boilerplateCommand.description( - spaceTrim(` + const boilerplateCommand = program.command("boilerplate"); + boilerplateCommand.description( + spaceTrim(` @@@ `), - ); + ); - boilerplateCommand.action(async () => { - // @@@ + boilerplateCommand.action(async () => { + // @@@ - console.error(colors.green(`@@@`)); + console.error(colors.green(`@@@`)); - - return process.exit(0); - }); + return process.exit(0); + }); } /** diff --git a/src/cli/cli-commands/about.ts b/src/cli/cli-commands/about.ts index 24e292ca6c..695e0041e9 100644 --- a/src/cli/cli-commands/about.ts +++ b/src/cli/cli-commands/about.ts @@ -1,15 +1,15 @@ -import colors from 'colors'; +import colors from "colors"; import type { - Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, -} from 'commander'; -import spaceTrim from 'spacetrim'; -import { CLAIM } from '../../config'; -import { $isRunningInBrowser } from '../../utils/environment/$isRunningInBrowser'; -import { $isRunningInJest } from '../../utils/environment/$isRunningInJest'; -import { $isRunningInNode } from '../../utils/environment/$isRunningInNode'; -import { $isRunningInWebWorker } from '../../utils/environment/$isRunningInWebWorker'; -import { BOOK_LANGUAGE_VERSION } from '../../version'; -import { PROMPTBOOK_ENGINE_VERSION } from '../../version'; + Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, +} from "commander"; +import spaceTrim from "spacetrim"; +import { CLAIM } from "../../config"; +import { $isRunningInBrowser } from "../../utils/environment/$isRunningInBrowser"; +import { $isRunningInJest } from "../../utils/environment/$isRunningInJest"; +import { $isRunningInNode } from "../../utils/environment/$isRunningInNode"; +import { $isRunningInWebWorker } from "../../utils/environment/$isRunningInWebWorker"; +import { BOOK_LANGUAGE_VERSION } from "../../version"; +import { PROMPTBOOK_ENGINE_VERSION } from "../../version"; /** * Initializes `about` command for Promptbook CLI utilities @@ -19,41 +19,45 @@ import { PROMPTBOOK_ENGINE_VERSION } from '../../version'; * @private internal function of `promptbookCli` */ export function $initializeAboutCommand(program: Program) { - const makeCommand = program.command('about'); - makeCommand.description( - spaceTrim(` + const makeCommand = program.command("about"); + makeCommand.description( + spaceTrim(` Tells about Promptbook CLI and its abilities `), - ); + ); - makeCommand.action(async () => { - console.info(colors.bold(colors.blue(`Promptbook: ${CLAIM}`))); - console.info(colors.cyan(`Book language version: ${BOOK_LANGUAGE_VERSION}`)); - console.info(colors.cyan(`Promptbook engine version: ${PROMPTBOOK_ENGINE_VERSION}`)); + makeCommand.action(async () => { + console.info(colors.bold(colors.blue(`Promptbook: ${CLAIM}`))); + console.info( + colors.cyan(`Book language version: ${BOOK_LANGUAGE_VERSION}`), + ); + console.info( + colors.cyan(`Promptbook engine version: ${PROMPTBOOK_ENGINE_VERSION}`), + ); - if ($isRunningInNode()) { - console.info(colors.cyan(`Environment: Node.js`)); - console.info(colors.cyan(`Node.js version: ${process.version}`)); - // <- TODO: [🧠][🎺] Make robust system to check platform requirements like browser/node environment, version of node, available memory, disk space, ... - console.info(colors.cyan(`Platform type: ${process.platform}`)); - console.info(colors.cyan(`Platform architecture: ${process.arch}`)); - // console.info(colors.cyan(`Available memory: ${process.availableMemory()}`)); - // <- TODO: [🧠] Should we show available memory / disk / ... - } else if ($isRunningInJest()) { - console.info(colors.cyan(`Environment: Jest (testing)`)); - } else if ($isRunningInBrowser()) { - // <- Note: This is unreliable because CLI itself is not running in browser but for future use - console.info(colors.cyan(`Environment: Browser`)); - } else if ($isRunningInWebWorker()) { - // <- Note: This is unreliable because CLI itself is not running in browser but for future use - console.info(colors.cyan(`Environment: Web Worker`)); - } + if ($isRunningInNode()) { + console.info(colors.cyan(`Environment: Node.js`)); + console.info(colors.cyan(`Node.js version: ${process.version}`)); + // <- TODO: [🧠][🎺] Make robust system to check platform requirements like browser/node environment, version of node, available memory, disk space, ... + console.info(colors.cyan(`Platform type: ${process.platform}`)); + console.info(colors.cyan(`Platform architecture: ${process.arch}`)); + // console.info(colors.cyan(`Available memory: ${process.availableMemory()}`)); + // <- TODO: [🧠] Should we show available memory / disk / ... + } else if ($isRunningInJest()) { + console.info(colors.cyan(`Environment: Jest (testing)`)); + } else if ($isRunningInBrowser()) { + // <- Note: This is unreliable because CLI itself is not running in browser but for future use + console.info(colors.cyan(`Environment: Browser`)); + } else if ($isRunningInWebWorker()) { + // <- Note: This is unreliable because CLI itself is not running in browser but for future use + console.info(colors.cyan(`Environment: Web Worker`)); + } - console.info(colors.gray(`https://github.com/webgptorg/promptbook`)); - console.info(colors.gray(`https://ptbk.io`)); + console.info(colors.gray(`https://github.com/webgptorg/promptbook`)); + console.info(colors.gray(`https://ptbk.io`)); - return process.exit(0); - }); + return process.exit(0); + }); } /** diff --git a/src/cli/cli-commands/hello.ts b/src/cli/cli-commands/hello.ts index 7915288a95..933c9abf70 100644 --- a/src/cli/cli-commands/hello.ts +++ b/src/cli/cli-commands/hello.ts @@ -1,9 +1,9 @@ -import colors from 'colors'; +import colors from "colors"; import type { - Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, -} from 'commander'; -import spaceTrim from 'spacetrim'; -import { forTime } from 'waitasecond'; + Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, +} from "commander"; +import spaceTrim from "spacetrim"; +import { forTime } from "waitasecond"; /** * Initializes testing `hello` command for Promptbook CLI utilities @@ -13,24 +13,24 @@ import { forTime } from 'waitasecond'; * @private internal function of `promptbookCli` */ export function $initializeHelloCommand(program: Program) { - const helloCommand = program.command('hello'); - helloCommand.description( - spaceTrim(` + const helloCommand = program.command("hello"); + helloCommand.description( + spaceTrim(` Just command for testing `), - ); + ); - helloCommand.alias('hi'); + helloCommand.alias("hi"); - helloCommand.argument('[name]', 'Your name', 'Paul'); - helloCommand.option('-g, --greeting ', `Greeting`, 'Hello'); + helloCommand.argument("[name]", "Your name", "Paul"); + helloCommand.option("-g, --greeting ", `Greeting`, "Hello"); - helloCommand.action(async (name, { greeting }) => { - console.info(colors.cyan(`${greeting} ${name}`)); - await forTime(1000); - console.info(colors.rainbow(`Nice to meet you!`)); - return process.exit(0); - }); + helloCommand.action(async (name, { greeting }) => { + console.info(colors.cyan(`${greeting} ${name}`)); + await forTime(1000); + console.info(colors.rainbow(`Nice to meet you!`)); + return process.exit(0); + }); } /** diff --git a/src/cli/cli-commands/list-models.ts b/src/cli/cli-commands/list-models.ts index 4943b25ee2..52fd448d4f 100644 --- a/src/cli/cli-commands/list-models.ts +++ b/src/cli/cli-commands/list-models.ts @@ -1,10 +1,10 @@ import type { - Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, -} from 'commander'; -import spaceTrim from 'spacetrim'; -import { $provideLlmToolsForWizzardOrCli } from '../../llm-providers/_common/register/$provideLlmToolsForWizzardOrCli'; -import { $registeredLlmToolsMessage } from '../../llm-providers/_common/register/$registeredLlmToolsMessage'; -import { $sideEffect } from '../../utils/organization/$sideEffect'; + Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, +} from "commander"; +import spaceTrim from "spacetrim"; +import { $provideLlmToolsForWizzardOrCli } from "../../llm-providers/_common/register/$provideLlmToolsForWizzardOrCli"; +import { $registeredLlmToolsMessage } from "../../llm-providers/_common/register/$registeredLlmToolsMessage"; +import { $sideEffect } from "../../utils/organization/$sideEffect"; /** * Initializes `list-models` command for Promptbook CLI utilities @@ -14,24 +14,24 @@ import { $sideEffect } from '../../utils/organization/$sideEffect'; * @private internal function of `promptbookCli` */ export function $initializeListModelsCommand(program: Program) { - const listModelsCommand = program.command('list-models'); - listModelsCommand.description( - spaceTrim(` + const listModelsCommand = program.command("list-models"); + listModelsCommand.description( + spaceTrim(` List all available and configured LLM models `), - ); + ); - listModelsCommand.alias('models'); - listModelsCommand.alias('llm'); + listModelsCommand.alias("models"); + listModelsCommand.alias("llm"); - listModelsCommand.action(async () => { - const llm = await $provideLlmToolsForWizzardOrCli({}); - $sideEffect(llm); - // <- Note: Providing LLM tools will make a side effect of registering all available LLM tools to show the message + listModelsCommand.action(async () => { + const llm = await $provideLlmToolsForWizzardOrCli({}); + $sideEffect(llm); + // <- Note: Providing LLM tools will make a side effect of registering all available LLM tools to show the message - console.info($registeredLlmToolsMessage()); - return process.exit(0); - }); + console.info($registeredLlmToolsMessage()); + return process.exit(0); + }); } /** diff --git a/src/cli/cli-commands/list-scrapers.ts b/src/cli/cli-commands/list-scrapers.ts index 06f6a4b55a..3f8f7b0d29 100644 --- a/src/cli/cli-commands/list-scrapers.ts +++ b/src/cli/cli-commands/list-scrapers.ts @@ -1,10 +1,10 @@ import type { - Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, -} from 'commander'; -import spaceTrim from 'spacetrim'; -import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode'; -import { $provideScrapersForNode } from '../../scrapers/_common/register/$provideScrapersForNode'; -import { $registeredScrapersMessage } from '../../scrapers/_common/register/$registeredScrapersMessage'; + Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, +} from "commander"; +import spaceTrim from "spacetrim"; +import { $provideExecutablesForNode } from "../../executables/$provideExecutablesForNode"; +import { $provideScrapersForNode } from "../../scrapers/_common/register/$provideScrapersForNode"; +import { $registeredScrapersMessage } from "../../scrapers/_common/register/$registeredScrapersMessage"; /** * Initializes `list-scrapers` command for Promptbook CLI utilities @@ -14,42 +14,50 @@ import { $registeredScrapersMessage } from '../../scrapers/_common/register/$reg * @private internal function of `promptbookCli` */ export function $initializeListScrapersCommand(program: Program) { - const listModelsCommand = program.command('list-scrapers'); - listModelsCommand.description( - spaceTrim(` + const listModelsCommand = program.command("list-scrapers"); + listModelsCommand.description( + spaceTrim(` List all available and configured scrapers and executables `), - ); + ); - listModelsCommand.alias('scrapers'); + listModelsCommand.alias("scrapers"); - listModelsCommand.action(async () => { - const scrapers = await $provideScrapersForNode({}); - const executables = await $provideExecutablesForNode(); + listModelsCommand.action(async () => { + const scrapers = await $provideScrapersForNode({}); + const executables = await $provideExecutablesForNode(); - console.info( - spaceTrim( - (block) => ` + console.info( + spaceTrim( + (block) => ` ${block($registeredScrapersMessage(scrapers))} All mime-types which can be scraped: ${block( - Array.from(new Set(Object.values(scrapers).flatMap(({ metadata }) => metadata.mimeTypes))) - .map((mimeType, i) => `${i + 1}) ${mimeType}`) - .join('\n'), - )} + Array.from( + new Set( + Object.values(scrapers).flatMap( + ({ metadata }) => metadata.mimeTypes, + ), + ), + ) + .map((mimeType, i) => `${i + 1}) ${mimeType}`) + .join("\n"), + )} Available executables: ${block( - Object.entries(executables) - .map(([name, path], i) => `${i + 1}) **${name}** ${path}`) - .join('\n'), - )} + Object.entries(executables) + .map( + ([name, path], i) => `${i + 1}) **${name}** ${path}`, + ) + .join("\n"), + )} `, - ), - ); - return process.exit(0); - }); + ), + ); + return process.exit(0); + }); } /** diff --git a/src/cli/cli-commands/make.ts b/src/cli/cli-commands/make.ts index 923b13439b..d43ba4b6a6 100644 --- a/src/cli/cli-commands/make.ts +++ b/src/cli/cli-commands/make.ts @@ -1,31 +1,31 @@ -import colors from 'colors'; +import { dirname, join } from "path"; +import colors from "colors"; import type { - Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, -} from 'commander'; -import { mkdir, writeFile } from 'fs/promises'; -import { dirname, join } from 'path'; -import spaceTrim from 'spacetrim'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import { collectionToJson } from '../../collection/collectionToJson'; -import { createCollectionFromDirectory } from '../../collection/constructors/createCollectionFromDirectory'; -import { DEFAULT_BOOKS_DIRNAME } from '../../config'; -import { DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME } from '../../config'; -import { DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME } from '../../config'; -import { GENERATOR_WARNING_BY_PROMPTBOOK_CLI } from '../../config'; -import { saveArchive } from '../../conversion/archive/saveArchive'; -import { validatePipeline } from '../../conversion/validation/validatePipeline'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import { usageToHuman } from '../../execution/utils/usageToHuman'; -import { $provideLlmToolsForWizzardOrCli } from '../../llm-providers/_common/register/$provideLlmToolsForWizzardOrCli'; -import { $provideFilesystemForNode } from '../../scrapers/_common/register/$provideFilesystemForNode'; -import { $provideScrapersForNode } from '../../scrapers/_common/register/$provideScrapersForNode'; -import type { string_file_extension } from '../../types/typeAliases'; -import { stringifyPipelineJson } from '../../utils/editable/utils/stringifyPipelineJson'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import { isValidJavascriptName } from '../../utils/validators/javascriptName/isValidJavascriptName'; -import { isValidUrl } from '../../utils/validators/url/isValidUrl'; + Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, +} from "commander"; +import { mkdir, writeFile } from "fs/promises"; +import spaceTrim from "spacetrim"; +import { collectionToJson } from "../../collection/collectionToJson"; +import { createCollectionFromDirectory } from "../../collection/constructors/createCollectionFromDirectory"; +import { DEFAULT_BOOKS_DIRNAME } from "../../config"; +import { DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME } from "../../config"; +import { DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME } from "../../config"; +import { GENERATOR_WARNING_BY_PROMPTBOOK_CLI } from "../../config"; +import { saveArchive } from "../../conversion/archive/saveArchive"; +import { validatePipeline } from "../../conversion/validation/validatePipeline"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import { $provideExecutablesForNode } from "../../executables/$provideExecutablesForNode"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import { usageToHuman } from "../../execution/utils/usageToHuman"; +import { $provideLlmToolsForWizzardOrCli } from "../../llm-providers/_common/register/$provideLlmToolsForWizzardOrCli"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import { $provideFilesystemForNode } from "../../scrapers/_common/register/$provideFilesystemForNode"; +import { $provideScrapersForNode } from "../../scrapers/_common/register/$provideScrapersForNode"; +import type { string_file_extension } from "../../types/typeAliases"; +import { stringifyPipelineJson } from "../../utils/editable/utils/stringifyPipelineJson"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import { isValidJavascriptName } from "../../utils/validators/javascriptName/isValidJavascriptName"; +import { isValidUrl } from "../../utils/validators/url/isValidUrl"; keepTypeImported(); @@ -37,223 +37,272 @@ keepTypeImported(); * @private internal function of `promptbookCli` */ export function $initializeMakeCommand(program: Program) { - const makeCommand = program.command('make'); - makeCommand.description( - spaceTrim(` + const makeCommand = program.command("make"); + makeCommand.description( + spaceTrim(` Makes a new pipeline collection in given folder `), - ); - - makeCommand.alias('compile'); - makeCommand.alias('prepare'); - makeCommand.alias('build'); - - // TODO: [🧅] DRY command arguments - - makeCommand.argument( - '[path]', - // <- TODO: [🧟‍♂️] Unite path to promptbook collection argument - 'Path to promptbook collection directory', - DEFAULT_BOOKS_DIRNAME, - ); - makeCommand.option('--project-name', `Name of the project for whom collection is`, 'Untitled Promptbook project'); - makeCommand.option('--root-url ', `Root URL of all pipelines to make`, undefined); - makeCommand.option( - '-f, --format ', - spaceTrim(` + ); + + makeCommand.alias("compile"); + makeCommand.alias("prepare"); + makeCommand.alias("build"); + + // TODO: [🧅] DRY command arguments + + makeCommand.argument( + "[path]", + // <- TODO: [🧟‍♂️] Unite path to promptbook collection argument + "Path to promptbook collection directory", + DEFAULT_BOOKS_DIRNAME, + ); + makeCommand.option( + "--project-name", + `Name of the project for whom collection is`, + "Untitled Promptbook project", + ); + makeCommand.option( + "--root-url ", + `Root URL of all pipelines to make`, + undefined, + ); + makeCommand.option( + "-f, --format ", + spaceTrim(` Output format of builded collection "bookc", "javascript", "typescript" or "json" Note: You can use multiple formats separated by comma `), - 'bookc' /* <- Note: [🏳‍🌈] */, - ); - makeCommand.option('--no-validation', `Do not validate logic of pipelines in collection`, true); - makeCommand.option( - '--validation', - `Types of validations separated by comma (options "logic","imports")`, - 'logic,imports', - ); - - makeCommand.option('-r, --reload', `Call LLM models even if same prompt with result is in the cache`, false); - makeCommand.option('-v, --verbose', `Is output verbose`, false); - makeCommand.option( - '-o, --output ', - spaceTrim(` + "bookc" /* <- Note: [🏳‍🌈] */, + ); + makeCommand.option( + "--no-validation", + `Do not validate logic of pipelines in collection`, + true, + ); + makeCommand.option( + "--validation", + `Types of validations separated by comma (options "logic","imports")`, + "logic,imports", + ); + + makeCommand.option( + "-r, --reload", + `Call LLM models even if same prompt with result is in the cache`, + false, + ); + makeCommand.option("-v, --verbose", `Is output verbose`, false); + makeCommand.option( + "-o, --output ", + spaceTrim(` Where to save the builded collection Note: If you keep it "${DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME}" it will be saved in the root of the promptbook directory If you set it to a path, it will be saved in that path BUT you can use only one format and set correct extension `), - DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, - ); - makeCommand.option( - '-fn, --function-name ', - spaceTrim(` + DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, + ); + makeCommand.option( + "-fn, --function-name ", + spaceTrim(` Name of the function to get pipeline collection Note: This can be used only with "javascript" or "typescript" format `), - DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME, - ); - - makeCommand.action( - async ( - path, - { - projectName, - rootUrl, - format, - functionName, - validation, - reload: isCacheReloaded, - verbose: isVerbose, - output, - }, - ) => { - if (!isValidJavascriptName(functionName)) { - console.error(colors.red(`Function name "${functionName}" is not valid javascript name`)); - return process.exit(1); - } - - if ( - rootUrl !== undefined && - !isValidUrl(rootUrl) /* <- Note: Not using `isValidPipelineUrl` because this is root url not book url */ - ) { - console.error(colors.red(`Root URL ${rootUrl} is not valid URL`)); - return process.exit(1); - } - - let formats = ((format as string | false) || '') - .split(',') - .map((_) => _.trim()) - .filter((_) => _ !== ''); - const validations = ((validation as string | false) || '') - .split(',') - .map((_) => _.trim()) - .filter((_) => _ !== ''); - - if (output !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) { - console.error(colors.red(`You can only use one format if you specify --out-file`)); - return process.exit(1); - } - - // TODO: DRY [◽] - const prepareAndScrapeOptions = { - isVerbose, - isCacheReloaded, - }; /* <- TODO: ` satisfies PrepareAndScrapeOptions` */ - const fs = $provideFilesystemForNode(prepareAndScrapeOptions); - const llm = await $provideLlmToolsForWizzardOrCli(prepareAndScrapeOptions); - const executables = await $provideExecutablesForNode(prepareAndScrapeOptions); - const tools = { - llm, - fs, - - scrapers: await $provideScrapersForNode({ fs, llm, executables }, prepareAndScrapeOptions), - script: [ - /*new JavascriptExecutionTools(options)*/ - ], - } satisfies ExecutionTools; - - // TODO: [🧟‍♂️][◽] DRY: - const collection = await createCollectionFromDirectory(path, tools, { - isVerbose, - rootUrl, - isRecursive: true, - isLazyLoaded: false, - isCrashedOnError: true, - // <- TODO: [🍖] Add `intermediateFilesStrategy` - }); - - const pipelinesUrls = await collection.listPipelines(); - - if (pipelinesUrls.length === 0) { - console.error(colors.red(`No books found in "${path}"`)); - return process.exit(1); - } - - for (const validation of validations) { - for (const pipelineUrl of pipelinesUrls) { - const pipeline = await collection.getPipelineByUrl(pipelineUrl); - - if (validation === 'logic') { - validatePipeline(pipeline); - - if (isVerbose) { - console.info(colors.cyan(`Validated logic of ${pipeline.pipelineUrl}`)); - } - } - - // TODO: Imports validation - } - } - - const collectionJson = await collectionToJson(collection); - - const collectionJsonString = stringifyPipelineJson(collectionJson).trim(); - const collectionJsonItems = (() => { - const firstChar = collectionJsonString.charAt(0); - - if (firstChar !== '[') { - throw new UnexpectedError( - `First character of serialized collection should be "[" not "${firstChar}"`, - ); - } - - const lastChar = collectionJsonString.charAt(collectionJsonString.length - 1); - if (lastChar !== ']') { - throw new UnexpectedError( - `Last character of serialized collection should be "]" not "${lastChar}"`, - ); - } - - return spaceTrim(collectionJsonString.substring(1, collectionJsonString.length - 1)); - })(); - - const saveFile = async ( - extension: string_file_extension, - content: string | ReadonlyArray, - ) => { - const filename = - output !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME - ? output - : join(path, `${DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME}.${extension}`); - - if (!output.endsWith(`.${extension}`)) { - console.warn(colors.yellow(`Warning: Extension of output file should be "${extension}"`)); - } - - await mkdir(dirname(filename), { recursive: true }); - - if (typeof content === 'string') { - await writeFile(filename, content, 'utf-8'); - } else { - await saveArchive(filename, content, fs); - } - - // Note: Log despite of verbose mode - console.info(colors.green(`Made ${filename.split('\\').join('/')}`)); - }; - - if (formats.includes('bookc')) { - formats = formats.filter((format) => format !== 'bookc'); - await saveFile('bookc', collectionJson); - } - - if (formats.includes('json')) { - formats = formats.filter((format) => format !== 'json'); - await saveFile('json', collectionJsonString); - // <- TODO: Add GENERATOR_WARNING_BY_PROMPTBOOK_CLI to package.json - } - - if (formats.includes('javascript') || formats.includes('js')) { - formats = formats.filter((format) => format !== 'javascript' && format !== 'js'); - (await saveFile( - 'js', - spaceTrim( - (block) => ` + DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME, + ); + + makeCommand.action( + async ( + path, + { + projectName, + rootUrl, + format, + functionName, + validation, + reload: isCacheReloaded, + verbose: isVerbose, + output, + }, + ) => { + if (!isValidJavascriptName(functionName)) { + console.error( + colors.red( + `Function name "${functionName}" is not valid javascript name`, + ), + ); + return process.exit(1); + } + + if ( + rootUrl !== undefined && + !isValidUrl( + rootUrl, + ) /* <- Note: Not using `isValidPipelineUrl` because this is root url not book url */ + ) { + console.error(colors.red(`Root URL ${rootUrl} is not valid URL`)); + return process.exit(1); + } + + let formats = ((format as string | false) || "") + .split(",") + .map((_) => _.trim()) + .filter((_) => _ !== ""); + const validations = ((validation as string | false) || "") + .split(",") + .map((_) => _.trim()) + .filter((_) => _ !== ""); + + if ( + output !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME && + formats.length !== 1 + ) { + console.error( + colors.red(`You can only use one format if you specify --out-file`), + ); + return process.exit(1); + } + + // TODO: DRY [◽] + const prepareAndScrapeOptions = { + isVerbose, + isCacheReloaded, + }; /* <- TODO: ` satisfies PrepareAndScrapeOptions` */ + const fs = $provideFilesystemForNode(prepareAndScrapeOptions); + const llm = await $provideLlmToolsForWizzardOrCli( + prepareAndScrapeOptions, + ); + const executables = await $provideExecutablesForNode( + prepareAndScrapeOptions, + ); + const tools = { + llm, + fs, + + scrapers: await $provideScrapersForNode( + { fs, llm, executables }, + prepareAndScrapeOptions, + ), + script: [ + /*new JavascriptExecutionTools(options)*/ + ], + } satisfies ExecutionTools; + + // TODO: [🧟‍♂️][◽] DRY: + const collection = await createCollectionFromDirectory(path, tools, { + isVerbose, + rootUrl, + isRecursive: true, + isLazyLoaded: false, + isCrashedOnError: true, + // <- TODO: [🍖] Add `intermediateFilesStrategy` + }); + + const pipelinesUrls = await collection.listPipelines(); + + if (pipelinesUrls.length === 0) { + console.error(colors.red(`No books found in "${path}"`)); + return process.exit(1); + } + + for (const validation of validations) { + for (const pipelineUrl of pipelinesUrls) { + const pipeline = await collection.getPipelineByUrl(pipelineUrl); + + if (validation === "logic") { + validatePipeline(pipeline); + + if (isVerbose) { + console.info( + colors.cyan(`Validated logic of ${pipeline.pipelineUrl}`), + ); + } + } + + // TODO: Imports validation + } + } + + const collectionJson = await collectionToJson(collection); + + const collectionJsonString = stringifyPipelineJson(collectionJson).trim(); + const collectionJsonItems = (() => { + const firstChar = collectionJsonString.charAt(0); + + if (firstChar !== "[") { + throw new UnexpectedError( + `First character of serialized collection should be "[" not "${firstChar}"`, + ); + } + + const lastChar = collectionJsonString.charAt( + collectionJsonString.length - 1, + ); + if (lastChar !== "]") { + throw new UnexpectedError( + `Last character of serialized collection should be "]" not "${lastChar}"`, + ); + } + + return spaceTrim( + collectionJsonString.substring(1, collectionJsonString.length - 1), + ); + })(); + + const saveFile = async ( + extension: string_file_extension, + content: string | ReadonlyArray, + ) => { + const filename = + output !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME + ? output + : join( + path, + `${DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME}.${extension}`, + ); + + if (!output.endsWith(`.${extension}`)) { + console.warn( + colors.yellow( + `Warning: Extension of output file should be "${extension}"`, + ), + ); + } + + await mkdir(dirname(filename), { recursive: true }); + + if (typeof content === "string") { + await writeFile(filename, content, "utf-8"); + } else { + await saveArchive(filename, content, fs); + } + + // Note: Log despite of verbose mode + console.info(colors.green(`Made ${filename.split("\\").join("/")}`)); + }; + + if (formats.includes("bookc")) { + formats = formats.filter((format) => format !== "bookc"); + await saveFile("bookc", collectionJson); + } + + if (formats.includes("json")) { + formats = formats.filter((format) => format !== "json"); + await saveFile("json", collectionJsonString); + // <- TODO: Add GENERATOR_WARNING_BY_PROMPTBOOK_CLI to package.json + } + + if (formats.includes("javascript") || formats.includes("js")) { + formats = formats.filter( + (format) => format !== "javascript" && format !== "js", + ); + (await saveFile( + "js", + spaceTrim( + (block) => ` // ${block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI)} import { createCollectionFromJson } from '@promptbook/core'; @@ -287,20 +336,22 @@ export function $initializeMakeCommand(program: Program) { return pipelineCollection; } `, - ), - // <- TODO: [0] DRY Javascript and typescript - // <- TODO: Prettify - // <- TODO: Convert inlined \n to spaceTrim - // <- Note: [🍡] - )) + '\n'; - } - - if (formats.includes('typescript') || formats.includes('ts')) { - formats = formats.filter((format) => format !== 'typescript' && format !== 'ts'); - await saveFile( - 'ts', - spaceTrim( - (block) => ` + ), + // <- TODO: [0] DRY Javascript and typescript + // <- TODO: Prettify + // <- TODO: Convert inlined \n to spaceTrim + // <- Note: [🍡] + )) + "\n"; + } + + if (formats.includes("typescript") || formats.includes("ts")) { + formats = formats.filter( + (format) => format !== "typescript" && format !== "ts", + ); + await saveFile( + "ts", + spaceTrim( + (block) => ` // ${block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI)} import { createCollectionFromJson } from '@promptbook/core'; @@ -338,26 +389,28 @@ export function $initializeMakeCommand(program: Program) { return pipelineCollection; } `, - ) + '\n', - // <- TODO: [0] DRY Javascript and typescript - // <- TODO: Prettify - // <- TODO: Convert inlined \n to spaceTrim - // <- Note: [🍡] - ); - } - - if (formats.length > 0) { - console.warn(colors.yellow(`Format ${formats.join(' and ')} is not supported`)); - } - - console.info(colors.green(`Collection builded successfully`)); - if (isVerbose) { - console.info(colors.cyan(usageToHuman(llm.getTotalUsage()))); - } - - return process.exit(0); - }, - ); + ) + "\n", + // <- TODO: [0] DRY Javascript and typescript + // <- TODO: Prettify + // <- TODO: Convert inlined \n to spaceTrim + // <- Note: [🍡] + ); + } + + if (formats.length > 0) { + console.warn( + colors.yellow(`Format ${formats.join(" and ")} is not supported`), + ); + } + + console.info(colors.green(`Collection builded successfully`)); + if (isVerbose) { + console.info(colors.cyan(usageToHuman(llm.getTotalUsage()))); + } + + return process.exit(0); + }, + ); } /** diff --git a/src/cli/cli-commands/prettify.ts b/src/cli/cli-commands/prettify.ts index 6ee01cb6a2..a1cf58adbf 100644 --- a/src/cli/cli-commands/prettify.ts +++ b/src/cli/cli-commands/prettify.ts @@ -1,12 +1,12 @@ -import colors from 'colors'; +import colors from "colors"; import type { - Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, -} from 'commander'; -import { readFile, writeFile } from 'fs/promises'; -import glob from 'glob-promise'; -import spaceTrim from 'spacetrim'; -import { prettifyPipelineString } from '../../conversion/prettify/prettifyPipelineString'; -import { validatePipelineString } from '../../pipeline/validatePipelineString'; + Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, +} from "commander"; +import { readFile, writeFile } from "fs/promises"; +import glob from "glob-promise"; +import spaceTrim from "spacetrim"; +import { prettifyPipelineString } from "../../conversion/prettify/prettifyPipelineString"; +import { validatePipelineString } from "../../pipeline/validatePipelineString"; /** * Initializes `prettify` command for Promptbook CLI utilities @@ -16,64 +16,66 @@ import { validatePipelineString } from '../../pipeline/validatePipelineString'; * @private internal function of `promptbookCli` */ export function $initializePrettifyCommand(program: Program) { - const prettifyCommand = program.command('prettify'); - prettifyCommand.description( - spaceTrim(` + const prettifyCommand = program.command("prettify"); + prettifyCommand.description( + spaceTrim(` Iterates over \`.book.md\` files and does multiple enhancing operations on them: 1) Adds Mermaid graph 2) Prettifies the markdown `), - ); + ); - prettifyCommand.argument( - '', - // <- TODO: [🧟‍♂️] Unite path to promptbook collection argument - 'Pipelines to prettify as glob pattern', - ); - prettifyCommand.option('-i, --ignore ', `Ignore as glob pattern`); - prettifyCommand.option('-v, --verbose', `Is output verbose`, false); + prettifyCommand.argument( + "", + // <- TODO: [🧟‍♂️] Unite path to promptbook collection argument + "Pipelines to prettify as glob pattern", + ); + prettifyCommand.option("-i, --ignore ", `Ignore as glob pattern`); + prettifyCommand.option("-v, --verbose", `Is output verbose`, false); - prettifyCommand.action(async (filesGlob, { ignore, verbose: isVerbose }) => { - const filenames = await glob(filesGlob!, { ignore }); - // <- TODO: [😶] + prettifyCommand.action(async (filesGlob, { ignore, verbose: isVerbose }) => { + const filenames = await glob(filesGlob!, { ignore }); + // <- TODO: [😶] - for (const filename of filenames) { - if (!filename.endsWith('.book') && isVerbose) { - console.info(colors.gray(`Skipping ${filename}`)); - continue; - } + for (const filename of filenames) { + if (!filename.endsWith(".book") && isVerbose) { + console.info(colors.gray(`Skipping ${filename}`)); + continue; + } - let pipelineMarkdown = validatePipelineString(await readFile(filename, 'utf-8')); + let pipelineMarkdown = validatePipelineString( + await readFile(filename, "utf-8"), + ); - try { - pipelineMarkdown = await prettifyPipelineString(pipelineMarkdown, { - isGraphAdded: true, - isPrettifyed: true, - // <- [🕌] - }); + try { + pipelineMarkdown = await prettifyPipelineString(pipelineMarkdown, { + isGraphAdded: true, + isPrettifyed: true, + // <- [🕌] + }); - await writeFile(filename, pipelineMarkdown); + await writeFile(filename, pipelineMarkdown); - if (isVerbose) { - console.info(colors.green(`Prettify ${filename}`)); - } - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } + if (isVerbose) { + console.info(colors.green(`Prettify ${filename}`)); + } + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } - console.info(colors.red(`Prettify ${error.name} ${filename}`)); - console.error(colors.bgRed(error.name /* <- 11:11 */)); - console.error(colors.red(error.stack || error.message)); + console.info(colors.red(`Prettify ${error.name} ${filename}`)); + console.error(colors.bgRed(error.name /* <- 11:11 */)); + console.error(colors.red(error.stack || error.message)); - return process.exit(1); - } - } + return process.exit(1); + } + } - console.info(colors.green(`All pipelines are prettified`)); - return process.exit(0); - }); + console.info(colors.green(`All pipelines are prettified`)); + return process.exit(0); + }); } /** diff --git a/src/cli/cli-commands/run.ts b/src/cli/cli-commands/run.ts index 46f254e45f..4264c6fc88 100644 --- a/src/cli/cli-commands/run.ts +++ b/src/cli/cli-commands/run.ts @@ -1,35 +1,35 @@ -import colors from 'colors'; +import { join } from "path"; +import colors from "colors"; import type { - Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, -} from 'commander'; -import { writeFile } from 'fs/promises'; -import { join } from 'path'; -import prompts from 'prompts'; -import spaceTrim from 'spacetrim'; -import { normalizeToKebabCase } from '../../utils/normalization/normalize-to-kebab-case'; -import { validatePipeline } from '../../conversion/validation/validatePipeline'; -import { ParseError } from '../../errors/ParseError'; -import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode'; -import { createPipelineExecutor } from '../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import { executionReportJsonToString } from '../../execution/execution-report/executionReportJsonToString'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import { usageToHuman } from '../../execution/utils/usageToHuman'; -import { $llmToolsMetadataRegister } from '../../llm-providers/_common/register/$llmToolsMetadataRegister'; -import { $provideLlmToolsForWizzardOrCli } from '../../llm-providers/_common/register/$provideLlmToolsForWizzardOrCli'; -import { $registeredLlmToolsMessage } from '../../llm-providers/_common/register/$registeredLlmToolsMessage'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import { $provideFilesystemForNode } from '../../scrapers/_common/register/$provideFilesystemForNode'; -import { $provideScrapersForNode } from '../../scrapers/_common/register/$provideScrapersForNode'; -import { scraperFetch } from '../../scrapers/_common/utils/scraperFetch'; -import type { string_parameter_name } from '../../types/typeAliases'; -import type { string_parameter_value } from '../../types/typeAliases'; -import { countLines } from '../../utils/expectation-counters/countLines'; -import { countWords } from '../../utils/expectation-counters/countWords'; -import { isFileExisting } from '../../utils/files/isFileExisting'; -import type { TODO_any } from '../../utils/organization/TODO_any'; -import { $getCompiledBook } from '../../wizzard/$getCompiledBook'; -import { runInteractiveChatbot } from './runInteractiveChatbot'; + Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, +} from "commander"; +import { writeFile } from "fs/promises"; +import prompts from "prompts"; +import spaceTrim from "spacetrim"; +import { validatePipeline } from "../../conversion/validation/validatePipeline"; +import { ParseError } from "../../errors/ParseError"; +import { $provideExecutablesForNode } from "../../executables/$provideExecutablesForNode"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import { createPipelineExecutor } from "../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { executionReportJsonToString } from "../../execution/execution-report/executionReportJsonToString"; +import { usageToHuman } from "../../execution/utils/usageToHuman"; +import { $llmToolsMetadataRegister } from "../../llm-providers/_common/register/$llmToolsMetadataRegister"; +import { $provideLlmToolsForWizzardOrCli } from "../../llm-providers/_common/register/$provideLlmToolsForWizzardOrCli"; +import { $registeredLlmToolsMessage } from "../../llm-providers/_common/register/$registeredLlmToolsMessage"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import { $provideFilesystemForNode } from "../../scrapers/_common/register/$provideFilesystemForNode"; +import { $provideScrapersForNode } from "../../scrapers/_common/register/$provideScrapersForNode"; +import { scraperFetch } from "../../scrapers/_common/utils/scraperFetch"; +import type { string_parameter_name } from "../../types/typeAliases"; +import type { string_parameter_value } from "../../types/typeAliases"; +import { countLines } from "../../utils/expectation-counters/countLines"; +import { countWords } from "../../utils/expectation-counters/countWords"; +import { isFileExisting } from "../../utils/files/isFileExisting"; +import { normalizeToKebabCase } from "../../utils/normalization/normalize-to-kebab-case"; +import type { TODO_any } from "../../utils/organization/TODO_any"; +import { $getCompiledBook } from "../../wizzard/$getCompiledBook"; +import { runInteractiveChatbot } from "./runInteractiveChatbot"; /** * Initializes `run` command for Promptbook CLI utilities @@ -39,89 +39,108 @@ import { runInteractiveChatbot } from './runInteractiveChatbot'; * @private internal function of `promptbookCli` */ export function $initializeRunCommand(program: Program) { - const runCommand = program.command('run', { isDefault: true }); - runCommand.description( - spaceTrim(` + const runCommand = program.command("run", { isDefault: true }); + runCommand.description( + spaceTrim(` Runs a pipeline `), - ); - - runCommand.alias('execute'); - - // TODO: [🧅] DRY command arguments - - runCommand.argument('[pipelineSource]', 'Path to book file OR URL to book file, if not provided it will be asked'); - runCommand.option('-r, --reload', `Call LLM models even if same prompt with result is in the cache`, false); - runCommand.option('-v, --verbose', `Is output verbose`, false); - runCommand.option( - '--no-interactive', - `Input is not interactive, if true you need to pass all the input parameters through --json`, - ); - runCommand.option( - '--no-formfactor', - `When set, behavior of the interactive mode is not changed by the formfactor of the pipeline`, - ); - runCommand.option( - '-j, --json ', - `Pass all or some input parameters as JSON record, if used the output is also returned as JSON`, - ); - runCommand.option('-s, --save-report ', `Save report to file`); - - runCommand.action(async (pipelineSource, options) => { - const { - reload: isCacheReloaded, - interactive: isInteractive, - formfactor: isFormfactorUsed, - json, - verbose: isVerbose, - saveReport, - } = options; - - if (pipelineSource.includes('-') && normalizeToKebabCase(pipelineSource) === pipelineSource) { - console.error(colors.red(`""${pipelineSource}" is not a valid command or book. See 'ptbk --help'.`)); - return process.exit(1); - } - - if (saveReport && !saveReport.endsWith('.json') && !saveReport.endsWith('.md')) { - console.error(colors.red(`Report file must be .json or .md`)); - return process.exit(1); - } - - let inputParameters: Record = {}; - - if (json) { - inputParameters = JSON.parse(json); - // <- TODO: Maybe check shape of passed JSON and if its valid parameters Record - } - - // TODO: DRY [◽] - const prepareAndScrapeOptions = { - isVerbose, - isCacheReloaded, - }; /* <- TODO: ` satisfies PrepareAndScrapeOptions` */ - - if (isVerbose) { - console.info(colors.gray('--- Preparing tools ---')); - } - - const fs = $provideFilesystemForNode(prepareAndScrapeOptions); - - let llm: LlmExecutionTools; - - try { - llm = await $provideLlmToolsForWizzardOrCli(prepareAndScrapeOptions); - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - if (!error.message.includes('No LLM tools')) { - throw error; - } - - console.error( - colors.red( - spaceTrim( - (block) => ` + ); + + runCommand.alias("execute"); + + // TODO: [🧅] DRY command arguments + + runCommand.argument( + "[pipelineSource]", + "Path to book file OR URL to book file, if not provided it will be asked", + ); + runCommand.option( + "-r, --reload", + `Call LLM models even if same prompt with result is in the cache`, + false, + ); + runCommand.option("-v, --verbose", `Is output verbose`, false); + runCommand.option( + "--no-interactive", + `Input is not interactive, if true you need to pass all the input parameters through --json`, + ); + runCommand.option( + "--no-formfactor", + `When set, behavior of the interactive mode is not changed by the formfactor of the pipeline`, + ); + runCommand.option( + "-j, --json ", + `Pass all or some input parameters as JSON record, if used the output is also returned as JSON`, + ); + runCommand.option("-s, --save-report ", `Save report to file`); + + runCommand.action(async (pipelineSource, options) => { + const { + reload: isCacheReloaded, + interactive: isInteractive, + formfactor: isFormfactorUsed, + json, + verbose: isVerbose, + saveReport, + } = options; + + if ( + pipelineSource.includes("-") && + normalizeToKebabCase(pipelineSource) === pipelineSource + ) { + console.error( + colors.red( + `""${pipelineSource}" is not a valid command or book. See 'ptbk --help'.`, + ), + ); + return process.exit(1); + } + + if ( + saveReport && + !saveReport.endsWith(".json") && + !saveReport.endsWith(".md") + ) { + console.error(colors.red(`Report file must be .json or .md`)); + return process.exit(1); + } + + let inputParameters: Record = + {}; + + if (json) { + inputParameters = JSON.parse(json); + // <- TODO: Maybe check shape of passed JSON and if its valid parameters Record + } + + // TODO: DRY [◽] + const prepareAndScrapeOptions = { + isVerbose, + isCacheReloaded, + }; /* <- TODO: ` satisfies PrepareAndScrapeOptions` */ + + if (isVerbose) { + console.info(colors.gray("--- Preparing tools ---")); + } + + const fs = $provideFilesystemForNode(prepareAndScrapeOptions); + + let llm: LlmExecutionTools; + + try { + llm = await $provideLlmToolsForWizzardOrCli(prepareAndScrapeOptions); + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + if (!error.message.includes("No LLM tools")) { + throw error; + } + + console.error( + colors.red( + spaceTrim( + (block) => ` You need to configure LLM tools first 1) Create .env file at the root of your project @@ -129,280 +148,326 @@ export function $initializeRunCommand(program: Program) { For example: ${block( - $llmToolsMetadataRegister - .list() - .map( - ({ title, envVariables }) => `- ${(envVariables || []).join(' + ')} (${title})`, - ) - .join('\n'), - )} + $llmToolsMetadataRegister + .list() + .map( + ({ title, envVariables }) => + `- ${(envVariables || []).join(" + ")} (${title})`, + ) + .join("\n"), + )} ${block($registeredLlmToolsMessage())} `, - ), - ), - ); - - // TODO: Maybe allow to sign-in as Promptbook.studio user here - - if (!(await isFileExisting('.env', fs))) { - await writeFile( - join(process.cwd(), '.env'), - $llmToolsMetadataRegister - .list() - .flatMap(({ title, envVariables }) => - envVariables === null - ? [] - : [`# ${title}`, ...envVariables.map((varname) => `# ${varname}=...`), ''], - ) - .join('\n'), - 'utf8', - ); - } - - // TODO: If the cwd is git repository, auto-create .gitignore and add .env to it - - return process.exit(1); - } - - if (!pipelineSource) { - const response = await prompts({ - type: 'text', - name: 'pipelineSource', - message: '', // <- TODO: [🧠] What is the message here - validate: (value) => (value.length > 0 ? true : 'Pipeline source is required'), - }); - - if (!response.pipelineSource) { - console.error(colors.red('Pipeline source is required')); - return process.exit(1); - } - - pipelineSource = response.pipelineSource; - } - - if (isVerbose) { - console.info(colors.gray('--- Getting the tools ---')); - } - - const executables = await $provideExecutablesForNode(prepareAndScrapeOptions); - const tools = { - llm, - fs, - fetch: scraperFetch, - scrapers: await $provideScrapersForNode({ fs, llm, executables }, prepareAndScrapeOptions), - script: [ - /*new JavascriptExecutionTools(options)*/ - ], - } satisfies ExecutionTools; - - if (isVerbose) { - console.info(colors.gray('--- Getting the book ---')); - } - - let pipeline: PipelineJson; - try { - pipeline = await $getCompiledBook(tools, pipelineSource, prepareAndScrapeOptions); - } catch (error) { - if (!(error instanceof ParseError)) { - throw error; - } - - console.error( - colors.red( - spaceTrim( - (block) => ` + ), + ), + ); + + // TODO: Maybe allow to sign-in as Promptbook.studio user here + + if (!(await isFileExisting(".env", fs))) { + await writeFile( + join(process.cwd(), ".env"), + $llmToolsMetadataRegister + .list() + .flatMap(({ title, envVariables }) => + envVariables === null + ? [] + : [ + `# ${title}`, + ...envVariables.map((varname) => `# ${varname}=...`), + "", + ], + ) + .join("\n"), + "utf8", + ); + } + + // TODO: If the cwd is git repository, auto-create .gitignore and add .env to it + + return process.exit(1); + } + + if (!pipelineSource) { + const response = await prompts({ + type: "text", + name: "pipelineSource", + message: "", // <- TODO: [🧠] What is the message here + validate: (value) => + value.length > 0 ? true : "Pipeline source is required", + }); + + if (!response.pipelineSource) { + console.error(colors.red("Pipeline source is required")); + return process.exit(1); + } + + pipelineSource = response.pipelineSource; + } + + if (isVerbose) { + console.info(colors.gray("--- Getting the tools ---")); + } + + const executables = await $provideExecutablesForNode( + prepareAndScrapeOptions, + ); + const tools = { + llm, + fs, + fetch: scraperFetch, + scrapers: await $provideScrapersForNode( + { fs, llm, executables }, + prepareAndScrapeOptions, + ), + script: [ + /*new JavascriptExecutionTools(options)*/ + ], + } satisfies ExecutionTools; + + if (isVerbose) { + console.info(colors.gray("--- Getting the book ---")); + } + + let pipeline: PipelineJson; + try { + pipeline = await $getCompiledBook( + tools, + pipelineSource, + prepareAndScrapeOptions, + ); + } catch (error) { + if (!(error instanceof ParseError)) { + throw error; + } + + console.error( + colors.red( + spaceTrim( + (block) => ` ${block((error as ParseError).message)} in ${pipelineSource} `, - ), - ), - ); - return process.exit(1); - } - - if (isVerbose) { - console.info(colors.gray('--- Validating pipeline ---')); - } - - // TODO: Same try-catch for LogicError - validatePipeline(pipeline); - - if (isVerbose) { - console.info(colors.gray('--- Creating executor ---')); - } - - const pipelineExecutor = createPipelineExecutor({ - pipeline, - tools, - isNotPreparedWarningSupressed: true, - maxExecutionAttempts: 3, // <- TODO: Pass via CLI argument - // <- TODO: Why "LLM execution failed undefinedx" - maxParallelCount: 1, // <- TODO: Pass CLI argument - }); - - // TODO: Make some better system for formfactors and interactive mode - here is just a quick hardcoded solution for chatbot - if (isInteractive === true && isFormfactorUsed === true && pipeline.formfactorName === 'CHATBOT') { - return /* not await */ runInteractiveChatbot({ pipeline, pipelineExecutor, isVerbose }); - } - - if (isVerbose) { - console.info(colors.gray('--- Getting input parameters ---')); - } - - const questions = pipeline.parameters - .filter(({ isInput }) => isInput) - .filter(({ name }) => typeof inputParameters[name] !== 'string') - .map(({ name, exampleValues }) => { - let message = name; - let initial = ''; - - if (exampleValues && exampleValues.length > 0) { - const exampleValuesFiltered = exampleValues.filter((exampleValue) => countLines(exampleValue) <= 1); - - if (exampleValuesFiltered.length !== 0) { - message += ` (e.g. ${exampleValuesFiltered.join(', ')})`; - } - - initial = exampleValues[0] || ''; - } - - return { - type: 'text', - name, - message, - initial, - // TODO: Maybe use> validate: value => value < 18 ? `Forbidden` : true - }; - }); - - if (isInteractive === false && questions.length !== 0) { - console.error( - colors.red( - spaceTrim( - (block) => ` + ), + ), + ); + return process.exit(1); + } + + if (isVerbose) { + console.info(colors.gray("--- Validating pipeline ---")); + } + + // TODO: Same try-catch for LogicError + validatePipeline(pipeline); + + if (isVerbose) { + console.info(colors.gray("--- Creating executor ---")); + } + + const pipelineExecutor = createPipelineExecutor({ + pipeline, + tools, + isNotPreparedWarningSupressed: true, + maxExecutionAttempts: 3, // <- TODO: Pass via CLI argument + // <- TODO: Why "LLM execution failed undefinedx" + maxParallelCount: 1, // <- TODO: Pass CLI argument + }); + + // TODO: Make some better system for formfactors and interactive mode - here is just a quick hardcoded solution for chatbot + if ( + isInteractive === true && + isFormfactorUsed === true && + pipeline.formfactorName === "CHATBOT" + ) { + return /* not await */ runInteractiveChatbot({ + pipeline, + pipelineExecutor, + isVerbose, + }); + } + + if (isVerbose) { + console.info(colors.gray("--- Getting input parameters ---")); + } + + const questions = pipeline.parameters + .filter(({ isInput }) => isInput) + .filter(({ name }) => typeof inputParameters[name] !== "string") + .map(({ name, exampleValues }) => { + let message = name; + let initial = ""; + + if (exampleValues && exampleValues.length > 0) { + const exampleValuesFiltered = exampleValues.filter( + (exampleValue) => countLines(exampleValue) <= 1, + ); + + if (exampleValuesFiltered.length !== 0) { + message += ` (e.g. ${exampleValuesFiltered.join(", ")})`; + } + + initial = exampleValues[0] || ""; + } + + return { + type: "text", + name, + message, + initial, + // TODO: Maybe use> validate: value => value < 18 ? `Forbidden` : true + }; + }); + + if (isInteractive === false && questions.length !== 0) { + console.error( + colors.red( + spaceTrim( + (block) => ` When using --no-interactive you need to pass all the input parameters through --json You are missing: ${block( - pipeline.parameters - .filter(({ isInput }) => isInput) - .filter( - ({ name: parameterName }) => - !questions.some( - ({ name: questionName }) => questionName === parameterName, - ), - ) - .map(({ name, description }) => `- **${name}** ${description}`) - .join('\n'), - )} + pipeline.parameters + .filter(({ isInput }) => isInput) + .filter( + ({ name: parameterName }) => + !questions.some( + ({ name: questionName }) => + questionName === parameterName, + ), + ) + .map( + ({ name, description }) => + `- **${name}** ${description}`, + ) + .join("\n"), + )} Example: --json '${JSON.stringify( - Object.fromEntries( - pipeline.parameters - .filter(({ isInput }) => isInput) - .map(({ name, exampleValues }) => [ - name, - inputParameters[name] || (exampleValues || [])[0] || '...', - ]), - ), - ) - .split("'") - .join("\\'")}' + Object.fromEntries( + pipeline.parameters + .filter(({ isInput }) => isInput) + .map(({ name, exampleValues }) => [ + name, + inputParameters[name] || + (exampleValues || [])[0] || + "...", + ]), + ), + ) + .split("'") + .join("\\'")}' `, - ), - ), - ); - return process.exit(1); - } - - const response = await prompts(questions as TODO_any); - // <- TODO: [🧠][🍼] Change behavior according to the formfactor - inputParameters = { ...inputParameters, ...response }; - - // TODO: Maybe do some validation of the response (and --json argument which is passed) - - if (isVerbose) { - console.info(colors.gray('--- Executing ---')); - } - - const executionTask = await pipelineExecutor(inputParameters); - - if (isVerbose) { - executionTask.asObservable().subscribe((partialResult) => { - console.info(colors.gray('--- Progress ---')); - console.info( - partialResult, - // <- TODO: Pretty print taskProgress - ); - }); - } - - const { isSuccessful, errors, warnings, outputParameters, executionReport, usage } = - await executionTask.asPromise({ - isCrashedOnError: false, - }); - - if (isVerbose) { - console.info(colors.gray('--- Detailed Result ---')); - - console.info( - { isSuccessful, errors, warnings, outputParameters, executionReport }, - // <- TODO: Pretty print taskProgress - ); - } - - if (saveReport && saveReport.endsWith('.json')) { - await writeFile(saveReport, JSON.stringify(executionReport, null, 4) + '\n', 'utf-8'); - } else if (saveReport && saveReport.endsWith('.md')) { - const executionReportString = executionReportJsonToString(executionReport); - await writeFile(saveReport, executionReportString, 'utf-8'); - } - - if (saveReport && isVerbose) { - console.info(colors.green(`Report saved to ${saveReport}`)); - } - - if (isVerbose) { - console.info(colors.gray('--- Usage ---')); - console.info(colors.cyan(usageToHuman(usage))); - } - - if (json === undefined || isVerbose === true) { - console.info(colors.gray('--- Result ---')); - } - - // TODO: [🧠] Should be errors or warnings shown first - - for (const error of errors || []) { - console.error(colors.red(colors.bold(error.name) + ': ' + error.message)); - } - - for (const warning of warnings || []) { - console.error(colors.red(colors.bold(warning.name) + ': ' + warning.message)); - } - - if (json === undefined) { - for (const key of Object.keys(outputParameters)) { - const value = outputParameters[key] || colors.grey(colors.italic('(nothing)')); - const separator = countLines(value) > 1 || countWords(value) > 100 ? ':\n' : ': '; - console.info(colors.green(colors.bold(key) + separator + value)); - } - } else { - console.info( - JSON.stringify( - outputParameters, - null, - 4, - // <- TODO: Allow to set --pretty - ), - ); - } - - return process.exit(0); - }); + ), + ), + ); + return process.exit(1); + } + + const response = await prompts(questions as TODO_any); + // <- TODO: [🧠][🍼] Change behavior according to the formfactor + inputParameters = { ...inputParameters, ...response }; + + // TODO: Maybe do some validation of the response (and --json argument which is passed) + + if (isVerbose) { + console.info(colors.gray("--- Executing ---")); + } + + const executionTask = await pipelineExecutor(inputParameters); + + if (isVerbose) { + executionTask.asObservable().subscribe((partialResult) => { + console.info(colors.gray("--- Progress ---")); + console.info( + partialResult, + // <- TODO: Pretty print taskProgress + ); + }); + } + + const { + isSuccessful, + errors, + warnings, + outputParameters, + executionReport, + usage, + } = await executionTask.asPromise({ + isCrashedOnError: false, + }); + + if (isVerbose) { + console.info(colors.gray("--- Detailed Result ---")); + + console.info( + { isSuccessful, errors, warnings, outputParameters, executionReport }, + // <- TODO: Pretty print taskProgress + ); + } + + if (saveReport && saveReport.endsWith(".json")) { + await writeFile( + saveReport, + JSON.stringify(executionReport, null, 4) + "\n", + "utf-8", + ); + } else if (saveReport && saveReport.endsWith(".md")) { + const executionReportString = + executionReportJsonToString(executionReport); + await writeFile(saveReport, executionReportString, "utf-8"); + } + + if (saveReport && isVerbose) { + console.info(colors.green(`Report saved to ${saveReport}`)); + } + + if (isVerbose) { + console.info(colors.gray("--- Usage ---")); + console.info(colors.cyan(usageToHuman(usage))); + } + + if (json === undefined || isVerbose === true) { + console.info(colors.gray("--- Result ---")); + } + + // TODO: [🧠] Should be errors or warnings shown first + + for (const error of errors || []) { + console.error(colors.red(colors.bold(error.name) + ": " + error.message)); + } + + for (const warning of warnings || []) { + console.error( + colors.red(colors.bold(warning.name) + ": " + warning.message), + ); + } + + if (json === undefined) { + for (const key of Object.keys(outputParameters)) { + const value = + outputParameters[key] || colors.grey(colors.italic("(nothing)")); + const separator = + countLines(value) > 1 || countWords(value) > 100 ? ":\n" : ": "; + console.info(colors.green(colors.bold(key) + separator + value)); + } + } else { + console.info( + JSON.stringify( + outputParameters, + null, + 4, + // <- TODO: Allow to set --pretty + ), + ); + } + + return process.exit(0); + }); } /** diff --git a/src/cli/cli-commands/runInteractiveChatbot.ts b/src/cli/cli-commands/runInteractiveChatbot.ts index 7d490692e8..74eb103f24 100644 --- a/src/cli/cli-commands/runInteractiveChatbot.ts +++ b/src/cli/cli-commands/runInteractiveChatbot.ts @@ -1,29 +1,29 @@ -import colors from 'colors'; -import prompts from 'prompts'; -import spaceTrim from 'spacetrim'; -import { forTime } from 'waitasecond'; -import type { PipelineExecutor } from '../../execution/PipelineExecutor'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import { just } from '../../utils/organization/just'; +import colors from "colors"; +import prompts from "prompts"; +import spaceTrim from "spacetrim"; +import { forTime } from "waitasecond"; +import type { PipelineExecutor } from "../../execution/PipelineExecutor"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import { just } from "../../utils/organization/just"; /** * Options for running the interactive chatbot */ type RunInteractiveChatbotOptions = { - /** - * Prepared pipeline to run - */ - pipeline: PipelineJson; - - /** - * Prepared pipeline executor - */ - pipelineExecutor: PipelineExecutor; - - /** - * Whether to show verbose output - */ - isVerbose: boolean; + /** + * Prepared pipeline to run + */ + pipeline: PipelineJson; + + /** + * Prepared pipeline executor + */ + pipelineExecutor: PipelineExecutor; + + /** + * Whether to show verbose output + */ + isVerbose: boolean; }; /** @@ -32,68 +32,74 @@ type RunInteractiveChatbotOptions = { * @returns Never-ending promise or process exit * @private internal function of `promptbookCli` and `initializeRunCommand` */ -export async function runInteractiveChatbot(options: RunInteractiveChatbotOptions): Promise { - const { pipeline, pipelineExecutor, isVerbose } = options; - - let ongoingParameters = { - /** - * Title of the conversation - */ - title: '', - - /** - * Summary of the conversation - */ - conversationSummary: '', - - /** - * Chatbot response - */ - chatbotResponse: '', - }; - - if (isVerbose) { - console.info(colors.gray('--- Running interactive chatbot ---')); - } - - const initialMessage = (pipeline.parameters.find(({ name }) => name === 'chatbotResponse')?.exampleValues || [])[0]; - - if (initialMessage) { - console.info(`\n`); - console.info( - spaceTrim( - (block) => ` - - ${colors.bold(colors.green('Chatbot:'))} +export async function runInteractiveChatbot( + options: RunInteractiveChatbotOptions, +): Promise { + const { pipeline, pipelineExecutor, isVerbose } = options; + + let ongoingParameters = { + /** + * Title of the conversation + */ + title: "", + + /** + * Summary of the conversation + */ + conversationSummary: "", + + /** + * Chatbot response + */ + chatbotResponse: "", + }; + + if (isVerbose) { + console.info(colors.gray("--- Running interactive chatbot ---")); + } + + const initialMessage = (pipeline.parameters.find( + ({ name }) => name === "chatbotResponse", + )?.exampleValues || [])[0]; + + if (initialMessage) { + console.info(`\n`); + console.info( + spaceTrim( + (block) => ` + + ${colors.bold(colors.green("Chatbot:"))} ${block(colors.green(initialMessage))} `, - ), - ); - } - - while (just(true)) { - try { - await forTime(100); - - const { title, conversationSummary } = ongoingParameters; - - console.info(`\n`); - if ( - title !== '' && - just(false) /* <- TODO: [⛲️] Some better way how to show the title of ongoing conversation */ - ) { - console.info(colors.gray(`--- ${title} ---`)); - } else { - console.info(colors.gray(`---`)); - } - - const response = await prompts({ - type: 'text', - name: 'userMessage', - message: 'User message', - hint: spaceTrim( - (block) => ` + ), + ); + } + + while (just(true)) { + try { + await forTime(100); + + const { title, conversationSummary } = ongoingParameters; + + console.info(`\n`); + if ( + title !== "" && + just( + false, + ) /* <- TODO: [⛲️] Some better way how to show the title of ongoing conversation */ + ) { + console.info(colors.gray(`--- ${title} ---`)); + } else { + console.info(colors.gray(`---`)); + } + + const response = await prompts({ + type: "text", + name: "userMessage", + message: "User message", + hint: spaceTrim( + (block) => ` Type "exit" to exit, previousTitle @@ -103,59 +109,63 @@ export async function runInteractiveChatbot(options: RunInteractiveChatbotOption ${block(conversationSummary)} `, - ), - }); + ), + }); - const { userMessage } = response; + const { userMessage } = response; - if (userMessage === 'exit' || userMessage === 'quit' || userMessage === undefined) { - return process.exit(0); - } + if ( + userMessage === "exit" || + userMessage === "quit" || + userMessage === undefined + ) { + return process.exit(0); + } - console.info(`\n`); - console.info( - spaceTrim( - (block) => ` + console.info(`\n`); + console.info( + spaceTrim( + (block) => ` - ${colors.bold(colors.blue('User:'))} + ${colors.bold(colors.blue("User:"))} ${block(colors.blue(userMessage))} `, - ), - ); + ), + ); - const inputParameters = { - previousTitle: title, + const inputParameters = { + previousTitle: title, - previousConversationSummary: conversationSummary, - userMessage, - }; + previousConversationSummary: conversationSummary, + userMessage, + }; - const result = await pipelineExecutor(inputParameters).asPromise(); + const result = await pipelineExecutor(inputParameters).asPromise(); - console.info(`\n`); - console.info( - spaceTrim( - (block) => ` + console.info(`\n`); + console.info( + spaceTrim( + (block) => ` - ${colors.bold(colors.green('Chatbot:'))} + ${colors.bold(colors.green("Chatbot:"))} ${block(colors.green(result.outputParameters.chatbotResponse!))} `, - ), - ); - - ongoingParameters = result.outputParameters as typeof ongoingParameters; - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - // TODO: Allow to ressurect the chatbot after an error - prompt the user to continue - console.error(colors.red(error.stack || error.message)); - return process.exit(1); - } - } + ), + ); + + ongoingParameters = result.outputParameters as typeof ongoingParameters; + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + // TODO: Allow to ressurect the chatbot after an error - prompt the user to continue + console.error(colors.red(error.stack || error.message)); + return process.exit(1); + } + } } /** diff --git a/src/cli/cli-commands/start-server.ts b/src/cli/cli-commands/start-server.ts index e1223efc1a..7ba0e06ad9 100644 --- a/src/cli/cli-commands/start-server.ts +++ b/src/cli/cli-commands/start-server.ts @@ -1,23 +1,23 @@ -import colors from 'colors'; +import colors from "colors"; import type { - Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, -} from 'commander'; -import spaceTrim from 'spacetrim'; -import { forEver } from 'waitasecond'; -import { createCollectionFromDirectory } from '../../collection/constructors/createCollectionFromDirectory'; -import { DEFAULT_BOOKS_DIRNAME } from '../../config'; -import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import { $provideLlmToolsForWizzardOrCli } from '../../llm-providers/_common/register/$provideLlmToolsForWizzardOrCli'; -import { startRemoteServer } from '../../remote-server/startRemoteServer'; -import { $provideFilesystemForNode } from '../../scrapers/_common/register/$provideFilesystemForNode'; -import { $provideScrapersForNode } from '../../scrapers/_common/register/$provideScrapersForNode'; -import type { number_port } from '../../types/typeAliases'; -import type { string_url } from '../../types/typeAliases'; -import { suffixUrl } from '../../utils/normalization/suffixUrl'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { isValidUrl } from '../../utils/validators/url/isValidUrl'; + Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, +} from "commander"; +import spaceTrim from "spacetrim"; +import { forEver } from "waitasecond"; +import { createCollectionFromDirectory } from "../../collection/constructors/createCollectionFromDirectory"; +import { DEFAULT_BOOKS_DIRNAME } from "../../config"; +import { $provideExecutablesForNode } from "../../executables/$provideExecutablesForNode"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import { $provideLlmToolsForWizzardOrCli } from "../../llm-providers/_common/register/$provideLlmToolsForWizzardOrCli"; +import { startRemoteServer } from "../../remote-server/startRemoteServer"; +import { $provideFilesystemForNode } from "../../scrapers/_common/register/$provideFilesystemForNode"; +import { $provideScrapersForNode } from "../../scrapers/_common/register/$provideScrapersForNode"; +import type { number_port } from "../../types/typeAliases"; +import type { string_url } from "../../types/typeAliases"; +import { suffixUrl } from "../../utils/normalization/suffixUrl"; +import { TODO_USE } from "../../utils/organization/TODO_USE"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { isValidUrl } from "../../utils/validators/url/isValidUrl"; /** * Initializes `start-server` command for Promptbook CLI utilities @@ -27,132 +27,151 @@ import { isValidUrl } from '../../utils/validators/url/isValidUrl'; * @private internal function of `promptbookCli` */ export function $initializeStartServerCommand(program: Program) { - const startServerCommand = program.command('start-server'); - - startServerCommand.argument( - '[path]', - // <- TODO: [🧟‍♂️] Unite path to promptbook collection argument - 'Path to promptbook collection directory', - DEFAULT_BOOKS_DIRNAME, - ); - startServerCommand.option('--port ', `Port to start the server on`, '4460'); - startServerCommand.option( - '-u, --url ', - spaceTrim(` + const startServerCommand = program.command("start-server"); + + startServerCommand.argument( + "[path]", + // <- TODO: [🧟‍♂️] Unite path to promptbook collection argument + "Path to promptbook collection directory", + DEFAULT_BOOKS_DIRNAME, + ); + startServerCommand.option( + "--port ", + `Port to start the server on`, + "4460", + ); + startServerCommand.option( + "-u, --url ", + spaceTrim(` Public root url of the server It is used for following purposes: 1) It is suffixed with /books and used as rootUrl for all served books 2) Path (if not just /) is used as rootPath for the server `), - ); - startServerCommand.option('--allow-anonymous', `Is anonymous mode allowed`, false); - startServerCommand.option('-r, --reload', `Call LLM models even if same prompt with result is in the cache`, false); - startServerCommand.option('-v, --verbose', `Is output verbose`, false); - - startServerCommand.description( - spaceTrim(` + ); + startServerCommand.option( + "--allow-anonymous", + `Is anonymous mode allowed`, + false, + ); + startServerCommand.option( + "-r, --reload", + `Call LLM models even if same prompt with result is in the cache`, + false, + ); + startServerCommand.option("-v, --verbose", `Is output verbose`, false); + + startServerCommand.description( + spaceTrim(` Starts a remote server to execute books `), - ); - - startServerCommand.alias('server'); - - startServerCommand.action( - async ( - path, - { - port: portRaw, - url: rawUrl, - allowAnonymous: isAnonymousModeAllowed, - reload: isCacheReloaded, - verbose: isVerbose, - }, - ) => { - if (rawUrl && !isValidUrl(rawUrl)) { - console.error(colors.red(`Invalid URL: ${rawUrl}`)); - return process.exit(1); - } - - const port: number_port = parseInt(portRaw, 10); - if (isNaN(port) || port <= 0 || port > 65535) { - console.error(colors.red(`Invalid port number: ${portRaw}`)); - return process.exit(1); - } - - const url = !rawUrl ? null : new URL(rawUrl); - - if (url !== null && url.port !== port.toString()) { - console.warn( - colors.yellow( - `Port in --url is different from --port which the server will listen on, this is ok only if you proxy from one port to another, for exaple via nginx or docker`, - ), - ); - } - - let rootUrl: string_url | undefined = undefined; - - if (url !== null) { - rootUrl = suffixUrl(url, '/books'); - } - - let rootPath = '/'; - - if (url !== null) { - rootPath = url.pathname; - } - - // TODO: DRY [◽] - const prepareAndScrapeOptions = { - isVerbose, - isCacheReloaded, - }; /* <- TODO: ` satisfies PrepareAndScrapeOptions` */ - const fs = $provideFilesystemForNode(prepareAndScrapeOptions); - const llm = await $provideLlmToolsForWizzardOrCli(prepareAndScrapeOptions); - const executables = await $provideExecutablesForNode(prepareAndScrapeOptions); - const tools = { - llm, - fs, - - scrapers: await $provideScrapersForNode({ fs, llm, executables }, prepareAndScrapeOptions), - script: [ - /*new JavascriptExecutionTools(options)*/ - ], - } satisfies ExecutionTools; - - // TODO: [🧟‍♂️][◽] DRY: - const collection = await createCollectionFromDirectory(path, tools, { - isVerbose, - rootUrl, - isRecursive: true, - isLazyLoaded: false, - isCrashedOnError: true, - // <- TODO: [🍖] Add `intermediateFilesStrategy` - }); - - // console.log(path, await collection.listPipelines()); - - const server = startRemoteServer({ - rootPath, - port, - isAnonymousModeAllowed, - isApplicationModeAllowed: true, - collection, - createLlmExecutionTools(options) { - const { appId, userId } = options; - TODO_USE({ appId, userId }); - return llm; - }, - }); - - keepUnused(server); - - // Note: Already logged by `startRemoteServer` - // console.error(colors.green(`Server started on port ${port}`)); - - return await forEver(); - }, - ); + ); + + startServerCommand.alias("server"); + + startServerCommand.action( + async ( + path, + { + port: portRaw, + url: rawUrl, + allowAnonymous: isAnonymousModeAllowed, + reload: isCacheReloaded, + verbose: isVerbose, + }, + ) => { + if (rawUrl && !isValidUrl(rawUrl)) { + console.error(colors.red(`Invalid URL: ${rawUrl}`)); + return process.exit(1); + } + + const port: number_port = Number.parseInt(portRaw, 10); + if (isNaN(port) || port <= 0 || port > 65535) { + console.error(colors.red(`Invalid port number: ${portRaw}`)); + return process.exit(1); + } + + const url = !rawUrl ? null : new URL(rawUrl); + + if (url !== null && url.port !== port.toString()) { + console.warn( + colors.yellow( + `Port in --url is different from --port which the server will listen on, this is ok only if you proxy from one port to another, for exaple via nginx or docker`, + ), + ); + } + + let rootUrl: string_url | undefined = undefined; + + if (url !== null) { + rootUrl = suffixUrl(url, "/books"); + } + + let rootPath = "/"; + + if (url !== null) { + rootPath = url.pathname; + } + + // TODO: DRY [◽] + const prepareAndScrapeOptions = { + isVerbose, + isCacheReloaded, + }; /* <- TODO: ` satisfies PrepareAndScrapeOptions` */ + const fs = $provideFilesystemForNode(prepareAndScrapeOptions); + const llm = await $provideLlmToolsForWizzardOrCli( + prepareAndScrapeOptions, + ); + const executables = await $provideExecutablesForNode( + prepareAndScrapeOptions, + ); + const tools = { + llm, + fs, + + scrapers: await $provideScrapersForNode( + { fs, llm, executables }, + prepareAndScrapeOptions, + ), + script: [ + /*new JavascriptExecutionTools(options)*/ + ], + } satisfies ExecutionTools; + + // TODO: [🧟‍♂️][◽] DRY: + const collection = await createCollectionFromDirectory(path, tools, { + isVerbose, + rootUrl, + isRecursive: true, + isLazyLoaded: false, + isCrashedOnError: true, + // <- TODO: [🍖] Add `intermediateFilesStrategy` + }); + + // console.log(path, await collection.listPipelines()); + + const server = startRemoteServer({ + rootPath, + port, + isAnonymousModeAllowed, + isApplicationModeAllowed: true, + collection, + createLlmExecutionTools(options) { + const { appId, userId } = options; + TODO_USE({ appId, userId }); + return llm; + }, + }); + + keepUnused(server); + + // Note: Already logged by `startRemoteServer` + // console.error(colors.green(`Server started on port ${port}`)); + + return await forEver(); + }, + ); } /** diff --git a/src/cli/cli-commands/test-command.ts b/src/cli/cli-commands/test-command.ts index 1a7ab998f9..535ce5a27e 100644 --- a/src/cli/cli-commands/test-command.ts +++ b/src/cli/cli-commands/test-command.ts @@ -1,19 +1,19 @@ -import colors from 'colors'; +import colors from "colors"; import type { - Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, -} from 'commander'; -import { readFile } from 'fs/promises'; -import glob from 'glob-promise'; -import spaceTrim from 'spacetrim'; -import { compilePipeline } from '../../conversion/compilePipeline'; -import { validatePipeline } from '../../conversion/validation/validatePipeline'; -import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import { $provideLlmToolsForWizzardOrCli } from '../../llm-providers/_common/register/$provideLlmToolsForWizzardOrCli'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import { validatePipelineString } from '../../pipeline/validatePipelineString'; -import { $provideFilesystemForNode } from '../../scrapers/_common/register/$provideFilesystemForNode'; -import { $provideScrapersForNode } from '../../scrapers/_common/register/$provideScrapersForNode'; + Command as Program /* <- Note: [🔸] Using Program because Command is misleading name */, +} from "commander"; +import { readFile } from "fs/promises"; +import glob from "glob-promise"; +import spaceTrim from "spacetrim"; +import { compilePipeline } from "../../conversion/compilePipeline"; +import { validatePipeline } from "../../conversion/validation/validatePipeline"; +import { $provideExecutablesForNode } from "../../executables/$provideExecutablesForNode"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import { $provideLlmToolsForWizzardOrCli } from "../../llm-providers/_common/register/$provideLlmToolsForWizzardOrCli"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import { validatePipelineString } from "../../pipeline/validatePipelineString"; +import { $provideFilesystemForNode } from "../../scrapers/_common/register/$provideFilesystemForNode"; +import { $provideScrapersForNode } from "../../scrapers/_common/register/$provideScrapersForNode"; /** * Initializes `test` command for Promptbook CLI utilities @@ -23,112 +23,139 @@ import { $provideScrapersForNode } from '../../scrapers/_common/register/$provid * @private internal function of `promptbookCli` */ export function $initializeTestCommand(program: Program) { - const testCommand = program.command('test'); - testCommand.description( - spaceTrim(` + const testCommand = program.command("test"); + testCommand.description( + spaceTrim(` Iterates over \`.book.md\` and \`.bookc\` and checks if they are parsable and logically valid `), - ); - - testCommand.argument( - '', - // <- TODO: [🧟‍♂️] Unite path to promptbook collection argument - 'Pipelines to test as glob pattern', - ); - testCommand.option('-i, --ignore ', `Ignore as glob patterns separated by comma`, ''); - - testCommand.option('--no-validation', `Do not validate logic of pipelines in collection`, true); - testCommand.option( - '--no-prepare', - `Do not prepare the pipelines, ideal when no LLM tools or scrapers available`, - true, - ); - - testCommand.option('-r, --reload', `Call LLM models even if same prompt with result is in the cache `, false); - testCommand.option('-v, --verbose', `Is output verbose`, false); - - testCommand.action( - async ( - filesGlob, - { - ignore: ignoreRaw = '', - validation: isValidated, - prepare: isPrepared, - reload: isCacheReloaded, - verbose: isVerbose, - }, - ) => { - let tools: Pick | undefined = undefined; - - if (isPrepared) { - // TODO: DRY [◽] - const prepareAndScrapeOptions = { - isVerbose, - isCacheReloaded, - }; /* <- TODO: ` satisfies PrepareAndScrapeOptions` */ - const fs = $provideFilesystemForNode(prepareAndScrapeOptions); - const llm = await $provideLlmToolsForWizzardOrCli(prepareAndScrapeOptions); - const executables = await $provideExecutablesForNode(prepareAndScrapeOptions); - tools = { - llm, - fs, - scrapers: await $provideScrapersForNode({ fs, llm, executables }, prepareAndScrapeOptions), - script: [ - /*new JavascriptExecutionTools(options)*/ - ], - } satisfies ExecutionTools; - } - - const ignore = (ignoreRaw as string).split(',').map((pattern) => pattern.trim()); - - const filenames = await glob(filesGlob!, { ignore }); - // <- TODO: [😶] - - // console.log({ filesGlob, ignore, filenames }); - // await forTime(1000000); - - pipelines: for (const filename of filenames) { - try { - let pipeline: PipelineJson; - - if (filename.endsWith('.book')) { - const pipelineMarkdown = validatePipelineString(await readFile(filename, 'utf-8')); - pipeline = await compilePipeline(pipelineMarkdown, tools); - - if (isVerbose) { - console.info(colors.green(`Parsable ${filename}`)); - } - } - if (filename.endsWith('.bookc')) { - pipeline = JSON.parse(await readFile(filename, 'utf-8')) as PipelineJson; - } else { - if (isVerbose) { - console.info(colors.gray(`Skipping ${filename}`)); - } - continue pipelines; - } - - if (isValidated) { - validatePipeline(pipeline); - console.info(colors.green(`Validated ${filename}`)); - } - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - console.info(colors.red(`Pipeline is not valid ${filename}`)); - console.error(colors.bgRed(error.name /* <- 11:11 */)); - console.error(colors.red(error.stack || error.message)); - - return process.exit(1); - } - } - - console.info(colors.green(`All pipelines are valid`)); - return process.exit(0); - }, - ); + ); + + testCommand.argument( + "", + // <- TODO: [🧟‍♂️] Unite path to promptbook collection argument + "Pipelines to test as glob pattern", + ); + testCommand.option( + "-i, --ignore ", + `Ignore as glob patterns separated by comma`, + "", + ); + + testCommand.option( + "--no-validation", + `Do not validate logic of pipelines in collection`, + true, + ); + testCommand.option( + "--no-prepare", + `Do not prepare the pipelines, ideal when no LLM tools or scrapers available`, + true, + ); + + testCommand.option( + "-r, --reload", + `Call LLM models even if same prompt with result is in the cache `, + false, + ); + testCommand.option("-v, --verbose", `Is output verbose`, false); + + testCommand.action( + async ( + filesGlob, + { + ignore: ignoreRaw = "", + validation: isValidated, + prepare: isPrepared, + reload: isCacheReloaded, + verbose: isVerbose, + }, + ) => { + let tools: + | Pick + | undefined = undefined; + + if (isPrepared) { + // TODO: DRY [◽] + const prepareAndScrapeOptions = { + isVerbose, + isCacheReloaded, + }; /* <- TODO: ` satisfies PrepareAndScrapeOptions` */ + const fs = $provideFilesystemForNode(prepareAndScrapeOptions); + const llm = await $provideLlmToolsForWizzardOrCli( + prepareAndScrapeOptions, + ); + const executables = await $provideExecutablesForNode( + prepareAndScrapeOptions, + ); + tools = { + llm, + fs, + scrapers: await $provideScrapersForNode( + { fs, llm, executables }, + prepareAndScrapeOptions, + ), + script: [ + /*new JavascriptExecutionTools(options)*/ + ], + } satisfies ExecutionTools; + } + + const ignore = (ignoreRaw as string) + .split(",") + .map((pattern) => pattern.trim()); + + const filenames = await glob(filesGlob!, { ignore }); + // <- TODO: [😶] + + // console.log({ filesGlob, ignore, filenames }); + // await forTime(1000000); + + for (const filename of filenames) { + try { + let pipeline: PipelineJson; + + if (filename.endsWith(".book")) { + const pipelineMarkdown = validatePipelineString( + await readFile(filename, "utf-8"), + ); + pipeline = await compilePipeline(pipelineMarkdown, tools); + + if (isVerbose) { + console.info(colors.green(`Parsable ${filename}`)); + } + } + if (filename.endsWith(".bookc")) { + pipeline = JSON.parse( + await readFile(filename, "utf-8"), + ) as PipelineJson; + } else { + if (isVerbose) { + console.info(colors.gray(`Skipping ${filename}`)); + } + continue; + } + + if (isValidated) { + validatePipeline(pipeline); + console.info(colors.green(`Validated ${filename}`)); + } + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + console.info(colors.red(`Pipeline is not valid ${filename}`)); + console.error(colors.bgRed(error.name /* <- 11:11 */)); + console.error(colors.red(error.stack || error.message)); + + return process.exit(1); + } + } + + console.info(colors.green(`All pipelines are valid`)); + return process.exit(0); + }, + ); } /** diff --git a/src/cli/main.ts b/src/cli/main.ts index 20d80c3259..2f7747be90 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -1,4 +1,4 @@ -import { promptbookCli } from './promptbookCli'; +import { promptbookCli } from "./promptbookCli"; /** * Note: [🔺] Purpose of this file is to export CLI for production environment @@ -10,8 +10,8 @@ import { promptbookCli } from './promptbookCli'; * @public exported from `@promptbook/cli` */ export const _CLI = { - // Note: [🥠] - _initialize_promptbookCli: promptbookCli, + // Note: [🥠] + _initialize_promptbookCli: promptbookCli, }; /** diff --git a/src/cli/promptbookCli.ts b/src/cli/promptbookCli.ts index 9ec0b93104..95b7f4088f 100644 --- a/src/cli/promptbookCli.ts +++ b/src/cli/promptbookCli.ts @@ -1,19 +1,19 @@ -import colors from 'colors'; -import commander from 'commander'; -import { spaceTrim } from 'spacetrim'; -import { CLAIM } from '../config'; -import { EnvironmentMismatchError } from '../errors/EnvironmentMismatchError'; -import { $isRunningInNode } from '../utils/environment/$isRunningInNode'; -import { PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { $initializeAboutCommand } from './cli-commands/about'; -import { $initializeHelloCommand } from './cli-commands/hello'; -import { $initializeListModelsCommand } from './cli-commands/list-models'; -import { $initializeListScrapersCommand } from './cli-commands/list-scrapers'; -import { $initializeMakeCommand } from './cli-commands/make'; -import { $initializePrettifyCommand } from './cli-commands/prettify'; -import { $initializeRunCommand } from './cli-commands/run'; -import { $initializeStartServerCommand } from './cli-commands/start-server'; -import { $initializeTestCommand } from './cli-commands/test-command'; +import colors from "colors"; +import commander from "commander"; +import { spaceTrim } from "spacetrim"; +import { CLAIM } from "../config"; +import { EnvironmentMismatchError } from "../errors/EnvironmentMismatchError"; +import { $isRunningInNode } from "../utils/environment/$isRunningInNode"; +import { PROMPTBOOK_ENGINE_VERSION } from "../version"; +import { $initializeAboutCommand } from "./cli-commands/about"; +import { $initializeHelloCommand } from "./cli-commands/hello"; +import { $initializeListModelsCommand } from "./cli-commands/list-models"; +import { $initializeListScrapersCommand } from "./cli-commands/list-scrapers"; +import { $initializeMakeCommand } from "./cli-commands/make"; +import { $initializePrettifyCommand } from "./cli-commands/prettify"; +import { $initializeRunCommand } from "./cli-commands/run"; +import { $initializeStartServerCommand } from "./cli-commands/start-server"; +import { $initializeTestCommand } from "./cli-commands/test-command"; /** * Runs CLI utilities of Promptbook package @@ -21,43 +21,47 @@ import { $initializeTestCommand } from './cli-commands/test-command'; * @private within the `@promptbook/cli` */ export async function promptbookCli(): Promise { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError( - spaceTrim(` + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + spaceTrim(` Function promptbookCli is initiator of CLI script and should be run in Node.js environment. - In browser use function exported from \`@promptbook/utils\` or \`@promptbook/core\` directly, for example \`prettifyPipelineString\`. `), - ); - } + ); + } - const isVerbose = process.argv.some((arg) => arg === '--verbose' || arg === '-v'); - // <- TODO: Can be this be done with commander before the commander commands are initialized? - if (isVerbose) { - console.info( - colors.gray(`Promptbook CLI version ${PROMPTBOOK_ENGINE_VERSION} in ${__filename.split('\\').join('/')}`), - ); - } + const isVerbose = process.argv.some( + (arg) => arg === "--verbose" || arg === "-v", + ); + // <- TODO: Can be this be done with commander before the commander commands are initialized? + if (isVerbose) { + console.info( + colors.gray( + `Promptbook CLI version ${PROMPTBOOK_ENGINE_VERSION} in ${__filename.split("\\").join("/")}`, + ), + ); + } - const program = new commander.Command(); - program.name('promptbook'); - program.alias('ptbk'); + const program = new commander.Command(); + program.name("promptbook"); + program.alias("ptbk"); - program.version(PROMPTBOOK_ENGINE_VERSION); - program.description(CLAIM); + program.version(PROMPTBOOK_ENGINE_VERSION); + program.description(CLAIM); - $initializeAboutCommand(program); - $initializeRunCommand(program); - $initializeHelloCommand(program); - $initializeMakeCommand(program); - $initializePrettifyCommand(program); - $initializeTestCommand(program); - $initializeListModelsCommand(program); - $initializeListScrapersCommand(program); - $initializeStartServerCommand(program); + $initializeAboutCommand(program); + $initializeRunCommand(program); + $initializeHelloCommand(program); + $initializeMakeCommand(program); + $initializePrettifyCommand(program); + $initializeTestCommand(program); + $initializeListModelsCommand(program); + $initializeListScrapersCommand(program); + $initializeStartServerCommand(program); - program.parse(process.argv); + program.parse(process.argv); } /** diff --git a/src/cli/test/ptbk.test.ts b/src/cli/test/ptbk.test.ts index e7b487cf78..0f7dca813a 100644 --- a/src/cli/test/ptbk.test.ts +++ b/src/cli/test/ptbk.test.ts @@ -1,25 +1,25 @@ -import { describe, expect, it } from '@jest/globals'; -import { $execCommand } from '../../utils/execCommand/$execCommand'; -import { PROMPTBOOK_ENGINE_VERSION } from '../../version'; +import { describe, expect, it } from "@jest/globals"; +import { $execCommand } from "../../utils/execCommand/$execCommand"; +import { PROMPTBOOK_ENGINE_VERSION } from "../../version"; -describe('how promptbookCli works', () => { - it('should initiate without errors', () => - expect( - $execCommand({ - command: 'ts-node src/cli/test/ptbk.ts --help', - crashOnError: false, - cwd: process.cwd(), - }), - ).resolves.toContain('Usage: promptbook|ptbk [options] [command]')); +describe("how promptbookCli works", () => { + it("should initiate without errors", () => + expect( + $execCommand({ + command: "ts-node src/cli/test/ptbk.ts --help", + crashOnError: false, + cwd: process.cwd(), + }), + ).resolves.toContain("Usage: promptbook|ptbk [options] [command]")); - it('should report version', () => - expect( - $execCommand({ - command: 'ts-node src/cli/test/ptbk.ts about', - crashOnError: false, - cwd: process.cwd(), - }), - ).resolves.toContain(PROMPTBOOK_ENGINE_VERSION)); + it("should report version", () => + expect( + $execCommand({ + command: "ts-node src/cli/test/ptbk.ts about", + crashOnError: false, + cwd: process.cwd(), + }), + ).resolves.toContain(PROMPTBOOK_ENGINE_VERSION)); - // TODO: Test each command + // TODO: Test each command }); diff --git a/src/cli/test/ptbk.ts b/src/cli/test/ptbk.ts index 2df3c33f18..2b225e0acb 100644 --- a/src/cli/test/ptbk.ts +++ b/src/cli/test/ptbk.ts @@ -4,7 +4,7 @@ * Note: [🔺] Purpose of this file is to test and use the current CLI in development environment */ -import '../../_packages/cli.index.ts'; // <- Note: Register all the LLM providers, scrapers, etc. by importing this file -import { promptbookCli } from '../promptbookCli'; +import "../../_packages/cli.index.ts"; // <- Note: Register all the LLM providers, scrapers, etc. by importing this file +import { promptbookCli } from "../promptbookCli"; promptbookCli(); diff --git a/src/cli/test/ptbk2.ts b/src/cli/test/ptbk2.ts index f7d6951f68..7914d89970 100644 --- a/src/cli/test/ptbk2.ts +++ b/src/cli/test/ptbk2.ts @@ -5,6 +5,6 @@ */ //import '../../_packages/cli.index.ts'; // <- Note: Register all the LLM providers, scrapers, etc. by importing this file -import { promptbookCli } from '../promptbookCli'; +import { promptbookCli } from "../promptbookCli"; promptbookCli(); diff --git a/src/cli/test/tsconfig.json b/src/cli/test/tsconfig.json index a28bc622f0..5e44c28e13 100644 --- a/src/cli/test/tsconfig.json +++ b/src/cli/test/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "ES2022", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "ES2022", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/collection/PipelineCollection.ts b/src/collection/PipelineCollection.ts index 718c6c4767..35c5b16964 100644 --- a/src/collection/PipelineCollection.ts +++ b/src/collection/PipelineCollection.ts @@ -1,7 +1,7 @@ -import type { Promisable } from 'type-fest'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import type { Prompt } from '../types/Prompt'; -import type { string_pipeline_url } from '../types/typeAliases'; +import type { Promisable } from "type-fest"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import type { Prompt } from "../types/Prompt"; +import type { string_pipeline_url } from "../types/typeAliases"; /** * Collection that groups together pipelines, knowledge, personas, tools and actions @@ -9,23 +9,23 @@ import type { string_pipeline_url } from '../types/typeAliases'; * @see @@@ https://github.com/webgptorg/pipeline#pipeline-collection */ export type PipelineCollection = { - /** - * Gets all pipelines in the collection - */ - listPipelines(): Promisable>; - // <- TODO: [🧠][👩🏾‍🤝‍🧑🏿] List `inputParameters` required for the execution + /** + * Gets all pipelines in the collection + */ + listPipelines(): Promisable>; + // <- TODO: [🧠][👩🏾‍🤝‍🧑🏿] List `inputParameters` required for the execution - /** - * Gets pipeline by its URL - * - * Note: This is not a direct fetching from the URL, but a lookup in the collection - */ - getPipelineByUrl(url: string_pipeline_url): Promisable; + /** + * Gets pipeline by its URL + * + * Note: This is not a direct fetching from the URL, but a lookup in the collection + */ + getPipelineByUrl(url: string_pipeline_url): Promisable; - /** - * Checks whether given prompt was defined in any pipeline in the collection - * - * @deprecated Make better mechanism for skimming the remote server - */ - isResponsibleForPrompt(prompt: Prompt): Promisable; + /** + * Checks whether given prompt was defined in any pipeline in the collection + * + * @deprecated Make better mechanism for skimming the remote server + */ + isResponsibleForPrompt(prompt: Prompt): Promisable; }; diff --git a/src/collection/SimplePipelineCollection.ts b/src/collection/SimplePipelineCollection.ts index 8dc76ae148..a2748e175f 100644 --- a/src/collection/SimplePipelineCollection.ts +++ b/src/collection/SimplePipelineCollection.ts @@ -1,13 +1,13 @@ -import { spaceTrim } from 'spacetrim'; -import { pipelineJsonToString } from '../conversion/pipelineJsonToString'; -import { validatePipeline } from '../conversion/validation/validatePipeline'; -import { NotFoundError } from '../errors/NotFoundError'; -import { PipelineUrlError } from '../errors/PipelineUrlError'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import { unpreparePipeline } from '../prepare/unpreparePipeline'; -import type { Prompt } from '../types/Prompt'; -import type { string_pipeline_url } from '../types/typeAliases'; -import type { PipelineCollection } from './PipelineCollection'; +import { spaceTrim } from "spacetrim"; +import { pipelineJsonToString } from "../conversion/pipelineJsonToString"; +import { validatePipeline } from "../conversion/validation/validatePipeline"; +import { NotFoundError } from "../errors/NotFoundError"; +import { PipelineUrlError } from "../errors/PipelineUrlError"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import { unpreparePipeline } from "../prepare/unpreparePipeline"; +import type { Prompt } from "../types/Prompt"; +import type { string_pipeline_url } from "../types/typeAliases"; +import type { PipelineCollection } from "./PipelineCollection"; /** * Library of pipelines that groups together pipelines for an application. @@ -17,122 +17,124 @@ import type { PipelineCollection } from './PipelineCollection'; * @see https://github.com/webgptorg/pipeline#pipeline-collection */ export class SimplePipelineCollection implements PipelineCollection { - private collection: Map; - - /** - * Constructs a pipeline collection from pipelines - * - * @param pipelines @@@ - * - * Note: During the construction logic of all pipelines are validated - * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead - */ - public constructor(...pipelines: ReadonlyArray) { - this.collection = new Map(); - for (const pipeline of pipelines) { - // TODO: [👠] DRY - if (pipeline.pipelineUrl === undefined) { - throw new PipelineUrlError( - spaceTrim(` + private collection: Map; + + /** + * Constructs a pipeline collection from pipelines + * + * @param pipelines @@@ + * + * Note: During the construction logic of all pipelines are validated + * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead + */ + public constructor(...pipelines: ReadonlyArray) { + this.collection = new Map(); + for (const pipeline of pipelines) { + // TODO: [👠] DRY + if (pipeline.pipelineUrl === undefined) { + throw new PipelineUrlError( + spaceTrim(` Pipeline with name "${pipeline.title}" does not have defined URL File: - ${pipeline.sourceFile || 'Unknown'} + ${pipeline.sourceFile || "Unknown"} Note: Pipelines without URLs are called anonymous pipelines They can be used as standalone pipelines, but they cannot be referenced by other pipelines And also they cannot be used in the pipeline collection `), - ); - } - - // Note: [🐨] - validatePipeline(pipeline); - - // TODO: [🦄] DRY - // Note: [🦄] - if ( - // TODO: [🐽] - this.collection.has(pipeline.pipelineUrl) && - pipelineJsonToString(unpreparePipeline(pipeline)) !== - pipelineJsonToString(unpreparePipeline(this.collection.get(pipeline.pipelineUrl)!)) - ) { - const existing = this.collection.get(pipeline.pipelineUrl)!; - - throw new PipelineUrlError( - spaceTrim(` + ); + } + + // Note: [🐨] + validatePipeline(pipeline); + + // TODO: [🦄] DRY + // Note: [🦄] + if ( + // TODO: [🐽] + this.collection.has(pipeline.pipelineUrl) && + pipelineJsonToString(unpreparePipeline(pipeline)) !== + pipelineJsonToString( + unpreparePipeline(this.collection.get(pipeline.pipelineUrl)!), + ) + ) { + const existing = this.collection.get(pipeline.pipelineUrl)!; + + throw new PipelineUrlError( + spaceTrim(` Pipeline with URL ${pipeline.pipelineUrl} is already in the collection 🍎 Conflicting files: - ${existing.sourceFile || 'Unknown'} - ${pipeline.sourceFile || 'Unknown'} + ${existing.sourceFile || "Unknown"} + ${pipeline.sourceFile || "Unknown"} Note: You have probably forgotten to run "ptbk make" to update the collection Note: Pipelines with the same URL are not allowed Only exepction is when the pipelines are identical `), - ); - } - - // Note: [🧠] Overwrite existing pipeline with the same URL - this.collection.set(pipeline.pipelineUrl, pipeline); - } - } - - /** - * Gets all pipelines in the collection - */ - public listPipelines(): ReadonlyArray { - return Array.from(this.collection.keys()); - } - - /** - * Gets pipeline by its URL - * - * Note: This is not a direct fetching from the URL, but a lookup in the collection - */ - public getPipelineByUrl(url: string_pipeline_url): PipelineJson { - const pipeline = this.collection.get(url); - if (!pipeline) { - if (this.listPipelines().length === 0) { - throw new NotFoundError( - spaceTrim( - ` + ); + } + + // Note: [🧠] Overwrite existing pipeline with the same URL + this.collection.set(pipeline.pipelineUrl, pipeline); + } + } + + /** + * Gets all pipelines in the collection + */ + public listPipelines(): ReadonlyArray { + return Array.from(this.collection.keys()); + } + + /** + * Gets pipeline by its URL + * + * Note: This is not a direct fetching from the URL, but a lookup in the collection + */ + public getPipelineByUrl(url: string_pipeline_url): PipelineJson { + const pipeline = this.collection.get(url); + if (!pipeline) { + if (this.listPipelines().length === 0) { + throw new NotFoundError( + spaceTrim( + ` Pipeline with url "${url}" not found No pipelines available `, - ), - ); - } + ), + ); + } - throw new NotFoundError( - spaceTrim( - (block) => ` + throw new NotFoundError( + spaceTrim( + (block) => ` Pipeline with url "${url}" not found Available pipelines: ${block( - this.listPipelines() - .map((pipelineUrl) => `- ${pipelineUrl}`) - .join('\n'), - )} + this.listPipelines() + .map((pipelineUrl) => `- ${pipelineUrl}`) + .join("\n"), + )} `, - ), - ); - } - return pipeline; - } - - /** - * Checks whether given prompt was defined in any pipeline in the collection - */ - public isResponsibleForPrompt(prompt: Prompt): boolean { - // TODO: [🍓][main] !!3 DO not hardcode this, really validate whether the prompt is in the collection - prompt; - return true; - } + ), + ); + } + return pipeline; + } + + /** + * Checks whether given prompt was defined in any pipeline in the collection + */ + public isResponsibleForPrompt(prompt: Prompt): boolean { + // TODO: [🍓][main] !!3 DO not hardcode this, really validate whether the prompt is in the collection + prompt; + return true; + } } diff --git a/src/collection/collectionToJson.test.ts b/src/collection/collectionToJson.test.ts index 2dea74a3ba..50c191e0e8 100644 --- a/src/collection/collectionToJson.test.ts +++ b/src/collection/collectionToJson.test.ts @@ -1,12 +1,12 @@ -import { describe, expect, it } from '@jest/globals'; -import spaceTrim from 'spacetrim'; -import { compilePipeline } from '../conversion/compilePipeline'; -import type { PipelineString } from '../pipeline/PipelineString'; -import { collectionToJson } from './collectionToJson'; -import { createCollectionFromJson } from './constructors/createCollectionFromJson'; - -describe('createCollectionFromJson', () => { - const pipelineString = spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import spaceTrim from "spacetrim"; +import { compilePipeline } from "../conversion/compilePipeline"; +import type { PipelineString } from "../pipeline/PipelineString"; +import { collectionToJson } from "./collectionToJson"; +import { createCollectionFromJson } from "./constructors/createCollectionFromJson"; + +describe("createCollectionFromJson", () => { + const pipelineString = spaceTrim(` # Example prompt Show how to use a simple completion prompt @@ -31,17 +31,17 @@ describe('createCollectionFromJson', () => { -> {response} `) as PipelineString; - // <- TODO: [📼] Use`book\`` string literal notation + // <- TODO: [📼] Use`book\`` string literal notation - it('should get pipeline by url from collection', async () => { - expect.assertions(1); - const pipeline = await compilePipeline(pipelineString); - const collection = createCollectionFromJson(pipeline); + it("should get pipeline by url from collection", async () => { + expect.assertions(1); + const pipeline = await compilePipeline(pipelineString); + const collection = createCollectionFromJson(pipeline); - // Note: This is the actual test: - const collectionJson = await collectionToJson(collection); - expect([pipeline]).toEqual(collectionJson); - }); + // Note: This is the actual test: + const collectionJson = await collectionToJson(collection); + expect([pipeline]).toEqual(collectionJson); + }); }); /** diff --git a/src/collection/collectionToJson.ts b/src/collection/collectionToJson.ts index fc90ce6944..20d40a1109 100644 --- a/src/collection/collectionToJson.ts +++ b/src/collection/collectionToJson.ts @@ -1,5 +1,5 @@ -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import type { PipelineCollection } from './PipelineCollection'; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import type { PipelineCollection } from "./PipelineCollection"; /** * Converts PipelineCollection to serialized JSON @@ -8,10 +8,14 @@ import type { PipelineCollection } from './PipelineCollection'; * * @public exported from `@promptbook/core` */ -export async function collectionToJson(collection: PipelineCollection): Promise> { - const pipelineUrls = await collection.listPipelines(); - const promptbooks = await Promise.all(pipelineUrls.map((url) => collection.getPipelineByUrl(url))); - return promptbooks; +export async function collectionToJson( + collection: PipelineCollection, +): Promise> { + const pipelineUrls = await collection.listPipelines(); + const promptbooks = await Promise.all( + pipelineUrls.map((url) => collection.getPipelineByUrl(url)), + ); + return promptbooks; } /** diff --git a/src/collection/constructors/createCollectionFromDirectory.test.ts b/src/collection/constructors/createCollectionFromDirectory.test.ts index 1952ac2177..85bdd332a3 100644 --- a/src/collection/constructors/createCollectionFromDirectory.test.ts +++ b/src/collection/constructors/createCollectionFromDirectory.test.ts @@ -1,111 +1,126 @@ -import { describe, expect, it } from '@jest/globals'; -import { unpreparePipeline } from '../../prepare/unpreparePipeline'; -import { $provideFilesystemForNode } from '../../scrapers/_common/register/$provideFilesystemForNode'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { createCollectionFromDirectory } from './createCollectionFromDirectory'; +import { describe, expect, it } from "@jest/globals"; +import { unpreparePipeline } from "../../prepare/unpreparePipeline"; +import { $provideFilesystemForNode } from "../../scrapers/_common/register/$provideFilesystemForNode"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { createCollectionFromDirectory } from "./createCollectionFromDirectory"; -describe('createCollectionFromDirectory', () => { - it('should get pipeline by url from collection', async () => { - expect.assertions(1); - const collection = await createCollectionFromDirectory( - './examples/pipelines', - { - fs: $provideFilesystemForNode(), - }, - { - isVerbose: true, - isRecursive: false, - isLazyLoaded: false, - }, - ); - let pipelineFromCollection = await collection.getPipelineByUrl( - 'https://promptbook.studio/examples/simple.book', - ); +describe("createCollectionFromDirectory", () => { + it("should get pipeline by url from collection", async () => { + expect.assertions(1); + const collection = await createCollectionFromDirectory( + "./examples/pipelines", + { + fs: $provideFilesystemForNode(), + }, + { + isVerbose: true, + isRecursive: false, + isLazyLoaded: false, + }, + ); + let pipelineFromCollection = await collection.getPipelineByUrl( + "https://promptbook.studio/examples/simple.book", + ); - pipelineFromCollection = unpreparePipeline(pipelineFromCollection); - pipelineFromCollection = { ...pipelineFromCollection, sourceFile: undefined }; + pipelineFromCollection = unpreparePipeline(pipelineFromCollection); + pipelineFromCollection = { + ...pipelineFromCollection, + sourceFile: undefined, + }; - expect(pipelineFromCollection).toMatchObject({ title: `✨ Example prompt with URL` }); - }); + expect(pipelineFromCollection).toMatchObject({ + title: `✨ Example prompt with URL`, + }); + }); - it('should get lazy-loaded pipeline by url from collection', async () => { - expect.assertions(1); + it("should get lazy-loaded pipeline by url from collection", async () => { + expect.assertions(1); - const collection = await createCollectionFromDirectory( - './examples/pipelines', - { fs: $provideFilesystemForNode() }, - { - isVerbose: true, - isRecursive: false, - isLazyLoaded: true, - }, - ); - let pipelineFromCollection = await collection.getPipelineByUrl( - 'https://promptbook.studio/examples/simple.book', - ); + const collection = await createCollectionFromDirectory( + "./examples/pipelines", + { fs: $provideFilesystemForNode() }, + { + isVerbose: true, + isRecursive: false, + isLazyLoaded: true, + }, + ); + let pipelineFromCollection = await collection.getPipelineByUrl( + "https://promptbook.studio/examples/simple.book", + ); - pipelineFromCollection = unpreparePipeline(pipelineFromCollection); - pipelineFromCollection = { ...pipelineFromCollection, sourceFile: undefined }; + pipelineFromCollection = unpreparePipeline(pipelineFromCollection); + pipelineFromCollection = { + ...pipelineFromCollection, + sourceFile: undefined, + }; - expect(pipelineFromCollection).toMatchObject({ title: `✨ Example prompt with URL` }); - }); + expect(pipelineFromCollection).toMatchObject({ + title: `✨ Example prompt with URL`, + }); + }); - it('should get different pipeline by url from collection', async () => { - expect.assertions(1); + it("should get different pipeline by url from collection", async () => { + expect.assertions(1); - const collection = await createCollectionFromDirectory( - './examples/pipelines', - { fs: $provideFilesystemForNode() }, - { - isVerbose: true, - isRecursive: false, - }, - ); - let pipelineFromCollection = await collection.getPipelineByUrl( - 'https://promptbook.studio/examples/jokers.book', - ); + const collection = await createCollectionFromDirectory( + "./examples/pipelines", + { fs: $provideFilesystemForNode() }, + { + isVerbose: true, + isRecursive: false, + }, + ); + let pipelineFromCollection = await collection.getPipelineByUrl( + "https://promptbook.studio/examples/jokers.book", + ); - pipelineFromCollection = unpreparePipeline(pipelineFromCollection); - pipelineFromCollection = { ...pipelineFromCollection, sourceFile: undefined }; + pipelineFromCollection = unpreparePipeline(pipelineFromCollection); + pipelineFromCollection = { + ...pipelineFromCollection, + sourceFile: undefined, + }; - expect(pipelineFromCollection).not.toMatchObject({ title: `✨ Example prompt with URL` }); - }); + expect(pipelineFromCollection).not.toMatchObject({ + title: `✨ Example prompt with URL`, + }); + }); - it('should NOT crash when include error pipelines but lazy-loaded', () => - expect( - (async () => { - const collection = await createCollectionFromDirectory( - './examples/pipelines', - { fs: $provideFilesystemForNode() }, - { - isVerbose: true, - // Note: Including subdirectories BUT lazy-loaded so it should not crash even if there are errors - isRecursive: true, - isLazyLoaded: true, - }, - ); - keepUnused(collection); - })(), - ).resolves.not.toThrow()); + it("should NOT crash when include error pipelines but lazy-loaded", () => + expect( + (async () => { + const collection = await createCollectionFromDirectory( + "./examples/pipelines", + { fs: $provideFilesystemForNode() }, + { + isVerbose: true, + // Note: Including subdirectories BUT lazy-loaded so it should not crash even if there are errors + isRecursive: true, + isLazyLoaded: true, + }, + ); + keepUnused(collection); + })(), + ).resolves.not.toThrow()); - it('should crash when include error pipelines', () => - expect( - (async () => { - const collection = await createCollectionFromDirectory( - './examples/pipelines', - { fs: $provideFilesystemForNode() }, - { - isVerbose: true, - // Note: Including subdirectories BUT lazy-loaded so it should not crash even if there are errors - isRecursive: true, - isLazyLoaded: false, - }, - ); - keepUnused(collection); - })(), - ).rejects.toThrowError(/^ParseError in pipeline examples.*/i)); + it("should crash when include error pipelines", () => + expect( + (async () => { + const collection = await createCollectionFromDirectory( + "./examples/pipelines", + { fs: $provideFilesystemForNode() }, + { + isVerbose: true, + // Note: Including subdirectories BUT lazy-loaded so it should not crash even if there are errors + isRecursive: true, + isLazyLoaded: false, + }, + ); + keepUnused(collection); + })(), + ).rejects.toThrowError(/^ParseError in pipeline examples.*/i)); - /* + /* TODO: Make separate folder for errors and enable this test it('should find pipeline in subdirectory', () => expect( diff --git a/src/collection/constructors/createCollectionFromDirectory.ts b/src/collection/constructors/createCollectionFromDirectory.ts index 7a504341b1..e57c2d5d37 100644 --- a/src/collection/constructors/createCollectionFromDirectory.ts +++ b/src/collection/constructors/createCollectionFromDirectory.ts @@ -1,30 +1,30 @@ -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { readFile } from 'fs/promises'; -import { dirname, join, relative } from 'path'; -import spaceTrim from 'spacetrim'; -import { DEFAULT_IS_VERBOSE } from '../../config'; -import { DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME } from '../../config'; -import { loadArchive } from '../../conversion/archive/loadArchive'; -import { compilePipeline } from '../../conversion/compilePipeline'; -import { pipelineJsonToString } from '../../conversion/pipelineJsonToString'; -import { validatePipeline } from '../../conversion/validation/validatePipeline'; -import { CollectionError } from '../../errors/CollectionError'; -import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError'; -import { PipelineUrlError } from '../../errors/PipelineUrlError'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import { $provideExecutionToolsForNode } from '../../execution/utils/$provideExecutionToolsForNode'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import { validatePipelineString } from '../../pipeline/validatePipelineString'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { unpreparePipeline } from '../../prepare/unpreparePipeline'; -import type { string_dirname } from '../../types/typeAliases'; -import type { string_filename } from '../../types/typeAliases'; -import type { string_pipeline_root_url } from '../../types/typeAliases'; -import type { string_pipeline_url } from '../../types/typeAliases'; -import { isFileExisting } from '../../utils/files/isFileExisting'; -import { listAllFiles } from '../../utils/files/listAllFiles'; -import type { PipelineCollection } from '../PipelineCollection'; -import { createCollectionFromPromise } from './createCollectionFromPromise'; +import { dirname, join, relative } from "path"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { readFile } from "fs/promises"; +import spaceTrim from "spacetrim"; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import { DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME } from "../../config"; +import { loadArchive } from "../../conversion/archive/loadArchive"; +import { compilePipeline } from "../../conversion/compilePipeline"; +import { pipelineJsonToString } from "../../conversion/pipelineJsonToString"; +import { validatePipeline } from "../../conversion/validation/validatePipeline"; +import { CollectionError } from "../../errors/CollectionError"; +import { EnvironmentMismatchError } from "../../errors/EnvironmentMismatchError"; +import { PipelineUrlError } from "../../errors/PipelineUrlError"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import { $provideExecutionToolsForNode } from "../../execution/utils/$provideExecutionToolsForNode"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import { validatePipelineString } from "../../pipeline/validatePipelineString"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { unpreparePipeline } from "../../prepare/unpreparePipeline"; +import type { string_dirname } from "../../types/typeAliases"; +import type { string_filename } from "../../types/typeAliases"; +import type { string_pipeline_root_url } from "../../types/typeAliases"; +import type { string_pipeline_url } from "../../types/typeAliases"; +import { isFileExisting } from "../../utils/files/isFileExisting"; +import { listAllFiles } from "../../utils/files/listAllFiles"; +import type { PipelineCollection } from "../PipelineCollection"; +import { createCollectionFromPromise } from "./createCollectionFromPromise"; /** * Options for `createCollectionFromDirectory` function @@ -32,48 +32,51 @@ import { createCollectionFromPromise } from './createCollectionFromPromise'; * Note: `rootDirname` is not needed because it is the folder in which `.book` or `.book` file is located * This is not same as `path` which is the first argument of `createCollectionFromDirectory` - it can be a subfolder */ -type CreatePipelineCollectionFromDirectoryOptions = Omit & { - /** - * If true, the directory is searched recursively for pipelines - * - * @default true - */ - isRecursive?: boolean; - - /** - * If true, the collection creation outputs information about each file it reads - * - * @default false - */ - isVerbose?: boolean; - - /** - * This will be used as a root URL for all pipelines in the collection - * - * It has 2 purposes: - * 1) Every pipeline in the collection is checked if it is a child of `rootUrl` - * 2) If the pipeline does not have a URL, it is created from the `rootUrl` and path to the pipeline - * - * @default false - */ - rootUrl?: string_pipeline_root_url; - - /** - * If true, directory will be scanned only when needed not during the construction - * - * @default false - */ - isLazyLoaded?: boolean; - - /** - * If true, whole collection creation crashes on error in any pipeline - * If true and isLazyLoaded is true, the error is thrown on first access to the pipeline - * - * @default true - */ - isCrashedOnError?: boolean; - - // <- TODO: [🍖] Add `intermediateFilesStrategy` +type CreatePipelineCollectionFromDirectoryOptions = Omit< + PrepareAndScrapeOptions, + "rootDirname" +> & { + /** + * If true, the directory is searched recursively for pipelines + * + * @default true + */ + isRecursive?: boolean; + + /** + * If true, the collection creation outputs information about each file it reads + * + * @default false + */ + isVerbose?: boolean; + + /** + * This will be used as a root URL for all pipelines in the collection + * + * It has 2 purposes: + * 1) Every pipeline in the collection is checked if it is a child of `rootUrl` + * 2) If the pipeline does not have a URL, it is created from the `rootUrl` and path to the pipeline + * + * @default false + */ + rootUrl?: string_pipeline_root_url; + + /** + * If true, directory will be scanned only when needed not during the construction + * + * @default false + */ + isLazyLoaded?: boolean; + + /** + * If true, whole collection creation crashes on error in any pipeline + * If true and isLazyLoaded is true, the error is thrown on first access to the pipeline + * + * @default true + */ + isCrashedOnError?: boolean; + + // <- TODO: [🍖] Add `intermediateFilesStrategy` }; /** @@ -88,31 +91,33 @@ type CreatePipelineCollectionFromDirectoryOptions = Omit, - options?: CreatePipelineCollectionFromDirectoryOptions, + rootPath: string_dirname, + tools?: Pick, + options?: CreatePipelineCollectionFromDirectoryOptions, ): Promise { - if (tools === undefined) { - tools = await $provideExecutionToolsForNode(); - } - - if (tools === undefined || tools.fs === undefined) { - throw new EnvironmentMismatchError('Can not create collection without filesystem tools'); - // <- TODO: [🧠] What is the best error type here` - } - - // TODO: [🍖] Allow to skip - - const madeLibraryFilePath = join( - rootPath, - `${ - DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME - // <- TODO: [🦒] Allow to override (pass different value into the function) - }.bookc`, - ); - - if (!(await isFileExisting(madeLibraryFilePath, tools.fs))) { - /* + if (tools === undefined) { + tools = await $provideExecutionToolsForNode(); + } + + if (tools === undefined || tools.fs === undefined) { + throw new EnvironmentMismatchError( + "Can not create collection without filesystem tools", + ); + // <- TODO: [🧠] What is the best error type here` + } + + // TODO: [🍖] Allow to skip + + const madeLibraryFilePath = join( + rootPath, + `${ + DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME + // <- TODO: [🦒] Allow to override (pass different value into the function) + }.bookc`, + ); + + if (!(await isFileExisting(madeLibraryFilePath, tools.fs))) { + /* TODO: [🌗][🧠] Should this message be here or just ignore console.info( colors.yellow( @@ -120,235 +125,267 @@ export async function createCollectionFromDirectory( ), ); */ - } else { - colors.green(`(In future, not implemented yet) Using your compiled pipeline collection ${madeLibraryFilePath}`); - // TODO: Implement; - // TODO: [🌗] - } - - const { - isRecursive = true, - isVerbose = DEFAULT_IS_VERBOSE, - isLazyLoaded = false, - isCrashedOnError = true, - rootUrl, - } = options || {}; - - const collection = createCollectionFromPromise(async () => { - if (isVerbose) { - console.info(colors.cyan(`Creating pipeline collection from path ${rootPath.split('\\').join('/')}`)); - } - - const fileNames = await listAllFiles(rootPath, isRecursive, tools!.fs!); - - // Note: First load compiled `.bookc` files and then source `.book` files - // `.bookc` are already compiled and can be used faster - fileNames.sort((a, b) => { - if ((a.endsWith('.bookc') || a.endsWith('.book.json')) && (b.endsWith('.book') || b.endsWith('.book.md'))) { - return -1; - } - if ((a.endsWith('.book') || a.endsWith('.book.md')) && (b.endsWith('.bookc') || b.endsWith('.book.json'))) { - return 1; - } - return 0; - }); - - const collection = new Map(); - const pipelinesWithFilenames: Array<{ - fileName: string_filename; - sourceFile: string_filename; - pipeline: PipelineJson; - }> = []; - - for (const fileName of fileNames) { - const sourceFile = './' + fileName.split('\\').join('/'); - const rootDirname = dirname(sourceFile).split('\\').join('/'); - - try { - if (fileName.endsWith('.book') || fileName.endsWith('.book.md')) { - const pipelineString = validatePipelineString(await readFile(fileName, 'utf-8')); - const pipeline = await compilePipeline(pipelineString, tools, { - rootDirname, - }); - pipelinesWithFilenames.push({ fileName, sourceFile, pipeline: { ...pipeline, sourceFile } }); - } else if (fileName.endsWith('.bookc') || fileName.endsWith('.book.json')) { - // TODO: Handle non-valid JSON files - - pipelinesWithFilenames.push( - ...(await loadArchive(fileName, tools!.fs!)).map((pipeline) => - // TODO: [🌗] - ({ fileName, sourceFile, pipeline: { ...pipeline, sourceFile } }), - ), - ); - } else { - if (isVerbose) { - console.info( - colors.gray(`Skipped file ${fileName.split('\\').join('/')} –⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠ Not a book`), - ); - } - } - - // --- - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - // TODO: [7] DRY - const wrappedErrorMessage = - spaceTrim( - (block) => ` - ${(error as Error).name} in pipeline ${fileName.split('\\').join('/')}⁠: + } else { + colors.green( + `(In future, not implemented yet) Using your compiled pipeline collection ${madeLibraryFilePath}`, + ); + // TODO: Implement; + // TODO: [🌗] + } + + const { + isRecursive = true, + isVerbose = DEFAULT_IS_VERBOSE, + isLazyLoaded = false, + isCrashedOnError = true, + rootUrl, + } = options || {}; + + const collection = createCollectionFromPromise(async () => { + if (isVerbose) { + console.info( + colors.cyan( + `Creating pipeline collection from path ${rootPath.split("\\").join("/")}`, + ), + ); + } + + const fileNames = await listAllFiles(rootPath, isRecursive, tools!.fs!); + + // Note: First load compiled `.bookc` files and then source `.book` files + // `.bookc` are already compiled and can be used faster + fileNames.sort((a, b) => { + if ( + (a.endsWith(".bookc") || a.endsWith(".book.json")) && + (b.endsWith(".book") || b.endsWith(".book.md")) + ) { + return -1; + } + if ( + (a.endsWith(".book") || a.endsWith(".book.md")) && + (b.endsWith(".bookc") || b.endsWith(".book.json")) + ) { + return 1; + } + return 0; + }); + + const collection = new Map(); + const pipelinesWithFilenames: Array<{ + fileName: string_filename; + sourceFile: string_filename; + pipeline: PipelineJson; + }> = []; + + for (const fileName of fileNames) { + const sourceFile = "./" + fileName.split("\\").join("/"); + const rootDirname = dirname(sourceFile).split("\\").join("/"); + + try { + if (fileName.endsWith(".book") || fileName.endsWith(".book.md")) { + const pipelineString = validatePipelineString( + await readFile(fileName, "utf-8"), + ); + const pipeline = await compilePipeline(pipelineString, tools, { + rootDirname, + }); + pipelinesWithFilenames.push({ + fileName, + sourceFile, + pipeline: { ...pipeline, sourceFile }, + }); + } else if ( + fileName.endsWith(".bookc") || + fileName.endsWith(".book.json") + ) { + // TODO: Handle non-valid JSON files + + pipelinesWithFilenames.push( + ...(await loadArchive(fileName, tools!.fs!)).map((pipeline) => + // TODO: [🌗] + ({ fileName, sourceFile, pipeline: { ...pipeline, sourceFile } }), + ), + ); + } else { + if (isVerbose) { + console.info( + colors.gray( + `Skipped file ${fileName.split("\\").join("/")} –⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠ Not a book`, + ), + ); + } + } + + // --- + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + // TODO: [7] DRY + const wrappedErrorMessage = + spaceTrim( + (block) => ` + ${(error as Error).name} in pipeline ${fileName.split("\\").join("/")}⁠: Original error message: ${block((error as Error).message)} Original stack trace: - ${block((error as Error).stack || '')} + ${block((error as Error).stack || "")} --- `, - ) + '\n'; - - if (isCrashedOnError) { - throw new CollectionError(wrappedErrorMessage); - } - - // TODO: [🟥] Detect browser / node and make it colorfull - console.error(wrappedErrorMessage); - } - } - - for (const pipelineWithFilenames of pipelinesWithFilenames) { - const { fileName, sourceFile } = pipelineWithFilenames; - let { pipeline } = pipelineWithFilenames; - - try { - if (rootUrl !== undefined) { - if (pipeline.pipelineUrl === undefined) { - const pipelineUrl = rootUrl + '/' + relative(rootPath, fileName).split('\\').join('/'); - - // console.log({ pipelineUrl, rootPath, rootUrl, fileName }); - - if (isVerbose) { - console.info( - colors.yellow( - `Implicitly set pipeline URL to ${pipelineUrl} from ${fileName - .split('\\') - .join('/')}`, - ), - ); - } - pipeline = { ...pipeline, pipelineUrl }; - } else if (!pipeline.pipelineUrl.startsWith(rootUrl)) { - throw new PipelineUrlError( - spaceTrim(` + ) + "\n"; + + if (isCrashedOnError) { + throw new CollectionError(wrappedErrorMessage); + } + + // TODO: [🟥] Detect browser / node and make it colorfull + console.error(wrappedErrorMessage); + } + } + + for (const pipelineWithFilenames of pipelinesWithFilenames) { + const { fileName, sourceFile } = pipelineWithFilenames; + let { pipeline } = pipelineWithFilenames; + + try { + if (rootUrl !== undefined) { + if (pipeline.pipelineUrl === undefined) { + const pipelineUrl = + rootUrl + + "/" + + relative(rootPath, fileName).split("\\").join("/"); + + // console.log({ pipelineUrl, rootPath, rootUrl, fileName }); + + if (isVerbose) { + console.info( + colors.yellow( + `Implicitly set pipeline URL to ${pipelineUrl} from ${fileName + .split("\\") + .join("/")}`, + ), + ); + } + pipeline = { ...pipeline, pipelineUrl }; + } else if (!pipeline.pipelineUrl.startsWith(rootUrl)) { + throw new PipelineUrlError( + spaceTrim(` Pipeline with URL ${pipeline.pipelineUrl} is not a child of the root URL ${rootUrl} 🍏 File: - ${sourceFile || 'Unknown'} + ${sourceFile || "Unknown"} `), - ); - } - } - - // TODO: [👠] DRY - if (pipeline.pipelineUrl === undefined) { - if (isVerbose) { - console.info( - colors.yellow( - `Can not load pipeline from ${fileName.split('\\').join('/')} because of missing URL`, - ), - ); - } - } else { - // Note: [🐨] Pipeline is checked multiple times - // TODO: Maybe once is enough BUT be sure to check it - better to check it multiple times than not at all - validatePipeline(pipeline); - - if ( - // TODO: [🐽] comparePipelines(pipeline1,pipeline2): 'IDENTICAL' |'IDENTICAL_UNPREPARED' | 'IDENTICAL_INTERFACE' | 'DIFFERENT' - !collection.has(pipeline.pipelineUrl) - ) { - if (isVerbose) { - console.info(colors.green(`Loaded pipeline ${fileName.split('\\').join('/')}⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠`)); - } - - // Note: [🦄] Pipeline with same url uniqueness will be double-checked automatically in SimplePipelineCollection - collection.set(pipeline.pipelineUrl, pipeline); - } else if ( - pipelineJsonToString(unpreparePipeline(pipeline)) === - pipelineJsonToString(unpreparePipeline(collection.get(pipeline.pipelineUrl)!)) - ) { - if (isVerbose) { - console.info( - colors.gray( - `Skipped pipeline ${fileName - .split('\\') - .join('/')} –⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠ Already identical pipeline in the collection`, - ), - ); - } - } else { - const existing = collection.get(pipeline.pipelineUrl)!; - - throw new PipelineUrlError( - spaceTrim(` + ); + } + } + + // TODO: [👠] DRY + if (pipeline.pipelineUrl === undefined) { + if (isVerbose) { + console.info( + colors.yellow( + `Can not load pipeline from ${fileName.split("\\").join("/")} because of missing URL`, + ), + ); + } + } else { + // Note: [🐨] Pipeline is checked multiple times + // TODO: Maybe once is enough BUT be sure to check it - better to check it multiple times than not at all + validatePipeline(pipeline); + + if ( + // TODO: [🐽] comparePipelines(pipeline1,pipeline2): 'IDENTICAL' |'IDENTICAL_UNPREPARED' | 'IDENTICAL_INTERFACE' | 'DIFFERENT' + !collection.has(pipeline.pipelineUrl) + ) { + if (isVerbose) { + console.info( + colors.green( + `Loaded pipeline ${fileName.split("\\").join("/")}⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠`, + ), + ); + } + + // Note: [🦄] Pipeline with same url uniqueness will be double-checked automatically in SimplePipelineCollection + collection.set(pipeline.pipelineUrl, pipeline); + } else if ( + pipelineJsonToString(unpreparePipeline(pipeline)) === + pipelineJsonToString( + unpreparePipeline(collection.get(pipeline.pipelineUrl)!), + ) + ) { + if (isVerbose) { + console.info( + colors.gray( + `Skipped pipeline ${fileName + .split("\\") + .join("/")} –⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠⁠ Already identical pipeline in the collection`, + ), + ); + } + } else { + const existing = collection.get(pipeline.pipelineUrl)!; + + throw new PipelineUrlError( + spaceTrim(` Pipeline with URL ${pipeline.pipelineUrl} is already in the collection 🍏 Conflicting files: - ${existing.sourceFile || 'Unknown'} - ${pipeline.sourceFile || 'Unknown'} + ${existing.sourceFile || "Unknown"} + ${pipeline.sourceFile || "Unknown"} Note: You have probably forgotten to run "ptbk make" to update the collection Note: Pipelines with the same URL are not allowed Only exepction is when the pipelines are identical `), - ); - } - } - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - // TODO: [7] DRY - const wrappedErrorMessage = - spaceTrim( - (block) => ` - ${(error as Error).name} in pipeline ${fileName.split('\\').join('/')}⁠: + ); + } + } + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + // TODO: [7] DRY + const wrappedErrorMessage = + spaceTrim( + (block) => ` + ${(error as Error).name} in pipeline ${fileName.split("\\").join("/")}⁠: Original error message: ${block((error as Error).message)} Original stack trace: - ${block((error as Error).stack || '')} + ${block((error as Error).stack || "")} --- `, - ) + '\n'; + ) + "\n"; - if (isCrashedOnError) { - throw new CollectionError(wrappedErrorMessage); - } + if (isCrashedOnError) { + throw new CollectionError(wrappedErrorMessage); + } - // TODO: [🟥] Detect browser / node and make it colorfull - console.error(wrappedErrorMessage); - } - } + // TODO: [🟥] Detect browser / node and make it colorfull + console.error(wrappedErrorMessage); + } + } - return Array.from(collection.values()); - }); + return Array.from(collection.values()); + }); - if (isLazyLoaded === false) { - await collection.listPipelines(); - } + if (isLazyLoaded === false) { + await collection.listPipelines(); + } - return collection; + return collection; } /** diff --git a/src/collection/constructors/createCollectionFromJson.test.ts b/src/collection/constructors/createCollectionFromJson.test.ts index 5813e46f36..cffdfb3fcd 100644 --- a/src/collection/constructors/createCollectionFromJson.test.ts +++ b/src/collection/constructors/createCollectionFromJson.test.ts @@ -1,11 +1,11 @@ -import { describe, expect, it } from '@jest/globals'; -import spaceTrim from 'spacetrim'; -import { compilePipeline } from '../../conversion/compilePipeline'; -import type { PipelineString } from '../../pipeline/PipelineString'; -import { createCollectionFromJson } from './createCollectionFromJson'; - -describe('createCollectionFromJson', () => { - const pipelineString = spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import spaceTrim from "spacetrim"; +import { compilePipeline } from "../../conversion/compilePipeline"; +import type { PipelineString } from "../../pipeline/PipelineString"; +import { createCollectionFromJson } from "./createCollectionFromJson"; + +describe("createCollectionFromJson", () => { + const pipelineString = spaceTrim(` # Example prompt Show how to use a simple completion prompt @@ -30,15 +30,17 @@ describe('createCollectionFromJson', () => { -> {response} `) as PipelineString; - // <- TODO: [📼] Use`book\`` string literal notation - - it('should get pipeline by url from collection', async () => { - expect.assertions(1); - const pipeline = await compilePipeline(pipelineString); - const collection = createCollectionFromJson(pipeline); - const pipelineFromCollection = await collection.getPipelineByUrl( - 'https://promptbook.studio/examples/pipeline.book', - ); - expect(pipelineFromCollection).toEqual(await compilePipeline(pipelineString)); - }); + // <- TODO: [📼] Use`book\`` string literal notation + + it("should get pipeline by url from collection", async () => { + expect.assertions(1); + const pipeline = await compilePipeline(pipelineString); + const collection = createCollectionFromJson(pipeline); + const pipelineFromCollection = await collection.getPipelineByUrl( + "https://promptbook.studio/examples/pipeline.book", + ); + expect(pipelineFromCollection).toEqual( + await compilePipeline(pipelineString), + ); + }); }); diff --git a/src/collection/constructors/createCollectionFromJson.ts b/src/collection/constructors/createCollectionFromJson.ts index 6893299c10..d8a590aaae 100644 --- a/src/collection/constructors/createCollectionFromJson.ts +++ b/src/collection/constructors/createCollectionFromJson.ts @@ -1,6 +1,6 @@ -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { PipelineCollection } from '../PipelineCollection'; -import { SimplePipelineCollection } from '../SimplePipelineCollection'; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { PipelineCollection } from "../PipelineCollection"; +import { SimplePipelineCollection } from "../SimplePipelineCollection"; /** * Creates PipelineCollection from array of PipelineJson or PipelineString @@ -12,6 +12,8 @@ import { SimplePipelineCollection } from '../SimplePipelineCollection'; * @returns PipelineCollection * @public exported from `@promptbook/core` */ -export function createCollectionFromJson(...promptbooks: ReadonlyArray): PipelineCollection { - return new SimplePipelineCollection(...promptbooks); +export function createCollectionFromJson( + ...promptbooks: ReadonlyArray +): PipelineCollection { + return new SimplePipelineCollection(...promptbooks); } diff --git a/src/collection/constructors/createCollectionFromPromise.test.ts b/src/collection/constructors/createCollectionFromPromise.test.ts index 8433f12740..51a320676c 100644 --- a/src/collection/constructors/createCollectionFromPromise.test.ts +++ b/src/collection/constructors/createCollectionFromPromise.test.ts @@ -1,12 +1,12 @@ -import { describe, expect, it } from '@jest/globals'; -import spaceTrim from 'spacetrim'; -import { forTime } from 'waitasecond'; -import { compilePipeline } from '../../conversion/compilePipeline'; -import type { PipelineString } from '../../pipeline/PipelineString'; -import { createCollectionFromPromise } from './createCollectionFromPromise'; - -describe('createCollectionFromPromise', () => { - const pipeline = spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import spaceTrim from "spacetrim"; +import { forTime } from "waitasecond"; +import { compilePipeline } from "../../conversion/compilePipeline"; +import type { PipelineString } from "../../pipeline/PipelineString"; +import { createCollectionFromPromise } from "./createCollectionFromPromise"; + +describe("createCollectionFromPromise", () => { + const pipeline = spaceTrim(` # Example prompt Show how to use a simple completion prompt @@ -31,18 +31,18 @@ describe('createCollectionFromPromise', () => { -> {response} `) as PipelineString; - // <- TODO: [📼] Use`book\`` string literal notation - - const collection = createCollectionFromPromise(async () => { - await forTime(100); - return [await compilePipeline(pipeline)]; - }); - - it('should get pipeline by url from collection', async () => { - expect.assertions(1); - const pipelineFromCollection = await collection.getPipelineByUrl( - 'https://promptbook.studio/examples/pipeline.book', - ); - expect(pipelineFromCollection).toEqual(await compilePipeline(pipeline)); - }); + // <- TODO: [📼] Use`book\`` string literal notation + + const collection = createCollectionFromPromise(async () => { + await forTime(100); + return [await compilePipeline(pipeline)]; + }); + + it("should get pipeline by url from collection", async () => { + expect.assertions(1); + const pipelineFromCollection = await collection.getPipelineByUrl( + "https://promptbook.studio/examples/pipeline.book", + ); + expect(pipelineFromCollection).toEqual(await compilePipeline(pipeline)); + }); }); diff --git a/src/collection/constructors/createCollectionFromPromise.ts b/src/collection/constructors/createCollectionFromPromise.ts index 6c7d853056..2b1b3032e3 100644 --- a/src/collection/constructors/createCollectionFromPromise.ts +++ b/src/collection/constructors/createCollectionFromPromise.ts @@ -1,8 +1,8 @@ -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { Prompt } from '../../types/Prompt'; -import type { string_pipeline_url } from '../../types/typeAliases'; -import type { PipelineCollection } from '../PipelineCollection'; -import { createCollectionFromJson } from './createCollectionFromJson'; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { Prompt } from "../../types/Prompt"; +import type { string_pipeline_url } from "../../types/typeAliases"; +import type { PipelineCollection } from "../PipelineCollection"; +import { createCollectionFromJson } from "./createCollectionFromJson"; /** * Constructs Promptbook from async sources @@ -25,40 +25,42 @@ import { createCollectionFromJson } from './createCollectionFromJson'; * @public exported from `@promptbook/core` */ export function createCollectionFromPromise( - promptbookSourcesPromiseOrFactory: - | Promise> - | (() => Promise>), + promptbookSourcesPromiseOrFactory: + | Promise> + | (() => Promise>), ): PipelineCollection { - let collection: PipelineCollection | null = null; + let collection: PipelineCollection | null = null; - async function load(): Promise { - if (collection !== null) { - return; - } - if (typeof promptbookSourcesPromiseOrFactory === 'function') { - // Note: Calling factory function only once despite multiple calls to resolveSources - promptbookSourcesPromiseOrFactory = promptbookSourcesPromiseOrFactory(); - } - const promptbookSources = await promptbookSourcesPromiseOrFactory; - collection = createCollectionFromJson(...promptbookSources); - } + async function load(): Promise { + if (collection !== null) { + return; + } + if (typeof promptbookSourcesPromiseOrFactory === "function") { + // Note: Calling factory function only once despite multiple calls to resolveSources + promptbookSourcesPromiseOrFactory = promptbookSourcesPromiseOrFactory(); + } + const promptbookSources = await promptbookSourcesPromiseOrFactory; + collection = createCollectionFromJson(...promptbookSources); + } - async function listPipelines(): Promise> { - await load(); - return /* not await */ collection!.listPipelines(); - } - async function getPipelineByUrl(url: string_pipeline_url): Promise { - await load(); - return /* not await */ collection!.getPipelineByUrl(url); - } - async function isResponsibleForPrompt(prompt: Prompt): Promise { - await load(); - return /* not await */ collection!.isResponsibleForPrompt(prompt); - } + async function listPipelines(): Promise> { + await load(); + return /* not await */ collection!.listPipelines(); + } + async function getPipelineByUrl( + url: string_pipeline_url, + ): Promise { + await load(); + return /* not await */ collection!.getPipelineByUrl(url); + } + async function isResponsibleForPrompt(prompt: Prompt): Promise { + await load(); + return /* not await */ collection!.isResponsibleForPrompt(prompt); + } - return { - listPipelines, - getPipelineByUrl, - isResponsibleForPrompt, - }; + return { + listPipelines, + getPipelineByUrl, + isResponsibleForPrompt, + }; } diff --git a/src/collection/constructors/createCollectionFromUrl.ts b/src/collection/constructors/createCollectionFromUrl.ts index 661e68e589..c8cf5bf8c9 100644 --- a/src/collection/constructors/createCollectionFromUrl.ts +++ b/src/collection/constructors/createCollectionFromUrl.ts @@ -1,25 +1,25 @@ -import { DEFAULT_IS_VERBOSE } from '../../config'; -import type { string_url } from '../../types/typeAliases'; -import type { PipelineCollection } from '../PipelineCollection'; -import { createCollectionFromPromise } from './createCollectionFromPromise'; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import type { string_url } from "../../types/typeAliases"; +import type { PipelineCollection } from "../PipelineCollection"; +import { createCollectionFromPromise } from "./createCollectionFromPromise"; /** * Options for `createCollectionFromDirectory` function */ type CreatePipelineCollectionFromUrlyOptions = { - /** - * If true, the collection creation outputs information about each file it reads - * - * @default false - */ - readonly isVerbose?: boolean; - - /** - * If true, directory will be scanned only when needed not during the construction - * - * @default false - */ - readonly isLazyLoaded?: boolean; + /** + * If true, the collection creation outputs information about each file it reads + * + * @default false + */ + readonly isVerbose?: boolean; + + /** + * If true, directory will be scanned only when needed not during the construction + * + * @default false + */ + readonly isLazyLoaded?: boolean; }; /** @@ -28,29 +28,30 @@ type CreatePipelineCollectionFromUrlyOptions = { * @public exported from `@promptbook/core` */ export async function createCollectionFromUrl( - url: string_url | URL, - options: CreatePipelineCollectionFromUrlyOptions, + url: string_url | URL, + options: CreatePipelineCollectionFromUrlyOptions, ): Promise { - const { isVerbose = DEFAULT_IS_VERBOSE, isLazyLoaded = false } = options || {}; + const { isVerbose = DEFAULT_IS_VERBOSE, isLazyLoaded = false } = + options || {}; - const collection = createCollectionFromPromise(async () => { - if (isVerbose) { - console.info(`Creating pipeline collection from url ${url.toString()}`); - } + const collection = createCollectionFromPromise(async () => { + if (isVerbose) { + console.info(`Creating pipeline collection from url ${url.toString()}`); + } - throw new Error('Not implemented yet'); - }); + throw new Error("Not implemented yet"); + }); - if (isLazyLoaded === false) { - await collection.listPipelines(); - } + if (isLazyLoaded === false) { + await collection.listPipelines(); + } - return collection; + return collection; - // TODO: [main] !!3 [🏳‍🌈] Allow variant with .json .js and .ts files - // TODO: [🧠][🏳‍🌈] .js and .ts files should create getter function of the collection - // TODO: Look at WebGPT "📖 Make Promptbook collection" and https://webgpt.cz/_books.json - // TODO: Implement via createCollectionFromPromise + // TODO: [main] !!3 [🏳‍🌈] Allow variant with .json .js and .ts files + // TODO: [🧠][🏳‍🌈] .js and .ts files should create getter function of the collection + // TODO: Look at WebGPT "📖 Make Promptbook collection" and https://webgpt.cz/_books.json + // TODO: Implement via createCollectionFromPromise } /** diff --git a/src/collection/constructors/createSubcollection.ts b/src/collection/constructors/createSubcollection.ts index fe7a7649ce..d22f507257 100644 --- a/src/collection/constructors/createSubcollection.ts +++ b/src/collection/constructors/createSubcollection.ts @@ -1,9 +1,9 @@ -import { spaceTrim } from 'spacetrim'; -import { NotFoundError } from '../../errors/NotFoundError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { Prompt } from '../../types/Prompt'; -import type { string_pipeline_url } from '../../types/typeAliases'; -import type { PipelineCollection } from '../PipelineCollection'; +import { spaceTrim } from "spacetrim"; +import { NotFoundError } from "../../errors/NotFoundError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { Prompt } from "../../types/Prompt"; +import type { string_pipeline_url } from "../../types/typeAliases"; +import type { PipelineCollection } from "../PipelineCollection"; /** * Creates PipelineCollection as a subset of another PipelineCollection @@ -16,45 +16,47 @@ import type { PipelineCollection } from '../PipelineCollection'; * @public exported from `@promptbook/core` */ export function createSubcollection( - collection: PipelineCollection, - predicate: (url: string_pipeline_url) => boolean, + collection: PipelineCollection, + predicate: (url: string_pipeline_url) => boolean, ): PipelineCollection { - async function listPipelines(): Promise> { - let promptbooks = await collection.listPipelines(); - promptbooks = promptbooks.filter(predicate); - return promptbooks; - } - async function getPipelineByUrl(url: string_pipeline_url): Promise { - if (!predicate(url)) { - throw new NotFoundError( - await spaceTrim( - async (block) => ` + async function listPipelines(): Promise> { + let promptbooks = await collection.listPipelines(); + promptbooks = promptbooks.filter(predicate); + return promptbooks; + } + async function getPipelineByUrl( + url: string_pipeline_url, + ): Promise { + if (!predicate(url)) { + throw new NotFoundError( + await spaceTrim( + async (block) => ` Promptbook with url "${url}" not found or not accessible Available promptbooks: - ${block((await listPipelines()).map((pipelineUrl) => `- ${pipelineUrl}`).join('\n'))} + ${block((await listPipelines()).map((pipelineUrl) => `- ${pipelineUrl}`).join("\n"))} All available promptbooks in parent collection: - ${block((await collection.listPipelines()).map((pipelineUrl) => `- ${pipelineUrl}`).join('\n'))} + ${block((await collection.listPipelines()).map((pipelineUrl) => `- ${pipelineUrl}`).join("\n"))} `, - ), - ); - } + ), + ); + } - const pipeline = await collection.getPipelineByUrl(url); + const pipeline = await collection.getPipelineByUrl(url); - return pipeline; - } - async function isResponsibleForPrompt(prompt: Prompt): Promise { - const isResponsible = await collection.isResponsibleForPrompt(prompt); - // TODO: Only if responsible, check if predicate is true - return isResponsible; - } + return pipeline; + } + async function isResponsibleForPrompt(prompt: Prompt): Promise { + const isResponsible = await collection.isResponsibleForPrompt(prompt); + // TODO: Only if responsible, check if predicate is true + return isResponsible; + } - return { - listPipelines, - getPipelineByUrl, - isResponsibleForPrompt, - }; + return { + listPipelines, + getPipelineByUrl, + isResponsibleForPrompt, + }; } diff --git a/src/commands/BOOK_VERSION/BookVersionCommand.ts b/src/commands/BOOK_VERSION/BookVersionCommand.ts index 71ca81e8c6..2f38bb8447 100644 --- a/src/commands/BOOK_VERSION/BookVersionCommand.ts +++ b/src/commands/BOOK_VERSION/BookVersionCommand.ts @@ -1,4 +1,4 @@ -import type { string_semantic_version } from '../../types/typeAliases'; +import type { string_semantic_version } from "../../types/typeAliases"; /** * Parsed BOOK_VERSION command @@ -7,6 +7,6 @@ import type { string_semantic_version } from '../../types/typeAliases'; * @public exported from `@promptbook/editable` */ export type BookVersionCommand = { - readonly type: 'BOOK_VERSION'; - readonly bookVersion: string_semantic_version; + readonly type: "BOOK_VERSION"; + readonly bookVersion: string_semantic_version; }; diff --git a/src/commands/BOOK_VERSION/bookVersionCommand.test.ts b/src/commands/BOOK_VERSION/bookVersionCommand.test.ts index 45e6baeda0..7b8d190d84 100644 --- a/src/commands/BOOK_VERSION/bookVersionCommand.test.ts +++ b/src/commands/BOOK_VERSION/bookVersionCommand.test.ts @@ -1,49 +1,62 @@ -import { describe, expect, it } from '@jest/globals'; -import { BOOK_LANGUAGE_VERSION } from '../../version'; -import { parseCommand } from '../_common/parseCommand'; -import { bookVersionCommandParser } from './bookVersionCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { BOOK_LANGUAGE_VERSION } from "../../version"; +import { parseCommand } from "../_common/parseCommand"; +import { bookVersionCommandParser } from "./bookVersionCommandParser"; -describe('how BOOK_VERSION command in .book.md files works', () => { - it('should parse BOOK_VERSION command', () => { - expect(parseCommand('promptbook version 0.62.0', 'PIPELINE_HEAD')).toEqual({ - type: 'BOOK_VERSION', - bookVersion: '0.62.0', - }); - expect(parseCommand('PTBK version 0.62.0', 'PIPELINE_HEAD')).toEqual({ - type: 'BOOK_VERSION', - bookVersion: '0.62.0', - }); +describe("how BOOK_VERSION command in .book.md files works", () => { + it("should parse BOOK_VERSION command", () => { + expect(parseCommand("promptbook version 0.62.0", "PIPELINE_HEAD")).toEqual({ + type: "BOOK_VERSION", + bookVersion: "0.62.0", + }); + expect(parseCommand("PTBK version 0.62.0", "PIPELINE_HEAD")).toEqual({ + type: "BOOK_VERSION", + bookVersion: "0.62.0", + }); - expect(parseCommand('PTBK version 0.62.0', 'PIPELINE_HEAD')).toEqual({ - type: 'BOOK_VERSION', - bookVersion: '0.62.0', - }); - expect(parseCommand(`PROMPTBOOK version ${BOOK_LANGUAGE_VERSION}`, 'PIPELINE_HEAD')).toEqual({ - type: 'BOOK_VERSION', - bookVersion: BOOK_LANGUAGE_VERSION, - }); - }); + expect(parseCommand("PTBK version 0.62.0", "PIPELINE_HEAD")).toEqual({ + type: "BOOK_VERSION", + bookVersion: "0.62.0", + }); + expect( + parseCommand( + `PROMPTBOOK version ${BOOK_LANGUAGE_VERSION}`, + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "BOOK_VERSION", + bookVersion: BOOK_LANGUAGE_VERSION, + }); + }); - it('should fail parsing BOOK_VERSION command', () => { - expect(() => parseCommand('PROMPTBOOK version', 'PIPELINE_HEAD')).toThrowError(/Version is required/i); - expect(() => parseCommand('PROMPTBOOK version ', 'PIPELINE_HEAD')).toThrowError(/Version is required/i); - expect(() => parseCommand('PROMPTBOOK version 0.25.0 0.26.0', 'PIPELINE_HEAD')).toThrowError( - /Can not have more than one Promptbook version/i, - ); - expect(() => parseCommand('PROMPTBOOK version 3.0.0', 'PIPELINE_HEAD')).toThrowError( - /Invalid Promptbook version/i, - ); - }); + it("should fail parsing BOOK_VERSION command", () => { + expect(() => + parseCommand("PROMPTBOOK version", "PIPELINE_HEAD"), + ).toThrowError(/Version is required/i); + expect(() => + parseCommand("PROMPTBOOK version ", "PIPELINE_HEAD"), + ).toThrowError(/Version is required/i); + expect(() => + parseCommand("PROMPTBOOK version 0.25.0 0.26.0", "PIPELINE_HEAD"), + ).toThrowError(/Can not have more than one Promptbook version/i); + expect(() => + parseCommand("PROMPTBOOK version 3.0.0", "PIPELINE_HEAD"), + ).toThrowError(/Invalid Promptbook version/i); + }); - it('should fail to prevent mismatch between version of pipeline and promptbook', () => { - expect(() => parseCommand('VERSION 0.62.0', 'PIPELINE_HEAD')).toThrowError(/Malformed or unknown command/i); - expect(() => parseCommand('V 0.62.0', 'PIPELINE_HEAD')).toThrowError(/Malformed or unknown command/i); - }); + it("should fail to prevent mismatch between version of pipeline and promptbook", () => { + expect(() => parseCommand("VERSION 0.62.0", "PIPELINE_HEAD")).toThrowError( + /Malformed or unknown command/i, + ); + expect(() => parseCommand("V 0.62.0", "PIPELINE_HEAD")).toThrowError( + /Malformed or unknown command/i, + ); + }); - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of bookVersionCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_HEAD')).not.toThrowError(); - } - }); + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of bookVersionCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_HEAD")).not.toThrowError(); + } + }); }); diff --git a/src/commands/BOOK_VERSION/bookVersionCommandParser.ts b/src/commands/BOOK_VERSION/bookVersionCommandParser.ts index cb3b1447d8..e3543d8d39 100644 --- a/src/commands/BOOK_VERSION/bookVersionCommandParser.ts +++ b/src/commands/BOOK_VERSION/bookVersionCommandParser.ts @@ -1,14 +1,14 @@ -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { ParseError } from '../../errors/ParseError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { isValidPromptbookVersion } from '../../utils/validators/semanticVersion/isValidPromptbookVersion'; -import { BOOK_LANGUAGE_VERSION } from '../../version'; -import type { $PipelineJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineHeadCommandParser } from '../_common/types/CommandParser'; -import type { BookVersionCommand } from './BookVersionCommand'; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { ParseError } from "../../errors/ParseError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { isValidPromptbookVersion } from "../../utils/validators/semanticVersion/isValidPromptbookVersion"; +import { BOOK_LANGUAGE_VERSION } from "../../version"; +import type { $PipelineJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineHeadCommandParser } from "../_common/types/CommandParser"; +import type { BookVersionCommand } from "./BookVersionCommand"; /** * Parses the BOOK_VERSION command @@ -16,88 +16,100 @@ import type { BookVersionCommand } from './BookVersionCommand'; * @see `documentationUrl` for more details * @public exported from `@promptbook/editable` */ -export const bookVersionCommandParser: PipelineHeadCommandParser = { - /** - * Name of the command - */ - name: 'BOOK_VERSION', +export const bookVersionCommandParser: PipelineHeadCommandParser = + { + /** + * Name of the command + */ + name: "BOOK_VERSION", - aliasNames: ['PTBK_VERSION', 'PROMPTBOOK_VERSION', 'BOOK'], + aliasNames: ["PTBK_VERSION", "PROMPTBOOK_VERSION", "BOOK"], - /** - * BOILERPLATE command can be used in: - */ - isUsedInPipelineHead: true, - isUsedInPipelineTask: false, + /** + * BOILERPLATE command can be used in: + */ + isUsedInPipelineHead: true, + isUsedInPipelineTask: false, - /** - * Description of the BOOK_VERSION command - */ - description: `Which version of the Book language is the .book.md using`, + /** + * Description of the BOOK_VERSION command + */ + description: `Which version of the Book language is the .book.md using`, - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/69', + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/69", - /** - * Example usages of the BOOK_VERSION command - */ - examples: [`BOOK VERSION ${BOOK_LANGUAGE_VERSION}`, `BOOK ${BOOK_LANGUAGE_VERSION}`], + /** + * Example usages of the BOOK_VERSION command + */ + examples: [ + `BOOK VERSION ${BOOK_LANGUAGE_VERSION}`, + `BOOK ${BOOK_LANGUAGE_VERSION}`, + ], - /** - * Parses the BOOK_VERSION command - */ - parse(input: CommandParserInput): BookVersionCommand { - const { args } = input; + /** + * Parses the BOOK_VERSION command + */ + parse(input: CommandParserInput): BookVersionCommand { + const { args } = input; - const bookVersion = args.pop(); + const bookVersion = args.pop(); - if (bookVersion === undefined) { - throw new ParseError(`Version is required`); - } + if (bookVersion === undefined) { + throw new ParseError(`Version is required`); + } - if (!isValidPromptbookVersion(bookVersion)) { - throw new ParseError(`Invalid Promptbook version "${bookVersion}"`); - } + if (!isValidPromptbookVersion(bookVersion)) { + throw new ParseError(`Invalid Promptbook version "${bookVersion}"`); + } - if (args.length > 0 && !(((args.length === 1 && args[0]) || '').toUpperCase() === 'VERSION')) { - throw new ParseError(`Can not have more than one Promptbook version`); - } + if ( + args.length > 0 && + !(((args.length === 1 && args[0]) || "").toUpperCase() === "VERSION") + ) { + throw new ParseError(`Can not have more than one Promptbook version`); + } - return { - type: 'BOOK_VERSION', - bookVersion: bookVersion, - } satisfies BookVersionCommand; - }, + return { + type: "BOOK_VERSION", + bookVersion: bookVersion, + } satisfies BookVersionCommand; + }, - /** - * Apply the BOOK_VERSION command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `pipelineJson` - */ - $applyToPipelineJson(command: BookVersionCommand, $pipelineJson: $PipelineJson): void { - // TODO: Warn if the version is overridden - $pipelineJson.bookVersion = command.bookVersion; - }, + /** + * Apply the BOOK_VERSION command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `pipelineJson` + */ + $applyToPipelineJson( + command: BookVersionCommand, + $pipelineJson: $PipelineJson, + ): void { + // TODO: Warn if the version is overridden + $pipelineJson.bookVersion = command.bookVersion; + }, - /** - * Converts the BOOK_VERSION command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: BookVersionCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, + /** + * Converts the BOOK_VERSION command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: BookVersionCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, - /** - * Reads the BOOK_VERSION command from the `PipelineJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray { - keepUnused(pipelineJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, -}; + /** + * Reads the BOOK_VERSION command from the `PipelineJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromPipelineJson( + pipelineJson: PipelineJson, + ): ReadonlyArray { + keepUnused(pipelineJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, + }; diff --git a/src/commands/EXPECT/ExpectCommand.ts b/src/commands/EXPECT/ExpectCommand.ts index b695920452..5a192d39e7 100644 --- a/src/commands/EXPECT/ExpectCommand.ts +++ b/src/commands/EXPECT/ExpectCommand.ts @@ -1,5 +1,5 @@ -import type { ExpectationAmount } from '../../pipeline/PipelineJson/Expectations'; -import type { ExpectationUnit } from '../../pipeline/PipelineJson/Expectations'; +import type { ExpectationAmount } from "../../pipeline/PipelineJson/Expectations"; +import type { ExpectationUnit } from "../../pipeline/PipelineJson/Expectations"; /** * Expect amount command describes the desired output of the task (after post-processing) @@ -8,10 +8,10 @@ import type { ExpectationUnit } from '../../pipeline/PipelineJson/Expectations'; * Note: LLMs work with tokens, not characters, but in Promptbooks we want to use some human-recognisable and cross-model interoperable units. */ export type ExpectCommand = { - readonly type: 'EXPECT'; - readonly sign: 'EXACTLY' | 'MINIMUM' | 'MAXIMUM'; - readonly unit: ExpectationUnit; - readonly amount: ExpectationAmount; + readonly type: "EXPECT"; + readonly sign: "EXACTLY" | "MINIMUM" | "MAXIMUM"; + readonly unit: ExpectationUnit; + readonly amount: ExpectationAmount; }; /** diff --git a/src/commands/EXPECT/expectCommand.test.ts b/src/commands/EXPECT/expectCommand.test.ts index a07fed6980..bb1c63e2f5 100644 --- a/src/commands/EXPECT/expectCommand.test.ts +++ b/src/commands/EXPECT/expectCommand.test.ts @@ -1,166 +1,198 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { expectCommandParser } from './expectCommandParser'; - -describe('how EXPECT command in .book.md files works', () => { - it('should parse EXPECT command', () => { - expect(parseCommand('Expect exactly 1 character', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'CHARACTERS', - amount: 1, - }); - - expect(parseCommand('Expect exactly 1 char', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'CHARACTERS', - amount: 1, - }); - - expect(parseCommand('Expect mininimum 1 character', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'MINIMUM', - unit: 'CHARACTERS', - amount: 1, - }); - - expect(parseCommand('Expect minimally 1 character', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'MINIMUM', - unit: 'CHARACTERS', - amount: 1, - }); - - expect(parseCommand('Expect min 1 char', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'MINIMUM', - unit: 'CHARACTERS', - amount: 1, - }); - - expect(parseCommand('Expect maximum 5 character', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'MAXIMUM', - unit: 'CHARACTERS', - amount: 5, - }); - - expect(parseCommand('Expect maximally 5 characters', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'MAXIMUM', - unit: 'CHARACTERS', - amount: 5, - }); - - expect(parseCommand('Expect max 5 CHARs', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'MAXIMUM', - unit: 'CHARACTERS', - amount: 5, - }); - - expect(parseCommand('Expect exact 1 word', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'WORDS', - amount: 1, - }); - - expect(parseCommand('Expect exactly 1 word', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'WORDS', - amount: 1, - }); - - expect(parseCommand('Expect eXactly 1 word', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'WORDS', - amount: 1, - }); - - expect(parseCommand('EXPECT EXACTLY 1 WORD', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'WORDS', - amount: 1, - }); - - expect(parseCommand('Expect exactly 2 words', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'WORDS', - amount: 2, - }); - - expect(parseCommand('Expect exactly 1 sentence', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'SENTENCES', - amount: 1, - }); - - expect(parseCommand('Expect exactly 2 sentences', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'SENTENCES', - amount: 2, - }); - - expect(parseCommand('Expect exactly 0 sentences', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'SENTENCES', - amount: 0, - }); - - expect(parseCommand('Expect exactly 1 paragraph', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'PARAGRAPHS', - amount: 1, - }); - - expect(parseCommand('Expect exactly 2 paragraphs', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'PARAGRAPHS', - amount: 2, - }); - - expect(parseCommand('Expect exactly 1 line', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'LINES', - amount: 1, - }); - - expect(parseCommand('Expect exactly 2 lines', 'PIPELINE_TASK')).toEqual({ - type: 'EXPECT', - sign: 'EXACTLY', - unit: 'LINES', - amount: 2, - }); - - // TODO: Add page test - }); - - it('should fail parsing expect command', () => { - expect(() => parseCommand('Expect foo 1 char', 'PIPELINE_TASK')).toThrowError(/Invalid EXPECT command/i); - expect(() => parseCommand('Expect min 1 vars', 'PIPELINE_TASK')).toThrowError(/Invalid EXPECT command/i); - expect(() => parseCommand('Expect min chars', 'PIPELINE_TASK')).toThrowError(/Invalid EXPECT command/i); - expect(() => parseCommand('Expect min xx chars', 'PIPELINE_TASK')).toThrowError(/Invalid EXPECT command/i); - expect(() => parseCommand('Expect exactly 2 p', 'PIPELINE_TASK')).toThrowError(/Ambiguous unit "p"/i); - expect(() => parseCommand('EXPECT', 'PIPELINE_TASK')).toThrowError(/Invalid EXPECT command/i); - expect(() => parseCommand('EXPECT brr', 'PIPELINE_TASK')).toThrowError(/Invalid EXPECT command/i); - }); - - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of expectCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_TASK')).not.toThrowError(); - } - }); +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { expectCommandParser } from "./expectCommandParser"; + +describe("how EXPECT command in .book.md files works", () => { + it("should parse EXPECT command", () => { + expect(parseCommand("Expect exactly 1 character", "PIPELINE_TASK")).toEqual( + { + type: "EXPECT", + sign: "EXACTLY", + unit: "CHARACTERS", + amount: 1, + }, + ); + + expect(parseCommand("Expect exactly 1 char", "PIPELINE_TASK")).toEqual({ + type: "EXPECT", + sign: "EXACTLY", + unit: "CHARACTERS", + amount: 1, + }); + + expect( + parseCommand("Expect mininimum 1 character", "PIPELINE_TASK"), + ).toEqual({ + type: "EXPECT", + sign: "MINIMUM", + unit: "CHARACTERS", + amount: 1, + }); + + expect( + parseCommand("Expect minimally 1 character", "PIPELINE_TASK"), + ).toEqual({ + type: "EXPECT", + sign: "MINIMUM", + unit: "CHARACTERS", + amount: 1, + }); + + expect(parseCommand("Expect min 1 char", "PIPELINE_TASK")).toEqual({ + type: "EXPECT", + sign: "MINIMUM", + unit: "CHARACTERS", + amount: 1, + }); + + expect(parseCommand("Expect maximum 5 character", "PIPELINE_TASK")).toEqual( + { + type: "EXPECT", + sign: "MAXIMUM", + unit: "CHARACTERS", + amount: 5, + }, + ); + + expect( + parseCommand("Expect maximally 5 characters", "PIPELINE_TASK"), + ).toEqual({ + type: "EXPECT", + sign: "MAXIMUM", + unit: "CHARACTERS", + amount: 5, + }); + + expect(parseCommand("Expect max 5 CHARs", "PIPELINE_TASK")).toEqual({ + type: "EXPECT", + sign: "MAXIMUM", + unit: "CHARACTERS", + amount: 5, + }); + + expect(parseCommand("Expect exact 1 word", "PIPELINE_TASK")).toEqual({ + type: "EXPECT", + sign: "EXACTLY", + unit: "WORDS", + amount: 1, + }); + + expect(parseCommand("Expect exactly 1 word", "PIPELINE_TASK")).toEqual({ + type: "EXPECT", + sign: "EXACTLY", + unit: "WORDS", + amount: 1, + }); + + expect(parseCommand("Expect eXactly 1 word", "PIPELINE_TASK")).toEqual({ + type: "EXPECT", + sign: "EXACTLY", + unit: "WORDS", + amount: 1, + }); + + expect(parseCommand("EXPECT EXACTLY 1 WORD", "PIPELINE_TASK")).toEqual({ + type: "EXPECT", + sign: "EXACTLY", + unit: "WORDS", + amount: 1, + }); + + expect(parseCommand("Expect exactly 2 words", "PIPELINE_TASK")).toEqual({ + type: "EXPECT", + sign: "EXACTLY", + unit: "WORDS", + amount: 2, + }); + + expect(parseCommand("Expect exactly 1 sentence", "PIPELINE_TASK")).toEqual({ + type: "EXPECT", + sign: "EXACTLY", + unit: "SENTENCES", + amount: 1, + }); + + expect(parseCommand("Expect exactly 2 sentences", "PIPELINE_TASK")).toEqual( + { + type: "EXPECT", + sign: "EXACTLY", + unit: "SENTENCES", + amount: 2, + }, + ); + + expect(parseCommand("Expect exactly 0 sentences", "PIPELINE_TASK")).toEqual( + { + type: "EXPECT", + sign: "EXACTLY", + unit: "SENTENCES", + amount: 0, + }, + ); + + expect(parseCommand("Expect exactly 1 paragraph", "PIPELINE_TASK")).toEqual( + { + type: "EXPECT", + sign: "EXACTLY", + unit: "PARAGRAPHS", + amount: 1, + }, + ); + + expect( + parseCommand("Expect exactly 2 paragraphs", "PIPELINE_TASK"), + ).toEqual({ + type: "EXPECT", + sign: "EXACTLY", + unit: "PARAGRAPHS", + amount: 2, + }); + + expect(parseCommand("Expect exactly 1 line", "PIPELINE_TASK")).toEqual({ + type: "EXPECT", + sign: "EXACTLY", + unit: "LINES", + amount: 1, + }); + + expect(parseCommand("Expect exactly 2 lines", "PIPELINE_TASK")).toEqual({ + type: "EXPECT", + sign: "EXACTLY", + unit: "LINES", + amount: 2, + }); + + // TODO: Add page test + }); + + it("should fail parsing expect command", () => { + expect(() => + parseCommand("Expect foo 1 char", "PIPELINE_TASK"), + ).toThrowError(/Invalid EXPECT command/i); + expect(() => + parseCommand("Expect min 1 vars", "PIPELINE_TASK"), + ).toThrowError(/Invalid EXPECT command/i); + expect(() => + parseCommand("Expect min chars", "PIPELINE_TASK"), + ).toThrowError(/Invalid EXPECT command/i); + expect(() => + parseCommand("Expect min xx chars", "PIPELINE_TASK"), + ).toThrowError(/Invalid EXPECT command/i); + expect(() => + parseCommand("Expect exactly 2 p", "PIPELINE_TASK"), + ).toThrowError(/Ambiguous unit "p"/i); + expect(() => parseCommand("EXPECT", "PIPELINE_TASK")).toThrowError( + /Invalid EXPECT command/i, + ); + expect(() => parseCommand("EXPECT brr", "PIPELINE_TASK")).toThrowError( + /Invalid EXPECT command/i, + ); + }); + + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of expectCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_TASK")).not.toThrowError(); + } + }); }); diff --git a/src/commands/EXPECT/expectCommandParser.ts b/src/commands/EXPECT/expectCommandParser.ts index a5cb08ae0d..080a4aed54 100644 --- a/src/commands/EXPECT/expectCommandParser.ts +++ b/src/commands/EXPECT/expectCommandParser.ts @@ -1,15 +1,15 @@ -import spaceTrim from 'spacetrim'; -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { ParseError } from '../../errors/ParseError'; -import type { ExpectationUnit } from '../../pipeline/PipelineJson/Expectations'; -import { EXPECTATION_UNITS } from '../../pipeline/PipelineJson/Expectations'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { parseNumber } from '../../utils/parseNumber'; -import type { $TaskJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineTaskCommandParser } from '../_common/types/CommandParser'; -import type { ExpectCommand } from './ExpectCommand'; +import spaceTrim from "spacetrim"; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { ParseError } from "../../errors/ParseError"; +import type { ExpectationUnit } from "../../pipeline/PipelineJson/Expectations"; +import { EXPECTATION_UNITS } from "../../pipeline/PipelineJson/Expectations"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { parseNumber } from "../../utils/parseNumber"; +import type { $TaskJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineTaskCommandParser } from "../_common/types/CommandParser"; +import type { ExpectCommand } from "./ExpectCommand"; /** * Parses the expect command @@ -18,167 +18,176 @@ import type { ExpectCommand } from './ExpectCommand'; * @public exported from `@promptbook/editable` */ export const expectCommandParser: PipelineTaskCommandParser = { - /** - * Name of the command - */ - name: 'EXPECT', - - /** - * BOILERPLATE command can be used in: - */ - isUsedInPipelineHead: false, - isUsedInPipelineTask: true, - - /** - * Description of the FORMAT command - */ - description: spaceTrim(` + /** + * Name of the command + */ + name: "EXPECT", + + /** + * BOILERPLATE command can be used in: + */ + isUsedInPipelineHead: false, + isUsedInPipelineTask: true, + + /** + * Description of the FORMAT command + */ + description: spaceTrim(` Expect command describes the desired output of the task *(after post-processing)* It can set limits for the maximum/minimum length of the output, measured in characters, words, sentences, paragraphs or some other shape of the output. `), - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/30', - - /** - * Example usages of the FORMAT command - */ - examples: [ - 'EXPECT MIN 100 Characters', - 'EXPECT MAX 10 Words', - 'EXPECT EXACTLY 3 Sentences', - 'EXPECT EXACTLY 1 Paragraph', - // <- TODO: 'EXPECT 1 Paragraph', - ], - - /** - * Parses the FORMAT command - */ - parse(input: CommandParserInput): ExpectCommand { - const { args } = input; - - try { - let sign: ExpectCommand['sign']; - const signRaw = args.shift()!; - if (/^exact/i.test(signRaw)) { - sign = 'EXACTLY'; - } else if (/^min/i.test(signRaw)) { - sign = 'MINIMUM'; - } else if (/^max/i.test(signRaw)) { - sign = 'MAXIMUM'; - } else { - throw new ParseError(`Invalid sign "${signRaw}", expected EXACTLY, MIN or MAX`); - } - - const amountRaw = args.shift()!; - const amount = parseNumber(amountRaw); - if (amount < 0) { - throw new ParseError('Amount must be positive number or zero'); - } - if (amount !== Math.floor(amount)) { - throw new ParseError('Amount must be whole number'); - } - - const unitRaw = args.shift()!; - let unit: ExpectCommand['unit'] | undefined = undefined; - for (const existingUnit of EXPECTATION_UNITS) { - let existingUnitText: string = existingUnit; - - existingUnitText = existingUnitText.substring(0, existingUnitText.length - 1); - if (existingUnitText === 'CHARACTER') { - existingUnitText = 'CHAR'; - } - - if ( - new RegExp(`^${existingUnitText.toLowerCase()}`).test(unitRaw.toLowerCase()) || - new RegExp(`^${unitRaw.toLowerCase()}`).test(existingUnitText.toLowerCase()) - ) { - if (unit !== undefined) { - throw new ParseError(`Ambiguous unit "${unitRaw}"`); - } - unit = existingUnit; - } - } - if (unit === undefined) { - throw new ParseError(`Invalid unit "${unitRaw}"`); - } - - return { - type: 'EXPECT', - sign, - unit, - amount, - } satisfies ExpectCommand; - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - throw new ParseError( - spaceTrim( - (block) => - ` + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/30", + + /** + * Example usages of the FORMAT command + */ + examples: [ + "EXPECT MIN 100 Characters", + "EXPECT MAX 10 Words", + "EXPECT EXACTLY 3 Sentences", + "EXPECT EXACTLY 1 Paragraph", + // <- TODO: 'EXPECT 1 Paragraph', + ], + + /** + * Parses the FORMAT command + */ + parse(input: CommandParserInput): ExpectCommand { + const { args } = input; + + try { + let sign: ExpectCommand["sign"]; + const signRaw = args.shift()!; + if (/^exact/i.test(signRaw)) { + sign = "EXACTLY"; + } else if (/^min/i.test(signRaw)) { + sign = "MINIMUM"; + } else if (/^max/i.test(signRaw)) { + sign = "MAXIMUM"; + } else { + throw new ParseError( + `Invalid sign "${signRaw}", expected EXACTLY, MIN or MAX`, + ); + } + + const amountRaw = args.shift()!; + const amount = parseNumber(amountRaw); + if (amount < 0) { + throw new ParseError("Amount must be positive number or zero"); + } + if (amount !== Math.floor(amount)) { + throw new ParseError("Amount must be whole number"); + } + + const unitRaw = args.shift()!; + let unit: ExpectCommand["unit"] | undefined = undefined; + for (const existingUnit of EXPECTATION_UNITS) { + let existingUnitText: string = existingUnit; + + existingUnitText = existingUnitText.substring( + 0, + existingUnitText.length - 1, + ); + if (existingUnitText === "CHARACTER") { + existingUnitText = "CHAR"; + } + + if ( + new RegExp(`^${existingUnitText.toLowerCase()}`).test( + unitRaw.toLowerCase(), + ) || + new RegExp(`^${unitRaw.toLowerCase()}`).test( + existingUnitText.toLowerCase(), + ) + ) { + if (unit !== undefined) { + throw new ParseError(`Ambiguous unit "${unitRaw}"`); + } + unit = existingUnit; + } + } + if (unit === undefined) { + throw new ParseError(`Invalid unit "${unitRaw}"`); + } + + return { + type: "EXPECT", + sign, + unit, + amount, + } satisfies ExpectCommand; + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + throw new ParseError( + spaceTrim( + (block) => + ` Invalid FORMAT command ${block((error as Error).message)}: `, - ), - ); - } - }, - - /** - * Apply the FORMAT command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `taskJson` - */ - $applyToTaskJson(command: ExpectCommand, $taskJson: $TaskJson): void { - // eslint-disable-next-line no-case-declarations - const unit = command.unit.toLowerCase() as Lowercase; - - $taskJson.expectations = $taskJson.expectations || {}; - $taskJson.expectations[unit] = $taskJson.expectations[unit] || {}; - - if (command.sign === 'MINIMUM' || command.sign === 'EXACTLY') { - if ($taskJson.expectations[unit]!.min !== undefined) { - throw new ParseError( - `Already defined minumum ${ - $taskJson.expectations![unit]!.min - } ${command.unit.toLowerCase()}, now trying to redefine it to ${command.amount}`, - ); - } - $taskJson.expectations[unit]!.min = command.amount; - } /* not else */ - if (command.sign === 'MAXIMUM' || command.sign === 'EXACTLY') { - if ($taskJson.expectations[unit]!.max !== undefined) { - throw new ParseError( - `Already defined maximum ${ - $taskJson.expectations![unit]!.max - } ${command.unit.toLowerCase()}, now trying to redefine it to ${command.amount}`, - ); - } - $taskJson.expectations[unit]!.max = command.amount; - } - }, - - /** - * Converts the FORMAT command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: ExpectCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, - - /** - * Reads the FORMAT command from the `TaskJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { - keepUnused($taskJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, + ), + ); + } + }, + + /** + * Apply the FORMAT command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `taskJson` + */ + $applyToTaskJson(command: ExpectCommand, $taskJson: $TaskJson): void { + // eslint-disable-next-line no-case-declarations + const unit = command.unit.toLowerCase() as Lowercase; + + $taskJson.expectations = $taskJson.expectations || {}; + $taskJson.expectations[unit] = $taskJson.expectations[unit] || {}; + + if (command.sign === "MINIMUM" || command.sign === "EXACTLY") { + if ($taskJson.expectations[unit]!.min !== undefined) { + throw new ParseError( + `Already defined minumum ${ + $taskJson.expectations![unit]!.min + } ${command.unit.toLowerCase()}, now trying to redefine it to ${command.amount}`, + ); + } + $taskJson.expectations[unit]!.min = command.amount; + } /* not else */ + if (command.sign === "MAXIMUM" || command.sign === "EXACTLY") { + if ($taskJson.expectations[unit]!.max !== undefined) { + throw new ParseError( + `Already defined maximum ${ + $taskJson.expectations![unit]!.max + } ${command.unit.toLowerCase()}, now trying to redefine it to ${command.amount}`, + ); + } + $taskJson.expectations[unit]!.max = command.amount; + } + }, + + /** + * Converts the FORMAT command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: ExpectCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, + + /** + * Reads the FORMAT command from the `TaskJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { + keepUnused($taskJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, }; diff --git a/src/commands/FOREACH/ForeachCommand.ts b/src/commands/FOREACH/ForeachCommand.ts index 601298a0fb..729eecc441 100644 --- a/src/commands/FOREACH/ForeachCommand.ts +++ b/src/commands/FOREACH/ForeachCommand.ts @@ -1,4 +1,4 @@ -import type { ForeachJson } from './ForeachJson'; +import type { ForeachJson } from "./ForeachJson"; /** * Parsed FOREACH command which is used to iterate over a table of values @@ -7,5 +7,5 @@ import type { ForeachJson } from './ForeachJson'; * @public exported from `@promptbook/editable` */ export type ForeachCommand = { - readonly type: 'FOREACH'; + readonly type: "FOREACH"; } & ForeachJson; diff --git a/src/commands/FOREACH/ForeachJson.ts b/src/commands/FOREACH/ForeachJson.ts index 0c5da5b259..0f82c2e8ef 100644 --- a/src/commands/FOREACH/ForeachJson.ts +++ b/src/commands/FOREACH/ForeachJson.ts @@ -1,33 +1,33 @@ -import type { string_parameter_name } from '../../types/typeAliases'; -import type { TODO_string } from '../../utils/organization/TODO_string'; +import type { string_parameter_name } from "../../types/typeAliases"; +import type { TODO_string } from "../../utils/organization/TODO_string"; /** * @@@ */ export type ForeachJson = { - /** - * @@@ - */ - readonly formatName: TODO_string; + /** + * @@@ + */ + readonly formatName: TODO_string; - /** - * @@@ - */ - readonly subformatName: TODO_string; + /** + * @@@ + */ + readonly subformatName: TODO_string; - /** - * @@@ - */ - readonly parameterName: string_parameter_name; + /** + * @@@ + */ + readonly parameterName: string_parameter_name; - /** - * @@@ - */ - readonly inputSubparameterNames: Array; - // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems + /** + * @@@ + */ + readonly inputSubparameterNames: Array; + // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems - /** - * @@@ - */ - readonly outputSubparameterName: string_parameter_name; + /** + * @@@ + */ + readonly outputSubparameterName: string_parameter_name; }; diff --git a/src/commands/FOREACH/foreachCommand.test.ts b/src/commands/FOREACH/foreachCommand.test.ts index 91fe206671..a07e4f5428 100644 --- a/src/commands/FOREACH/foreachCommand.test.ts +++ b/src/commands/FOREACH/foreachCommand.test.ts @@ -1,91 +1,127 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { foreachCommandParser } from './foreachCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { foreachCommandParser } from "./foreachCommandParser"; -describe('how FOREACH command in .book.md files works', () => { - it('should parse FOREACH command in PIPELINE_TASK', () => { - expect(parseCommand('FOREACH Text Line `{customers}` -> `{customer}`', 'PIPELINE_TASK')).toEqual({ - type: 'FOREACH', - formatName: 'TEXT', - subformatName: 'LINE', - parameterName: 'customers', - inputSubparameterNames: ['customer'], - outputSubparameterName: 'newLine', - }); - }); - - it('should parse FOREACH command in PIPELINE_TASK with multiple inputSubparameterNames', () => { - expect(parseCommand('FOREACH Text Line `{customers}` -> `{firstName}`, `{lastName}`', 'PIPELINE_TASK')).toEqual( - { - type: 'FOREACH', - formatName: 'TEXT', - subformatName: 'LINE', - parameterName: 'customers', - inputSubparameterNames: ['firstName', 'lastName'], - outputSubparameterName: 'newLine', - }, - ); - }); - - it('should parse FOREACH command in PIPELINE_TASK with specified outputSubparameterName', () => { - expect( - parseCommand('FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`', 'PIPELINE_TASK'), - ).toEqual({ - type: 'FOREACH', - formatName: 'CSV', - subformatName: 'ROW', - parameterName: 'customers', - inputSubparameterNames: ['firstName', 'lastName'], - outputSubparameterName: 'email', - }); - }); +describe("how FOREACH command in .book.md files works", () => { + it("should parse FOREACH command in PIPELINE_TASK", () => { + expect( + parseCommand( + "FOREACH Text Line `{customers}` -> `{customer}`", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "TEXT", + subformatName: "LINE", + parameterName: "customers", + inputSubparameterNames: ["customer"], + outputSubparameterName: "newLine", + }); + }); - it('should parse FOREACH command in multiple formats', () => { - expect(parseCommand('FOREACH Text Line `{customers}` -> `{customer}`', 'PIPELINE_TASK')).toEqual({ - type: 'FOREACH', - formatName: 'TEXT', - subformatName: 'LINE', - parameterName: 'customers', - inputSubparameterNames: ['customer'], - outputSubparameterName: 'newLine', - }); - expect(parseCommand('FOREACH Text Line {customers} -> {customer}', 'PIPELINE_TASK')).toEqual({ - type: 'FOREACH', - formatName: 'TEXT', - subformatName: 'LINE', - parameterName: 'customers', - inputSubparameterNames: ['customer'], - outputSubparameterName: 'newLine', - }); + it("should parse FOREACH command in PIPELINE_TASK with multiple inputSubparameterNames", () => { + expect( + parseCommand( + "FOREACH Text Line `{customers}` -> `{firstName}`, `{lastName}`", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "TEXT", + subformatName: "LINE", + parameterName: "customers", + inputSubparameterNames: ["firstName", "lastName"], + outputSubparameterName: "newLine", + }); + }); - expect(parseCommand('FOREACH Text Line `{customers} -> {customer}`', 'PIPELINE_TASK')).toEqual({ - type: 'FOREACH', - formatName: 'TEXT', - subformatName: 'LINE', - parameterName: 'customers', - inputSubparameterNames: ['customer'], - outputSubparameterName: 'newLine', - }); + it("should parse FOREACH command in PIPELINE_TASK with specified outputSubparameterName", () => { + expect( + parseCommand( + "FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "CSV", + subformatName: "ROW", + parameterName: "customers", + inputSubparameterNames: ["firstName", "lastName"], + outputSubparameterName: "email", + }); + }); - expect(parseCommand('EACH Text Line {customers} -> {customer} ', 'PIPELINE_TASK')).toEqual({ - type: 'FOREACH', - formatName: 'TEXT', - subformatName: 'LINE', - parameterName: 'customers', - inputSubparameterNames: ['customer'], - outputSubparameterName: 'newLine', - }); + it("should parse FOREACH command in multiple formats", () => { + expect( + parseCommand( + "FOREACH Text Line `{customers}` -> `{customer}`", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "TEXT", + subformatName: "LINE", + parameterName: "customers", + inputSubparameterNames: ["customer"], + outputSubparameterName: "newLine", + }); + expect( + parseCommand( + "FOREACH Text Line {customers} -> {customer}", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "TEXT", + subformatName: "LINE", + parameterName: "customers", + inputSubparameterNames: ["customer"], + outputSubparameterName: "newLine", + }); - expect(parseCommand('EACH Text Line customers -> customer ', 'PIPELINE_TASK')).toEqual({ - type: 'FOREACH', - formatName: 'TEXT', - subformatName: 'LINE', - parameterName: 'customers', - inputSubparameterNames: ['customer'], - outputSubparameterName: 'newLine', - }); + expect( + parseCommand( + "FOREACH Text Line `{customers} -> {customer}`", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "TEXT", + subformatName: "LINE", + parameterName: "customers", + inputSubparameterNames: ["customer"], + outputSubparameterName: "newLine", + }); + + expect( + parseCommand( + "EACH Text Line {customers} -> {customer} ", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "TEXT", + subformatName: "LINE", + parameterName: "customers", + inputSubparameterNames: ["customer"], + outputSubparameterName: "newLine", + }); - /* + expect( + parseCommand( + "EACH Text Line customers -> customer ", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "TEXT", + subformatName: "LINE", + parameterName: "customers", + inputSubparameterNames: ["customer"], + outputSubparameterName: "newLine", + }); + + /* TODO: This should work expect(parseCommand('FOREACH TEXT LINE `{customers}` ->`{customer}`', 'PIPELINE_TASK')).toEqual({ type: 'FOREACH', @@ -97,97 +133,118 @@ describe('how FOREACH command in .book.md files works', () => { }); */ - expect( - parseCommand('FOREACH Csv Row `{customers}` -> {firstName} {lastName} +{email}', 'PIPELINE_TASK'), - ).toEqual({ - type: 'FOREACH', - formatName: 'CSV', - subformatName: 'ROW', - parameterName: 'customers', - inputSubparameterNames: ['firstName', 'lastName'], - outputSubparameterName: 'email', - }); - expect(parseCommand('FOREACH Csv Row {customers} -> {firstName} {lastName} +{email}', 'PIPELINE_TASK')).toEqual( - { - type: 'FOREACH', - formatName: 'CSV', - subformatName: 'ROW', - parameterName: 'customers', - inputSubparameterNames: ['firstName', 'lastName'], - outputSubparameterName: 'email', - }, - ); - - expect( - parseCommand('FOREACH Csv Row `{customers} -> {firstName}, {lastName} +{email}`', 'PIPELINE_TASK'), - ).toEqual({ - type: 'FOREACH', - formatName: 'CSV', - subformatName: 'ROW', - parameterName: 'customers', - inputSubparameterNames: ['firstName', 'lastName'], - outputSubparameterName: 'email', - }); + expect( + parseCommand( + "FOREACH Csv Row `{customers}` -> {firstName} {lastName} +{email}", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "CSV", + subformatName: "ROW", + parameterName: "customers", + inputSubparameterNames: ["firstName", "lastName"], + outputSubparameterName: "email", + }); + expect( + parseCommand( + "FOREACH Csv Row {customers} -> {firstName} {lastName} +{email}", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "CSV", + subformatName: "ROW", + parameterName: "customers", + inputSubparameterNames: ["firstName", "lastName"], + outputSubparameterName: "email", + }); - expect( - parseCommand('FOREACH Csv Row `{customers} -> {firstName} , {lastName} +{email}`', 'PIPELINE_TASK'), - ).toEqual({ - type: 'FOREACH', - formatName: 'CSV', - subformatName: 'ROW', - parameterName: 'customers', - inputSubparameterNames: ['firstName', 'lastName'], - outputSubparameterName: 'email', - }); + expect( + parseCommand( + "FOREACH Csv Row `{customers} -> {firstName}, {lastName} +{email}`", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "CSV", + subformatName: "ROW", + parameterName: "customers", + inputSubparameterNames: ["firstName", "lastName"], + outputSubparameterName: "email", + }); - expect( - parseCommand( - 'EACH Csv Row {customers} -> {firstName} {lastName} +{email}', - 'PIPELINE_TASK', - ), - ).toEqual({ - type: 'FOREACH', - formatName: 'CSV', - subformatName: 'ROW', - parameterName: 'customers', - inputSubparameterNames: ['firstName', 'lastName'], - outputSubparameterName: 'email', - }); + expect( + parseCommand( + "FOREACH Csv Row `{customers} -> {firstName} , {lastName} +{email}`", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "CSV", + subformatName: "ROW", + parameterName: "customers", + inputSubparameterNames: ["firstName", "lastName"], + outputSubparameterName: "email", + }); - expect( - parseCommand('FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`', 'PIPELINE_TASK'), - ).toEqual({ - type: 'FOREACH', - formatName: 'CSV', - subformatName: 'ROW', - parameterName: 'customers', - inputSubparameterNames: ['firstName', 'lastName'], - outputSubparameterName: 'email', - }); + expect( + parseCommand( + "EACH Csv Row {customers} -> {firstName} {lastName} +{email}", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "CSV", + subformatName: "ROW", + parameterName: "customers", + inputSubparameterNames: ["firstName", "lastName"], + outputSubparameterName: "email", + }); - expect( - parseCommand('FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, +`{email}`', 'PIPELINE_TASK'), - ).toEqual({ - type: 'FOREACH', - formatName: 'CSV', - subformatName: 'ROW', - parameterName: 'customers', - inputSubparameterNames: ['firstName', 'lastName'], - outputSubparameterName: 'email', - }); + expect( + parseCommand( + "FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "CSV", + subformatName: "ROW", + parameterName: "customers", + inputSubparameterNames: ["firstName", "lastName"], + outputSubparameterName: "email", + }); - expect( - parseCommand('FOREACH Csv Row `{customers}` -> {firstName}, {lastName}, +{email}', 'PIPELINE_TASK'), - ).toEqual({ - type: 'FOREACH', - formatName: 'CSV', - subformatName: 'ROW', - parameterName: 'customers', - inputSubparameterNames: ['firstName', 'lastName'], - outputSubparameterName: 'email', - }); + expect( + parseCommand( + "FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, +`{email}`", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "CSV", + subformatName: "ROW", + parameterName: "customers", + inputSubparameterNames: ["firstName", "lastName"], + outputSubparameterName: "email", + }); - /* + expect( + parseCommand( + "FOREACH Csv Row `{customers}` -> {firstName}, {lastName}, +{email}", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "CSV", + subformatName: "ROW", + parameterName: "customers", + inputSubparameterNames: ["firstName", "lastName"], + outputSubparameterName: "email", + }); + + /* TODO: This should work expect(parseCommand('FOREACH CSV ROW `{customers}` ->`{firstName}``{lastName}`', 'PIPELINE_TASK')).toEqual({ type: 'FOREACH', @@ -198,49 +255,67 @@ describe('how FOREACH command in .book.md files works', () => { outputSubparameterName: 'newLine', }); */ - }); + }); - it('should parse FOREACH command in shortcut form', () => { - expect(parseCommand('EACH CSV CELL `{customers} -> `{customer}`', 'PIPELINE_TASK')).toEqual({ - type: 'FOREACH', - formatName: 'CSV', - subformatName: 'CELL', - parameterName: 'customers', - inputSubparameterNames: ['customer'], - outputSubparameterName: 'newCell', - }); - }); - - it('should fail parsing FOREACH command', () => { - expect(() => parseCommand('FOREACH brr', 'PIPELINE_TASK')).toThrowError(/Unsupported format "BRR"/i); - expect(() => parseCommand('FOREACH Text', 'PIPELINE_TASK')).toThrowError(/Invalid FOREACH command/i); - expect(() => parseCommand('FOREACH Text Line', 'PIPELINE_TASK')).toThrowError(/Invalid FOREACH command/i); - expect(() => parseCommand('FOREACH Text Line `{customer}`', 'PIPELINE_TASK')).toThrowError( - /Invalid FOREACH command/i, - ); - expect(() => parseCommand('FOREACH Text Line -> `{customer}`', 'PIPELINE_TASK')).toThrowError( - /Invalid FOREACH command/i, - ); - expect(() => parseCommand('FOREACH Csv Row `{customer}` ->', 'PIPELINE_TASK')).toThrowError( - /FOREACH command must have at least one input subparameter/i, - ); - - expect(() => - parseCommand('FOREACH Csv Row `{customer}` -> {firstName}, {lastName}', 'PIPELINE_TASK'), - ).toThrowError(/FOREACH CSV ROW must specify output subparameter/i); - - expect(() => - parseCommand( - 'FOREACH Csv Row `{customer}` -> {firstName}, {lastName}, +{email1}, +{email2}', - 'PIPELINE_TASK', - ), - ).toThrowError(/FOREACH command can not have more than one output subparameter/i); - }); - - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of foreachCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_TASK')).not.toThrowError(); - } - }); + it("should parse FOREACH command in shortcut form", () => { + expect( + parseCommand( + "EACH CSV CELL `{customers} -> `{customer}`", + "PIPELINE_TASK", + ), + ).toEqual({ + type: "FOREACH", + formatName: "CSV", + subformatName: "CELL", + parameterName: "customers", + inputSubparameterNames: ["customer"], + outputSubparameterName: "newCell", + }); + }); + + it("should fail parsing FOREACH command", () => { + expect(() => parseCommand("FOREACH brr", "PIPELINE_TASK")).toThrowError( + /Unsupported format "BRR"/i, + ); + expect(() => parseCommand("FOREACH Text", "PIPELINE_TASK")).toThrowError( + /Invalid FOREACH command/i, + ); + expect(() => + parseCommand("FOREACH Text Line", "PIPELINE_TASK"), + ).toThrowError(/Invalid FOREACH command/i); + expect(() => + parseCommand("FOREACH Text Line `{customer}`", "PIPELINE_TASK"), + ).toThrowError(/Invalid FOREACH command/i); + expect(() => + parseCommand("FOREACH Text Line -> `{customer}`", "PIPELINE_TASK"), + ).toThrowError(/Invalid FOREACH command/i); + expect(() => + parseCommand("FOREACH Csv Row `{customer}` ->", "PIPELINE_TASK"), + ).toThrowError( + /FOREACH command must have at least one input subparameter/i, + ); + + expect(() => + parseCommand( + "FOREACH Csv Row `{customer}` -> {firstName}, {lastName}", + "PIPELINE_TASK", + ), + ).toThrowError(/FOREACH CSV ROW must specify output subparameter/i); + + expect(() => + parseCommand( + "FOREACH Csv Row `{customer}` -> {firstName}, {lastName}, +{email1}, +{email2}", + "PIPELINE_TASK", + ), + ).toThrowError( + /FOREACH command can not have more than one output subparameter/i, + ); + }); + + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of foreachCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_TASK")).not.toThrowError(); + } + }); }); diff --git a/src/commands/FOREACH/foreachCommandParser.ts b/src/commands/FOREACH/foreachCommandParser.ts index dabef4fcf7..18ca3f4b07 100644 --- a/src/commands/FOREACH/foreachCommandParser.ts +++ b/src/commands/FOREACH/foreachCommandParser.ts @@ -1,17 +1,17 @@ -import spaceTrim from 'spacetrim'; -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { ParseError } from '../../errors/ParseError'; -import { FORMAT_DEFINITIONS } from '../../formats/index'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_parameter_name } from '../../types/typeAliases'; -import { normalizeTo_SCREAMING_CASE } from '../../utils/normalization/normalizeTo_SCREAMING_CASE'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { validateParameterName } from '../../utils/validators/parameterName/validateParameterName'; -import type { $PipelineJson } from '../_common/types/CommandParser'; -import type { $TaskJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineTaskCommandParser } from '../_common/types/CommandParser'; -import type { ForeachCommand } from './ForeachCommand'; +import spaceTrim from "spacetrim"; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { ParseError } from "../../errors/ParseError"; +import { FORMAT_DEFINITIONS } from "../../formats/index"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_parameter_name } from "../../types/typeAliases"; +import { normalizeTo_SCREAMING_CASE } from "../../utils/normalization/normalizeTo_SCREAMING_CASE"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { validateParameterName } from "../../utils/validators/parameterName/validateParameterName"; +import type { $PipelineJson } from "../_common/types/CommandParser"; +import type { $TaskJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineTaskCommandParser } from "../_common/types/CommandParser"; +import type { ForeachCommand } from "./ForeachCommand"; /** * Parses the foreach command @@ -22,212 +22,239 @@ import type { ForeachCommand } from './ForeachCommand'; * @public exported from `@promptbook/editable` */ export const foreachCommandParser: PipelineTaskCommandParser = { - /** - * Name of the command - */ - name: 'FOREACH', - - /** - * Aliases for the FOREACH command - */ - aliasNames: ['FOR', 'EACH'], - - /** - * FOREACH command can be used in: - */ - isUsedInPipelineHead: false, - isUsedInPipelineTask: true, - - /** - * Description of the FOREACH command - */ - description: `@@`, - - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/148', - - /** - * Example usages of the FOREACH command - */ - examples: [ - 'FOREACH Text Line `{customers}` -> `{customer}`', - 'FOREACH Csv Cell `{customers}` -> `{cell}`', - 'FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`', - 'FOR Text Line `{customers}` -> `{customer}`', - 'EACH Text Line `{customers}` -> `{customer}`', - ], - - /** - * Parses the FOREACH command - */ - parse(input: CommandParserInput): ForeachCommand { - const { args } = input; - - const formatName = normalizeTo_SCREAMING_CASE(args[0] || ''); - const subformatName = normalizeTo_SCREAMING_CASE(args[1] || ''); - const parameterNameArg = args[2] || ''; - const assignSign = args[3]; - - const formatDefinition = FORMAT_DEFINITIONS.find( - (formatDefinition) => - [formatDefinition.formatName, ...(formatDefinition.aliases || [])].includes(formatName), - - // <- Note: [⛷] - // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as dynamic plugins - ); - - if (formatDefinition === undefined) { - throw new ParseError( - spaceTrim( - (block) => ` + /** + * Name of the command + */ + name: "FOREACH", + + /** + * Aliases for the FOREACH command + */ + aliasNames: ["FOR", "EACH"], + + /** + * FOREACH command can be used in: + */ + isUsedInPipelineHead: false, + isUsedInPipelineTask: true, + + /** + * Description of the FOREACH command + */ + description: `@@`, + + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/148", + + /** + * Example usages of the FOREACH command + */ + examples: [ + "FOREACH Text Line `{customers}` -> `{customer}`", + "FOREACH Csv Cell `{customers}` -> `{cell}`", + "FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`", + "FOR Text Line `{customers}` -> `{customer}`", + "EACH Text Line `{customers}` -> `{customer}`", + ], + + /** + * Parses the FOREACH command + */ + parse(input: CommandParserInput): ForeachCommand { + const { args } = input; + + const formatName = normalizeTo_SCREAMING_CASE(args[0] || ""); + const subformatName = normalizeTo_SCREAMING_CASE(args[1] || ""); + const parameterNameArg = args[2] || ""; + const assignSign = args[3]; + + const formatDefinition = FORMAT_DEFINITIONS.find( + (formatDefinition) => + [ + formatDefinition.formatName, + ...(formatDefinition.aliases || []), + ].includes(formatName), + + // <- Note: [⛷] + // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as dynamic plugins + ); + + if (formatDefinition === undefined) { + throw new ParseError( + spaceTrim( + (block) => ` Unsupported format "${formatName}" Available formats: ${block( - FORMAT_DEFINITIONS.map((formatDefinition) => formatDefinition.formatName) - .map((formatName) => `- ${formatName}`) - .join('\n'), - )} + FORMAT_DEFINITIONS.map( + (formatDefinition) => formatDefinition.formatName, + ) + .map((formatName) => `- ${formatName}`) + .join("\n"), + )} `, - ), - ); - // <- TODO: [🏢] List all supported format names - } - - const subvalueDefinition = formatDefinition.subvalueDefinitions.find( - (subvalueDefinition) => - [subvalueDefinition.subvalueName, ...(subvalueDefinition.aliases || [])].includes(subformatName), - // <- Note: [⛷] - // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as dynamic plugins - ); - - if (subvalueDefinition === undefined) { - throw new ParseError( - spaceTrim( - (block) => ` + ), + ); + // <- TODO: [🏢] List all supported format names + } + + const subvalueDefinition = formatDefinition.subvalueDefinitions.find( + (subvalueDefinition) => + [ + subvalueDefinition.subvalueName, + ...(subvalueDefinition.aliases || []), + ].includes(subformatName), + // <- Note: [⛷] + // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as dynamic plugins + ); + + if (subvalueDefinition === undefined) { + throw new ParseError( + spaceTrim( + (block) => ` Unsupported subformat name "${subformatName}" for format "${formatName}" Available subformat names for format "${formatDefinition.formatName}": ${block( - formatDefinition.subvalueDefinitions - .map((subvalueDefinition) => subvalueDefinition.subvalueName) - .map((subvalueName) => `- ${subvalueName}`) - .join('\n'), - )} + formatDefinition.subvalueDefinitions + .map( + (subvalueDefinition) => + subvalueDefinition.subvalueName, + ) + .map((subvalueName) => `- ${subvalueName}`) + .join("\n"), + )} `, - ), - ); - // <- TODO: [🏢] List all supported subformat names for the format - } - - if (assignSign !== '->') { - throw new ParseError(`FOREACH command must have '->' to assign the value to the parameter`); - } - - const parameterName = validateParameterName(parameterNameArg); - - let outputSubparameterName: string_parameter_name | null = null; - - // TODO: [4] DRY - const inputSubparameterNames = args - .slice(4) - .map((parameterName) => parameterName.split(',').join(' ').trim()) - .filter((parameterName) => !parameterName.includes('+')) - .filter((parameterName) => parameterName !== '') - .map(validateParameterName); - - // TODO: [4] DRY - const outputSubparameterNames = args - .slice(4) - .map((parameterName) => parameterName.split(',').join(' ').trim()) - .filter((parameterName) => parameterName.includes('+')) - .map((parameterName) => parameterName.split('+').join('')) - .map(validateParameterName); - - if (outputSubparameterNames.length === 1) { - outputSubparameterName = outputSubparameterNames[0]!; - } else if (outputSubparameterNames.length > 1) { - throw new ParseError(`FOREACH command can not have more than one output subparameter`); - } - - if (inputSubparameterNames.length === 0) { - throw new ParseError(`FOREACH command must have at least one input subparameter`); - } - - if (outputSubparameterName === null) { - // TODO: Following code should be unhardcoded from here and moved to the format definition - if (formatName === 'CSV' && subformatName === 'CELL') { - outputSubparameterName = 'newCell'; - } else if (formatName === 'TEXT' && subformatName === 'LINE') { - outputSubparameterName = 'newLine'; - } else { - throw new ParseError( - spaceTrim(` + ), + ); + // <- TODO: [🏢] List all supported subformat names for the format + } + + if (assignSign !== "->") { + throw new ParseError( + `FOREACH command must have '->' to assign the value to the parameter`, + ); + } + + const parameterName = validateParameterName(parameterNameArg); + + let outputSubparameterName: string_parameter_name | null = null; + + // TODO: [4] DRY + const inputSubparameterNames = args + .slice(4) + .map((parameterName) => parameterName.split(",").join(" ").trim()) + .filter((parameterName) => !parameterName.includes("+")) + .filter((parameterName) => parameterName !== "") + .map(validateParameterName); + + // TODO: [4] DRY + const outputSubparameterNames = args + .slice(4) + .map((parameterName) => parameterName.split(",").join(" ").trim()) + .filter((parameterName) => parameterName.includes("+")) + .map((parameterName) => parameterName.split("+").join("")) + .map(validateParameterName); + + if (outputSubparameterNames.length === 1) { + outputSubparameterName = outputSubparameterNames[0]!; + } else if (outputSubparameterNames.length > 1) { + throw new ParseError( + `FOREACH command can not have more than one output subparameter`, + ); + } + + if (inputSubparameterNames.length === 0) { + throw new ParseError( + `FOREACH command must have at least one input subparameter`, + ); + } + + if (outputSubparameterName === null) { + // TODO: Following code should be unhardcoded from here and moved to the format definition + if (formatName === "CSV" && subformatName === "CELL") { + outputSubparameterName = "newCell"; + } else if (formatName === "TEXT" && subformatName === "LINE") { + outputSubparameterName = "newLine"; + } else { + throw new ParseError( + spaceTrim(` FOREACH ${formatName} ${subformatName} must specify output subparameter Correct example: - FOREACH ${formatName} ${subformatName} {${parameterName}} -> {inputSubparameterName1}, {inputSubparameterName2}, +{outputSubparameterName} `), - ); - } - } - - return { - type: 'FOREACH', - formatName, - subformatName, - parameterName, - inputSubparameterNames, - outputSubparameterName, - } satisfies ForeachCommand; - }, - - /** - * Apply the FOREACH command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `taskJson` - */ - $applyToTaskJson(command: ForeachCommand, $taskJson: $TaskJson, $pipelineJson: $PipelineJson): void { - const { formatName, subformatName, parameterName, inputSubparameterNames, outputSubparameterName } = command; - - // TODO: [🍭] Detect double use - // TODO: [🍭] Detect usage with JOKER and don't allow it - - $taskJson.foreach = { - formatName, - subformatName, - parameterName, - inputSubparameterNames, - outputSubparameterName, - }; - - keepUnused($pipelineJson); // <- TODO: [🧠] Maybe register subparameter from foreach into parameters of the pipeline - - // Note: [🍭] FOREACH apply has some sideeffects on different places in codebase - }, - - /** - * Converts the FOREACH command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: ForeachCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, - - /** - * Reads the FOREACH command from the `TaskJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { - keepUnused($taskJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, + ); + } + } + + return { + type: "FOREACH", + formatName, + subformatName, + parameterName, + inputSubparameterNames, + outputSubparameterName, + } satisfies ForeachCommand; + }, + + /** + * Apply the FOREACH command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `taskJson` + */ + $applyToTaskJson( + command: ForeachCommand, + $taskJson: $TaskJson, + $pipelineJson: $PipelineJson, + ): void { + const { + formatName, + subformatName, + parameterName, + inputSubparameterNames, + outputSubparameterName, + } = command; + + // TODO: [🍭] Detect double use + // TODO: [🍭] Detect usage with JOKER and don't allow it + + $taskJson.foreach = { + formatName, + subformatName, + parameterName, + inputSubparameterNames, + outputSubparameterName, + }; + + keepUnused($pipelineJson); // <- TODO: [🧠] Maybe register subparameter from foreach into parameters of the pipeline + + // Note: [🍭] FOREACH apply has some sideeffects on different places in codebase + }, + + /** + * Converts the FOREACH command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: ForeachCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, + + /** + * Reads the FOREACH command from the `TaskJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { + keepUnused($taskJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, }; /** diff --git a/src/commands/FORMAT/FormatCommand.ts b/src/commands/FORMAT/FormatCommand.ts index 56b5c3a406..4842727a9c 100644 --- a/src/commands/FORMAT/FormatCommand.ts +++ b/src/commands/FORMAT/FormatCommand.ts @@ -4,8 +4,8 @@ * Note: [🚉] This is fully serializable as JSON */ export type FormatCommand = { - readonly type: 'FORMAT'; - readonly format: 'JSON'; // <- TODO: [🏢] Change to `formatName` + readonly type: "FORMAT"; + readonly format: "JSON"; // <- TODO: [🏢] Change to `formatName` }; // <- TODO: [🦽] Why this is constantly removed by repair-imports.ts diff --git a/src/commands/FORMAT/formatCommand.test.ts b/src/commands/FORMAT/formatCommand.test.ts index 0940b61d89..03ea4ad3a4 100644 --- a/src/commands/FORMAT/formatCommand.test.ts +++ b/src/commands/FORMAT/formatCommand.test.ts @@ -1,28 +1,36 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { formatCommandParser } from './formatCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { formatCommandParser } from "./formatCommandParser"; -describe('how FORMAT command in .book.md files works', () => { - it('should parse FORMAT command', () => { - expect(parseCommand('FORMAT JSON', 'PIPELINE_TASK')).toEqual({ - type: 'FORMAT', - format: 'JSON', - }); +describe("how FORMAT command in .book.md files works", () => { + it("should parse FORMAT command", () => { + expect(parseCommand("FORMAT JSON", "PIPELINE_TASK")).toEqual({ + type: "FORMAT", + format: "JSON", + }); - // [🥤] - Test here relative and absolute schema reference - }); + // [🥤] - Test here relative and absolute schema reference + }); - it('should fail parsing format command', () => { - expect(() => parseCommand('Format 2x JSON', 'PIPELINE_TASK')).toThrowError(/Invalid FORMAT command/i); - expect(() => parseCommand('Format PNG', 'PIPELINE_TASK')).toThrowError(/Invalid FORMAT command/i); - expect(() => parseCommand('FORMAT', 'PIPELINE_TASK')).toThrowError(/Invalid FORMAT command/i); - expect(() => parseCommand('FORMAT brr', 'PIPELINE_TASK')).toThrowError(/Invalid FORMAT command/i); - }); + it("should fail parsing format command", () => { + expect(() => parseCommand("Format 2x JSON", "PIPELINE_TASK")).toThrowError( + /Invalid FORMAT command/i, + ); + expect(() => parseCommand("Format PNG", "PIPELINE_TASK")).toThrowError( + /Invalid FORMAT command/i, + ); + expect(() => parseCommand("FORMAT", "PIPELINE_TASK")).toThrowError( + /Invalid FORMAT command/i, + ); + expect(() => parseCommand("FORMAT brr", "PIPELINE_TASK")).toThrowError( + /Invalid FORMAT command/i, + ); + }); - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of formatCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_TASK')).not.toThrowError(); - } - }); + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of formatCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_TASK")).not.toThrowError(); + } + }); }); diff --git a/src/commands/FORMAT/formatCommandParser.ts b/src/commands/FORMAT/formatCommandParser.ts index 6789cb0c18..d1aab60e7f 100644 --- a/src/commands/FORMAT/formatCommandParser.ts +++ b/src/commands/FORMAT/formatCommandParser.ts @@ -1,12 +1,12 @@ -import spaceTrim from 'spacetrim'; -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { ParseError } from '../../errors/ParseError'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import type { $TaskJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineTaskCommandParser } from '../_common/types/CommandParser'; -import type { FormatCommand } from './FormatCommand'; +import spaceTrim from "spacetrim"; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { ParseError } from "../../errors/ParseError"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import type { $TaskJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineTaskCommandParser } from "../_common/types/CommandParser"; +import type { FormatCommand } from "./FormatCommand"; /** * Parses the format command @@ -15,84 +15,86 @@ import type { FormatCommand } from './FormatCommand'; * @public exported from `@promptbook/editable` */ export const formatCommandParser: PipelineTaskCommandParser = { - /** - * Name of the command - */ - name: 'FORMAT', + /** + * Name of the command + */ + name: "FORMAT", - /** - * BOILERPLATE command can be used in: - */ - isUsedInPipelineHead: false, - isUsedInPipelineTask: true, + /** + * BOILERPLATE command can be used in: + */ + isUsedInPipelineHead: false, + isUsedInPipelineTask: true, - /** - * Description of the FORMAT command - */ - description: spaceTrim(` + /** + * Description of the FORMAT command + */ + description: spaceTrim(` Format command describes the desired output of the task (after post-processing) It can set limits for the maximum/minimum length of the output, measured in characters, words, sentences, paragraphs or some other shape of the output. `), - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/30', + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/30", - /** - * Example usages of the FORMAT command - */ - examples: ['FORMAT JSON'], + /** + * Example usages of the FORMAT command + */ + examples: ["FORMAT JSON"], - /** - * Parses the FORMAT command - */ - parse(input: CommandParserInput): FormatCommand { - const { normalized } = input; + /** + * Parses the FORMAT command + */ + parse(input: CommandParserInput): FormatCommand { + const { normalized } = input; - if (!normalized.startsWith('FORMAT_JSON')) { - throw new ParseError(`For now only JSON format is supported, in future we will support more formats`); - } + if (!normalized.startsWith("FORMAT_JSON")) { + throw new ParseError( + `For now only JSON format is supported, in future we will support more formats`, + ); + } - return { - type: 'FORMAT', - format: 'JSON', - } satisfies FormatCommand; - // <- TODO: [🦽] Why this is constantly removed by repair-imports.ts + return { + type: "FORMAT", + format: "JSON", + } satisfies FormatCommand; + // <- TODO: [🦽] Why this is constantly removed by repair-imports.ts - // [🥤] - }, + // [🥤] + }, - /** - * Apply the FORMAT command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `taskJson` - */ - $applyToTaskJson(command: FormatCommand, $taskJson: $TaskJson): void { - if ($taskJson.format !== undefined && command.format !== $taskJson.format) { - throw new ParseError(`Format format is already defined to "${$taskJson.format}". + /** + * Apply the FORMAT command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `taskJson` + */ + $applyToTaskJson(command: FormatCommand, $taskJson: $TaskJson): void { + if ($taskJson.format !== undefined && command.format !== $taskJson.format) { + throw new ParseError(`Format format is already defined to "${$taskJson.format}". Now you try to redefine it by "${command.format}"`); - } - $taskJson.format = command.format; - }, + } + $taskJson.format = command.format; + }, - /** - * Converts the FORMAT command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: FormatCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, + /** + * Converts the FORMAT command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: FormatCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, - /** - * Reads the FORMAT command from the `TaskJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { - keepUnused($taskJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, + /** + * Reads the FORMAT command from the `TaskJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { + keepUnused($taskJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, }; diff --git a/src/commands/FORMFACTOR/FormfactorCommand.ts b/src/commands/FORMFACTOR/FormfactorCommand.ts index 49144e4219..893734c31e 100644 --- a/src/commands/FORMFACTOR/FormfactorCommand.ts +++ b/src/commands/FORMFACTOR/FormfactorCommand.ts @@ -1,4 +1,4 @@ -import type { string_formfactor_name } from '../../formfactors/_common/string_formfactor_name'; +import type { string_formfactor_name } from "../../formfactors/_common/string_formfactor_name"; /** * Parsed FORMFACTOR command @@ -7,6 +7,6 @@ import type { string_formfactor_name } from '../../formfactors/_common/string_fo * @public exported from `@promptbook/editable` */ export type FormfactorCommand = { - readonly type: 'FORMFACTOR'; - readonly formfactorName: string_formfactor_name; + readonly type: "FORMFACTOR"; + readonly formfactorName: string_formfactor_name; }; diff --git a/src/commands/FORMFACTOR/formfactorCommand.test.ts b/src/commands/FORMFACTOR/formfactorCommand.test.ts index ca74f86e7f..e2677126a2 100644 --- a/src/commands/FORMFACTOR/formfactorCommand.test.ts +++ b/src/commands/FORMFACTOR/formfactorCommand.test.ts @@ -1,39 +1,43 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { formfactorCommandParser } from './formfactorCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { formfactorCommandParser } from "./formfactorCommandParser"; -describe('how FORMFACTOR command in .book.md files works', () => { - it('should parse FORMFACTOR command in PIPELINE_HEAD', () => { - expect(parseCommand('FORMFACTOR chatbot', 'PIPELINE_HEAD')).toEqual({ - type: 'FORMFACTOR', - formfactorName: 'CHATBOT', - }); - expect(parseCommand('FORMFACTOR `ChatBot`', 'PIPELINE_HEAD')).toEqual({ - type: 'FORMFACTOR', - formfactorName: 'CHATBOT', - }); - expect(parseCommand('FORMFACTOR `CHATBOT`', 'PIPELINE_HEAD')).toEqual({ - type: 'FORMFACTOR', - formfactorName: 'CHATBOT', - }); - }); +describe("how FORMFACTOR command in .book.md files works", () => { + it("should parse FORMFACTOR command in PIPELINE_HEAD", () => { + expect(parseCommand("FORMFACTOR chatbot", "PIPELINE_HEAD")).toEqual({ + type: "FORMFACTOR", + formfactorName: "CHATBOT", + }); + expect(parseCommand("FORMFACTOR `ChatBot`", "PIPELINE_HEAD")).toEqual({ + type: "FORMFACTOR", + formfactorName: "CHATBOT", + }); + expect(parseCommand("FORMFACTOR `CHATBOT`", "PIPELINE_HEAD")).toEqual({ + type: "FORMFACTOR", + formfactorName: "CHATBOT", + }); + }); - it('should parse FORMFACTOR command in shortcut form', () => { - expect(parseCommand('FF Chat', 'PIPELINE_HEAD')).toEqual({ - type: 'FORMFACTOR', - formfactorName: 'CHATBOT', - }); - }); + it("should parse FORMFACTOR command in shortcut form", () => { + expect(parseCommand("FF Chat", "PIPELINE_HEAD")).toEqual({ + type: "FORMFACTOR", + formfactorName: "CHATBOT", + }); + }); - it('should fail parsing FORMFACTOR command', () => { - expect(() => parseCommand('FORMFACTOR', 'PIPELINE_HEAD')).toThrowError(/requires exactly one argument/i); - expect(() => parseCommand('FORMFACTOR Malfsdxsxed', 'PIPELINE_HEAD')).toThrowError(/Unknown formfactor name/i); - }); + it("should fail parsing FORMFACTOR command", () => { + expect(() => parseCommand("FORMFACTOR", "PIPELINE_HEAD")).toThrowError( + /requires exactly one argument/i, + ); + expect(() => + parseCommand("FORMFACTOR Malfsdxsxed", "PIPELINE_HEAD"), + ).toThrowError(/Unknown formfactor name/i); + }); - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of formfactorCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_HEAD')).not.toThrowError(); - } - }); + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of formfactorCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_HEAD")).not.toThrowError(); + } + }); }); diff --git a/src/commands/FORMFACTOR/formfactorCommandParser.ts b/src/commands/FORMFACTOR/formfactorCommandParser.ts index eb07d10b05..eb0eb8aa86 100644 --- a/src/commands/FORMFACTOR/formfactorCommandParser.ts +++ b/src/commands/FORMFACTOR/formfactorCommandParser.ts @@ -1,13 +1,13 @@ -import spaceTrim from 'spacetrim'; -import { ParseError } from '../../errors/ParseError'; -import { FORMFACTOR_DEFINITIONS } from '../../formfactors/index'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { really_any } from '../../utils/organization/really_any'; -import type { $PipelineJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineHeadCommandParser } from '../_common/types/CommandParser'; -import type { FormfactorCommand } from './FormfactorCommand'; +import spaceTrim from "spacetrim"; +import { ParseError } from "../../errors/ParseError"; +import { FORMFACTOR_DEFINITIONS } from "../../formfactors/index"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { really_any } from "../../utils/organization/really_any"; +import type { $PipelineJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineHeadCommandParser } from "../_common/types/CommandParser"; +import type { FormfactorCommand } from "./FormfactorCommand"; /** * Parses the formfactor command @@ -17,116 +17,128 @@ import type { FormfactorCommand } from './FormfactorCommand'; * @see `documentationUrl` for more details * @public exported from `@promptbook/editable` */ -export const formfactorCommandParser: PipelineHeadCommandParser = { - /** - * Name of the command - */ - name: 'FORMFACTOR', - - /** - * Aliases for the FORMFACTOR command - */ - aliasNames: ['FORM', 'FF'], - - /** - * FORMFACTOR command can be used in: - */ - isUsedInPipelineHead: true, - isUsedInPipelineTask: false, - - /** - * Description of the FORMFACTOR command - */ - description: `@@`, - - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/168', - - /** - * Example usages of the FORMFACTOR command - */ - examples: ['FORMFACTOR Chatbot', 'FF Chat'], - - /** - * Parses the FORMFACTOR command - */ - parse(input: CommandParserInput): FormfactorCommand { - const { args } = input; - - if (args.length !== 1) { - throw new ParseError(`FORMFACTOR command requires exactly one argument`); - } - - const formfactorNameCandidate = args[0]!.toUpperCase(); - - const formfactor = FORMFACTOR_DEFINITIONS.find((definition) => - [definition.name, ...{ aliasNames: [], ...definition }.aliasNames].includes( - formfactorNameCandidate as really_any, - ), - ); - - if (formfactor === undefined) { - throw new ParseError( - spaceTrim( - (block) => ` +export const formfactorCommandParser: PipelineHeadCommandParser = + { + /** + * Name of the command + */ + name: "FORMFACTOR", + + /** + * Aliases for the FORMFACTOR command + */ + aliasNames: ["FORM", "FF"], + + /** + * FORMFACTOR command can be used in: + */ + isUsedInPipelineHead: true, + isUsedInPipelineTask: false, + + /** + * Description of the FORMFACTOR command + */ + description: `@@`, + + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/168", + + /** + * Example usages of the FORMFACTOR command + */ + examples: ["FORMFACTOR Chatbot", "FF Chat"], + + /** + * Parses the FORMFACTOR command + */ + parse(input: CommandParserInput): FormfactorCommand { + const { args } = input; + + if (args.length !== 1) { + throw new ParseError( + `FORMFACTOR command requires exactly one argument`, + ); + } + + const formfactorNameCandidate = args[0]!.toUpperCase(); + + const formfactor = FORMFACTOR_DEFINITIONS.find((definition) => + [ + definition.name, + ...{ aliasNames: [], ...definition }.aliasNames, + ].includes(formfactorNameCandidate as really_any), + ); + + if (formfactor === undefined) { + throw new ParseError( + spaceTrim( + (block) => ` Unknown formfactor name "${formfactorNameCandidate}" Available formfactors: - ${block(FORMFACTOR_DEFINITIONS.map(({ name }) => `- ${name}`).join('\n'))} + ${block(FORMFACTOR_DEFINITIONS.map(({ name }) => `- ${name}`).join("\n"))} `, - ), - ); - } - - return { - type: 'FORMFACTOR', - formfactorName: formfactor.name, - } satisfies FormfactorCommand; - }, - - /** - * Apply the FORMFACTOR command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `pipelineJson` - */ - $applyToPipelineJson(command: FormfactorCommand, $pipelineJson: $PipelineJson): void { - if ($pipelineJson.formfactorName !== undefined && $pipelineJson.formfactorName !== command.formfactorName) { - throw new ParseError( - spaceTrim(` + ), + ); + } + + return { + type: "FORMFACTOR", + formfactorName: formfactor.name, + } satisfies FormfactorCommand; + }, + + /** + * Apply the FORMFACTOR command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `pipelineJson` + */ + $applyToPipelineJson( + command: FormfactorCommand, + $pipelineJson: $PipelineJson, + ): void { + if ( + $pipelineJson.formfactorName !== undefined && + $pipelineJson.formfactorName !== command.formfactorName + ) { + throw new ParseError( + spaceTrim(` Redefinition of \`FORMFACTOR\` in the pipeline head You have used: 1) FORMFACTOR \`${$pipelineJson.formfactorName}\` 2) FORMFACTOR \`${command.formfactorName}\` `), - ); - } - - $pipelineJson.formfactorName = command.formfactorName; - }, - - /** - * Converts the FORMFACTOR command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: FormfactorCommand): string_markdown_text { - return `FORMFACTOR ${command.formfactorName}`; - }, - - /** - * Reads the FORMFACTOR command from the `PipelineJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray { - return [ - { - type: 'FORMFACTOR', - formfactorName: pipelineJson.formfactorName, - }, - ]; - }, -}; + ); + } + + $pipelineJson.formfactorName = command.formfactorName; + }, + + /** + * Converts the FORMFACTOR command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: FormfactorCommand): string_markdown_text { + return `FORMFACTOR ${command.formfactorName}`; + }, + + /** + * Reads the FORMFACTOR command from the `PipelineJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromPipelineJson( + pipelineJson: PipelineJson, + ): ReadonlyArray { + return [ + { + type: "FORMFACTOR", + formfactorName: pipelineJson.formfactorName, + }, + ]; + }, + }; diff --git a/src/commands/JOKER/JokerCommand.ts b/src/commands/JOKER/JokerCommand.ts index ac73d1861a..77f771bf35 100644 --- a/src/commands/JOKER/JokerCommand.ts +++ b/src/commands/JOKER/JokerCommand.ts @@ -1,4 +1,4 @@ -import type { string_name } from '../../types/typeAliases'; +import type { string_name } from "../../types/typeAliases"; /** * Parsed JOKER command @@ -7,6 +7,6 @@ import type { string_name } from '../../types/typeAliases'; * @public exported from `@promptbook/editable` */ export type JokerCommand = { - readonly type: 'JOKER'; - readonly parameterName: string_name; + readonly type: "JOKER"; + readonly parameterName: string_name; }; diff --git a/src/commands/JOKER/jokerCommand.test.ts b/src/commands/JOKER/jokerCommand.test.ts index aa15e64046..e37f382104 100644 --- a/src/commands/JOKER/jokerCommand.test.ts +++ b/src/commands/JOKER/jokerCommand.test.ts @@ -1,29 +1,33 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { jokerCommandParser } from './jokerCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { jokerCommandParser } from "./jokerCommandParser"; -describe('how JOKER command in .book.md files works', () => { - it('should parse JOKER command', () => { - expect(parseCommand('joker {name}', 'PIPELINE_TASK')).toEqual({ - type: 'JOKER', - parameterName: 'name', - }); - expect(parseCommand('JOKER {woooow}', 'PIPELINE_TASK')).toEqual({ - type: 'JOKER', - parameterName: 'woooow', - }); - }); - it('should fail parsing JOKER command', () => { - expect(() => parseCommand('JOKER', 'PIPELINE_TASK')).toThrowError(/Invalid joker/i); - expect(() => parseCommand('JOKER {foo} {bar}', 'PIPELINE_TASK')).toThrowError( - /Invalid joker/i /* <- TODO: /JOKER must reference a parameter/i */, - ); - }); +describe("how JOKER command in .book.md files works", () => { + it("should parse JOKER command", () => { + expect(parseCommand("joker {name}", "PIPELINE_TASK")).toEqual({ + type: "JOKER", + parameterName: "name", + }); + expect(parseCommand("JOKER {woooow}", "PIPELINE_TASK")).toEqual({ + type: "JOKER", + parameterName: "woooow", + }); + }); + it("should fail parsing JOKER command", () => { + expect(() => parseCommand("JOKER", "PIPELINE_TASK")).toThrowError( + /Invalid joker/i, + ); + expect(() => + parseCommand("JOKER {foo} {bar}", "PIPELINE_TASK"), + ).toThrowError( + /Invalid joker/i /* <- TODO: /JOKER must reference a parameter/i */, + ); + }); - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of jokerCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_TASK')).not.toThrowError(); - } - }); + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of jokerCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_TASK")).not.toThrowError(); + } + }); }); diff --git a/src/commands/JOKER/jokerCommandParser.ts b/src/commands/JOKER/jokerCommandParser.ts index a68be80ad6..61b5628732 100644 --- a/src/commands/JOKER/jokerCommandParser.ts +++ b/src/commands/JOKER/jokerCommandParser.ts @@ -1,12 +1,12 @@ -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { ParseError } from '../../errors/ParseError'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { validateParameterName } from '../../utils/validators/parameterName/validateParameterName'; -import type { $TaskJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineTaskCommandParser } from '../_common/types/CommandParser'; -import type { JokerCommand } from './JokerCommand'; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { ParseError } from "../../errors/ParseError"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { validateParameterName } from "../../utils/validators/parameterName/validateParameterName"; +import type { $TaskJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineTaskCommandParser } from "../_common/types/CommandParser"; +import type { JokerCommand } from "./JokerCommand"; /** * Parses the joker command @@ -15,79 +15,79 @@ import type { JokerCommand } from './JokerCommand'; * @public exported from `@promptbook/editable` */ export const jokerCommandParser: PipelineTaskCommandParser = { - /** - * Name of the command - */ - name: 'JOKER', + /** + * Name of the command + */ + name: "JOKER", - /** - * BOILERPLATE command can be used in: - */ - isUsedInPipelineHead: false, - isUsedInPipelineTask: true, + /** + * BOILERPLATE command can be used in: + */ + isUsedInPipelineHead: false, + isUsedInPipelineTask: true, - /** - * Description of the JOKER command - */ - description: `Joker parameter is used instead of executing the task result if jokers value meets the expectations requirements`, + /** + * Description of the JOKER command + */ + description: `Joker parameter is used instead of executing the task result if jokers value meets the expectations requirements`, - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/66', + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/66", - /** - * Example usages of the JOKER command - */ - examples: ['JOKER {documentTitle}'], + /** + * Example usages of the JOKER command + */ + examples: ["JOKER {documentTitle}"], - /** - * Parses the JOKER command - */ - parse(input: CommandParserInput): JokerCommand { - const { args } = input; + /** + * Parses the JOKER command + */ + parse(input: CommandParserInput): JokerCommand { + const { args } = input; - if (args.length !== 1) { - throw new ParseError(`JOKE command expects exactly one parameter name`); - } + if (args.length !== 1) { + throw new ParseError(`JOKE command expects exactly one parameter name`); + } - const parameterNameArg = args[0] || ''; + const parameterNameArg = args[0] || ""; - const parameterName = validateParameterName(parameterNameArg); + const parameterName = validateParameterName(parameterNameArg); - return { - type: 'JOKER', - parameterName, - } satisfies JokerCommand; - }, + return { + type: "JOKER", + parameterName, + } satisfies JokerCommand; + }, - /** - * Apply the JOKER command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `taskJson` - */ - $applyToTaskJson(command: JokerCommand, $taskJson: $TaskJson): void { - $taskJson.jokerParameterNames = $taskJson.jokerParameterNames || []; - $taskJson.jokerParameterNames.push(command.parameterName); - }, + /** + * Apply the JOKER command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `taskJson` + */ + $applyToTaskJson(command: JokerCommand, $taskJson: $TaskJson): void { + $taskJson.jokerParameterNames = $taskJson.jokerParameterNames || []; + $taskJson.jokerParameterNames.push(command.parameterName); + }, - /** - * Converts the JOKER command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: JokerCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, + /** + * Converts the JOKER command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: JokerCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, - /** - * Reads the JOKER command from the `TaskJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { - keepUnused($taskJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, + /** + * Reads the JOKER command from the `TaskJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { + keepUnused($taskJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, }; diff --git a/src/commands/KNOWLEDGE/KnowledgeCommand.ts b/src/commands/KNOWLEDGE/KnowledgeCommand.ts index 20a1d6dc6f..da60c23402 100644 --- a/src/commands/KNOWLEDGE/KnowledgeCommand.ts +++ b/src/commands/KNOWLEDGE/KnowledgeCommand.ts @@ -1,4 +1,4 @@ -import type { string_knowledge_source_content } from '../../types/typeAliases'; +import type { string_knowledge_source_content } from "../../types/typeAliases"; /** * Parsed KNOWLEDGE command @@ -7,6 +7,6 @@ import type { string_knowledge_source_content } from '../../types/typeAliases'; * @public exported from `@promptbook/editable` */ export type KnowledgeCommand = { - readonly type: 'KNOWLEDGE'; - readonly knowledgeSourceContent: string_knowledge_source_content; + readonly type: "KNOWLEDGE"; + readonly knowledgeSourceContent: string_knowledge_source_content; }; diff --git a/src/commands/KNOWLEDGE/knowledgeCommand.test.ts b/src/commands/KNOWLEDGE/knowledgeCommand.test.ts index a4e469bb30..8e2385c2d1 100644 --- a/src/commands/KNOWLEDGE/knowledgeCommand.test.ts +++ b/src/commands/KNOWLEDGE/knowledgeCommand.test.ts @@ -1,53 +1,62 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { knowledgeCommandParser } from './knowledgeCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { knowledgeCommandParser } from "./knowledgeCommandParser"; -describe('how KNOWLEDGE command in .book.md files works', () => { - it('should parse KNOWLEDGE command', () => { - expect(parseCommand('KNOWLEDGE https://www.pavolhejny.com/', 'PIPELINE_HEAD')).toEqual({ - type: 'KNOWLEDGE', - knowledgeSourceContent: 'https://www.pavolhejny.com/', - }); - expect(parseCommand('KNOWLEDGE ./hejny-cv.pdf', 'PIPELINE_HEAD')).toEqual({ - // <- TODO: [😿] Allow ONLY files scoped in the (sub)directory NOT ../ and test it - type: 'KNOWLEDGE', - knowledgeSourceContent: './hejny-cv.pdf', - }); - }); +describe("how KNOWLEDGE command in .book.md files works", () => { + it("should parse KNOWLEDGE command", () => { + expect( + parseCommand("KNOWLEDGE https://www.pavolhejny.com/", "PIPELINE_HEAD"), + ).toEqual({ + type: "KNOWLEDGE", + knowledgeSourceContent: "https://www.pavolhejny.com/", + }); + expect(parseCommand("KNOWLEDGE ./hejny-cv.pdf", "PIPELINE_HEAD")).toEqual({ + // <- TODO: [😿] Allow ONLY files scoped in the (sub)directory NOT ../ and test it + type: "KNOWLEDGE", + knowledgeSourceContent: "./hejny-cv.pdf", + }); + }); - it('should fail parsing KNOWLEDGE command', () => { - expect(() => parseCommand('KNOWLEDGE', 'PIPELINE_HEAD')).toThrowError(/Source is not defined/i); - expect(() => parseCommand('KNOWLEDGE brr', 'PIPELINE_HEAD')).toThrowError(/Source not valid/i); - expect(() => parseCommand('KNOWLEDGE http://www.pavolhejny.com/', 'PIPELINE_HEAD')).toThrowError( - /Source is not secure/i, - ); - expect(() => parseCommand('KNOWLEDGE ../hejny-cv.pdf', 'PIPELINE_HEAD')).toThrowError( - /Source cannot be outside .* folder/i, - ); - expect(() => parseCommand('KNOWLEDGE /hejny-cv.pdf', 'PIPELINE_HEAD')).toThrowError( - /Source cannot be outside .* folder/i, - ); - expect(() => parseCommand('KNOWLEDGE /etc/system-folder/hejny-cv.pdf', 'PIPELINE_HEAD')).toThrowError( - /Source cannot be outside .* folder/i, - ); - expect(() => parseCommand('KNOWLEDGE C:/hejny-cv.pdf', 'PIPELINE_HEAD')).toThrowError( - /Source cannot be outside .* folder/i, - ); - expect(() => parseCommand('KNOWLEDGE C://hejny-cv.pdf', 'PIPELINE_HEAD')).toThrowError( - /Source cannot be outside .* folder/i, - ); - expect(() => parseCommand('KNOWLEDGE C:\\hejny-cv.pdf', 'PIPELINE_HEAD')).toThrowError( - /Source cannot be outside .* folder/i, - ); - expect(() => parseCommand('KNOWLEDGE C:\\\\hejny-cv.pdf', 'PIPELINE_HEAD')).toThrowError( - /Source cannot be outside .* folder/i, - ); - }); + it("should fail parsing KNOWLEDGE command", () => { + expect(() => parseCommand("KNOWLEDGE", "PIPELINE_HEAD")).toThrowError( + /Source is not defined/i, + ); + expect(() => parseCommand("KNOWLEDGE brr", "PIPELINE_HEAD")).toThrowError( + /Source not valid/i, + ); + expect(() => + parseCommand("KNOWLEDGE http://www.pavolhejny.com/", "PIPELINE_HEAD"), + ).toThrowError(/Source is not secure/i); + expect(() => + parseCommand("KNOWLEDGE ../hejny-cv.pdf", "PIPELINE_HEAD"), + ).toThrowError(/Source cannot be outside .* folder/i); + expect(() => + parseCommand("KNOWLEDGE /hejny-cv.pdf", "PIPELINE_HEAD"), + ).toThrowError(/Source cannot be outside .* folder/i); + expect(() => + parseCommand( + "KNOWLEDGE /etc/system-folder/hejny-cv.pdf", + "PIPELINE_HEAD", + ), + ).toThrowError(/Source cannot be outside .* folder/i); + expect(() => + parseCommand("KNOWLEDGE C:/hejny-cv.pdf", "PIPELINE_HEAD"), + ).toThrowError(/Source cannot be outside .* folder/i); + expect(() => + parseCommand("KNOWLEDGE C://hejny-cv.pdf", "PIPELINE_HEAD"), + ).toThrowError(/Source cannot be outside .* folder/i); + expect(() => + parseCommand("KNOWLEDGE C:\\hejny-cv.pdf", "PIPELINE_HEAD"), + ).toThrowError(/Source cannot be outside .* folder/i); + expect(() => + parseCommand("KNOWLEDGE C:\\\\hejny-cv.pdf", "PIPELINE_HEAD"), + ).toThrowError(/Source cannot be outside .* folder/i); + }); - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of knowledgeCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_HEAD')).not.toThrowError(); - } - }); + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of knowledgeCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_HEAD")).not.toThrowError(); + } + }); }); diff --git a/src/commands/KNOWLEDGE/knowledgeCommandParser.ts b/src/commands/KNOWLEDGE/knowledgeCommandParser.ts index b4eda6b72d..8ea2367b7d 100644 --- a/src/commands/KNOWLEDGE/knowledgeCommandParser.ts +++ b/src/commands/KNOWLEDGE/knowledgeCommandParser.ts @@ -1,16 +1,16 @@ -import spaceTrim from 'spacetrim'; -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { ParseError } from '../../errors/ParseError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { isValidFilePath } from '../../utils/validators/filePath/isValidFilePath'; -import { isValidUrl } from '../../utils/validators/url/isValidUrl'; -import type { $PipelineJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineHeadCommandParser } from '../_common/types/CommandParser'; -import type { KnowledgeCommand } from './KnowledgeCommand'; -import { knowledgeSourceContentToName } from './utils/knowledgeSourceContentToName'; +import spaceTrim from "spacetrim"; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { ParseError } from "../../errors/ParseError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { isValidFilePath } from "../../utils/validators/filePath/isValidFilePath"; +import { isValidUrl } from "../../utils/validators/url/isValidUrl"; +import type { $PipelineJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineHeadCommandParser } from "../_common/types/CommandParser"; +import type { KnowledgeCommand } from "./KnowledgeCommand"; +import { knowledgeSourceContentToName } from "./utils/knowledgeSourceContentToName"; /** * Parses the knowledge command @@ -18,110 +18,121 @@ import { knowledgeSourceContentToName } from './utils/knowledgeSourceContentToNa * @see `documentationUrl` for more details * @public exported from `@promptbook/editable` */ -export const knowledgeCommandParser: PipelineHeadCommandParser = { - /** - * Name of the command - */ - name: 'KNOWLEDGE', - - /** - * BOILERPLATE command can be used in: - */ - isUsedInPipelineHead: true, - isUsedInPipelineTask: false, // <- [👙] Maybe allow to use here and make relevant for just this task - - /** - * Description of the KNOWLEDGE command - */ - description: `Tells promptbook which external knowledge to use`, - - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/41', - - /** - * Example usages of the KNOWLEDGE command - */ - examples: [ - 'KNOWLEDGE https://www.pavolhejny.com/', - 'KNOWLEDGE ./hejny-cv.txt', - 'KNOWLEDGE ./hejny-cv.md', - 'KNOWLEDGE ./hejny-cv.pdf', - 'KNOWLEDGE ./hejny-cv.docx', - // <- TODO: [😿] Allow ONLY files scoped in the (sub)directory NOT ../ and test it - ], - - /** - * Parses the KNOWLEDGE command - */ - parse(input: CommandParserInput): KnowledgeCommand { - const { args } = input; - - const knowledgeSourceContent = spaceTrim(args[0] || ''); - - if (knowledgeSourceContent === '') { - throw new ParseError(`Source is not defined`); - } - - // TODO: [main] !!4 Following checks should be applied every link in the `sourceContent` - - if (knowledgeSourceContent.startsWith('http://')) { - throw new ParseError(`Source is not secure`); - } - - if (!(isValidFilePath(knowledgeSourceContent) || isValidUrl(knowledgeSourceContent))) { - throw new ParseError(`Source not valid`); - } - - if ( - knowledgeSourceContent.startsWith('../') || - knowledgeSourceContent.startsWith('/') || - /^[A-Z]:[\\/]+/i.test(knowledgeSourceContent) - ) { - throw new ParseError(`Source cannot be outside of the .book.md folder`); - } - - return { - type: 'KNOWLEDGE', - knowledgeSourceContent, - } satisfies KnowledgeCommand; - }, - - /** - * Apply the KNOWLEDGE command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `pipelineJson` - */ - $applyToPipelineJson(command: KnowledgeCommand, $pipelineJson: $PipelineJson): void { - const { knowledgeSourceContent } = command; - - $pipelineJson.knowledgeSources.push({ - name: knowledgeSourceContentToName(knowledgeSourceContent), - knowledgeSourceContent, - }); - }, - - /** - * Converts the KNOWLEDGE command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: KnowledgeCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, - - /** - * Reads the KNOWLEDGE command from the `PipelineJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray { - keepUnused(pipelineJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, -}; +export const knowledgeCommandParser: PipelineHeadCommandParser = + { + /** + * Name of the command + */ + name: "KNOWLEDGE", + + /** + * BOILERPLATE command can be used in: + */ + isUsedInPipelineHead: true, + isUsedInPipelineTask: false, // <- [👙] Maybe allow to use here and make relevant for just this task + + /** + * Description of the KNOWLEDGE command + */ + description: `Tells promptbook which external knowledge to use`, + + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/41", + + /** + * Example usages of the KNOWLEDGE command + */ + examples: [ + "KNOWLEDGE https://www.pavolhejny.com/", + "KNOWLEDGE ./hejny-cv.txt", + "KNOWLEDGE ./hejny-cv.md", + "KNOWLEDGE ./hejny-cv.pdf", + "KNOWLEDGE ./hejny-cv.docx", + // <- TODO: [😿] Allow ONLY files scoped in the (sub)directory NOT ../ and test it + ], + + /** + * Parses the KNOWLEDGE command + */ + parse(input: CommandParserInput): KnowledgeCommand { + const { args } = input; + + const knowledgeSourceContent = spaceTrim(args[0] || ""); + + if (knowledgeSourceContent === "") { + throw new ParseError(`Source is not defined`); + } + + // TODO: [main] !!4 Following checks should be applied every link in the `sourceContent` + + if (knowledgeSourceContent.startsWith("http://")) { + throw new ParseError(`Source is not secure`); + } + + if ( + !( + isValidFilePath(knowledgeSourceContent) || + isValidUrl(knowledgeSourceContent) + ) + ) { + throw new ParseError(`Source not valid`); + } + + if ( + knowledgeSourceContent.startsWith("../") || + knowledgeSourceContent.startsWith("/") || + /^[A-Z]:[\\/]+/i.test(knowledgeSourceContent) + ) { + throw new ParseError(`Source cannot be outside of the .book.md folder`); + } + + return { + type: "KNOWLEDGE", + knowledgeSourceContent, + } satisfies KnowledgeCommand; + }, + + /** + * Apply the KNOWLEDGE command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `pipelineJson` + */ + $applyToPipelineJson( + command: KnowledgeCommand, + $pipelineJson: $PipelineJson, + ): void { + const { knowledgeSourceContent } = command; + + $pipelineJson.knowledgeSources.push({ + name: knowledgeSourceContentToName(knowledgeSourceContent), + knowledgeSourceContent, + }); + }, + + /** + * Converts the KNOWLEDGE command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: KnowledgeCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, + + /** + * Reads the KNOWLEDGE command from the `PipelineJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromPipelineJson( + pipelineJson: PipelineJson, + ): ReadonlyArray { + keepUnused(pipelineJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, + }; /** * Note: [⛱] There are two types of KNOWLEDGE commands *...(read more in [⛱])* diff --git a/src/commands/KNOWLEDGE/utils/knowledgeSourceContentToName.test.ts b/src/commands/KNOWLEDGE/utils/knowledgeSourceContentToName.test.ts index 3855c5e026..ed435f72af 100644 --- a/src/commands/KNOWLEDGE/utils/knowledgeSourceContentToName.test.ts +++ b/src/commands/KNOWLEDGE/utils/knowledgeSourceContentToName.test.ts @@ -1,24 +1,30 @@ -import { describe, expect, it } from '@jest/globals'; -import { knowledgeSourceContentToName } from './knowledgeSourceContentToName'; +import { describe, expect, it } from "@jest/globals"; +import { knowledgeSourceContentToName } from "./knowledgeSourceContentToName"; -describe('how `sourceContentToName` works', () => { - it('should work file source', () => { - expect(knowledgeSourceContentToName('/path/to/file.doc')).toBe('source-path-to-file-doc-94997379c25e0cfe5f0b'); - }); +describe("how `sourceContentToName` works", () => { + it("should work file source", () => { + expect(knowledgeSourceContentToName("/path/to/file.doc")).toBe( + "source-path-to-file-doc-94997379c25e0cfe5f0b", + ); + }); - it('should work url source', () => { - expect(knowledgeSourceContentToName('https://promptbook.studio/file.doc')).toBe( - 'source-https-promptbook-s-9d4e414fe118421d73b1', - ); - }); + it("should work url source", () => { + expect( + knowledgeSourceContentToName("https://promptbook.studio/file.doc"), + ).toBe("source-https-promptbook-s-9d4e414fe118421d73b1"); + }); - it('should work explicit source', () => { - expect(knowledgeSourceContentToName('Promptbook is a TypeScript framework')).toBe( - 'source-promptbook-is-a-type-166bbd70c4d34342cb68', - ); - expect(knowledgeSourceContentToName('')).toBe('source-6e340b9cffb37a989ca5'); - expect(knowledgeSourceContentToName('-')).toBe('source-c465d6811e5a78386e88'); - }); + it("should work explicit source", () => { + expect( + knowledgeSourceContentToName("Promptbook is a TypeScript framework"), + ).toBe("source-promptbook-is-a-type-166bbd70c4d34342cb68"); + expect(knowledgeSourceContentToName("")).toBe( + "source-6e340b9cffb37a989ca5", + ); + expect(knowledgeSourceContentToName("-")).toBe( + "source-c465d6811e5a78386e88", + ); + }); }); /** diff --git a/src/commands/KNOWLEDGE/utils/knowledgeSourceContentToName.ts b/src/commands/KNOWLEDGE/utils/knowledgeSourceContentToName.ts index 378ac43135..39cd444c7c 100644 --- a/src/commands/KNOWLEDGE/utils/knowledgeSourceContentToName.ts +++ b/src/commands/KNOWLEDGE/utils/knowledgeSourceContentToName.ts @@ -1,28 +1,32 @@ -import { SHA256 as sha256 } from 'crypto-js'; -import hexEncoder from 'crypto-js/enc-hex'; -import type { string_knowledge_source_content } from '../../../types/typeAliases'; -import type { string_name } from '../../../types/typeAliases'; -import { normalizeToKebabCase } from '../../../utils/normalization/normalize-to-kebab-case'; +import { SHA256 as sha256 } from "crypto-js"; +import hexEncoder from "crypto-js/enc-hex"; +import type { string_knowledge_source_content } from "../../../types/typeAliases"; +import type { string_name } from "../../../types/typeAliases"; +import { normalizeToKebabCase } from "../../../utils/normalization/normalize-to-kebab-case"; /** * Creates unique name for the source * * @public exported from `@promptbook/editable` */ -export function knowledgeSourceContentToName(knowledgeSourceContent: string_knowledge_source_content): string_name { - const hash = sha256(hexEncoder.parse(JSON.stringify(knowledgeSourceContent))) - // <- TODO: [🥬] Encapsulate sha256 to some private utility function - .toString(/* hex */) - .substring(0, 20); - // <- TODO: [🥬] Make some system for hashes and ids of promptbook - const semanticName = normalizeToKebabCase(knowledgeSourceContent.substring(0, 20)); +export function knowledgeSourceContentToName( + knowledgeSourceContent: string_knowledge_source_content, +): string_name { + const hash = sha256(hexEncoder.parse(JSON.stringify(knowledgeSourceContent))) + // <- TODO: [🥬] Encapsulate sha256 to some private utility function + .toString(/* hex */) + .substring(0, 20); + // <- TODO: [🥬] Make some system for hashes and ids of promptbook + const semanticName = normalizeToKebabCase( + knowledgeSourceContent.substring(0, 20), + ); - const pieces = ['source', semanticName, hash].filter((piece) => piece !== ''); + const pieces = ["source", semanticName, hash].filter((piece) => piece !== ""); - const name = pieces.join('-').split('--').join('-'); - // <- TODO: Use MAX_FILENAME_LENGTH + const name = pieces.join("-").split("--").join("-"); + // <- TODO: Use MAX_FILENAME_LENGTH - return name; + return name; } /** diff --git a/src/commands/MODEL/ModelCommand.ts b/src/commands/MODEL/ModelCommand.ts index a923208bf2..13d0a07f6e 100644 --- a/src/commands/MODEL/ModelCommand.ts +++ b/src/commands/MODEL/ModelCommand.ts @@ -1,5 +1,5 @@ -import type { ModelRequirements } from '../../types/ModelRequirements'; -import type { TODO_any } from '../../utils/organization/TODO_any'; +import type { ModelRequirements } from "../../types/ModelRequirements"; +import type { TODO_any } from "../../utils/organization/TODO_any"; /** * Parsed MODEL command @@ -9,7 +9,7 @@ import type { TODO_any } from '../../utils/organization/TODO_any'; * @public exported from `@promptbook/editable` */ export type ModelCommand = { - readonly type: 'MODEL'; - readonly key: keyof ModelRequirements; - readonly value: TODO_any /* <- TODO: Infer from used key, can it be done in TypeScript */; + readonly type: "MODEL"; + readonly key: keyof ModelRequirements; + readonly value: TODO_any /* <- TODO: Infer from used key, can it be done in TypeScript */; }; diff --git a/src/commands/MODEL/modelCommand.test.ts b/src/commands/MODEL/modelCommand.test.ts index b75dc25680..b936d897c9 100644 --- a/src/commands/MODEL/modelCommand.test.ts +++ b/src/commands/MODEL/modelCommand.test.ts @@ -1,58 +1,68 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { modelCommandParser } from './modelCommandParser'; - -describe('how MODEL command in .book.md files works', () => { - it('should parse MODEL command', () => { - expect(parseCommand('MODEL VARIANT Completion', 'PIPELINE_TASK')).toEqual({ - type: 'MODEL', - key: 'modelVariant', - value: 'COMPLETION', - }); - - expect(parseCommand('MODEL VARIANT Chat', 'PIPELINE_TASK')).toEqual({ - type: 'MODEL', - key: 'modelVariant', - value: 'CHAT', - }); - - expect(parseCommand('MODEL VARIANT Completion ', 'PIPELINE_TASK')).toEqual({ - type: 'MODEL', - key: 'modelVariant', - value: 'COMPLETION', - }); - - // <- Note: [🤖] - - expect(parseCommand('MODEL VARIANT `CHAT`', 'PIPELINE_TASK')).toEqual({ - type: 'MODEL', - key: 'modelVariant', - value: 'CHAT', - }); - - expect(parseCommand('MODEL NAME gpt-4-1106-preview', 'PIPELINE_TASK')).toEqual({ - type: 'MODEL', - key: 'modelName', - value: 'gpt-4-1106-preview', - }); - - expect(parseCommand('MODEL NAME gpt-3.5-turbo-instruct', 'PIPELINE_TASK')).toEqual({ - type: 'MODEL', - key: 'modelName', - value: 'gpt-3.5-turbo-instruct', - }); - }); - - it('should fail parsing MODEL VARIANT command', () => { - expect(() => parseCommand('MODEL wet', 'PIPELINE_TASK')).toThrowError(/Unknown model key/i); - expect(() => parseCommand('MODEL {script}', 'PIPELINE_TASK')).toThrowError(/Unknown model key/i); - }); - - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of modelCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_HEAD')).not.toThrowError(); - expect(() => parseCommand(example, 'PIPELINE_TASK')).not.toThrowError(); - } - }); +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { modelCommandParser } from "./modelCommandParser"; + +describe("how MODEL command in .book.md files works", () => { + it("should parse MODEL command", () => { + expect(parseCommand("MODEL VARIANT Completion", "PIPELINE_TASK")).toEqual({ + type: "MODEL", + key: "modelVariant", + value: "COMPLETION", + }); + + expect(parseCommand("MODEL VARIANT Chat", "PIPELINE_TASK")).toEqual({ + type: "MODEL", + key: "modelVariant", + value: "CHAT", + }); + + expect( + parseCommand("MODEL VARIANT Completion ", "PIPELINE_TASK"), + ).toEqual({ + type: "MODEL", + key: "modelVariant", + value: "COMPLETION", + }); + + // <- Note: [🤖] + + expect(parseCommand("MODEL VARIANT `CHAT`", "PIPELINE_TASK")).toEqual({ + type: "MODEL", + key: "modelVariant", + value: "CHAT", + }); + + expect( + parseCommand("MODEL NAME gpt-4-1106-preview", "PIPELINE_TASK"), + ).toEqual({ + type: "MODEL", + key: "modelName", + value: "gpt-4-1106-preview", + }); + + expect( + parseCommand("MODEL NAME gpt-3.5-turbo-instruct", "PIPELINE_TASK"), + ).toEqual({ + type: "MODEL", + key: "modelName", + value: "gpt-3.5-turbo-instruct", + }); + }); + + it("should fail parsing MODEL VARIANT command", () => { + expect(() => parseCommand("MODEL wet", "PIPELINE_TASK")).toThrowError( + /Unknown model key/i, + ); + expect(() => parseCommand("MODEL {script}", "PIPELINE_TASK")).toThrowError( + /Unknown model key/i, + ); + }); + + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of modelCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_HEAD")).not.toThrowError(); + expect(() => parseCommand(example, "PIPELINE_TASK")).not.toThrowError(); + } + }); }); diff --git a/src/commands/MODEL/modelCommandParser.ts b/src/commands/MODEL/modelCommandParser.ts index acdcd1408b..e55a348ecc 100644 --- a/src/commands/MODEL/modelCommandParser.ts +++ b/src/commands/MODEL/modelCommandParser.ts @@ -1,16 +1,16 @@ -import spaceTrim from 'spacetrim'; -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { ParseError } from '../../errors/ParseError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { ModelRequirements } from '../../types/ModelRequirements'; -import { MODEL_VARIANTS } from '../../types/ModelVariant'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import type { $PipelineJson } from '../_common/types/CommandParser'; -import type { $TaskJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineBothCommandParser } from '../_common/types/CommandParser'; -import type { ModelCommand } from './ModelCommand'; +import spaceTrim from "spacetrim"; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { ParseError } from "../../errors/ParseError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { ModelRequirements } from "../../types/ModelRequirements"; +import { MODEL_VARIANTS } from "../../types/ModelVariant"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import type { $PipelineJson } from "../_common/types/CommandParser"; +import type { $TaskJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineBothCommandParser } from "../_common/types/CommandParser"; +import type { ModelCommand } from "./ModelCommand"; /** * Parses the model command @@ -20,185 +20,202 @@ import type { ModelCommand } from './ModelCommand'; * @public exported from `@promptbook/editable` */ export const modelCommandParser: PipelineBothCommandParser = { - /** - * Name of the command - */ - name: 'MODEL', - - /** - * BOILERPLATE command can be used in: - */ - isUsedInPipelineHead: true, // <- TODO: [🧠][❔] Should there be possibility to set MODEL for entire pipeline? - isUsedInPipelineTask: true, - - /** - * Description of the MODEL command - */ - description: `Tells which \`modelRequirements\` (for example which model) to use for the prompt task execution`, - - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/67', - - /** - * Example usages of the MODEL command - */ - examples: ['MODEL VARIANT Chat', 'MODEL NAME `gpt-4`'], - - /** - * Parses the MODEL command - */ - parse(input: CommandParserInput): ModelCommand { - const { args, normalized } = input; - - const availableVariantsMessage = spaceTrim( - (block) => ` + /** + * Name of the command + */ + name: "MODEL", + + /** + * BOILERPLATE command can be used in: + */ + isUsedInPipelineHead: true, // <- TODO: [🧠][❔] Should there be possibility to set MODEL for entire pipeline? + isUsedInPipelineTask: true, + + /** + * Description of the MODEL command + */ + description: `Tells which \`modelRequirements\` (for example which model) to use for the prompt task execution`, + + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/67", + + /** + * Example usages of the MODEL command + */ + examples: ["MODEL VARIANT Chat", "MODEL NAME `gpt-4`"], + + /** + * Parses the MODEL command + */ + parse(input: CommandParserInput): ModelCommand { + const { args, normalized } = input; + + const availableVariantsMessage = spaceTrim( + (block) => ` Available variants are: ${block( - MODEL_VARIANTS.map( - (variantName) => - `- ${variantName}${variantName !== 'EMBEDDING' ? '' : ' (Not available in pipeline)'}`, - ).join('\n'), - )} + MODEL_VARIANTS.map( + (variantName) => + `- ${variantName}${variantName !== "EMBEDDING" ? "" : " (Not available in pipeline)"}`, + ).join("\n"), + )} `, - ); - - // TODO: Make this more elegant and dynamically - if (normalized.startsWith('MODEL_VARIANT')) { - if (normalized === 'MODEL_VARIANT_CHAT') { - return { - type: 'MODEL', - key: 'modelVariant', - value: 'CHAT', - } satisfies ModelCommand; - } else if (normalized === 'MODEL_VARIANT_COMPLETION') { - return { - type: 'MODEL', - key: 'modelVariant', - value: 'COMPLETION', - } satisfies ModelCommand; - // <- Note: [🤖] - } else if (normalized.startsWith('MODEL_VARIANT_EMBED')) { - spaceTrim( - (block) => ` + ); + + // TODO: Make this more elegant and dynamically + if (normalized.startsWith("MODEL_VARIANT")) { + if (normalized === "MODEL_VARIANT_CHAT") { + return { + type: "MODEL", + key: "modelVariant", + value: "CHAT", + } satisfies ModelCommand; + } else if (normalized === "MODEL_VARIANT_COMPLETION") { + return { + type: "MODEL", + key: "modelVariant", + value: "COMPLETION", + } satisfies ModelCommand; + // <- Note: [🤖] + } else if (normalized.startsWith("MODEL_VARIANT_EMBED")) { + spaceTrim( + (block) => ` Embedding model can not be used in pipeline ${block(availableVariantsMessage)} `, - ); - } else { - throw new ParseError( - spaceTrim( - (block) => ` + ); + } else { + throw new ParseError( + spaceTrim( + (block) => ` Unknown model variant in command: ${block(availableVariantsMessage)} `, - ), - ); - } - } - if (normalized.startsWith('MODEL_NAME')) { - return { - type: 'MODEL', - key: 'modelName', - value: args.pop()!, - } satisfies ModelCommand; - } else { - throw new ParseError( - spaceTrim( - (block) => ` + ), + ); + } + } + if (normalized.startsWith("MODEL_NAME")) { + return { + type: "MODEL", + key: "modelName", + value: args.pop()!, + } satisfies ModelCommand; + } else { + throw new ParseError( + spaceTrim( + (block) => ` Unknown model key in command. Supported model keys are: - ${block(['variant', 'name'].join(', '))} + ${block(["variant", "name"].join(", "))} Example: - MODEL VARIANT Chat - MODEL NAME gpt-4 `, - ), - ); - } - }, - - /** - * Apply the MODEL command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `pipelineJson` - */ - $applyToPipelineJson(command: ModelCommand, $pipelineJson: $PipelineJson): void { - $pipelineJson.defaultModelRequirements = $pipelineJson.defaultModelRequirements || {}; - - // TODO: [🚜] DRY - if ($pipelineJson.defaultModelRequirements[command.key] !== undefined) { - if ($pipelineJson.defaultModelRequirements[command.key] === command.value) { - console.warn(`Multiple commands \`MODEL ${command.key} ${command.value}\` in the pipeline head`); - // <- TODO: [🚎][💩] Some better way how to get warnings from pipeline parsing / logic - } else { - throw new ParseError( - spaceTrim(` + ), + ); + } + }, + + /** + * Apply the MODEL command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `pipelineJson` + */ + $applyToPipelineJson( + command: ModelCommand, + $pipelineJson: $PipelineJson, + ): void { + $pipelineJson.defaultModelRequirements = + $pipelineJson.defaultModelRequirements || {}; + + // TODO: [🚜] DRY + if ($pipelineJson.defaultModelRequirements[command.key] !== undefined) { + if ( + $pipelineJson.defaultModelRequirements[command.key] === command.value + ) { + console.warn( + `Multiple commands \`MODEL ${command.key} ${command.value}\` in the pipeline head`, + ); + // <- TODO: [🚎][💩] Some better way how to get warnings from pipeline parsing / logic + } else { + throw new ParseError( + spaceTrim(` Redefinition of \`MODEL ${command.key}\` in the pipeline head You have used: 1) \`MODEL ${command.key} ${$pipelineJson.defaultModelRequirements[command.key]}\` 2) \`MODEL ${command.key} ${command.value}\` `), - ); - } - } - - $pipelineJson.defaultModelRequirements[command.key] = command.value; - }, - - /** - * Apply the MODEL command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `taskJson` - */ - $applyToTaskJson(command: ModelCommand, $taskJson: $TaskJson, $pipelineJson: $PipelineJson): void { - if ($taskJson.taskType !== 'PROMPT_TASK') { - throw new ParseError(`MODEL command can only be used in PROMPT_TASK block`); - } - - $taskJson.modelRequirements = $taskJson.modelRequirements || {}; - - // TODO: [🚜] DRY - if ($taskJson.modelRequirements[command.key] !== undefined) { - if ($taskJson.modelRequirements[command.key] === command.value) { - console.warn( - `Multiple commands \`MODEL ${ - ( - { - modelName: 'NAME', - modelVariant: 'VARIANT', - maxTokens: '???', - } as Record - )[command.key] - } ${command.value}\` in the task "${$taskJson.title || $taskJson.name}"`, - ); - } else { - throw new ParseError( - spaceTrim(` + ); + } + } + + $pipelineJson.defaultModelRequirements[command.key] = command.value; + }, + + /** + * Apply the MODEL command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `taskJson` + */ + $applyToTaskJson( + command: ModelCommand, + $taskJson: $TaskJson, + $pipelineJson: $PipelineJson, + ): void { + if ($taskJson.taskType !== "PROMPT_TASK") { + throw new ParseError( + `MODEL command can only be used in PROMPT_TASK block`, + ); + } + + $taskJson.modelRequirements = $taskJson.modelRequirements || {}; + + // TODO: [🚜] DRY + if ($taskJson.modelRequirements[command.key] !== undefined) { + if ($taskJson.modelRequirements[command.key] === command.value) { + console.warn( + `Multiple commands \`MODEL ${ + ( + { + modelName: "NAME", + modelVariant: "VARIANT", + maxTokens: "???", + } as Record + )[command.key] + } ${command.value}\` in the task "${$taskJson.title || $taskJson.name}"`, + ); + } else { + throw new ParseError( + spaceTrim(` Redefinition of MODEL \`${command.key}\` in the task "${ - $taskJson.title || $taskJson.name - }" + $taskJson.title || $taskJson.name + }" You have used: - MODEL ${command.key} ${$taskJson.modelRequirements[command.key]} - MODEL ${command.key} ${command.value} `), - ); - } - } + ); + } + } - if (command.value === ($pipelineJson.defaultModelRequirements || {})[command.key]) { - console.log( - spaceTrim(` + if ( + command.value === + ($pipelineJson.defaultModelRequirements || {})[command.key] + ) { + console.log( + spaceTrim(` Setting MODEL \`${command.key}\` in the task "${ - $taskJson.title || $taskJson.name - }" to the same value as in the pipeline head + $taskJson.title || $taskJson.name + }" to the same value as in the pipeline head In pipeline head: - MODEL ${command.key} ${($pipelineJson.defaultModelRequirements || {})[command.key]} @@ -206,39 +223,41 @@ export const modelCommandParser: PipelineBothCommandParser = { But same value is used in the task: - MODEL ${command.key} ${command.value} `), - ); - } - - $taskJson.modelRequirements[command.key] = command.value; - }, - - /** - * Converts the MODEL command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: ModelCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, - - /** - * Reads the MODEL command from the `PipelineJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray { - keepUnused(pipelineJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, - - /** - * Reads the MODEL command from the `TaskJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { - keepUnused($taskJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, + ); + } + + $taskJson.modelRequirements[command.key] = command.value; + }, + + /** + * Converts the MODEL command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: ModelCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, + + /** + * Reads the MODEL command from the `PipelineJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromPipelineJson( + pipelineJson: PipelineJson, + ): ReadonlyArray { + keepUnused(pipelineJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, + + /** + * Reads the MODEL command from the `TaskJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { + keepUnused($taskJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, }; diff --git a/src/commands/PARAMETER/ParameterCommand.ts b/src/commands/PARAMETER/ParameterCommand.ts index 11631214d6..a5fd4b109d 100644 --- a/src/commands/PARAMETER/ParameterCommand.ts +++ b/src/commands/PARAMETER/ParameterCommand.ts @@ -1,5 +1,5 @@ -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_name } from '../../types/typeAliases'; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_name } from "../../types/typeAliases"; /** * Parsed PARAMETER command @@ -8,9 +8,9 @@ import type { string_name } from '../../types/typeAliases'; * @public exported from `@promptbook/editable` */ export type ParameterCommand = { - readonly type: 'PARAMETER'; - readonly isInput: boolean; - readonly isOutput: boolean; - readonly parameterName: string_name; - readonly parameterDescription: string_markdown_text | null; + readonly type: "PARAMETER"; + readonly isInput: boolean; + readonly isOutput: boolean; + readonly parameterName: string_name; + readonly parameterDescription: string_markdown_text | null; }; diff --git a/src/commands/PARAMETER/parameterCommand.test.ts b/src/commands/PARAMETER/parameterCommand.test.ts index 3aeb60692f..c5b3fdf1f4 100644 --- a/src/commands/PARAMETER/parameterCommand.test.ts +++ b/src/commands/PARAMETER/parameterCommand.test.ts @@ -1,81 +1,104 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { parameterCommandParser } from './parameterCommandParser'; - -describe('how PARAMETER command in .book.md files works', () => { - it('should parse PARAMETER command', () => { - expect(parseCommand('parameter {name} Name for the hero1', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, - isOutput: false, - parameterName: 'name', - parameterDescription: 'Name for the hero1', - }); - - // Note: [🦈] - expect(parseCommand('{name} Name for the hero2', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, - isOutput: false, - parameterName: 'name', - parameterDescription: 'Name for the hero2', - }); - - // Note: [🦈] and [🐡] - expect(parseCommand('> {name} Name for the hero3', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, - isOutput: false, - parameterName: 'name', - parameterDescription: 'Name for the hero3', - }); - - // Note: [🦈] - expect(parseCommand('{name} Input for the hero4', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, - isOutput: false, - parameterName: 'name', - parameterDescription: 'Input for the hero4', - }); - expect(parseCommand('input parameter {name} Name for the hero5', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: true, - isOutput: false, - parameterName: 'name', - parameterDescription: 'Name for the hero5', - }); - expect(parseCommand('input parameter {name}', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: true, - isOutput: false, - parameterName: 'name', - parameterDescription: null, - }); - expect(parseCommand('input parameter {name} ', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: true, - isOutput: false, - parameterName: 'name', - parameterDescription: null, - }); - - expect(parseCommand('OUTPUT parameter {name} Name for the hero6', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, - isOutput: true, - parameterName: 'name', - parameterDescription: 'Name for the hero6', - }); - expect(parseCommand(' parameter {name} Name for the hero7 ', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, - isOutput: false, - parameterName: 'name', - parameterDescription: 'Name for the hero7', - }); - - /* +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { parameterCommandParser } from "./parameterCommandParser"; + +describe("how PARAMETER command in .book.md files works", () => { + it("should parse PARAMETER command", () => { + expect( + parseCommand("parameter {name} Name for the hero1", "PIPELINE_HEAD"), + ).toEqual({ + type: "PARAMETER", + isInput: false, + isOutput: false, + parameterName: "name", + parameterDescription: "Name for the hero1", + }); + + // Note: [🦈] + expect(parseCommand("{name} Name for the hero2", "PIPELINE_HEAD")).toEqual({ + type: "PARAMETER", + isInput: false, + isOutput: false, + parameterName: "name", + parameterDescription: "Name for the hero2", + }); + + // Note: [🦈] and [🐡] + expect( + parseCommand("> {name} Name for the hero3", "PIPELINE_HEAD"), + ).toEqual({ + type: "PARAMETER", + isInput: false, + isOutput: false, + parameterName: "name", + parameterDescription: "Name for the hero3", + }); + + // Note: [🦈] + expect(parseCommand("{name} Input for the hero4", "PIPELINE_HEAD")).toEqual( + { + type: "PARAMETER", + isInput: false, + isOutput: false, + parameterName: "name", + parameterDescription: "Input for the hero4", + }, + ); + expect( + parseCommand( + "input parameter {name} Name for the hero5", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PARAMETER", + isInput: true, + isOutput: false, + parameterName: "name", + parameterDescription: "Name for the hero5", + }); + expect(parseCommand("input parameter {name}", "PIPELINE_HEAD")).toEqual({ + type: "PARAMETER", + isInput: true, + isOutput: false, + parameterName: "name", + parameterDescription: null, + }); + expect( + parseCommand("input parameter {name} ", "PIPELINE_HEAD"), + ).toEqual({ + type: "PARAMETER", + isInput: true, + isOutput: false, + parameterName: "name", + parameterDescription: null, + }); + + expect( + parseCommand( + "OUTPUT parameter {name} Name for the hero6", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PARAMETER", + isInput: false, + isOutput: true, + parameterName: "name", + parameterDescription: "Name for the hero6", + }); + expect( + parseCommand( + " parameter {name} Name for the hero7 ", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PARAMETER", + isInput: false, + isOutput: false, + parameterName: "name", + parameterDescription: "Name for the hero7", + }); + + /* TODO: When [🥶] fixed, allow formatting in parameters expect(parseCommand('parameter {name} **Name** for the hero8', 'PIPELINE_HEAD')).toEqual({ type: 'PARAMETER', @@ -92,101 +115,132 @@ describe('how PARAMETER command in .book.md files works', () => { parameterDescription: '**Name** for `the` {', }); */ - }); - - it('should not be confused by input/output word in parameter name or description', () => { - // Note: [🐡] - expect(parseCommand('> {inputText} The input text', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, // <- Note: Not input despite the word input in the parameter name - isOutput: false, - parameterName: 'inputText', - parameterDescription: 'The input text', - }); - - // Note: [🐡] - expect(parseCommand('> {outputText} The output text', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, - isOutput: false, // <- Note: Not output despite the word output in the parameter name - parameterName: 'outputText', - parameterDescription: 'The output text', - }); - - expect(parseCommand('PARAMETER {inputText} The input text', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, // <- Note: Not input despite the word input in the parameter name - isOutput: false, - parameterName: 'inputText', - parameterDescription: 'The input text', - }); - - expect(parseCommand('PARAMETER {outputText} The output text', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, - isOutput: false, // <- Note: Not output despite the word output in the parameter name - parameterName: 'outputText', - parameterDescription: 'The output text', - }); - - expect(parseCommand('OUTPUT PARAMETER {inputText} The input text', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, // <- Note: Not input despite the word input in the parameter name - isOutput: true, - parameterName: 'inputText', - parameterDescription: 'The input text', - }); - - expect(parseCommand('INPUT PARAMETER {outputText} The output text', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: true, - isOutput: false, // <- Note: Not output despite the word output in the parameter name - parameterName: 'outputText', - parameterDescription: 'The output text', - }); - - expect(parseCommand('parameter name', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, - isOutput: false, - parameterName: 'name', - parameterDescription: null, - }); - - expect(parseCommand('parameter {jméno}', 'PIPELINE_HEAD')).toEqual({ - type: 'PARAMETER', - isInput: false, - isOutput: false, - parameterName: 'jmeno', - parameterDescription: null, - }); - }); - - it('should fail parsing PARAMETER command', () => { - expect(() => parseCommand('parameter {}', 'PIPELINE_HEAD')).toThrowError(/Invalid parameter/i); - expect(() => parseCommand('parameter { name }', 'PIPELINE_HEAD')).toThrowError(/Invalid parameter/i); - expect(() => parseCommand('parameter {name} {name}', 'PIPELINE_HEAD')).toThrowError( - /Can not contain another parameter in description/i, - ); - expect(() => parseCommand('parameter {name} {name} Name for the hero9', 'PIPELINE_HEAD')).toThrowError( - /Can not contain another parameter in description/i, - ); - expect(() => parseCommand('parameter {name} Name for the hero9 {name}', 'PIPELINE_HEAD')).toThrowError( - /Can not contain another parameter in description/i, - ); - expect(() => - parseCommand('parameter {name} Name for the hero {name} Name for the herox', 'PIPELINE_HEAD'), - ).toThrowError(/Can not contain another parameter in description/i); - expect(() => parseCommand('parmeter {name} Name for the heroy', 'PIPELINE_HEAD')).toThrowError( - /Unknown command/i, - ); - }); - - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of parameterCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_HEAD')).not.toThrowError(); - expect(() => parseCommand(example, 'PIPELINE_TASK')).not.toThrowError(); - } - }); + }); + + it("should not be confused by input/output word in parameter name or description", () => { + // Note: [🐡] + expect( + parseCommand("> {inputText} The input text", "PIPELINE_HEAD"), + ).toEqual({ + type: "PARAMETER", + isInput: false, // <- Note: Not input despite the word input in the parameter name + isOutput: false, + parameterName: "inputText", + parameterDescription: "The input text", + }); + + // Note: [🐡] + expect( + parseCommand("> {outputText} The output text", "PIPELINE_HEAD"), + ).toEqual({ + type: "PARAMETER", + isInput: false, + isOutput: false, // <- Note: Not output despite the word output in the parameter name + parameterName: "outputText", + parameterDescription: "The output text", + }); + + expect( + parseCommand("PARAMETER {inputText} The input text", "PIPELINE_HEAD"), + ).toEqual({ + type: "PARAMETER", + isInput: false, // <- Note: Not input despite the word input in the parameter name + isOutput: false, + parameterName: "inputText", + parameterDescription: "The input text", + }); + + expect( + parseCommand("PARAMETER {outputText} The output text", "PIPELINE_HEAD"), + ).toEqual({ + type: "PARAMETER", + isInput: false, + isOutput: false, // <- Note: Not output despite the word output in the parameter name + parameterName: "outputText", + parameterDescription: "The output text", + }); + + expect( + parseCommand( + "OUTPUT PARAMETER {inputText} The input text", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PARAMETER", + isInput: false, // <- Note: Not input despite the word input in the parameter name + isOutput: true, + parameterName: "inputText", + parameterDescription: "The input text", + }); + + expect( + parseCommand( + "INPUT PARAMETER {outputText} The output text", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PARAMETER", + isInput: true, + isOutput: false, // <- Note: Not output despite the word output in the parameter name + parameterName: "outputText", + parameterDescription: "The output text", + }); + + expect(parseCommand("parameter name", "PIPELINE_HEAD")).toEqual({ + type: "PARAMETER", + isInput: false, + isOutput: false, + parameterName: "name", + parameterDescription: null, + }); + + expect(parseCommand("parameter {jméno}", "PIPELINE_HEAD")).toEqual({ + type: "PARAMETER", + isInput: false, + isOutput: false, + parameterName: "jmeno", + parameterDescription: null, + }); + }); + + it("should fail parsing PARAMETER command", () => { + expect(() => parseCommand("parameter {}", "PIPELINE_HEAD")).toThrowError( + /Invalid parameter/i, + ); + expect(() => + parseCommand("parameter { name }", "PIPELINE_HEAD"), + ).toThrowError(/Invalid parameter/i); + expect(() => + parseCommand("parameter {name} {name}", "PIPELINE_HEAD"), + ).toThrowError(/Can not contain another parameter in description/i); + expect(() => + parseCommand( + "parameter {name} {name} Name for the hero9", + "PIPELINE_HEAD", + ), + ).toThrowError(/Can not contain another parameter in description/i); + expect(() => + parseCommand( + "parameter {name} Name for the hero9 {name}", + "PIPELINE_HEAD", + ), + ).toThrowError(/Can not contain another parameter in description/i); + expect(() => + parseCommand( + "parameter {name} Name for the hero {name} Name for the herox", + "PIPELINE_HEAD", + ), + ).toThrowError(/Can not contain another parameter in description/i); + expect(() => + parseCommand("parmeter {name} Name for the heroy", "PIPELINE_HEAD"), + ).toThrowError(/Unknown command/i); + }); + + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of parameterCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_HEAD")).not.toThrowError(); + expect(() => parseCommand(example, "PIPELINE_TASK")).not.toThrowError(); + } + }); }); diff --git a/src/commands/PARAMETER/parameterCommandParser.ts b/src/commands/PARAMETER/parameterCommandParser.ts index 6bd848b591..2294c71657 100644 --- a/src/commands/PARAMETER/parameterCommandParser.ts +++ b/src/commands/PARAMETER/parameterCommandParser.ts @@ -1,15 +1,15 @@ -import spaceTrim from 'spacetrim'; -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { ParseError } from '../../errors/ParseError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { validateParameterName } from '../../utils/validators/parameterName/validateParameterName'; -import type { $PipelineJson } from '../_common/types/CommandParser'; -import type { $TaskJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineBothCommandParser } from '../_common/types/CommandParser'; -import type { ParameterCommand } from './ParameterCommand'; +import spaceTrim from "spacetrim"; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { ParseError } from "../../errors/ParseError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { validateParameterName } from "../../utils/validators/parameterName/validateParameterName"; +import type { $PipelineJson } from "../_common/types/CommandParser"; +import type { $TaskJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineBothCommandParser } from "../_common/types/CommandParser"; +import type { ParameterCommand } from "./ParameterCommand"; /** * Parses the parameter command @@ -17,133 +17,157 @@ import type { ParameterCommand } from './ParameterCommand'; * @see `documentationUrl` for more details * @public exported from `@promptbook/editable` */ -export const parameterCommandParser: PipelineBothCommandParser = { - /** - * Name of the command - */ - name: 'PARAMETER', - - /** - * Aliases for the PARAMETER command - */ - aliasNames: ['PARAM', 'INPUT_PARAM', 'OUTPUT_PARAM', 'INPUT_PARAMETER', 'OUTPUT_PARAMETER'], - - /* +export const parameterCommandParser: PipelineBothCommandParser = + { + /** + * Name of the command + */ + name: "PARAMETER", + + /** + * Aliases for the PARAMETER command + */ + aliasNames: [ + "PARAM", + "INPUT_PARAM", + "OUTPUT_PARAM", + "INPUT_PARAMETER", + "OUTPUT_PARAMETER", + ], + + /* Note: [🦈] No need for extra alias name because it is already preprocessed */ - /** - * BOILERPLATE command can be used in: - */ - isUsedInPipelineHead: true, - isUsedInPipelineTask: true, - - /** - * Description of the PARAMETER command - */ - description: `Describes one parameter of the task`, - - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/68', - - /** - * Example usages of the PARAMETER command - */ - examples: ['PARAMETER {title} Title of the book', 'OUTPUT PARAMETER {websiteContent} Content of the book'], - - /** - * Parses the PARAMETER command - */ - parse(input: CommandParserInput): ParameterCommand { - const { normalized, args, raw } = input; - - const parameterNameRaw = args.shift() || ''; - const parameterDescriptionRaw = args.join(' '); - // <- TODO: When [🥶] fixed, change to: - // > const parameterDescriptionRaw = rawArgs.split(parameterNameRaw).join('').trim(); - - if (parameterDescriptionRaw && parameterDescriptionRaw.match(/\{(?[a-z0-9_]+)\}/im)) { - throw new ParseError( - spaceTrim( - (block) => ` + /** + * BOILERPLATE command can be used in: + */ + isUsedInPipelineHead: true, + isUsedInPipelineTask: true, + + /** + * Description of the PARAMETER command + */ + description: `Describes one parameter of the task`, + + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/68", + + /** + * Example usages of the PARAMETER command + */ + examples: [ + "PARAMETER {title} Title of the book", + "OUTPUT PARAMETER {websiteContent} Content of the book", + ], + + /** + * Parses the PARAMETER command + */ + parse(input: CommandParserInput): ParameterCommand { + const { normalized, args, raw } = input; + + const parameterNameRaw = args.shift() || ""; + const parameterDescriptionRaw = args.join(" "); + // <- TODO: When [🥶] fixed, change to: + // > const parameterDescriptionRaw = rawArgs.split(parameterNameRaw).join('').trim(); + + if ( + parameterDescriptionRaw && + parameterDescriptionRaw.match( + /\{(?[a-z0-9_]+)\}/im, + ) + ) { + throw new ParseError( + spaceTrim( + (block) => ` Parameter \`{${parameterNameRaw}}\` can not contain another parameter in description The description: ${block(parameterDescriptionRaw)} `, - ), - ); - } - - let isInput = normalized.startsWith('INPUT'); - let isOutput = normalized.startsWith('OUTPUT'); - - if (raw.startsWith('> {')) { - isInput = false; - isOutput = false; - } - - const parameterName = validateParameterName(parameterNameRaw); - const parameterDescription = parameterDescriptionRaw.trim() || null; - - return { - type: 'PARAMETER', - parameterName, - parameterDescription, - isInput, - isOutput, - } satisfies ParameterCommand; - }, - - /** - * Apply the PARAMETER command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `pipelineJson` - */ - $applyToPipelineJson(command: ParameterCommand, $pipelineJson: $PipelineJson): void { - keepUnused(command, $pipelineJson); - // Note: [🍣] Do nothing, its application is implemented separately in `parsePipeline` - }, - - /** - * Apply the PARAMETER command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `taskJson` - */ - $applyToTaskJson(command: ParameterCommand, $taskJson: $TaskJson, $pipelineJson: $PipelineJson): void { - keepUnused(command, $taskJson, $pipelineJson); - // Note: [🍣] Do nothing, its application is implemented separately in `parsePipeline` - }, - - /** - * Converts the PARAMETER command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: ParameterCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, - - /** - * Reads the PARAMETER command from the `PipelineJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray { - keepUnused(pipelineJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, - - /** - * Reads the PARAMETER command from the `TaskJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { - keepUnused($taskJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, -}; + ), + ); + } + + let isInput = normalized.startsWith("INPUT"); + let isOutput = normalized.startsWith("OUTPUT"); + + if (raw.startsWith("> {")) { + isInput = false; + isOutput = false; + } + + const parameterName = validateParameterName(parameterNameRaw); + const parameterDescription = parameterDescriptionRaw.trim() || null; + + return { + type: "PARAMETER", + parameterName, + parameterDescription, + isInput, + isOutput, + } satisfies ParameterCommand; + }, + + /** + * Apply the PARAMETER command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `pipelineJson` + */ + $applyToPipelineJson( + command: ParameterCommand, + $pipelineJson: $PipelineJson, + ): void { + keepUnused(command, $pipelineJson); + // Note: [🍣] Do nothing, its application is implemented separately in `parsePipeline` + }, + + /** + * Apply the PARAMETER command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `taskJson` + */ + $applyToTaskJson( + command: ParameterCommand, + $taskJson: $TaskJson, + $pipelineJson: $PipelineJson, + ): void { + keepUnused(command, $taskJson, $pipelineJson); + // Note: [🍣] Do nothing, its application is implemented separately in `parsePipeline` + }, + + /** + * Converts the PARAMETER command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: ParameterCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, + + /** + * Reads the PARAMETER command from the `PipelineJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromPipelineJson( + pipelineJson: PipelineJson, + ): ReadonlyArray { + keepUnused(pipelineJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, + + /** + * Reads the PARAMETER command from the `TaskJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { + keepUnused($taskJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, + }; diff --git a/src/commands/PERSONA/PersonaCommand.ts b/src/commands/PERSONA/PersonaCommand.ts index 5ea513b867..be350c5a76 100644 --- a/src/commands/PERSONA/PersonaCommand.ts +++ b/src/commands/PERSONA/PersonaCommand.ts @@ -1,5 +1,5 @@ -import type { string_name } from '../../types/typeAliases'; -import type { string_persona_description } from '../../types/typeAliases'; +import type { string_name } from "../../types/typeAliases"; +import type { string_persona_description } from "../../types/typeAliases"; /** * Parsed PERSONA command @@ -8,7 +8,7 @@ import type { string_persona_description } from '../../types/typeAliases'; * @public exported from `@promptbook/editable` */ export type PersonaCommand = { - readonly type: 'PERSONA'; - readonly personaName: string_name; - readonly personaDescription: string_persona_description | null; + readonly type: "PERSONA"; + readonly personaName: string_name; + readonly personaDescription: string_persona_description | null; }; diff --git a/src/commands/PERSONA/personaCommand.test.ts b/src/commands/PERSONA/personaCommand.test.ts index bdc924dfd2..dd2b62ab79 100644 --- a/src/commands/PERSONA/personaCommand.test.ts +++ b/src/commands/PERSONA/personaCommand.test.ts @@ -1,102 +1,146 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { personaCommandParser } from './personaCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { personaCommandParser } from "./personaCommandParser"; -describe('how PERSONA command in .book.md files works', () => { - it('should parse PERSONA command in PIPELINE_HEAD', () => { - expect(parseCommand('PERSONA John', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: 'John', - personaDescription: null, - }); - expect(parseCommand('PERSONA Joe', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: 'Joe', - personaDescription: null, - }); - expect(parseCommand('PERSONA Jiří', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: 'Jiří', - personaDescription: null, - }); - expect(parseCommand('PERSONA Іван', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: 'Іван', - personaDescription: null, - }); - expect(parseCommand('PERSONA 田中', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: '田中', - personaDescription: null, - }); - expect(parseCommand('PERSONA محمد مصدق', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: 'محمد مصدق', - personaDescription: null, - }); - }); +describe("how PERSONA command in .book.md files works", () => { + it("should parse PERSONA command in PIPELINE_HEAD", () => { + expect(parseCommand("PERSONA John", "PIPELINE_HEAD")).toEqual({ + type: "PERSONA", + personaName: "John", + personaDescription: null, + }); + expect(parseCommand("PERSONA Joe", "PIPELINE_HEAD")).toEqual({ + type: "PERSONA", + personaName: "Joe", + personaDescription: null, + }); + expect(parseCommand("PERSONA Jiří", "PIPELINE_HEAD")).toEqual({ + type: "PERSONA", + personaName: "Jiří", + personaDescription: null, + }); + expect(parseCommand("PERSONA Іван", "PIPELINE_HEAD")).toEqual({ + type: "PERSONA", + personaName: "Іван", + personaDescription: null, + }); + expect(parseCommand("PERSONA 田中", "PIPELINE_HEAD")).toEqual({ + type: "PERSONA", + personaName: "田中", + personaDescription: null, + }); + expect(parseCommand("PERSONA محمد مصدق", "PIPELINE_HEAD")).toEqual({ + type: "PERSONA", + personaName: "محمد مصدق", + personaDescription: null, + }); + }); - it('should parse PERSONA command in PIPELINE_TASK', () => { - expect(parseCommand('PERSONA John', 'PIPELINE_TASK')).toEqual({ - type: 'PERSONA', - personaName: 'John', - personaDescription: null, - }); - }); + it("should parse PERSONA command in PIPELINE_TASK", () => { + expect(parseCommand("PERSONA John", "PIPELINE_TASK")).toEqual({ + type: "PERSONA", + personaName: "John", + personaDescription: null, + }); + }); - it('should parse PERSONA with description', () => { - expect(parseCommand('PERSONA John, male 38 years old programmer', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: 'John', - personaDescription: 'male 38 years old programmer', - }); - expect(parseCommand('PERSONA John; male 38 years old programmer', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: 'John', - personaDescription: 'male 38 years old programmer', - }); - expect(parseCommand('PERSONA John: male 38 years old programmer', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: 'John', - personaDescription: 'male 38 years old programmer', - }); - expect(parseCommand('PERSONA Joe, male 38 years old programmer', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: 'Joe', - personaDescription: 'male 38 years old programmer', - }); - expect(parseCommand('PERSONA Jiří, male 38 years old programmer', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: 'Jiří', - personaDescription: 'male 38 years old programmer', - }); - expect(parseCommand('PERSONA Іван, male 38 years old programmer', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: 'Іван', - personaDescription: 'male 38 years old programmer', - }); - expect(parseCommand('PERSONA 田中, male 38 years old programmer', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: '田中', - personaDescription: 'male 38 years old programmer', - }); - expect(parseCommand('PERSONA محمد مصدق, male 38 years old programmer', 'PIPELINE_HEAD')).toEqual({ - type: 'PERSONA', - personaName: 'محمد مصدق', - personaDescription: 'male 38 years old programmer', - }); - }); + it("should parse PERSONA with description", () => { + expect( + parseCommand( + "PERSONA John, male 38 years old programmer", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PERSONA", + personaName: "John", + personaDescription: "male 38 years old programmer", + }); + expect( + parseCommand( + "PERSONA John; male 38 years old programmer", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PERSONA", + personaName: "John", + personaDescription: "male 38 years old programmer", + }); + expect( + parseCommand( + "PERSONA John: male 38 years old programmer", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PERSONA", + personaName: "John", + personaDescription: "male 38 years old programmer", + }); + expect( + parseCommand( + "PERSONA Joe, male 38 years old programmer", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PERSONA", + personaName: "Joe", + personaDescription: "male 38 years old programmer", + }); + expect( + parseCommand( + "PERSONA Jiří, male 38 years old programmer", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PERSONA", + personaName: "Jiří", + personaDescription: "male 38 years old programmer", + }); + expect( + parseCommand( + "PERSONA Іван, male 38 years old programmer", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PERSONA", + personaName: "Іван", + personaDescription: "male 38 years old programmer", + }); + expect( + parseCommand( + "PERSONA 田中, male 38 years old programmer", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PERSONA", + personaName: "田中", + personaDescription: "male 38 years old programmer", + }); + expect( + parseCommand( + "PERSONA محمد مصدق, male 38 years old programmer", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "PERSONA", + personaName: "محمد مصدق", + personaDescription: "male 38 years old programmer", + }); + }); - it('should fail parsing PERSONA command', () => { - expect(() => parseCommand('PERSONA', 'PIPELINE_HEAD')).toThrowError(/You must set name for the persona/i); - expect(() => parseCommand('PERSONA ,', 'PIPELINE_HEAD')).toThrowError(/You must set name for the persona/i); - }); + it("should fail parsing PERSONA command", () => { + expect(() => parseCommand("PERSONA", "PIPELINE_HEAD")).toThrowError( + /You must set name for the persona/i, + ); + expect(() => parseCommand("PERSONA ,", "PIPELINE_HEAD")).toThrowError( + /You must set name for the persona/i, + ); + }); - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of personaCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_HEAD')).not.toThrowError(); - expect(() => parseCommand(example, 'PIPELINE_TASK')).not.toThrowError(); - } - }); + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of personaCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_HEAD")).not.toThrowError(); + expect(() => parseCommand(example, "PIPELINE_TASK")).not.toThrowError(); + } + }); }); diff --git a/src/commands/PERSONA/personaCommandParser.ts b/src/commands/PERSONA/personaCommandParser.ts index f2e06c5278..748fc13742 100644 --- a/src/commands/PERSONA/personaCommandParser.ts +++ b/src/commands/PERSONA/personaCommandParser.ts @@ -1,14 +1,14 @@ -import spaceTrim from 'spacetrim'; -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { ParseError } from '../../errors/ParseError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import type { $PipelineJson } from '../_common/types/CommandParser'; -import type { $TaskJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineBothCommandParser } from '../_common/types/CommandParser'; -import type { PersonaCommand } from './PersonaCommand'; +import spaceTrim from "spacetrim"; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { ParseError } from "../../errors/ParseError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import type { $PipelineJson } from "../_common/types/CommandParser"; +import type { $TaskJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineBothCommandParser } from "../_common/types/CommandParser"; +import type { PersonaCommand } from "./PersonaCommand"; /** * Parses the persona command @@ -17,104 +17,114 @@ import type { PersonaCommand } from './PersonaCommand'; * @public exported from `@promptbook/editable` */ export const personaCommandParser: PipelineBothCommandParser = { - /** - * Name of the command - */ - name: 'PERSONA', - - /** - * Aliases for the PERSONA command - */ - aliasNames: ['PERSON'], - - /** - * PERSONA command can be used in: - */ - isUsedInPipelineHead: true, - isUsedInPipelineTask: true, - - /** - * Description of the PERSONA command - */ - description: `Persona command is used to specify who the system is, it will be transformed into system message, top_t,...`, - - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/22', - - /** - * Example usages of the PERSONA command - */ - examples: ['PERSONA Jane, skilled copywriter', 'PERSONA Joe, male 28 years old, programmer'], - - /** - * Parses the PERSONA command - */ - parse(input: CommandParserInput): PersonaCommand { - const { rawArgs } = input; - - const [personaNameRaw, personaDescriptionRaw] = rawArgs.split(/[,;:]/, 2); - - const personaName = (personaNameRaw || '').trim(); - - if (personaName === '') { - throw new ParseError(`You must set name for the persona`); - } - - let personaDescription: string | null = (personaDescriptionRaw || '').trim(); - - if (personaDescription === '') { - personaDescription = null; - } - - return { - type: 'PERSONA', - personaName, - personaDescription, - } satisfies PersonaCommand; - }, - - /** - * Apply the PERSONA command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `pipelineJson` - */ - $applyToPipelineJson(command: PersonaCommand, $pipelineJson: $PipelineJson): void { - $applyToTaskJson(command, null, $pipelineJson); - }, - - $applyToTaskJson, - - /** - * Converts the PERSONA command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: PersonaCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, - - /** - * Reads the PERSONA command from the `PipelineJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray { - keepUnused(pipelineJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, - - /** - * Reads the PERSONA command from the `TaskJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { - keepUnused($taskJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, + /** + * Name of the command + */ + name: "PERSONA", + + /** + * Aliases for the PERSONA command + */ + aliasNames: ["PERSON"], + + /** + * PERSONA command can be used in: + */ + isUsedInPipelineHead: true, + isUsedInPipelineTask: true, + + /** + * Description of the PERSONA command + */ + description: `Persona command is used to specify who the system is, it will be transformed into system message, top_t,...`, + + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/22", + + /** + * Example usages of the PERSONA command + */ + examples: [ + "PERSONA Jane, skilled copywriter", + "PERSONA Joe, male 28 years old, programmer", + ], + + /** + * Parses the PERSONA command + */ + parse(input: CommandParserInput): PersonaCommand { + const { rawArgs } = input; + + const [personaNameRaw, personaDescriptionRaw] = rawArgs.split(/[,;:]/, 2); + + const personaName = (personaNameRaw || "").trim(); + + if (personaName === "") { + throw new ParseError(`You must set name for the persona`); + } + + let personaDescription: string | null = ( + personaDescriptionRaw || "" + ).trim(); + + if (personaDescription === "") { + personaDescription = null; + } + + return { + type: "PERSONA", + personaName, + personaDescription, + } satisfies PersonaCommand; + }, + + /** + * Apply the PERSONA command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `pipelineJson` + */ + $applyToPipelineJson( + command: PersonaCommand, + $pipelineJson: $PipelineJson, + ): void { + $applyToTaskJson(command, null, $pipelineJson); + }, + + $applyToTaskJson, + + /** + * Converts the PERSONA command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: PersonaCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, + + /** + * Reads the PERSONA command from the `PipelineJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromPipelineJson( + pipelineJson: PipelineJson, + ): ReadonlyArray { + keepUnused(pipelineJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, + + /** + * Reads the PERSONA command from the `TaskJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { + keepUnused($taskJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, }; /** @@ -122,44 +132,52 @@ export const personaCommandParser: PipelineBothCommandParser = { * * Note: `$` is used to indicate that this function mutates given `taskJson` */ -function $applyToTaskJson(command: PersonaCommand, $taskJson: $TaskJson | null, $pipelineJson: $PipelineJson): void { - const { personaName, personaDescription } = command; - - if ($taskJson !== null) { - if ($taskJson.taskType !== 'PROMPT_TASK') { - throw new ParseError(`PERSONA command can be used only in PROMPT_TASK block`); - } - - $taskJson.personaName = personaName; - } else { - // TODO: [🛳] Default PERSONA for the pipeline `defaultPersonaName` (same as `defaultModelRequirements`) - } - - const persona = $pipelineJson.personas.find((persona) => persona.name === personaName); - - if (persona === undefined) { - $pipelineJson.personas.push({ - name: personaName, - description: personaDescription || '', - }); - return; - } - - if (persona.description === personaDescription) { - return; - } - - if (personaDescription === null) { - return; - } - - if (persona.description === '') { - persona.description = personaDescription; - return; - } - - console.warn( - spaceTrim(` +function $applyToTaskJson( + command: PersonaCommand, + $taskJson: $TaskJson | null, + $pipelineJson: $PipelineJson, +): void { + const { personaName, personaDescription } = command; + + if ($taskJson !== null) { + if ($taskJson.taskType !== "PROMPT_TASK") { + throw new ParseError( + `PERSONA command can be used only in PROMPT_TASK block`, + ); + } + + $taskJson.personaName = personaName; + } else { + // TODO: [🛳] Default PERSONA for the pipeline `defaultPersonaName` (same as `defaultModelRequirements`) + } + + const persona = $pipelineJson.personas.find( + (persona) => persona.name === personaName, + ); + + if (persona === undefined) { + $pipelineJson.personas.push({ + name: personaName, + description: personaDescription || "", + }); + return; + } + + if (persona.description === personaDescription) { + return; + } + + if (personaDescription === null) { + return; + } + + if (persona.description === "") { + persona.description = personaDescription; + return; + } + + console.warn( + spaceTrim(` Persona "${personaName}" is defined multiple times with different description: @@ -170,9 +188,9 @@ function $applyToTaskJson(command: PersonaCommand, $taskJson: $TaskJson | null, ${personaDescription} `), - // <- TODO: [🚞] - // <- TODO: [🧠] What is the propper way of theese `compilePipeline` warnings - ); + // <- TODO: [🚞] + // <- TODO: [🧠] What is the propper way of theese `compilePipeline` warnings + ); - persona.description += spaceTrim('\n\n' + personaDescription); + persona.description += spaceTrim("\n\n" + personaDescription); } diff --git a/src/commands/POSTPROCESS/PostprocessCommand.ts b/src/commands/POSTPROCESS/PostprocessCommand.ts index 9d476210fb..3a3b4f40fb 100644 --- a/src/commands/POSTPROCESS/PostprocessCommand.ts +++ b/src/commands/POSTPROCESS/PostprocessCommand.ts @@ -1,4 +1,4 @@ -import type { string_name } from '../../types/typeAliases'; +import type { string_name } from "../../types/typeAliases"; /** * Parsed POSTPROCESS command @@ -7,6 +7,6 @@ import type { string_name } from '../../types/typeAliases'; * @public exported from `@promptbook/editable` */ export type PostprocessCommand = { - readonly type: 'POSTPROCESS'; - readonly functionName: string_name; + readonly type: "POSTPROCESS"; + readonly functionName: string_name; }; diff --git a/src/commands/POSTPROCESS/postprocessCommand.test.ts b/src/commands/POSTPROCESS/postprocessCommand.test.ts index 38432c8e52..c73ff639d2 100644 --- a/src/commands/POSTPROCESS/postprocessCommand.test.ts +++ b/src/commands/POSTPROCESS/postprocessCommand.test.ts @@ -1,57 +1,59 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { postprocessCommandParser } from './postprocessCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { postprocessCommandParser } from "./postprocessCommandParser"; -describe('how POSTPROCESS command in .book.md files works', () => { - it('should parse POSTPROCESS command', () => { - expect(parseCommand('Postprocess spaceTrim', 'PIPELINE_TASK')).toEqual({ - type: 'POSTPROCESS', - functionName: 'spaceTrim', - }); - expect(parseCommand('Postprocess `spaceTrim`', 'PIPELINE_TASK')).toEqual({ - type: 'POSTPROCESS', - functionName: 'spaceTrim', - }); - expect(parseCommand('Postprocess **spaceTrim**', 'PIPELINE_TASK')).toEqual({ - type: 'POSTPROCESS', - functionName: 'spaceTrim', - }); - expect(parseCommand('Postprocess unwrapResult', 'PIPELINE_TASK')).toEqual({ - type: 'POSTPROCESS', - functionName: 'unwrapResult', - }); - }); +describe("how POSTPROCESS command in .book.md files works", () => { + it("should parse POSTPROCESS command", () => { + expect(parseCommand("Postprocess spaceTrim", "PIPELINE_TASK")).toEqual({ + type: "POSTPROCESS", + functionName: "spaceTrim", + }); + expect(parseCommand("Postprocess `spaceTrim`", "PIPELINE_TASK")).toEqual({ + type: "POSTPROCESS", + functionName: "spaceTrim", + }); + expect(parseCommand("Postprocess **spaceTrim**", "PIPELINE_TASK")).toEqual({ + type: "POSTPROCESS", + functionName: "spaceTrim", + }); + expect(parseCommand("Postprocess unwrapResult", "PIPELINE_TASK")).toEqual({ + type: "POSTPROCESS", + functionName: "unwrapResult", + }); + }); - it('should parse POSTPROCESS command in shortcut form', () => { - expect(parseCommand('PP unwrapResult', 'PIPELINE_TASK')).toEqual({ - type: 'POSTPROCESS', - functionName: 'unwrapResult', - }); - }); + it("should parse POSTPROCESS command in shortcut form", () => { + expect(parseCommand("PP unwrapResult", "PIPELINE_TASK")).toEqual({ + type: "POSTPROCESS", + functionName: "unwrapResult", + }); + }); - it('should fail parsing POSTPROCESS command', () => { - expect(() => parseCommand('Postprocess spaceTrim unwrapResult', 'PIPELINE_TASK')).toThrowError( - /Can not have more than one postprocess function/i, - ); - expect(() => parseCommand('POSTPROCESS @#$%%', 'PIPELINE_TASK')).toThrowError( - /Invalid postprocess function name/i, - ); - expect(() => parseCommand('Process spaceTrim', 'PIPELINE_TASK')).toThrowError(/Unknown command/i); - expect(() => parseCommand('Postprocess', 'PIPELINE_TASK')).toThrowError( - /Postprocess function name is required/i, - ); - expect(() => parseCommand('POSTPROCESS', 'PIPELINE_TASK')).toThrowError( - /Postprocess function name is required/i, - ); - expect(() => parseCommand('POSTPROCESS as^fadf', 'PIPELINE_TASK')).toThrowError( - /Invalid postprocess function name/i, - ); - }); + it("should fail parsing POSTPROCESS command", () => { + expect(() => + parseCommand("Postprocess spaceTrim unwrapResult", "PIPELINE_TASK"), + ).toThrowError(/Can not have more than one postprocess function/i); + expect(() => + parseCommand("POSTPROCESS @#$%%", "PIPELINE_TASK"), + ).toThrowError(/Invalid postprocess function name/i); + expect(() => + parseCommand("Process spaceTrim", "PIPELINE_TASK"), + ).toThrowError(/Unknown command/i); + expect(() => parseCommand("Postprocess", "PIPELINE_TASK")).toThrowError( + /Postprocess function name is required/i, + ); + expect(() => parseCommand("POSTPROCESS", "PIPELINE_TASK")).toThrowError( + /Postprocess function name is required/i, + ); + expect(() => + parseCommand("POSTPROCESS as^fadf", "PIPELINE_TASK"), + ).toThrowError(/Invalid postprocess function name/i); + }); - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of postprocessCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_TASK')).not.toThrowError(); - } - }); + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of postprocessCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_TASK")).not.toThrowError(); + } + }); }); diff --git a/src/commands/POSTPROCESS/postprocessCommandParser.ts b/src/commands/POSTPROCESS/postprocessCommandParser.ts index 31c878be6e..9d88d4c9af 100644 --- a/src/commands/POSTPROCESS/postprocessCommandParser.ts +++ b/src/commands/POSTPROCESS/postprocessCommandParser.ts @@ -1,12 +1,12 @@ -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { ParseError } from '../../errors/ParseError'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { isValidJavascriptName } from '../../utils/validators/javascriptName/isValidJavascriptName'; -import type { $TaskJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineTaskCommandParser } from '../_common/types/CommandParser'; -import type { PostprocessCommand } from './PostprocessCommand'; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { ParseError } from "../../errors/ParseError"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { isValidJavascriptName } from "../../utils/validators/javascriptName/isValidJavascriptName"; +import type { $TaskJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineTaskCommandParser } from "../_common/types/CommandParser"; +import type { PostprocessCommand } from "./PostprocessCommand"; /** * Parses the postprocess command @@ -14,90 +14,94 @@ import type { PostprocessCommand } from './PostprocessCommand'; * @see `documentationUrl` for more details * @public exported from `@promptbook/editable` */ -export const postprocessCommandParser: PipelineTaskCommandParser = { - /** - * Name of the command - */ - name: 'POSTPROCESS', +export const postprocessCommandParser: PipelineTaskCommandParser = + { + /** + * Name of the command + */ + name: "POSTPROCESS", - aliasNames: ['POSTPROCESSING', 'PP'], + aliasNames: ["POSTPROCESSING", "PP"], - /** - * BOILERPLATE command can be used in: - */ - isUsedInPipelineHead: false, - isUsedInPipelineTask: true, + /** + * BOILERPLATE command can be used in: + */ + isUsedInPipelineHead: false, + isUsedInPipelineTask: true, - /** - * Description of the POSTPROCESS command - */ - description: `Defines the postprocess function to be used on the result from LLM and before the result is validated`, + /** + * Description of the POSTPROCESS command + */ + description: `Defines the postprocess function to be used on the result from LLM and before the result is validated`, - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/31', + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/31", - /** - * Example usages of the POSTPROCESS command - */ - examples: [ - 'POSTPROCESS unwrapResult' /* <- TODO: Make it `POSTPROCESS` examples dynamic, load from all possible postprocess functions */, - ], + /** + * Example usages of the POSTPROCESS command + */ + examples: [ + "POSTPROCESS unwrapResult" /* <- TODO: Make it `POSTPROCESS` examples dynamic, load from all possible postprocess functions */, + ], - /** - * Parses the POSTPROCESS command - */ - parse(input: CommandParserInput): PostprocessCommand { - const { args } = input; + /** + * Parses the POSTPROCESS command + */ + parse(input: CommandParserInput): PostprocessCommand { + const { args } = input; - const functionName = args.pop()!; + const functionName = args.pop()!; - if (functionName === undefined) { - throw new ParseError(`Postprocess function name is required`); - } + if (functionName === undefined) { + throw new ParseError(`Postprocess function name is required`); + } - if (!isValidJavascriptName(functionName)) { - throw new ParseError(`Invalid postprocess function name "${functionName}"`); - } + if (!isValidJavascriptName(functionName)) { + throw new ParseError( + `Invalid postprocess function name "${functionName}"`, + ); + } - if (args.length > 0) { - throw new ParseError(`Can not have more than one postprocess function`); - } + if (args.length > 0) { + throw new ParseError(`Can not have more than one postprocess function`); + } - return { - type: 'POSTPROCESS', - functionName, - } satisfies PostprocessCommand; - }, + return { + type: "POSTPROCESS", + functionName, + } satisfies PostprocessCommand; + }, - /** - * Apply the POSTPROCESS command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `taskJson` - */ - $applyToTaskJson(command: PostprocessCommand, $taskJson: $TaskJson): void { - $taskJson.postprocessingFunctionNames = $taskJson.postprocessingFunctionNames || []; - $taskJson.postprocessingFunctionNames.push(command.functionName); - }, + /** + * Apply the POSTPROCESS command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `taskJson` + */ + $applyToTaskJson(command: PostprocessCommand, $taskJson: $TaskJson): void { + $taskJson.postprocessingFunctionNames = + $taskJson.postprocessingFunctionNames || []; + $taskJson.postprocessingFunctionNames.push(command.functionName); + }, - /** - * Converts the POSTPROCESS command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: PostprocessCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, + /** + * Converts the POSTPROCESS command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: PostprocessCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, - /** - * Reads the POSTPROCESS command from the `TaskJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { - keepUnused($taskJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, -}; + /** + * Reads the POSTPROCESS command from the `TaskJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { + keepUnused($taskJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, + }; diff --git a/src/commands/SECTION/SectionCommand.ts b/src/commands/SECTION/SectionCommand.ts index 6abcbc49b4..656bee14f4 100644 --- a/src/commands/SECTION/SectionCommand.ts +++ b/src/commands/SECTION/SectionCommand.ts @@ -1,4 +1,4 @@ -import type { SectionType } from '../../types/SectionType'; +import type { SectionType } from "../../types/SectionType"; /** * Parsed SECTION command @@ -7,6 +7,6 @@ import type { SectionType } from '../../types/SectionType'; * @public exported from `@promptbook/editable` */ export type SectionCommand = { - readonly type: 'SECTION'; - readonly taskType: SectionType; + readonly type: "SECTION"; + readonly taskType: SectionType; }; diff --git a/src/commands/SECTION/sectionCommand.test.ts b/src/commands/SECTION/sectionCommand.test.ts index fbe257e672..f5c9b872a9 100644 --- a/src/commands/SECTION/sectionCommand.test.ts +++ b/src/commands/SECTION/sectionCommand.test.ts @@ -1,47 +1,49 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { sectionCommandParser } from './sectionCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { sectionCommandParser } from "./sectionCommandParser"; -describe('how SECTION command in .book.md files works', () => { - it('should parse SECTION command in recommended form', () => { - expect(parseCommand('PROMPT SECTION', 'PIPELINE_TASK')).toEqual({ - type: 'SECTION', - taskType: 'PROMPT_TASK', - }); +describe("how SECTION command in .book.md files works", () => { + it("should parse SECTION command in recommended form", () => { + expect(parseCommand("PROMPT SECTION", "PIPELINE_TASK")).toEqual({ + type: "SECTION", + taskType: "PROMPT_TASK", + }); - // Note: No need to test all types, because it is tested from `sectionCommandParser.examples` - }); + // Note: No need to test all types, because it is tested from `sectionCommandParser.examples` + }); - it('should work with deprecated EXECUTE command', () => { - expect(parseCommand('EXECUTE Prompt', 'PIPELINE_TASK')).toEqual({ - type: 'SECTION', - taskType: 'PROMPT_TASK', - }); - expect(parseCommand('EXECUTE simple', 'PIPELINE_TASK')).toEqual({ - type: 'SECTION', - taskType: 'SIMPLE_TASK', - }); - expect(parseCommand('EXECUTE script', 'PIPELINE_TASK')).toEqual({ - type: 'SECTION', - taskType: 'SCRIPT_TASK', - }); - expect(parseCommand('EXECUTE dialog', 'PIPELINE_TASK')).toEqual({ - type: 'SECTION', - taskType: 'DIALOG_TASK', - }); - }); + it("should work with deprecated EXECUTE command", () => { + expect(parseCommand("EXECUTE Prompt", "PIPELINE_TASK")).toEqual({ + type: "SECTION", + taskType: "PROMPT_TASK", + }); + expect(parseCommand("EXECUTE simple", "PIPELINE_TASK")).toEqual({ + type: "SECTION", + taskType: "SIMPLE_TASK", + }); + expect(parseCommand("EXECUTE script", "PIPELINE_TASK")).toEqual({ + type: "SECTION", + taskType: "SCRIPT_TASK", + }); + expect(parseCommand("EXECUTE dialog", "PIPELINE_TASK")).toEqual({ + type: "SECTION", + taskType: "DIALOG_TASK", + }); + }); - it('should fail parsing SECTION command', () => { - expect(() => parseCommand('section fooo', 'PIPELINE_TASK')).toThrowError(/Unknown section type/i); - expect(() => parseCommand('section script prompt', 'PIPELINE_TASK')).toThrowError( - /Unknown section type/i, - ); - }); + it("should fail parsing SECTION command", () => { + expect(() => parseCommand("section fooo", "PIPELINE_TASK")).toThrowError( + /Unknown section type/i, + ); + expect(() => + parseCommand("section script prompt", "PIPELINE_TASK"), + ).toThrowError(/Unknown section type/i); + }); - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of sectionCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_TASK')).not.toThrowError(); - } - }); + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of sectionCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_TASK")).not.toThrowError(); + } + }); }); diff --git a/src/commands/SECTION/sectionCommandParser.ts b/src/commands/SECTION/sectionCommandParser.ts index 7976d5b365..e732639ea0 100644 --- a/src/commands/SECTION/sectionCommandParser.ts +++ b/src/commands/SECTION/sectionCommandParser.ts @@ -1,18 +1,18 @@ -import spaceTrim from 'spacetrim'; -import type { WritableDeep } from 'type-fest'; -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { ParseError } from '../../errors/ParseError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { TaskJson } from '../../pipeline/PipelineJson/TaskJson'; -import { SectionTypes } from '../../types/SectionType'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { knowledgeCommandParser } from '../KNOWLEDGE/knowledgeCommandParser'; -import type { $PipelineJson } from '../_common/types/CommandParser'; -import type { $TaskJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineTaskCommandParser } from '../_common/types/CommandParser'; -import type { SectionCommand } from './SectionCommand'; +import spaceTrim from "spacetrim"; +import type { WritableDeep } from "type-fest"; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { ParseError } from "../../errors/ParseError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { TaskJson } from "../../pipeline/PipelineJson/TaskJson"; +import { SectionTypes } from "../../types/SectionType"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { knowledgeCommandParser } from "../KNOWLEDGE/knowledgeCommandParser"; +import type { $PipelineJson } from "../_common/types/CommandParser"; +import type { $TaskJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineTaskCommandParser } from "../_common/types/CommandParser"; +import type { SectionCommand } from "./SectionCommand"; /** * Parses the section command @@ -21,225 +21,237 @@ import type { SectionCommand } from './SectionCommand'; * @public exported from `@promptbook/editable` */ export const sectionCommandParser: PipelineTaskCommandParser = { - /** - * Name of the command - */ - name: 'SECTION', - - /** - * Aliases for the SECTION command - */ - aliasNames: [ - 'PROMPT', - 'SIMPLE', - 'SCRIPT', - 'DIALOG', - 'SAMPLE', - 'EXAMPLE', - 'KNOWLEDGE', // <- Note: [⛱] - 'INSTRUMENT', // <- Note: [⛱] - 'ACTION', // <- Note: [⛱] - ], - - /** - * Aliases for the SECTION command - */ - deprecatedNames: ['TEMPLATE', 'BLOCK', 'EXECUTE'], - - /** - * BOILERPLATE command can be used in: - */ - isUsedInPipelineHead: false, - isUsedInPipelineTask: true, - - /** - * Description of the SECTION command - */ - description: `Defines the purpose of the markdown section - if its a task and which type or something else`, - - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/64', - - /** - * Example usages of the SECTION command - */ - examples: [ - // Short form: - 'PROMPT', - 'SIMPLE', - 'SCRIPT', - 'DIALOG', - // <- [🅱] - 'EXAMPLE', - 'KNOWLEDGE', // <- Note: [⛱] Thare can not be confusion with KNOWLEDGE command because KNOWLEDGE command is not used in tasks but in pipeline head - 'INSTRUMENT', // <- Note: [⛱] -- || -- - 'ACTION', // <- Note: [⛱] -- || -- - - // ----------------- - // Recommended (reversed) form: - 'PROMPT SECTION', - 'SIMPLE SECTION', - 'SCRIPT SECTION', - 'DIALOG SECTION', - // <- [🅱] - 'EXAMPLE SECTION', - 'KNOWLEDGE SECTION', - 'INSTRUMENT SECTION', - 'ACTION SECTION', - - // ----------------- - // Standard form: - 'SECTION PROMPT', - 'SECTION SIMPLE', - 'SECTION SCRIPT', - 'SECTION DIALOG', - // <- [🅱] - 'SECTION EXAMPLE', - 'SECTION KNOWLEDGE', - 'SECTION INSTRUMENT', - 'SECTION ACTION', - ], - - // TODO: [♓️] order: -10 /* <- Note: Putting before other commands */ - - /** - * Parses the SECTION command - */ - parse(input: CommandParserInput): SectionCommand { - let { normalized } = input; - - normalized = normalized.split('SAMPLE').join('EXAMPLE'); - normalized = normalized.split('EXECUTE_').join(''); - normalized = normalized.split('DIALOGUE').join('DIALOG'); - - const taskTypes = SectionTypes.filter((sectionType) => - normalized.includes(sectionType.split('_TASK').join('')), - ); - - if (taskTypes.length !== 1) { - throw new ParseError( - spaceTrim( - (block) => ` + /** + * Name of the command + */ + name: "SECTION", + + /** + * Aliases for the SECTION command + */ + aliasNames: [ + "PROMPT", + "SIMPLE", + "SCRIPT", + "DIALOG", + "SAMPLE", + "EXAMPLE", + "KNOWLEDGE", // <- Note: [⛱] + "INSTRUMENT", // <- Note: [⛱] + "ACTION", // <- Note: [⛱] + ], + + /** + * Aliases for the SECTION command + */ + deprecatedNames: ["TEMPLATE", "BLOCK", "EXECUTE"], + + /** + * BOILERPLATE command can be used in: + */ + isUsedInPipelineHead: false, + isUsedInPipelineTask: true, + + /** + * Description of the SECTION command + */ + description: `Defines the purpose of the markdown section - if its a task and which type or something else`, + + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/64", + + /** + * Example usages of the SECTION command + */ + examples: [ + // Short form: + "PROMPT", + "SIMPLE", + "SCRIPT", + "DIALOG", + // <- [🅱] + "EXAMPLE", + "KNOWLEDGE", // <- Note: [⛱] Thare can not be confusion with KNOWLEDGE command because KNOWLEDGE command is not used in tasks but in pipeline head + "INSTRUMENT", // <- Note: [⛱] -- || -- + "ACTION", // <- Note: [⛱] -- || -- + + // ----------------- + // Recommended (reversed) form: + "PROMPT SECTION", + "SIMPLE SECTION", + "SCRIPT SECTION", + "DIALOG SECTION", + // <- [🅱] + "EXAMPLE SECTION", + "KNOWLEDGE SECTION", + "INSTRUMENT SECTION", + "ACTION SECTION", + + // ----------------- + // Standard form: + "SECTION PROMPT", + "SECTION SIMPLE", + "SECTION SCRIPT", + "SECTION DIALOG", + // <- [🅱] + "SECTION EXAMPLE", + "SECTION KNOWLEDGE", + "SECTION INSTRUMENT", + "SECTION ACTION", + ], + + // TODO: [♓️] order: -10 /* <- Note: Putting before other commands */ + + /** + * Parses the SECTION command + */ + parse(input: CommandParserInput): SectionCommand { + let { normalized } = input; + + normalized = normalized.split("SAMPLE").join("EXAMPLE"); + normalized = normalized.split("EXECUTE_").join(""); + normalized = normalized.split("DIALOGUE").join("DIALOG"); + + const taskTypes = SectionTypes.filter((sectionType) => + normalized.includes(sectionType.split("_TASK").join("")), + ); + + if (taskTypes.length !== 1) { + throw new ParseError( + spaceTrim( + (block) => ` Unknown section type "${normalized}" Supported section types are: - ${block(SectionTypes.join(', '))} + ${block(SectionTypes.join(", "))} `, // <- TODO: [🚞] - ), - ); - } - - const taskType = taskTypes[0]!; - - return { - type: 'SECTION', - taskType, - } satisfies SectionCommand; - }, - - /** - * Apply the SECTION command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `taskJson` - */ - $applyToTaskJson(command: SectionCommand, $taskJson: $TaskJson, $pipelineJson: $PipelineJson): void { - if ($taskJson.isSectionTypeSet === true) { - throw new ParseError( - spaceTrim(` + ), + ); + } + + const taskType = taskTypes[0]!; + + return { + type: "SECTION", + taskType, + } satisfies SectionCommand; + }, + + /** + * Apply the SECTION command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `taskJson` + */ + $applyToTaskJson( + command: SectionCommand, + $taskJson: $TaskJson, + $pipelineJson: $PipelineJson, + ): void { + if ($taskJson.isSectionTypeSet === true) { + throw new ParseError( + spaceTrim(` Section type is already defined in the section. It can be defined only once. `), - ); - } - - $taskJson.isSectionTypeSet = true; - - // TODO: [🍧][💩] Rearrange better - but at bottom and unwrap from function - const expectResultingParameterName = () => { - if ($taskJson.resultingParameterName) { - return; - } - - throw new ParseError(`Task section and example section must end with return statement -> {parameterName}`); - }; - - if ($taskJson.content === undefined) { - throw new UnexpectedError( - `Content is missing in the taskJson - probbably commands are applied in wrong order`, - ); - } - - if (command.taskType === 'EXAMPLE') { - expectResultingParameterName(); - - const parameter = $pipelineJson.parameters.find((param) => param.name === $taskJson.resultingParameterName); - if (parameter === undefined) { - // TODO: !!6 Change to logic error for higher level abstraction of chatbot to work - throw new ParseError( - `Parameter \`{${$taskJson.resultingParameterName}}\` is not defined so can not define example value of it`, - ); - } - parameter.exampleValues = parameter.exampleValues || []; - parameter.exampleValues.push($taskJson.content); - - $taskJson.isTask = false; - return; - } - - if (command.taskType === 'KNOWLEDGE') { - knowledgeCommandParser.$applyToPipelineJson( - { - type: 'KNOWLEDGE', - knowledgeSourceContent: $taskJson.content, // <- TODO: [🐝][main] !!3 Work with KNOWLEDGE which not referring to the source file or website, but its content itself - }, - $pipelineJson, - ); - - $taskJson.isTask = false; - return; - } - - if (command.taskType === 'ACTION') { - console.error(new NotYetImplementedError('Actions are not implemented yet')); - - $taskJson.isTask = false; - return; - } - - if (command.taskType === 'INSTRUMENT') { - console.error(new NotYetImplementedError('Instruments are not implemented yet')); - - $taskJson.isTask = false; - return; - } - - expectResultingParameterName(); - ($taskJson as WritableDeep).taskType = command.taskType; - - $taskJson.isTask = true; - }, - - /** - * Converts the SECTION command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: SectionCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, - - /** - * Reads the SECTION command from the `TaskJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { - keepUnused($taskJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, + ); + } + + $taskJson.isSectionTypeSet = true; + + // TODO: [🍧][💩] Rearrange better - but at bottom and unwrap from function + const expectResultingParameterName = () => { + if ($taskJson.resultingParameterName) { + return; + } + + throw new ParseError( + `Task section and example section must end with return statement -> {parameterName}`, + ); + }; + + if ($taskJson.content === undefined) { + throw new UnexpectedError( + `Content is missing in the taskJson - probbably commands are applied in wrong order`, + ); + } + + if (command.taskType === "EXAMPLE") { + expectResultingParameterName(); + + const parameter = $pipelineJson.parameters.find( + (param) => param.name === $taskJson.resultingParameterName, + ); + if (parameter === undefined) { + // TODO: !!6 Change to logic error for higher level abstraction of chatbot to work + throw new ParseError( + `Parameter \`{${$taskJson.resultingParameterName}}\` is not defined so can not define example value of it`, + ); + } + parameter.exampleValues = parameter.exampleValues || []; + parameter.exampleValues.push($taskJson.content); + + $taskJson.isTask = false; + return; + } + + if (command.taskType === "KNOWLEDGE") { + knowledgeCommandParser.$applyToPipelineJson( + { + type: "KNOWLEDGE", + knowledgeSourceContent: $taskJson.content, // <- TODO: [🐝][main] !!3 Work with KNOWLEDGE which not referring to the source file or website, but its content itself + }, + $pipelineJson, + ); + + $taskJson.isTask = false; + return; + } + + if (command.taskType === "ACTION") { + console.error( + new NotYetImplementedError("Actions are not implemented yet"), + ); + + $taskJson.isTask = false; + return; + } + + if (command.taskType === "INSTRUMENT") { + console.error( + new NotYetImplementedError("Instruments are not implemented yet"), + ); + + $taskJson.isTask = false; + return; + } + + expectResultingParameterName(); + ($taskJson as WritableDeep).taskType = command.taskType; + + $taskJson.isTask = true; + }, + + /** + * Converts the SECTION command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: SectionCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, + + /** + * Reads the SECTION command from the `TaskJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { + keepUnused($taskJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, }; /** diff --git a/src/commands/URL/UrlCommand.ts b/src/commands/URL/UrlCommand.ts index 3b2ceaff41..b4e99f21bb 100644 --- a/src/commands/URL/UrlCommand.ts +++ b/src/commands/URL/UrlCommand.ts @@ -5,6 +5,6 @@ * @public exported from `@promptbook/editable` */ export type UrlCommand = { - readonly type: 'URL'; - readonly pipelineUrl: URL; + readonly type: "URL"; + readonly pipelineUrl: URL; }; diff --git a/src/commands/URL/urlCommand.test.ts b/src/commands/URL/urlCommand.test.ts index 287b08b500..167c30ae52 100644 --- a/src/commands/URL/urlCommand.test.ts +++ b/src/commands/URL/urlCommand.test.ts @@ -1,38 +1,55 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { urlCommandParser } from './urlCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { urlCommandParser } from "./urlCommandParser"; -describe('how URL command in .book.md files works', () => { - it('should parse URL command', () => { - expect(parseCommand('https://promptbook.studio/webgpt/write-website-content-cs.book', 'PIPELINE_HEAD')).toEqual( - { - type: 'URL', - pipelineUrl: new URL('https://promptbook.studio/webgpt/write-website-content-cs.book'), - }, - ); - expect( - parseCommand( - ' https://promptbook.studio/webgpt/write-website-content-cs.book.md ', - 'PIPELINE_HEAD', - ), - ).toEqual({ - type: 'URL', - pipelineUrl: new URL('https://promptbook.studio/webgpt/write-website-content-cs.book.md'), - }); - expect( - parseCommand('url https://promptbook.studio/webgpt/write-website-content-cs.book', 'PIPELINE_HEAD'), - ).toEqual({ - type: 'URL', - pipelineUrl: new URL('https://promptbook.studio/webgpt/write-website-content-cs.book'), - }); - expect( - parseCommand('URL https://promptbook.studio/webgpt/write-website-content-cs.book', 'PIPELINE_HEAD'), - ).toEqual({ - type: 'URL', - pipelineUrl: new URL('https://promptbook.studio/webgpt/write-website-content-cs.book'), - }); +describe("how URL command in .book.md files works", () => { + it("should parse URL command", () => { + expect( + parseCommand( + "https://promptbook.studio/webgpt/write-website-content-cs.book", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "URL", + pipelineUrl: new URL( + "https://promptbook.studio/webgpt/write-website-content-cs.book", + ), + }); + expect( + parseCommand( + " https://promptbook.studio/webgpt/write-website-content-cs.book.md ", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "URL", + pipelineUrl: new URL( + "https://promptbook.studio/webgpt/write-website-content-cs.book.md", + ), + }); + expect( + parseCommand( + "url https://promptbook.studio/webgpt/write-website-content-cs.book", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "URL", + pipelineUrl: new URL( + "https://promptbook.studio/webgpt/write-website-content-cs.book", + ), + }); + expect( + parseCommand( + "URL https://promptbook.studio/webgpt/write-website-content-cs.book", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "URL", + pipelineUrl: new URL( + "https://promptbook.studio/webgpt/write-website-content-cs.book", + ), + }); - /* + /* TODO: [🧠][🌘] Should this work: expect( parseCommand( @@ -55,14 +72,19 @@ describe('how URL command in .book.md files works', () => { }); */ - expect( - parseCommand('URL https://promptbook.studio/webgpt/write-website-content-cs.book', 'PIPELINE_HEAD'), - ).toEqual({ - type: 'URL', - pipelineUrl: new URL('https://promptbook.studio/webgpt/write-website-content-cs.book'), - }); + expect( + parseCommand( + "URL https://promptbook.studio/webgpt/write-website-content-cs.book", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "URL", + pipelineUrl: new URL( + "https://promptbook.studio/webgpt/write-website-content-cs.book", + ), + }); - /* + /* TODO: [🧠][🌘] Should this work: expect( parseCommand( @@ -84,21 +106,31 @@ describe('how URL command in .book.md files works', () => { }); */ - expect( - parseCommand('url *https://promptbook.studio/webgpt/write-website-content-cs.book*', 'PIPELINE_HEAD'), - ).toEqual({ - type: 'URL', - pipelineUrl: new URL('https://promptbook.studio/webgpt/write-website-content-cs.book'), - }); - expect( - parseCommand('`https://promptbook.studio/webgpt/write-website-content-cs.book`', 'PIPELINE_HEAD'), - ).toEqual({ - type: 'URL', - pipelineUrl: new URL('https://promptbook.studio/webgpt/write-website-content-cs.book'), - }); - }); + expect( + parseCommand( + "url *https://promptbook.studio/webgpt/write-website-content-cs.book*", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "URL", + pipelineUrl: new URL( + "https://promptbook.studio/webgpt/write-website-content-cs.book", + ), + }); + expect( + parseCommand( + "`https://promptbook.studio/webgpt/write-website-content-cs.book`", + "PIPELINE_HEAD", + ), + ).toEqual({ + type: "URL", + pipelineUrl: new URL( + "https://promptbook.studio/webgpt/write-website-content-cs.book", + ), + }); + }); - /* + /* TODO: [🧠][🌘] Should this work: it('should parse URL command in shortcut form', () => { @@ -129,18 +161,22 @@ describe('how URL command in .book.md files works', () => { }); */ - it('should fail parsing URL command', () => { - expect(() => parseCommand('URL', 'PIPELINE_HEAD')).toThrowError(/URL is required/i); - expect(() => - parseCommand( - 'URL https://promptbook.studio/webgpt/write-website-content-cs.book https://promptbook.studio/webgpt/write-website-content-cs.book', - 'PIPELINE_HEAD', - ), - ).toThrowError(/Can not have more than one pipeline URL/i); + it("should fail parsing URL command", () => { + expect(() => parseCommand("URL", "PIPELINE_HEAD")).toThrowError( + /URL is required/i, + ); + expect(() => + parseCommand( + "URL https://promptbook.studio/webgpt/write-website-content-cs.book https://promptbook.studio/webgpt/write-website-content-cs.book", + "PIPELINE_HEAD", + ), + ).toThrowError(/Can not have more than one pipeline URL/i); - expect(() => parseCommand('url http:^404', 'PIPELINE_HEAD')).toThrowError(/Invalid pipeline URL/i); + expect(() => parseCommand("url http:^404", "PIPELINE_HEAD")).toThrowError( + /Invalid pipeline URL/i, + ); - /* + /* Note: [👣] expect(() => parseCommand('url http://promptbook.studio/write-website-content', 'PIPELINE_HEAD')).toThrowError( /Invalid pipeline URL/i, @@ -148,21 +184,21 @@ describe('how URL command in .book.md files works', () => { ); */ - expect(() => - parseCommand( - 'url https://promptbook.studio/webgpt/write-website-content-cs.book#keywords', - 'PIPELINE_HEAD', - ), - ).toThrowError( - /Invalid pipeline URL/i, - // <- TODO: [🐠] /URL must not contain hash/i - ); - }); + expect(() => + parseCommand( + "url https://promptbook.studio/webgpt/write-website-content-cs.book#keywords", + "PIPELINE_HEAD", + ), + ).toThrowError( + /Invalid pipeline URL/i, + // <- TODO: [🐠] /URL must not contain hash/i + ); + }); - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of urlCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_HEAD')).not.toThrowError(); - } - }); + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of urlCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_HEAD")).not.toThrowError(); + } + }); }); diff --git a/src/commands/URL/urlCommandParser.ts b/src/commands/URL/urlCommandParser.ts index d10fa7d7d5..81d9a5c5f2 100644 --- a/src/commands/URL/urlCommandParser.ts +++ b/src/commands/URL/urlCommandParser.ts @@ -1,13 +1,13 @@ -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { ParseError } from '../../errors/ParseError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { isValidPipelineUrl } from '../../utils/validators/url/isValidPipelineUrl'; -import type { $PipelineJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineHeadCommandParser } from '../_common/types/CommandParser'; -import type { UrlCommand } from './UrlCommand'; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { ParseError } from "../../errors/ParseError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { isValidPipelineUrl } from "../../utils/validators/url/isValidPipelineUrl"; +import type { $PipelineJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineHeadCommandParser } from "../_common/types/CommandParser"; +import type { UrlCommand } from "./UrlCommand"; /** * Parses the url command @@ -16,65 +16,65 @@ import type { UrlCommand } from './UrlCommand'; * @public exported from `@promptbook/editable` */ export const urlCommandParser: PipelineHeadCommandParser = { - /** - * Name of the command - */ - name: 'URL', + /** + * Name of the command + */ + name: "URL", - aliasNames: ['PIPELINE_URL'], + aliasNames: ["PIPELINE_URL"], - /* + /* Note: [🛵] No need for this alias name because it is already preprocessed aliasNames: ['HTTPS'], */ - /** - * BOILERPLATE command can be used in: - */ - isUsedInPipelineHead: true, - isUsedInPipelineTask: false, - - /** - * Description of the URL command - */ - description: `Declares unique URL for the pipeline`, - - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/70', - - /** - * Example usages of the URL command - */ - examples: [ - 'PIPELINE URL https://promptbook.studio/library/write-cv.book', - 'URL https://promptbook.studio/library/write-cv.book', - 'https://promptbook.studio/library/write-cv.book', - ], - - /** - * Parses the URL command - */ - parse(input: CommandParserInput): UrlCommand { - const { args } = input; - - const pipelineUrl = args.pop()!; - - if (pipelineUrl === undefined) { - throw new ParseError(`URL is required`); - } - - // TODO: [🧠][🚲] This should be maybe tested as logic not parse - if (!isValidPipelineUrl(pipelineUrl)) { - throw new ParseError(`Invalid pipeline URL "${pipelineUrl}"`); - } - - if (args.length > 0) { - throw new ParseError(`Can not have more than one pipeline URL`); - } - - /* + /** + * BOILERPLATE command can be used in: + */ + isUsedInPipelineHead: true, + isUsedInPipelineTask: false, + + /** + * Description of the URL command + */ + description: `Declares unique URL for the pipeline`, + + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/70", + + /** + * Example usages of the URL command + */ + examples: [ + "PIPELINE URL https://promptbook.studio/library/write-cv.book", + "URL https://promptbook.studio/library/write-cv.book", + "https://promptbook.studio/library/write-cv.book", + ], + + /** + * Parses the URL command + */ + parse(input: CommandParserInput): UrlCommand { + const { args } = input; + + const pipelineUrl = args.pop()!; + + if (pipelineUrl === undefined) { + throw new ParseError(`URL is required`); + } + + // TODO: [🧠][🚲] This should be maybe tested as logic not parse + if (!isValidPipelineUrl(pipelineUrl)) { + throw new ParseError(`Invalid pipeline URL "${pipelineUrl}"`); + } + + if (args.length > 0) { + throw new ParseError(`Can not have more than one pipeline URL`); + } + + /* TODO: [🐠 Maybe more info from `isValidPipelineUrl`: if (pipelineUrl.protocol !== 'https:') { throw new ParseError(`Protocol must be HTTPS`); @@ -92,38 +92,41 @@ export const urlCommandParser: PipelineHeadCommandParser = { } */ - return { - type: 'URL', - pipelineUrl: new URL(pipelineUrl), - } satisfies UrlCommand; - }, - - /** - * Apply the URL command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `pipelineJson` - */ - $applyToPipelineJson(command: UrlCommand, $pipelineJson: $PipelineJson): void { - $pipelineJson.pipelineUrl = command.pipelineUrl.href; - }, - - /** - * Converts the URL command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: UrlCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, - - /** - * Reads the URL command from the `PipelineJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray { - keepUnused(pipelineJson); - throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement - }, + return { + type: "URL", + pipelineUrl: new URL(pipelineUrl), + } satisfies UrlCommand; + }, + + /** + * Apply the URL command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `pipelineJson` + */ + $applyToPipelineJson( + command: UrlCommand, + $pipelineJson: $PipelineJson, + ): void { + $pipelineJson.pipelineUrl = command.pipelineUrl.href; + }, + + /** + * Converts the URL command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: UrlCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, + + /** + * Reads the URL command from the `PipelineJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray { + keepUnused(pipelineJson); + throw new NotYetImplementedError(`[🛋] Not implemented yet`); // <- TODO: [🛋] Implement + }, }; diff --git a/src/commands/X_ACTION/ActionCommand.ts b/src/commands/X_ACTION/ActionCommand.ts index 42d4fd4e75..c3a48d815a 100644 --- a/src/commands/X_ACTION/ActionCommand.ts +++ b/src/commands/X_ACTION/ActionCommand.ts @@ -5,5 +5,5 @@ * @public exported from `@promptbook/editable` */ export type ActionCommand = { - readonly type: 'ACTION'; + readonly type: "ACTION"; }; diff --git a/src/commands/X_ACTION/actionCommand.test.ts b/src/commands/X_ACTION/actionCommand.test.ts index 7ee3b04eb4..2cfd372548 100644 --- a/src/commands/X_ACTION/actionCommand.test.ts +++ b/src/commands/X_ACTION/actionCommand.test.ts @@ -1,19 +1,19 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { actionCommandParser } from './actionCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { actionCommandParser } from "./actionCommandParser"; -describe('how ACTION command in .book.md files works', () => { - it('should parse ACTION command in PIPELINE_HEAD', () => { - expect(parseCommand('ACTION', 'PIPELINE_HEAD')).toEqual({ - type: 'ACTION', - }); - }); +describe("how ACTION command in .book.md files works", () => { + it("should parse ACTION command in PIPELINE_HEAD", () => { + expect(parseCommand("ACTION", "PIPELINE_HEAD")).toEqual({ + type: "ACTION", + }); + }); - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of actionCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_HEAD')).not.toThrowError(); - expect(() => parseCommand(example, 'PIPELINE_TASK')).not.toThrowError(); - } - }); + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of actionCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_HEAD")).not.toThrowError(); + expect(() => parseCommand(example, "PIPELINE_TASK")).not.toThrowError(); + } + }); }); diff --git a/src/commands/X_ACTION/actionCommandParser.ts b/src/commands/X_ACTION/actionCommandParser.ts index aff3f75801..4191b5cbdf 100644 --- a/src/commands/X_ACTION/actionCommandParser.ts +++ b/src/commands/X_ACTION/actionCommandParser.ts @@ -1,12 +1,12 @@ -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; -import type { $PipelineJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineHeadCommandParser } from '../_common/types/CommandParser'; -import type { ActionCommand } from './ActionCommand'; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { TODO_USE } from "../../utils/organization/TODO_USE"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import type { $PipelineJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineHeadCommandParser } from "../_common/types/CommandParser"; +import type { ActionCommand } from "./ActionCommand"; /** * Parses the action command @@ -15,74 +15,81 @@ import type { ActionCommand } from './ActionCommand'; * @public exported from `@promptbook/editable` */ export const actionCommandParser: PipelineHeadCommandParser = { - /** - * Name of the command - */ - name: 'ACTION', + /** + * Name of the command + */ + name: "ACTION", - /** - * ACTION command can be used in: - */ - isUsedInPipelineHead: true, - isUsedInPipelineTask: false, // <- [👙] Maybe allow to use here and make relevant for just this task + /** + * ACTION command can be used in: + */ + isUsedInPipelineHead: true, + isUsedInPipelineTask: false, // <- [👙] Maybe allow to use here and make relevant for just this task - /** - * Description of the ACTION command - */ - description: `Actions influences from the pipeline or task into external world. Like turning on a light, sending an email, etc.`, + /** + * Description of the ACTION command + */ + description: `Actions influences from the pipeline or task into external world. Like turning on a light, sending an email, etc.`, - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/72', + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/72", - /** - * Example usages of the ACTION command - */ - examples: ['ACTION'], + /** + * Example usages of the ACTION command + */ + examples: ["ACTION"], - /** - * Parses the ACTION command - */ - parse(input: CommandParserInput): ActionCommand { - const { args } = input; + /** + * Parses the ACTION command + */ + parse(input: CommandParserInput): ActionCommand { + const { args } = input; - TODO_USE(args); + TODO_USE(args); - return { - type: 'ACTION', - } satisfies ActionCommand; - }, + return { + type: "ACTION", + } satisfies ActionCommand; + }, - /** - * Apply the ACTION command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `pipelineJson` - */ - $applyToPipelineJson(command: ActionCommand, $pipelineJson: $PipelineJson): void { - keepUnused(command, $pipelineJson); - console.error(new NotYetImplementedError('[🛠] Actions are not implemented yet')); - }, + /** + * Apply the ACTION command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `pipelineJson` + */ + $applyToPipelineJson( + command: ActionCommand, + $pipelineJson: $PipelineJson, + ): void { + keepUnused(command, $pipelineJson); + console.error( + new NotYetImplementedError("[🛠] Actions are not implemented yet"), + ); + }, - /** - * Converts the ACTION command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: ActionCommand): string_markdown_text { - keepUnused(command); - throw new NotYetImplementedError('[🛠] Actions are not implemented yet'); - }, + /** + * Converts the ACTION command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: ActionCommand): string_markdown_text { + keepUnused(command); + throw new NotYetImplementedError("[🛠] Actions are not implemented yet"); + }, - /** - * Reads the ACTION command from the `PipelineJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray { - keepUnused(pipelineJson); - throw new NotYetImplementedError('[🛠] Actions are not implemented yet'); - }, + /** + * Reads the ACTION command from the `PipelineJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromPipelineJson( + pipelineJson: PipelineJson, + ): ReadonlyArray { + keepUnused(pipelineJson); + throw new NotYetImplementedError("[🛠] Actions are not implemented yet"); + }, }; /** diff --git a/src/commands/X_INSTRUMENT/InstrumentCommand.ts b/src/commands/X_INSTRUMENT/InstrumentCommand.ts index 364e374e85..ef8bc920e7 100644 --- a/src/commands/X_INSTRUMENT/InstrumentCommand.ts +++ b/src/commands/X_INSTRUMENT/InstrumentCommand.ts @@ -5,5 +5,5 @@ * @public exported from `@promptbook/editable` */ export type InstrumentCommand = { - readonly type: 'INSTRUMENT'; + readonly type: "INSTRUMENT"; }; diff --git a/src/commands/X_INSTRUMENT/instrumentCommand.test.ts b/src/commands/X_INSTRUMENT/instrumentCommand.test.ts index a51c02f292..544d04ea1f 100644 --- a/src/commands/X_INSTRUMENT/instrumentCommand.test.ts +++ b/src/commands/X_INSTRUMENT/instrumentCommand.test.ts @@ -1,19 +1,19 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { instrumentCommandParser } from './instrumentCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { instrumentCommandParser } from "./instrumentCommandParser"; -describe('how INSTRUMENT command in .book.md files works', () => { - it('should parse INSTRUMENT command in PIPELINE_HEAD', () => { - expect(parseCommand('INSTRUMENT', 'PIPELINE_HEAD')).toEqual({ - type: 'INSTRUMENT', - }); - }); +describe("how INSTRUMENT command in .book.md files works", () => { + it("should parse INSTRUMENT command in PIPELINE_HEAD", () => { + expect(parseCommand("INSTRUMENT", "PIPELINE_HEAD")).toEqual({ + type: "INSTRUMENT", + }); + }); - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of instrumentCommandParser.examples) { - expect(() => parseCommand(example, 'PIPELINE_HEAD')).not.toThrowError(); - expect(() => parseCommand(example, 'PIPELINE_TASK')).not.toThrowError(); - } - }); + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of instrumentCommandParser.examples) { + expect(() => parseCommand(example, "PIPELINE_HEAD")).not.toThrowError(); + expect(() => parseCommand(example, "PIPELINE_TASK")).not.toThrowError(); + } + }); }); diff --git a/src/commands/X_INSTRUMENT/instrumentCommandParser.ts b/src/commands/X_INSTRUMENT/instrumentCommandParser.ts index d658b4fe6d..c5a42d87c9 100644 --- a/src/commands/X_INSTRUMENT/instrumentCommandParser.ts +++ b/src/commands/X_INSTRUMENT/instrumentCommandParser.ts @@ -1,12 +1,12 @@ -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; -import type { $PipelineJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineHeadCommandParser } from '../_common/types/CommandParser'; -import type { InstrumentCommand } from './InstrumentCommand'; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { TODO_USE } from "../../utils/organization/TODO_USE"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import type { $PipelineJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineHeadCommandParser } from "../_common/types/CommandParser"; +import type { InstrumentCommand } from "./InstrumentCommand"; /** * Parses the instrument command @@ -14,77 +14,89 @@ import type { InstrumentCommand } from './InstrumentCommand'; * @see `documentationUrl` for more details * @public exported from `@promptbook/editable` */ -export const instrumentCommandParser: PipelineHeadCommandParser = { - /** - * Name of the command - */ - name: 'INSTRUMENT', +export const instrumentCommandParser: PipelineHeadCommandParser = + { + /** + * Name of the command + */ + name: "INSTRUMENT", - /** - * INSTRUMENT command can be used in: - */ - isUsedInPipelineHead: true, - isUsedInPipelineTask: false, // <- [👙] Maybe allow to use here and make relevant for just this task + /** + * INSTRUMENT command can be used in: + */ + isUsedInPipelineHead: true, + isUsedInPipelineTask: false, // <- [👙] Maybe allow to use here and make relevant for just this task - /** - * Description of the INSTRUMENT command - */ - description: `Instrument command is used to specify the instrument to be used in the pipeline or task like search, calculate, etc.`, + /** + * Description of the INSTRUMENT command + */ + description: `Instrument command is used to specify the instrument to be used in the pipeline or task like search, calculate, etc.`, - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/71', + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/71", - /** - * Example usages of the INSTRUMENT command - */ - examples: ['INSTRUMENT'], + /** + * Example usages of the INSTRUMENT command + */ + examples: ["INSTRUMENT"], - /** - * Parses the INSTRUMENT command - */ - parse(input: CommandParserInput): InstrumentCommand { - const { args } = input; + /** + * Parses the INSTRUMENT command + */ + parse(input: CommandParserInput): InstrumentCommand { + const { args } = input; - // TODO: [🛠] Implement - TODO_USE(args); + // TODO: [🛠] Implement + TODO_USE(args); - return { - type: 'INSTRUMENT', - } satisfies InstrumentCommand; - }, + return { + type: "INSTRUMENT", + } satisfies InstrumentCommand; + }, - /** - * Apply the INSTRUMENT command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `pipelineJson` - */ - $applyToPipelineJson(command: InstrumentCommand, $pipelineJson: $PipelineJson): void { - keepUnused(command, $pipelineJson); - console.error(new NotYetImplementedError('[🛠] Instruments are not implemented yet')); - }, + /** + * Apply the INSTRUMENT command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `pipelineJson` + */ + $applyToPipelineJson( + command: InstrumentCommand, + $pipelineJson: $PipelineJson, + ): void { + keepUnused(command, $pipelineJson); + console.error( + new NotYetImplementedError("[🛠] Instruments are not implemented yet"), + ); + }, - /** - * Converts the INSTRUMENT command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: InstrumentCommand): string_markdown_text { - keepUnused(command); - throw new NotYetImplementedError('[🛠] Instruments are not implemented yet'); - }, + /** + * Converts the INSTRUMENT command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: InstrumentCommand): string_markdown_text { + keepUnused(command); + throw new NotYetImplementedError( + "[🛠] Instruments are not implemented yet", + ); + }, - /** - * Reads the INSTRUMENT command from the `PipelineJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray { - keepUnused(pipelineJson); - throw new NotYetImplementedError('[🛠] Instruments are not implemented yet'); - }, -}; + /** + * Reads the INSTRUMENT command from the `PipelineJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromPipelineJson( + pipelineJson: PipelineJson, + ): ReadonlyArray { + keepUnused(pipelineJson); + throw new NotYetImplementedError( + "[🛠] Instruments are not implemented yet", + ); + }, + }; /** * Note: [⛱] There are two types of INSTRUMENT commands *...(read more in [⛱])* diff --git a/src/commands/_BOILERPLATE/BoilerplateCommand.ts b/src/commands/_BOILERPLATE/BoilerplateCommand.ts index da73084dec..885f84c21c 100644 --- a/src/commands/_BOILERPLATE/BoilerplateCommand.ts +++ b/src/commands/_BOILERPLATE/BoilerplateCommand.ts @@ -5,6 +5,6 @@ * @private within the commands folder */ export type BoilerplateCommand = { - readonly type: 'BOILERPLATE'; - readonly value: string; + readonly type: "BOILERPLATE"; + readonly value: string; }; diff --git a/src/commands/_BOILERPLATE/boilerplateCommand.test.ts b/src/commands/_BOILERPLATE/boilerplateCommand.test.ts index 9f9f3ab14a..c09e93a7da 100644 --- a/src/commands/_BOILERPLATE/boilerplateCommand.test.ts +++ b/src/commands/_BOILERPLATE/boilerplateCommand.test.ts @@ -1,54 +1,56 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseCommand } from '../_common/parseCommand'; -import { boilerplateCommandParser } from './boilerplateCommandParser'; +import { describe, expect, it } from "@jest/globals"; +import { parseCommand } from "../_common/parseCommand"; +import { boilerplateCommandParser } from "./boilerplateCommandParser"; -describe('how BOILERPLATE command in .book.md files works', () => { - it('should parse BOILERPLATE command in PIPELINE_HEAD', () => { - expect(parseCommand('BOILERPLATE foo', 'PIPELINE_HEAD')).toEqual({ - type: 'BOILERPLATE', - value: 'foo', - }); - expect(parseCommand('BOILERPLATE bar', 'PIPELINE_HEAD')).toEqual({ - type: 'BOILERPLATE', - value: 'bar', - }); - }); +describe("how BOILERPLATE command in .book.md files works", () => { + it("should parse BOILERPLATE command in PIPELINE_HEAD", () => { + expect(parseCommand("BOILERPLATE foo", "PIPELINE_HEAD")).toEqual({ + type: "BOILERPLATE", + value: "foo", + }); + expect(parseCommand("BOILERPLATE bar", "PIPELINE_HEAD")).toEqual({ + type: "BOILERPLATE", + value: "bar", + }); + }); - it('should parse BOILERPLATE command in PIPELINE_TASK', () => { - expect(parseCommand('BOILERPLATE foo', 'PIPELINE_TASK')).toEqual({ - type: 'BOILERPLATE', - value: 'foo', - }); - expect(parseCommand('BOILERPLATE bar', 'PIPELINE_TASK')).toEqual({ - type: 'BOILERPLATE', - value: 'bar', - }); - }); + it("should parse BOILERPLATE command in PIPELINE_TASK", () => { + expect(parseCommand("BOILERPLATE foo", "PIPELINE_TASK")).toEqual({ + type: "BOILERPLATE", + value: "foo", + }); + expect(parseCommand("BOILERPLATE bar", "PIPELINE_TASK")).toEqual({ + type: "BOILERPLATE", + value: "bar", + }); + }); - it('should parse BOILERPLATE command in shortcut form', () => { - expect(parseCommand('BP foo', 'PIPELINE_HEAD')).toEqual({ - type: 'BOILERPLATE', - value: 'foo', - }); - expect(parseCommand('BP bar', 'PIPELINE_TASK')).toEqual({ - type: 'BOILERPLATE', - value: 'bar', - }); - }); + it("should parse BOILERPLATE command in shortcut form", () => { + expect(parseCommand("BP foo", "PIPELINE_HEAD")).toEqual({ + type: "BOILERPLATE", + value: "foo", + }); + expect(parseCommand("BP bar", "PIPELINE_TASK")).toEqual({ + type: "BOILERPLATE", + value: "bar", + }); + }); - it('should fail parsing BOILERPLATE command', () => { - expect(() => parseCommand('BOILERPLATE', 'PIPELINE_HEAD')).toThrowError(/requires exactly one argument/i); - expect(() => parseCommand('BOILERPLATE brr', 'PIPELINE_HEAD')).toThrowError( - /BOILERPLATE value can not contain brr/i, - ); - }); + it("should fail parsing BOILERPLATE command", () => { + expect(() => parseCommand("BOILERPLATE", "PIPELINE_HEAD")).toThrowError( + /requires exactly one argument/i, + ); + expect(() => parseCommand("BOILERPLATE brr", "PIPELINE_HEAD")).toThrowError( + /BOILERPLATE value can not contain brr/i, + ); + }); - it(`should work with all examples`, () => { - // Note: This is tested also in the common test file parseCommand.test.ts - for (const example of boilerplateCommandParser.examples) { - // TODO: @@ Remove places not using the command: - expect(() => parseCommand(example, 'PIPELINE_HEAD')).not.toThrowError(); - expect(() => parseCommand(example, 'PIPELINE_TASK')).not.toThrowError(); - } - }); + it(`should work with all examples`, () => { + // Note: This is tested also in the common test file parseCommand.test.ts + for (const example of boilerplateCommandParser.examples) { + // TODO: @@ Remove places not using the command: + expect(() => parseCommand(example, "PIPELINE_HEAD")).not.toThrowError(); + expect(() => parseCommand(example, "PIPELINE_TASK")).not.toThrowError(); + } + }); }); diff --git a/src/commands/_BOILERPLATE/boilerplateCommandParser.ts b/src/commands/_BOILERPLATE/boilerplateCommandParser.ts index 86b96ab79e..e016d8100f 100644 --- a/src/commands/_BOILERPLATE/boilerplateCommandParser.ts +++ b/src/commands/_BOILERPLATE/boilerplateCommandParser.ts @@ -1,12 +1,12 @@ -import { ParseError } from '../../errors/ParseError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import type { $PipelineJson } from '../_common/types/CommandParser'; -import type { $TaskJson } from '../_common/types/CommandParser'; -import type { CommandParserInput } from '../_common/types/CommandParser'; -import type { PipelineBothCommandParser } from '../_common/types/CommandParser'; -import type { BoilerplateCommand } from './BoilerplateCommand'; +import { ParseError } from "../../errors/ParseError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import type { $PipelineJson } from "../_common/types/CommandParser"; +import type { $TaskJson } from "../_common/types/CommandParser"; +import type { CommandParserInput } from "../_common/types/CommandParser"; +import type { PipelineBothCommandParser } from "../_common/types/CommandParser"; +import type { BoilerplateCommand } from "./BoilerplateCommand"; /** * Parses the boilerplate command @@ -16,115 +16,127 @@ import type { BoilerplateCommand } from './BoilerplateCommand'; * @see `documentationUrl` for more details * @private within the commands folder */ -export const boilerplateCommandParser: PipelineBothCommandParser = { - /** - * Name of the command - */ - name: 'BOILERPLATE', - - /** - * Aliases for the BOILERPLATE command - */ - aliasNames: ['BP'], - - /** - * BOILERPLATE command can be used in: - */ - isUsedInPipelineHead: true, - isUsedInPipelineTask: true, - - /** - * Description of the BOILERPLATE command - */ - description: `@@`, - - /** - * Link to documentation - */ - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@', - - /** - * Example usages of the BOILERPLATE command - */ - examples: ['BOILERPLATE foo', 'BOILERPLATE bar', 'BP foo', 'BP bar'], - - /** - * Parses the BOILERPLATE command - */ - parse(input: CommandParserInput): BoilerplateCommand { - const { args } = input; - - if (args.length !== 1) { - throw new ParseError(`BOILERPLATE command requires exactly one argument`); - } - - const value = args[0]!.toLowerCase(); - - if (value.includes('brr')) { - throw new ParseError(`BOILERPLATE value can not contain brr`); - } - - return { - type: 'BOILERPLATE', - value, - } satisfies BoilerplateCommand; - }, - - /** - * Apply the BOILERPLATE command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `pipelineJson` - */ - $applyToPipelineJson(command: BoilerplateCommand, $pipelineJson: $PipelineJson): void { - keepUnused(command, $pipelineJson); - throw new ParseError( - `BOILERPLATE command is only for testing purposes and should not be used in the .book.md file`, - ); - }, - - /** - * Apply the BOILERPLATE command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `taskJson` - */ - $applyToTaskJson(command: BoilerplateCommand, $taskJson: $TaskJson, $pipelineJson: $PipelineJson): void { - keepUnused(command, $taskJson, $pipelineJson); - throw new ParseError( - `BOILERPLATE command is only for testing purposes and should not be used in the .book.md file`, - ); - }, - - /** - * Converts the BOILERPLATE command back to string - * - * Note: This is used in `pipelineJsonToString` utility - */ - stringify(command: BoilerplateCommand): string_markdown_text { - keepUnused(command); - return `---`; // <- TODO: [🛋] Implement - }, - - /** - * Reads the BOILERPLATE command from the `PipelineJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray { - keepUnused(pipelineJson); - throw new ParseError( - `BOILERPLATE command is only for testing purposes and should not be used in the .book.md file`, - ); - }, - - /** - * Reads the BOILERPLATE command from the `TaskJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { - keepUnused($taskJson); - throw new ParseError( - `BOILERPLATE command is only for testing purposes and should not be used in the .book.md file`, - ); - }, -}; +export const boilerplateCommandParser: PipelineBothCommandParser = + { + /** + * Name of the command + */ + name: "BOILERPLATE", + + /** + * Aliases for the BOILERPLATE command + */ + aliasNames: ["BP"], + + /** + * BOILERPLATE command can be used in: + */ + isUsedInPipelineHead: true, + isUsedInPipelineTask: true, + + /** + * Description of the BOILERPLATE command + */ + description: `@@`, + + /** + * Link to documentation + */ + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@", + + /** + * Example usages of the BOILERPLATE command + */ + examples: ["BOILERPLATE foo", "BOILERPLATE bar", "BP foo", "BP bar"], + + /** + * Parses the BOILERPLATE command + */ + parse(input: CommandParserInput): BoilerplateCommand { + const { args } = input; + + if (args.length !== 1) { + throw new ParseError( + `BOILERPLATE command requires exactly one argument`, + ); + } + + const value = args[0]!.toLowerCase(); + + if (value.includes("brr")) { + throw new ParseError(`BOILERPLATE value can not contain brr`); + } + + return { + type: "BOILERPLATE", + value, + } satisfies BoilerplateCommand; + }, + + /** + * Apply the BOILERPLATE command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `pipelineJson` + */ + $applyToPipelineJson( + command: BoilerplateCommand, + $pipelineJson: $PipelineJson, + ): void { + keepUnused(command, $pipelineJson); + throw new ParseError( + `BOILERPLATE command is only for testing purposes and should not be used in the .book.md file`, + ); + }, + + /** + * Apply the BOILERPLATE command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `taskJson` + */ + $applyToTaskJson( + command: BoilerplateCommand, + $taskJson: $TaskJson, + $pipelineJson: $PipelineJson, + ): void { + keepUnused(command, $taskJson, $pipelineJson); + throw new ParseError( + `BOILERPLATE command is only for testing purposes and should not be used in the .book.md file`, + ); + }, + + /** + * Converts the BOILERPLATE command back to string + * + * Note: This is used in `pipelineJsonToString` utility + */ + stringify(command: BoilerplateCommand): string_markdown_text { + keepUnused(command); + return `---`; // <- TODO: [🛋] Implement + }, + + /** + * Reads the BOILERPLATE command from the `PipelineJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromPipelineJson( + pipelineJson: PipelineJson, + ): ReadonlyArray { + keepUnused(pipelineJson); + throw new ParseError( + `BOILERPLATE command is only for testing purposes and should not be used in the .book.md file`, + ); + }, + + /** + * Reads the BOILERPLATE command from the `TaskJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray { + keepUnused($taskJson); + throw new ParseError( + `BOILERPLATE command is only for testing purposes and should not be used in the .book.md file`, + ); + }, + }; diff --git a/src/commands/_common/getParserForCommand.test.ts b/src/commands/_common/getParserForCommand.test.ts index bd31bbf87a..9585b61bbd 100644 --- a/src/commands/_common/getParserForCommand.test.ts +++ b/src/commands/_common/getParserForCommand.test.ts @@ -1,19 +1,27 @@ -import { describe, expect, it } from '@jest/globals'; -import type { really_any } from '../../utils/organization/really_any'; -import { boilerplateCommandParser } from '../_BOILERPLATE/boilerplateCommandParser'; -import { modelCommandParser } from '../MODEL/modelCommandParser'; -import { getParserForCommand } from './getParserForCommand'; -import { stringifyCommand } from './stringifyCommand'; +import { describe, expect, it } from "@jest/globals"; +import type { really_any } from "../../utils/organization/really_any"; +import { modelCommandParser } from "../MODEL/modelCommandParser"; +import { boilerplateCommandParser } from "../_BOILERPLATE/boilerplateCommandParser"; +import { getParserForCommand } from "./getParserForCommand"; +import { stringifyCommand } from "./stringifyCommand"; -describe('getParserForCommand', () => { - it('should get the parser', () => { - expect(getParserForCommand({ type: 'BOILERPLATE', value: 'foo' })).toBe(boilerplateCommandParser); - expect(getParserForCommand({ type: 'MODEL', key: 'modelVariant', value: 'COMPLETION' })).toBe( - modelCommandParser, - ); - }); +describe("getParserForCommand", () => { + it("should get the parser", () => { + expect(getParserForCommand({ type: "BOILERPLATE", value: "foo" })).toBe( + boilerplateCommandParser, + ); + expect( + getParserForCommand({ + type: "MODEL", + key: "modelVariant", + value: "COMPLETION", + }), + ).toBe(modelCommandParser); + }); - it('should fail getting parser for unknown command', () => { - expect(() => stringifyCommand({} as really_any)).toThrowError(/parser is not found/i); - }); + it("should fail getting parser for unknown command", () => { + expect(() => stringifyCommand({} as really_any)).toThrowError( + /parser is not found/i, + ); + }); }); diff --git a/src/commands/_common/getParserForCommand.ts b/src/commands/_common/getParserForCommand.ts index 77c5d86466..d1fbcb4221 100644 --- a/src/commands/_common/getParserForCommand.ts +++ b/src/commands/_common/getParserForCommand.ts @@ -1,8 +1,8 @@ -import { spaceTrim } from 'spacetrim'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import { COMMANDS } from '../index'; -import type { Command } from './types/Command'; -import type { CommandParser } from './types/CommandParser'; +import { spaceTrim } from "spacetrim"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import { COMMANDS } from "../index"; +import type { Command } from "./types/Command"; +import type { CommandParser } from "./types/CommandParser"; /** * Gets the parser for the command @@ -12,25 +12,29 @@ import type { CommandParser } from './types/CommandParser'; * * @public exported from `@promptbook/editable` */ -export function getParserForCommand(command: TCommand): CommandParser { - const commandParser = COMMANDS.find((commandParser) => commandParser.name === command.type); +export function getParserForCommand( + command: TCommand, +): CommandParser { + const commandParser = COMMANDS.find( + (commandParser) => commandParser.name === command.type, + ); - if (commandParser === undefined) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + if (commandParser === undefined) { + throw new UnexpectedError( + spaceTrim( + (block) => ` Command ${command.type} parser is not found ${block( - JSON.stringify(command, null, 4) - .split('\n') - .map((line) => `> ${line}`) - .join('\n'), - )} + JSON.stringify(command, null, 4) + .split("\n") + .map((line) => `> ${line}`) + .join("\n"), + )} `, - ), - ); - } + ), + ); + } - return commandParser as CommandParser; + return commandParser as CommandParser; } diff --git a/src/commands/_common/parseCommand.test.ts b/src/commands/_common/parseCommand.test.ts index 0b006bee58..551ef261d8 100644 --- a/src/commands/_common/parseCommand.test.ts +++ b/src/commands/_common/parseCommand.test.ts @@ -1,46 +1,63 @@ -import { describe, expect, it } from '@jest/globals'; -import { just } from '../../utils/organization/just'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { COMMANDS } from '../index'; -import { parseCommand } from './parseCommand'; -import { CommandUsagePlaces } from './types/CommandUsagePlaces'; +import { describe, expect, it } from "@jest/globals"; +import { just } from "../../utils/organization/just"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { COMMANDS } from "../index"; +import { parseCommand } from "./parseCommand"; +import { CommandUsagePlaces } from "./types/CommandUsagePlaces"; -describe('parsing the commands', () => { - // Note: Other working cases and better tests for each command is in the corresponding command test file +describe("parsing the commands", () => { + // Note: Other working cases and better tests for each command is in the corresponding command test file - for (const { name, isUsedInPipelineHead, isUsedInPipelineTask, examples } of COMMANDS) { - for (const usagePlace of CommandUsagePlaces) { - if (just(false)) { - keepUnused(/* for better indentation */); - } else if (usagePlace === 'PIPELINE_HEAD' && !isUsedInPipelineHead) { - continue; - } else if (usagePlace === 'PIPELINE_TASK' && !isUsedInPipelineTask) { - continue; - } + for (const { + name, + isUsedInPipelineHead, + isUsedInPipelineTask, + examples, + } of COMMANDS) { + for (const usagePlace of CommandUsagePlaces) { + if (just(false)) { + keepUnused(/* for better indentation */); + } else if (usagePlace === "PIPELINE_HEAD" && !isUsedInPipelineHead) { + continue; + } else if (usagePlace === "PIPELINE_TASK" && !isUsedInPipelineTask) { + continue; + } - it(`should parse command ${name} in ${usagePlace} without errors`, () => { - for (const example of examples) { - expect(() => parseCommand(example, usagePlace)).not.toThrowError(); - } - }); + it(`should parse command ${name} in ${usagePlace} without errors`, () => { + for (const example of examples) { + expect(() => parseCommand(example, usagePlace)).not.toThrowError(); + } + }); - it(`should parse command ${name} in ${usagePlace} and return parsed command \`{"type": "${name}"}\``, () => { - for (const example of examples) { - expect(parseCommand(example, usagePlace).type).toBe(name); - } - }); - } - } + it(`should parse command ${name} in ${usagePlace} and return parsed command \`{"type": "${name}"}\``, () => { + for (const example of examples) { + expect(parseCommand(example, usagePlace).type).toBe(name); + } + }); + } + } - it('should fail parsing multi-line command', () => { - expect(() => parseCommand('\nprompt section', 'PIPELINE_HEAD')).toThrowError(/Can not contain new line/i); - expect(() => parseCommand('prompt section\n', 'PIPELINE_HEAD')).toThrowError(/Can not contain new line/i); - }); + it("should fail parsing multi-line command", () => { + expect(() => + parseCommand("\nprompt section", "PIPELINE_HEAD"), + ).toThrowError(/Can not contain new line/i); + expect(() => + parseCommand("prompt section\n", "PIPELINE_HEAD"), + ).toThrowError(/Can not contain new line/i); + }); - it('should fail parsing unknown command', () => { - expect(() => parseCommand('', 'PIPELINE_HEAD')).toThrowError(/Malformed command/i); - expect(() => parseCommand('afasf ddd', 'PIPELINE_HEAD')).toThrowError(/Malformed or unknown command/i); - expect(() => parseCommand('nothing to get', 'PIPELINE_HEAD')).toThrowError(/Malformed or unknown command/i); - expect(() => parseCommand('prameter {name}', 'PIPELINE_HEAD')).toThrowError(/Malformed or unknown command/i); - }); + it("should fail parsing unknown command", () => { + expect(() => parseCommand("", "PIPELINE_HEAD")).toThrowError( + /Malformed command/i, + ); + expect(() => parseCommand("afasf ddd", "PIPELINE_HEAD")).toThrowError( + /Malformed or unknown command/i, + ); + expect(() => parseCommand("nothing to get", "PIPELINE_HEAD")).toThrowError( + /Malformed or unknown command/i, + ); + expect(() => parseCommand("prameter {name}", "PIPELINE_HEAD")).toThrowError( + /Malformed or unknown command/i, + ); + }); }); diff --git a/src/commands/_common/parseCommand.ts b/src/commands/_common/parseCommand.ts index 77e968c0c1..ca421b5929 100644 --- a/src/commands/_common/parseCommand.ts +++ b/src/commands/_common/parseCommand.ts @@ -1,18 +1,18 @@ -import { spaceTrim } from 'spacetrim'; -import { ParseError } from '../../errors/ParseError'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; -import { removeMarkdownFormatting } from '../../utils/markdown/removeMarkdownFormatting'; -import { normalizeTo_SCREAMING_CASE } from '../../utils/normalization/normalizeTo_SCREAMING_CASE'; -import { just } from '../../utils/organization/just'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import type { really_unknown } from '../../utils/organization/really_unknown'; -import { COMMANDS } from '../index'; -import type { Command } from './types/Command'; -import type { CommandBase } from './types/CommandParser'; -import type { CommandParser } from './types/CommandParser'; -import type { CommandParserInput } from './types/CommandParser'; -import type { CommandUsagePlace } from './types/CommandUsagePlaces'; +import { spaceTrim } from "spacetrim"; +import { ParseError } from "../../errors/ParseError"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; +import { removeMarkdownFormatting } from "../../utils/markdown/removeMarkdownFormatting"; +import { normalizeTo_SCREAMING_CASE } from "../../utils/normalization/normalizeTo_SCREAMING_CASE"; +import { just } from "../../utils/organization/just"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import type { really_unknown } from "../../utils/organization/really_unknown"; +import { COMMANDS } from "../index"; +import type { Command } from "./types/Command"; +import type { CommandBase } from "./types/CommandParser"; +import type { CommandParser } from "./types/CommandParser"; +import type { CommandParserInput } from "./types/CommandParser"; +import type { CommandUsagePlace } from "./types/CommandUsagePlaces"; /** * Parses one line of ul/ol to command @@ -22,54 +22,59 @@ import type { CommandUsagePlace } from './types/CommandUsagePlaces'; * * @public exported from `@promptbook/editable` */ -export function parseCommand(raw: string_markdown_text, usagePlace: CommandUsagePlace): Command { - if (raw.includes('\n') || raw.includes('\r')) { - throw new ParseError('Command can not contain new line characters' /* <- TODO: [🚞] */); - } - - // TODO: Unit test all this processing and parsing - let normalized = raw.trim(); - normalized = normalized.split('`').join(''); - normalized = normalized.split('"').join(''); - normalized = normalized.split("'").join(''); - normalized = normalized.split('~').join(''); - normalized = normalized.split('[').join(''); - normalized = normalized.split(']').join(''); - normalized = normalized.split('(').join(''); - normalized = normalized.split(')').join(''); - normalized = normalizeTo_SCREAMING_CASE(normalized); - - const items = raw - .trim() - // Note: [🐡] - .split(/^>/) - .join('') - // --- - .trim() - .split(/["'`]/) - .join('') - .trim() - - // Note: [🛵]: - .split(/^http/) - .join('URL http') - // --- - // Note: [🦈] - .split(/^{/) - .join('PARAMETER {') - // --- - .split(' ') - .map((part) => part.trim()) - .filter((item) => item !== '') - - .map(removeMarkdownFormatting) - .map((item) => item.trim()); - - if (items.length === 0 || items[0] === '') { - throw new ParseError( - spaceTrim( - (block) => - ` +export function parseCommand( + raw: string_markdown_text, + usagePlace: CommandUsagePlace, +): Command { + if (raw.includes("\n") || raw.includes("\r")) { + throw new ParseError( + "Command can not contain new line characters" /* <- TODO: [🚞] */, + ); + } + + // TODO: Unit test all this processing and parsing + let normalized = raw.trim(); + normalized = normalized.split("`").join(""); + normalized = normalized.split('"').join(""); + normalized = normalized.split("'").join(""); + normalized = normalized.split("~").join(""); + normalized = normalized.split("[").join(""); + normalized = normalized.split("]").join(""); + normalized = normalized.split("(").join(""); + normalized = normalized.split(")").join(""); + normalized = normalizeTo_SCREAMING_CASE(normalized); + + const items = raw + .trim() + // Note: [🐡] + .split(/^>/) + .join("") + // --- + .trim() + .split(/["'`]/) + .join("") + .trim() + + // Note: [🛵]: + .split(/^http/) + .join("URL http") + // --- + // Note: [🦈] + .split(/^{/) + .join("PARAMETER {") + // --- + .split(" ") + .map((part) => part.trim()) + .filter((item) => item !== "") + + .map(removeMarkdownFormatting) + .map((item) => item.trim()); + + if (items.length === 0 || items[0] === "") { + throw new ParseError( + spaceTrim( + (block) => + ` Malformed command: - ${raw} @@ -77,58 +82,74 @@ export function parseCommand(raw: string_markdown_text, usagePlace: CommandUsage ${block(getSupportedCommandsMessage())} `, - ), - ); - } - - // Note: Taking command name from beginning of the line - // FOO | BAR Arg1 Arg2 Arg3 - // FOO BAR | Arg1 Arg2 Arg3 - for ( - let commandNameSegmentsCount = 0; - commandNameSegmentsCount < Math.min(items.length, 3); - commandNameSegmentsCount++ - ) { - const commandNameRaw = items.slice(0, commandNameSegmentsCount + 1).join('_'); - const args = items.slice(commandNameSegmentsCount + 1); - - const rawArgs = raw - .substring( - commandNameRaw.length, - // <- TODO: [🥶] This is not correct in case [🐡] and [🦈] - ) - .trim(); - const command = parseCommandVariant({ usagePlace, raw, rawArgs, normalized, args, commandNameRaw }); - - if (command !== null) { - return command; - } - } - - // Note: Taking command name from end of the line - // Arg1 Arg2 Arg3 | FOO - { - const commandNameRaw = items.slice(-1).join('_'); - const args = items.slice(0, -1); // <- Note: This is probbably not correct - - const rawArgs = raw - .substring( - 0, - raw.length - commandNameRaw.length, - // <- TODO: [🥶] This is MAYBE not correct in case [🐡] and [🦈] - ) - .trim(); - const command = parseCommandVariant({ usagePlace, raw, rawArgs, normalized, args, commandNameRaw }); - - if (command !== null) { - return command; - } - } - - throw new ParseError( - spaceTrim( - (block) => - ` + ), + ); + } + + // Note: Taking command name from beginning of the line + // FOO | BAR Arg1 Arg2 Arg3 + // FOO BAR | Arg1 Arg2 Arg3 + for ( + let commandNameSegmentsCount = 0; + commandNameSegmentsCount < Math.min(items.length, 3); + commandNameSegmentsCount++ + ) { + const commandNameRaw = items + .slice(0, commandNameSegmentsCount + 1) + .join("_"); + const args = items.slice(commandNameSegmentsCount + 1); + + const rawArgs = raw + .substring( + commandNameRaw.length, + // <- TODO: [🥶] This is not correct in case [🐡] and [🦈] + ) + .trim(); + const command = parseCommandVariant({ + usagePlace, + raw, + rawArgs, + normalized, + args, + commandNameRaw, + }); + + if (command !== null) { + return command; + } + } + + // Note: Taking command name from end of the line + // Arg1 Arg2 Arg3 | FOO + { + const commandNameRaw = items.slice(-1).join("_"); + const args = items.slice(0, -1); // <- Note: This is probbably not correct + + const rawArgs = raw + .substring( + 0, + raw.length - commandNameRaw.length, + // <- TODO: [🥶] This is MAYBE not correct in case [🐡] and [🦈] + ) + .trim(); + const command = parseCommandVariant({ + usagePlace, + raw, + rawArgs, + normalized, + args, + commandNameRaw, + }); + + if (command !== null) { + return command; + } + } + + throw new ParseError( + spaceTrim( + (block) => + ` Malformed or unknown command: - ${raw} @@ -136,56 +157,75 @@ export function parseCommand(raw: string_markdown_text, usagePlace: CommandUsage ${block(getSupportedCommandsMessage())} `, - ), - ); + ), + ); } /** * @@@ */ function getSupportedCommandsMessage(): string_markdown { - return COMMANDS.flatMap(({ name, aliasNames, description, documentationUrl }: CommandParser) => - // <- Note: [🦦] Its strange that this type assertion is needed - [ - `- **${name}** ${description}, see [discussion](${documentationUrl})`, - ...(aliasNames || []).map((aliasName) => ` - **${aliasName}** Alias for **${name}**`), - ], - ).join('\n'); + return COMMANDS.flatMap( + ({ + name, + aliasNames, + description, + documentationUrl, + }: CommandParser) => + // <- Note: [🦦] Its strange that this type assertion is needed + [ + `- **${name}** ${description}, see [discussion](${documentationUrl})`, + ...(aliasNames || []).map( + (aliasName) => ` - **${aliasName}** Alias for **${name}**`, + ), + ], + ).join("\n"); } /** * @@@ */ -function parseCommandVariant(input: CommandParserInput & { commandNameRaw: string }): Command | null { - const { commandNameRaw, usagePlace, normalized, args, raw, rawArgs } = input; - - const commandName = normalizeTo_SCREAMING_CASE(commandNameRaw); - - for (const commandParser of COMMANDS as really_unknown as ReadonlyArray>) { - // <- Note: [🦦] Its strange that this type assertion is needed - const { name, isUsedInPipelineHead, isUsedInPipelineTask, aliasNames, deprecatedNames, parse } = commandParser; - - if (just(false)) { - keepUnused(/* for better indentation */); - } else if (usagePlace === 'PIPELINE_HEAD' && !isUsedInPipelineHead) { - continue; - } else if (usagePlace === 'PIPELINE_TASK' && !isUsedInPipelineTask) { - continue; - } - - const names = [name, ...(aliasNames || []), ...(deprecatedNames || [])]; - if (names.includes(commandName)) { - try { - return parse({ usagePlace, raw, rawArgs, normalized, args }) as Command; // <- Note: [🦦] - } catch (error) { - if (!(error instanceof ParseError)) { - throw error; - } - - throw new ParseError( - spaceTrim( - (block) => - ` +function parseCommandVariant( + input: CommandParserInput & { commandNameRaw: string }, +): Command | null { + const { commandNameRaw, usagePlace, normalized, args, raw, rawArgs } = input; + + const commandName = normalizeTo_SCREAMING_CASE(commandNameRaw); + + for (const commandParser of COMMANDS as really_unknown as ReadonlyArray< + CommandParser + >) { + // <- Note: [🦦] Its strange that this type assertion is needed + const { + name, + isUsedInPipelineHead, + isUsedInPipelineTask, + aliasNames, + deprecatedNames, + parse, + } = commandParser; + + if (just(false)) { + keepUnused(/* for better indentation */); + } else if (usagePlace === "PIPELINE_HEAD" && !isUsedInPipelineHead) { + continue; + } else if (usagePlace === "PIPELINE_TASK" && !isUsedInPipelineTask) { + continue; + } + + const names = [name, ...(aliasNames || []), ...(deprecatedNames || [])]; + if (names.includes(commandName)) { + try { + return parse({ usagePlace, raw, rawArgs, normalized, args }) as Command; // <- Note: [🦦] + } catch (error) { + if (!(error instanceof ParseError)) { + throw error; + } + + throw new ParseError( + spaceTrim( + (block) => + ` Invalid ${commandName} command: Your command: @@ -195,17 +235,17 @@ function parseCommandVariant(input: CommandParserInput & { commandNameRaw: strin ${block((error as ParseError).message)} Usage of ${commandName}: - ${block(commandParser.examples.map((example) => `- ${example}`).join('\n'))} + ${block(commandParser.examples.map((example) => `- ${example}`).join("\n"))} All supported commands are: ${block(getSupportedCommandsMessage())} `, - ), - ); - } - } - } + ), + ); + } + } + } - return null; + return null; } diff --git a/src/commands/_common/stringifyCommand.test.ts b/src/commands/_common/stringifyCommand.test.ts index 287d00c826..f50125fa93 100644 --- a/src/commands/_common/stringifyCommand.test.ts +++ b/src/commands/_common/stringifyCommand.test.ts @@ -1,9 +1,9 @@ -import { describe, expect, it } from '@jest/globals'; -import type { really_any } from '../../utils/organization/really_any'; -import { stringifyCommand } from './stringifyCommand'; +import { describe, expect, it } from "@jest/globals"; +import type { really_any } from "../../utils/organization/really_any"; +import { stringifyCommand } from "./stringifyCommand"; -describe('stringify the commands', () => { - /* +describe("stringify the commands", () => { + /* TODO: [🛋] Uncomment for (const { name, isUsedInPipelineHead, isUsedInPipelineTask, examples } of COMMANDS) { for (const usagePlace of CommandUsagePlaces) { @@ -24,13 +24,15 @@ describe('stringify the commands', () => { } */ - it('should fail stringifying unknown command', () => { - expect(() => stringifyCommand({} as really_any)).toThrowError(/parser is not found/i); - expect(() => stringifyCommand({ type: 'UNKNOWN' } as really_any)).toThrowError( - /Command UNKNOWN parser is not found/i, - ); - expect(() => stringifyCommand({ type: 'NOTHING' } as really_any)).toThrowError( - /Command NOTHING parser is not found/i, - ); - }); + it("should fail stringifying unknown command", () => { + expect(() => stringifyCommand({} as really_any)).toThrowError( + /parser is not found/i, + ); + expect(() => + stringifyCommand({ type: "UNKNOWN" } as really_any), + ).toThrowError(/Command UNKNOWN parser is not found/i); + expect(() => + stringifyCommand({ type: "NOTHING" } as really_any), + ).toThrowError(/Command NOTHING parser is not found/i); + }); }); diff --git a/src/commands/_common/stringifyCommand.ts b/src/commands/_common/stringifyCommand.ts index 9657f23b65..2c14e72495 100644 --- a/src/commands/_common/stringifyCommand.ts +++ b/src/commands/_common/stringifyCommand.ts @@ -1,6 +1,6 @@ -import type { string_markdown_text } from '../../types/typeAliases'; -import { getParserForCommand } from './getParserForCommand'; -import type { Command } from './types/Command'; +import type { string_markdown_text } from "../../types/typeAliases"; +import { getParserForCommand } from "./getParserForCommand"; +import type { Command } from "./types/Command"; /** * Stringifies the command @@ -12,6 +12,6 @@ import type { Command } from './types/Command'; * @private within the compilePipeline */ export function stringifyCommand(command: Command): string_markdown_text { - const commandParser = getParserForCommand(command); - return commandParser.stringify(command); + const commandParser = getParserForCommand(command); + return commandParser.stringify(command); } diff --git a/src/commands/_common/types/Command.ts b/src/commands/_common/types/Command.ts index 357ce87b1b..1b5921f385 100644 --- a/src/commands/_common/types/Command.ts +++ b/src/commands/_common/types/Command.ts @@ -1,7 +1,7 @@ -import { COMMANDS } from '../../index'; +import type { COMMANDS } from "../../index"; /** * Command is one piece of the book file section which adds some logic to the task or the whole pipeline. * It is parsed from the markdown from ul/ol items - one command per one item. */ -export type Command = ReturnType; +export type Command = ReturnType<(typeof COMMANDS)[number]["parse"]>; diff --git a/src/commands/_common/types/CommandParser.ts b/src/commands/_common/types/CommandParser.ts index 8a77728cee..f55fbb62b3 100644 --- a/src/commands/_common/types/CommandParser.ts +++ b/src/commands/_common/types/CommandParser.ts @@ -1,12 +1,12 @@ -import type { SetOptional, WritableDeep } from 'type-fest'; -import type { PipelineJson } from '../../../pipeline/PipelineJson/PipelineJson'; -import type { TaskJson } from '../../../pipeline/PipelineJson/TaskJson'; -import type { string_markdown_text } from '../../../types/typeAliases'; -import type { string_name } from '../../../types/typeAliases'; -import type { string_promptbook_documentation_url } from '../../../types/typeAliases'; -import type { string_SCREAMING_CASE } from '../../../utils/normalization/normalizeTo_SCREAMING_CASE'; -import type { ___and___ } from '../../../utils/organization/___and___'; -import type { CommandUsagePlace } from './CommandUsagePlaces'; +import type { SetOptional, WritableDeep } from "type-fest"; +import type { PipelineJson } from "../../../pipeline/PipelineJson/PipelineJson"; +import type { TaskJson } from "../../../pipeline/PipelineJson/TaskJson"; +import type { string_markdown_text } from "../../../types/typeAliases"; +import type { string_name } from "../../../types/typeAliases"; +import type { string_promptbook_documentation_url } from "../../../types/typeAliases"; +import type { string_SCREAMING_CASE } from "../../../utils/normalization/normalizeTo_SCREAMING_CASE"; +import type { ___and___ } from "../../../utils/organization/___and___"; +import type { CommandUsagePlace } from "./CommandUsagePlaces"; /** * @@@ @@ -21,9 +21,9 @@ export type CommandBase = { type: string_name & string_SCREAMING_CASE }; * @public exported from `@promptbook/editable` */ export type CommandParser = - | PipelineHeadCommandParser - | PipelineTaskCommandParser - | PipelineBothCommandParser; + | PipelineHeadCommandParser + | PipelineTaskCommandParser + | PipelineBothCommandParser; /** * @@@ @@ -31,60 +31,60 @@ export type CommandParser = * @private just abstract the common properties of the command parsers */ export type CommonCommandParser = { - /** - * @@@ - */ - readonly name: string_name & string_SCREAMING_CASE; - - /** - * @@@ - */ - readonly isUsedInPipelineHead: boolean; - - /** - * @@@ - */ - readonly isUsedInPipelineTask: boolean; - - /** - * @@@ - */ - readonly aliasNames?: ReadonlyArray; - // <- TODO: [🧘] Make it non-optional - - /** - * @@@ - */ - readonly deprecatedNames?: ReadonlyArray; - // <- TODO: [🧘] Make it non-optional - - /** - * @@@ - */ - readonly description: string_markdown_text; - - /** - * @@@ - */ - readonly documentationUrl: string_promptbook_documentation_url; - - /** - * @@@ - */ - readonly examples: ReadonlyArray; - - /** - * @throws {ParseError} if the parsing fails - */ - parse(input: CommandParserInput): TCommand; - - /** - * Converts the command back to string - * - * Note: This is used in `pipelineJsonToString` utility - * @deprecated TODO: [🥍][🧠] Backup original files in `PipelineJson` same as in Promptbook.studio - */ - stringify(command: TCommand): string_markdown_text; + /** + * @@@ + */ + readonly name: string_name & string_SCREAMING_CASE; + + /** + * @@@ + */ + readonly isUsedInPipelineHead: boolean; + + /** + * @@@ + */ + readonly isUsedInPipelineTask: boolean; + + /** + * @@@ + */ + readonly aliasNames?: ReadonlyArray; + // <- TODO: [🧘] Make it non-optional + + /** + * @@@ + */ + readonly deprecatedNames?: ReadonlyArray; + // <- TODO: [🧘] Make it non-optional + + /** + * @@@ + */ + readonly description: string_markdown_text; + + /** + * @@@ + */ + readonly documentationUrl: string_promptbook_documentation_url; + + /** + * @@@ + */ + readonly examples: ReadonlyArray; + + /** + * @throws {ParseError} if the parsing fails + */ + parse(input: CommandParserInput): TCommand; + + /** + * Converts the command back to string + * + * Note: This is used in `pipelineJsonToString` utility + * @deprecated TODO: [🥍][🧠] Backup original files in `PipelineJson` same as in Promptbook.studio + */ + stringify(command: TCommand): string_markdown_text; }; /** @@ -92,72 +92,79 @@ export type CommonCommandParser = { * * @public exported from `@promptbook/editable` */ -export type PipelineBothCommandParser = ___and___ & - Omit, 'isUsedInPipelineTask'> & - Omit, 'isUsedInPipelineHead'>; +export type PipelineBothCommandParser = + ___and___ & + Omit, "isUsedInPipelineTask"> & + Omit, "isUsedInPipelineHead">; /** * @@@ * * @public exported from `@promptbook/editable` */ -export type PipelineHeadCommandParser = CommonCommandParser & { - /** - * @@@ - */ - readonly isUsedInPipelineHead: true; - - /** - * @@@ - */ - readonly isUsedInPipelineTask: false; - - /** - * Apply the command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `pipelineJson` - */ - $applyToPipelineJson(command: TCommand, $pipelineJson: $PipelineJson): void; - - /** - * Reads the command from the `PipelineJson` - * - * Note: This is used in `pipelineJsonToString` utility - * @deprecated TODO: [🥍][🧠] Backup original files in `PipelineJson` same as in Promptbook.studio - */ - takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray; -}; +export type PipelineHeadCommandParser = + CommonCommandParser & { + /** + * @@@ + */ + readonly isUsedInPipelineHead: true; + + /** + * @@@ + */ + readonly isUsedInPipelineTask: false; + + /** + * Apply the command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `pipelineJson` + */ + $applyToPipelineJson(command: TCommand, $pipelineJson: $PipelineJson): void; + + /** + * Reads the command from the `PipelineJson` + * + * Note: This is used in `pipelineJsonToString` utility + * @deprecated TODO: [🥍][🧠] Backup original files in `PipelineJson` same as in Promptbook.studio + */ + takeFromPipelineJson(pipelineJson: PipelineJson): ReadonlyArray; + }; /** * @@@ * * @public exported from `@promptbook/editable` */ -export type PipelineTaskCommandParser = CommonCommandParser & { - /** - * @@@ - */ - readonly isUsedInPipelineHead: false; - - /** - * @@@ - */ - readonly isUsedInPipelineTask: true; - - /** - * Apply the command to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `taskJson` and/or `pipelineJson` - */ - $applyToTaskJson(command: TCommand, $taskJson: $TaskJson, $pipelineJson: $PipelineJson): void; - - /** - * Reads the command from the `TaskJson` - * - * Note: This is used in `pipelineJsonToString` utility - */ - takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray; -}; +export type PipelineTaskCommandParser = + CommonCommandParser & { + /** + * @@@ + */ + readonly isUsedInPipelineHead: false; + + /** + * @@@ + */ + readonly isUsedInPipelineTask: true; + + /** + * Apply the command to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `taskJson` and/or `pipelineJson` + */ + $applyToTaskJson( + command: TCommand, + $taskJson: $TaskJson, + $pipelineJson: $PipelineJson, + ): void; + + /** + * Reads the command from the `TaskJson` + * + * Note: This is used in `pipelineJsonToString` utility + */ + takeFromTaskJson($taskJson: $TaskJson): ReadonlyArray; + }; /** * @@@ @@ -167,8 +174,8 @@ export type PipelineTaskCommandParser = CommonComm * @private internal helper for command parsers */ export type $TaskJson = { - isSectionTypeSet: boolean; - isTask: boolean; + isSectionTypeSet: boolean; + isTask: boolean; } & Partial>; // <- TODO: [🧠] `Partial>; +export type $PipelineJson = WritableDeep< + SetOptional +>; /** * @@@ */ export type CommandParserInput = { - /** - * @@@ - * - * @example 'PIPELINE_HEAD' - * @example 'PIPELINE_TASK' - */ - readonly usagePlace: CommandUsagePlace; - - /** - * @@@ - * - * @example 'promptbook version 0.62.0' - * @example 'FOREACH Text Line `{customers}` -> `{customer}`' - */ - readonly raw: string_markdown_text; - - /** - * @@@ - * - * @example '0.62.0' - * @example 'List Line `{customers}` -> `{customer}`' - */ - readonly rawArgs: string_markdown_text; - - /** - * @@@ - * - * @example 'PROMPTBOOK_ENGINE_VERSION_0_62_0' - * @example 'FOREACH_LIST_LINE_CUSTOMERS_CUSTOMER' - */ - readonly normalized: string_name & string_SCREAMING_CASE; - - /** - * @@@ - * - * @example [ '0.62.0' ] - * @example [ 'List', 'Line', '{customers}', '', '{customer}' ] - */ - readonly args: Array; + /** + * @@@ + * + * @example 'PIPELINE_HEAD' + * @example 'PIPELINE_TASK' + */ + readonly usagePlace: CommandUsagePlace; + + /** + * @@@ + * + * @example 'promptbook version 0.62.0' + * @example 'FOREACH Text Line `{customers}` -> `{customer}`' + */ + readonly raw: string_markdown_text; + + /** + * @@@ + * + * @example '0.62.0' + * @example 'List Line `{customers}` -> `{customer}`' + */ + readonly rawArgs: string_markdown_text; + + /** + * @@@ + * + * @example 'PROMPTBOOK_ENGINE_VERSION_0_62_0' + * @example 'FOREACH_LIST_LINE_CUSTOMERS_CUSTOMER' + */ + readonly normalized: string_name & string_SCREAMING_CASE; + + /** + * @@@ + * + * @example [ '0.62.0' ] + * @example [ 'List', 'Line', '{customers}', '', '{customer}' ] + */ + readonly args: Array; }; /** diff --git a/src/commands/_common/types/CommandType.ts b/src/commands/_common/types/CommandType.ts index bcf6b4de0b..8473715a5e 100644 --- a/src/commands/_common/types/CommandType.ts +++ b/src/commands/_common/types/CommandType.ts @@ -1,4 +1,4 @@ -import type { Command } from './Command'; +import type { Command } from "./Command"; /** * Command is one piece of the book file section which adds some logic to the task or the whole pipeline. @@ -6,7 +6,7 @@ import type { Command } from './Command'; * * This is a type of the command like "KNOWLEDGE" or "PERSONA" */ -export type CommandType = Command['type']; +export type CommandType = Command["type"]; /** * Command is one piece of the book file section which adds some logic to the task or the whole pipeline. diff --git a/src/commands/_common/types/CommandUsagePlaces.ts b/src/commands/_common/types/CommandUsagePlaces.ts index 3d39bbf1f3..138dbf385d 100644 --- a/src/commands/_common/types/CommandUsagePlaces.ts +++ b/src/commands/_common/types/CommandUsagePlaces.ts @@ -1,4 +1,4 @@ -import type { TupleToUnion } from 'type-fest'; +import type { TupleToUnion } from "type-fest"; /** * Where the command can be used @@ -12,4 +12,4 @@ export type CommandUsagePlace = TupleToUnion; * * @private internal base for `CommandUsagePlace` */ -export const CommandUsagePlaces = ['PIPELINE_HEAD', 'PIPELINE_TASK'] as const; +export const CommandUsagePlaces = ["PIPELINE_HEAD", "PIPELINE_TASK"] as const; diff --git a/src/commands/index.ts b/src/commands/index.ts index 5920abab60..f5fa454493 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -1,19 +1,19 @@ -import { boilerplateCommandParser } from './_BOILERPLATE/boilerplateCommandParser'; -import { bookVersionCommandParser } from './BOOK_VERSION/bookVersionCommandParser'; -import { expectCommandParser } from './EXPECT/expectCommandParser'; -import { foreachCommandParser } from './FOREACH/foreachCommandParser'; -import { formatCommandParser } from './FORMAT/formatCommandParser'; -import { formfactorCommandParser } from './FORMFACTOR/formfactorCommandParser'; -import { jokerCommandParser } from './JOKER/jokerCommandParser'; -import { knowledgeCommandParser } from './KNOWLEDGE/knowledgeCommandParser'; -import { modelCommandParser } from './MODEL/modelCommandParser'; -import { parameterCommandParser } from './PARAMETER/parameterCommandParser'; -import { personaCommandParser } from './PERSONA/personaCommandParser'; -import { postprocessCommandParser } from './POSTPROCESS/postprocessCommandParser'; -import { sectionCommandParser } from './SECTION/sectionCommandParser'; -import { urlCommandParser } from './URL/urlCommandParser'; -import { actionCommandParser } from './X_ACTION/actionCommandParser'; -import { instrumentCommandParser } from './X_INSTRUMENT/instrumentCommandParser'; +import { bookVersionCommandParser } from "./BOOK_VERSION/bookVersionCommandParser"; +import { expectCommandParser } from "./EXPECT/expectCommandParser"; +import { foreachCommandParser } from "./FOREACH/foreachCommandParser"; +import { formatCommandParser } from "./FORMAT/formatCommandParser"; +import { formfactorCommandParser } from "./FORMFACTOR/formfactorCommandParser"; +import { jokerCommandParser } from "./JOKER/jokerCommandParser"; +import { knowledgeCommandParser } from "./KNOWLEDGE/knowledgeCommandParser"; +import { modelCommandParser } from "./MODEL/modelCommandParser"; +import { parameterCommandParser } from "./PARAMETER/parameterCommandParser"; +import { personaCommandParser } from "./PERSONA/personaCommandParser"; +import { postprocessCommandParser } from "./POSTPROCESS/postprocessCommandParser"; +import { sectionCommandParser } from "./SECTION/sectionCommandParser"; +import { urlCommandParser } from "./URL/urlCommandParser"; +import { actionCommandParser } from "./X_ACTION/actionCommandParser"; +import { instrumentCommandParser } from "./X_INSTRUMENT/instrumentCommandParser"; +import { boilerplateCommandParser } from "./_BOILERPLATE/boilerplateCommandParser"; /** * All available command parsers @@ -21,24 +21,24 @@ import { instrumentCommandParser } from './X_INSTRUMENT/instrumentCommandParser' * @public exported from `@promptbook/editable` */ export const COMMANDS = [ - sectionCommandParser, - expectCommandParser, - formatCommandParser, - jokerCommandParser, - modelCommandParser, - parameterCommandParser, - postprocessCommandParser, - bookVersionCommandParser, - formfactorCommandParser, - urlCommandParser, - knowledgeCommandParser, - actionCommandParser, - instrumentCommandParser, - personaCommandParser, - foreachCommandParser, - boilerplateCommandParser, // <- TODO: Only in development, remove in production + sectionCommandParser, + expectCommandParser, + formatCommandParser, + jokerCommandParser, + modelCommandParser, + parameterCommandParser, + postprocessCommandParser, + bookVersionCommandParser, + formfactorCommandParser, + urlCommandParser, + knowledgeCommandParser, + actionCommandParser, + instrumentCommandParser, + personaCommandParser, + foreachCommandParser, + boilerplateCommandParser, // <- TODO: Only in development, remove in production - // <- Note: [♓️][💩] This is the order of the commands in the pipeline, BUT its not used in parsing and before usage maybe it should be done better + // <- Note: [♓️][💩] This is the order of the commands in the pipeline, BUT its not used in parsing and before usage maybe it should be done better ] as const; /** diff --git a/src/config.test.ts b/src/config.test.ts index 3f7e5263bc..7b0faf3b5a 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -1,16 +1,16 @@ -import { describe, expect, it } from '@jest/globals'; -import { IS_PIPELINE_LOGIC_VALIDATED } from './config'; +import { describe, expect, it } from "@jest/globals"; +import { IS_PIPELINE_LOGIC_VALIDATED } from "./config"; -describe('configuration', () => { - /*/ +describe("configuration", () => { + /*/ it('prevents accidental costs', () => { expect(IS_COST_PREVENTED).toBe(true); }); /**/ - it('checks examples logic', () => { - expect(IS_PIPELINE_LOGIC_VALIDATED).toBe(true); - }); + it("checks examples logic", () => { + expect(IS_PIPELINE_LOGIC_VALIDATED).toBe(true); + }); }); /** diff --git a/src/config.ts b/src/config.ts index 6a734d5d80..a0d66e7bc1 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,9 +1,9 @@ -import type { CsvSettings } from './formats/csv/CsvSettings'; -import type { IntermediateFilesStrategy } from './types/IntermediateFilesStrategy'; -import type { string_email } from './types/typeAliases'; -import type { string_name } from './types/typeAliases'; -import type { string_url_image } from './types/typeAliases'; -import { just } from './utils/organization/just'; +import type { CsvSettings } from "./formats/csv/CsvSettings"; +import type { IntermediateFilesStrategy } from "./types/IntermediateFilesStrategy"; +import type { string_email } from "./types/typeAliases"; +import type { string_name } from "./types/typeAliases"; +import type { string_url_image } from "./types/typeAliases"; +import { just } from "./utils/organization/just"; /** * Warning message for the generated sections and files files @@ -26,14 +26,14 @@ export const NAME = `Promptbook`; * * @public exported from `@promptbook/core` */ -export const ADMIN_EMAIL: string_email = 'pavol@ptbk.io'; +export const ADMIN_EMAIL: string_email = "pavol@ptbk.io"; /** * Name of the responsible person for the Promptbook on GitHub * * @public exported from `@promptbook/core` */ -export const ADMIN_GITHUB_NAME: string_name = 'hejny'; +export const ADMIN_GITHUB_NAME: string_name = "hejny"; /** * Claim for the Promptbook @@ -89,7 +89,7 @@ export const DEFAULT_PROMPT_TASK_TITLE = `Prompt`; * * @public exported from `@promptbook/core` */ -export const DEFAULT_BOOK_OUTPUT_PARAMETER_NAME = 'result'; +export const DEFAULT_BOOK_OUTPUT_PARAMETER_NAME = "result"; /** * Maximum file size limit @@ -127,15 +127,15 @@ export const CHARACTER_LOOP_LIMIT = 100000; * @public exported from `@promptbook/utils` */ export const VALUE_STRINGS = { - empty: '(nothing; empty string)', - null: '(no value; null)', - undefined: '(unknown value; undefined)', + empty: "(nothing; empty string)", + null: "(no value; null)", + undefined: "(unknown value; undefined)", - nan: '(not a number; NaN)', - infinity: '(infinity; ∞)', - negativeInfinity: '(negative infinity; -∞)', + nan: "(not a number; NaN)", + infinity: "(infinity; ∞)", + negativeInfinity: "(negative infinity; -∞)", - unserializable: '(unserializable value)', + unserializable: "(unserializable value)", } as const; /** @@ -180,7 +180,8 @@ export const MAX_FILENAME_LENGTH = 30; * * @public exported from `@promptbook/core` */ -export const DEFAULT_INTERMEDIATE_FILES_STRATEGY: IntermediateFilesStrategy = 'HIDE_AND_KEEP'; +export const DEFAULT_INTERMEDIATE_FILES_STRATEGY: IntermediateFilesStrategy = + "HIDE_AND_KEEP"; // <- TODO: [😡] Change to 'VISIBLE' /** @@ -219,7 +220,7 @@ export const DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = 200; * * @public exported from `@promptbook/core` */ -export const DEFAULT_BOOKS_DIRNAME = './books'; +export const DEFAULT_BOOKS_DIRNAME = "./books"; // <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES` /** @@ -229,7 +230,7 @@ export const DEFAULT_BOOKS_DIRNAME = './books'; * * @public exported from `@promptbook/core` */ -export const DEFAULT_DOWNLOAD_CACHE_DIRNAME = './.promptbook/download-cache'; +export const DEFAULT_DOWNLOAD_CACHE_DIRNAME = "./.promptbook/download-cache"; /** * Where to store the cache of executions for promptbook CLI @@ -238,7 +239,7 @@ export const DEFAULT_DOWNLOAD_CACHE_DIRNAME = './.promptbook/download-cache'; * * @public exported from `@promptbook/core` */ -export const DEFAULT_EXECUTION_CACHE_DIRNAME = './.promptbook/execution-cache'; +export const DEFAULT_EXECUTION_CACHE_DIRNAME = "./.promptbook/execution-cache"; /** * Where to store the scrape cache @@ -247,7 +248,7 @@ export const DEFAULT_EXECUTION_CACHE_DIRNAME = './.promptbook/execution-cache'; * * @public exported from `@promptbook/core` */ -export const DEFAULT_SCRAPE_CACHE_DIRNAME = './.promptbook/scrape-cache'; +export const DEFAULT_SCRAPE_CACHE_DIRNAME = "./.promptbook/scrape-cache"; /** * The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory` @@ -263,7 +264,7 @@ export const DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = `index`; * @private within the repository - too low-level in comparison with other constants */ export const MOMENT_ARG_THRESHOLDS = { - ss: 3, // <- least number of seconds to be counted in seconds, minus 1. Must be set after setting the `s` unit or without setting the `s` unit. + ss: 3, // <- least number of seconds to be counted in seconds, minus 1. Must be set after setting the `s` unit or without setting the `s` unit. } as const; /** @@ -271,14 +272,14 @@ export const MOMENT_ARG_THRESHOLDS = { * * @public exported from `@promptbook/core` */ -export const DEFAULT_REMOTE_URL = 'https://api.pavolhejny.com/'; +export const DEFAULT_REMOTE_URL = "https://api.pavolhejny.com/"; /** * @@@ * * @public exported from `@promptbook/core` */ -export const DEFAULT_REMOTE_URL_PATH = '/promptbook/socket.io'; +export const DEFAULT_REMOTE_URL_PATH = "/promptbook/socket.io"; // <- TODO: [🧜‍♂️] @@ -288,10 +289,10 @@ export const DEFAULT_REMOTE_URL_PATH = '/promptbook/socket.io'; * @public exported from `@promptbook/core` */ export const DEFAULT_CSV_SETTINGS: CsvSettings = Object.freeze({ - delimiter: ',', - quoteChar: '"', - newline: '\n', - skipEmptyLines: true, + delimiter: ",", + quoteChar: '"', + newline: "\n", + skipEmptyLines: true, }); /** @@ -309,7 +310,7 @@ export let DEFAULT_IS_VERBOSE = false; * @public exported from `@promptbook/core` */ export function SET_IS_VERBOSE(isVerbose: boolean): void { - DEFAULT_IS_VERBOSE = isVerbose; + DEFAULT_IS_VERBOSE = isVerbose; } /** @@ -332,19 +333,19 @@ export const DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME = `getPipelineCollect * @private within the repository */ export const IS_PIPELINE_LOGIC_VALIDATED: boolean = just( - /**/ - // Note: In normal situations, we check the pipeline logic: - true, - /**/ + /**/ + // Note: In normal situations, we check the pipeline logic: + true, + /**/ - /*/ + /*/ // When working on some new features, you can temporarily turn off the validation: false, /**/ - // Commit message: - // [🔑] Temporarily **disable** pipeline validation - // [🔒] **Enable** pipeline validation + // Commit message: + // [🔑] Temporarily **disable** pipeline validation + // [🔒] **Enable** pipeline validation ); /** @@ -353,19 +354,19 @@ export const IS_PIPELINE_LOGIC_VALIDATED: boolean = just( * @private within the repository */ export const IS_COST_PREVENTED: boolean = just( - /*/ + /*/ // Note: In normal situations, we prevent ability to use real API keys in tests: true, /**/ - /**/ - // When working on preparations, you can temporarily turn off the prevention: - false, - /**/ + /**/ + // When working on preparations, you can temporarily turn off the prevention: + false, + /**/ - // Commit message: - // [🔑] Temporarily **disable** cost-prevention - // [🔒] **Enable** cost-prevention + // Commit message: + // [🔑] Temporarily **disable** cost-prevention + // [🔒] **Enable** cost-prevention ); /** diff --git a/src/constants.ts b/src/constants.ts index 79f46b169b..ca3141046e 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,50 +1,49 @@ -import type { PipelineJson } from './pipeline/PipelineJson/PipelineJson'; -import type { ExportJsonOptions } from './utils/serialization/exportJson'; -import { exportJson } from './utils/serialization/exportJson'; +import type { PipelineJson } from "./pipeline/PipelineJson/PipelineJson"; +import type { ExportJsonOptions } from "./utils/serialization/exportJson"; +import { exportJson } from "./utils/serialization/exportJson"; /** * Order of keys in the pipeline JSON * * @public exported from `@promptbook/core` */ -export const ORDER_OF_PIPELINE_JSON: ExportJsonOptions['order'] = [ - // Note: [🍙] In this order will be pipeline serialized - 'title', - 'pipelineUrl', - 'bookVersion', - 'description', - 'formfactorName', - 'parameters', - 'tasks', - 'personas', - 'preparations', - 'knowledgeSources', - 'knowledgePieces', - 'sources', // <- TODO: [🧠] Where should the `sources` be -]; +export const ORDER_OF_PIPELINE_JSON: ExportJsonOptions["order"] = + [ + // Note: [🍙] In this order will be pipeline serialized + "title", + "pipelineUrl", + "bookVersion", + "description", + "formfactorName", + "parameters", + "tasks", + "personas", + "preparations", + "knowledgeSources", + "knowledgePieces", + "sources", // <- TODO: [🧠] Where should the `sources` be + ]; /** * Nonce which is used for replacing things in strings * * @private within the repository */ -export const REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW'; - - +export const REPLACING_NONCE = "ptbkauk42kV2dzao34faw7FudQUHYPtW"; /** * @@@ * * @private within the repository */ -export const RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE; +export const RESERVED_PARAMETER_MISSING_VALUE = "MISSING-" + REPLACING_NONCE; /** * @@@ * * @private within the repository */ -export const RESERVED_PARAMETER_RESTRICTED = 'RESTRICTED-' + REPLACING_NONCE; +export const RESERVED_PARAMETER_RESTRICTED = "RESTRICTED-" + REPLACING_NONCE; /** * The names of the parameters that are reserved for special purposes @@ -52,20 +51,20 @@ export const RESERVED_PARAMETER_RESTRICTED = 'RESTRICTED-' + REPLACING_NONCE; * @public exported from `@promptbook/core` */ export const RESERVED_PARAMETER_NAMES = exportJson({ - name: 'RESERVED_PARAMETER_NAMES', - message: `The names of the parameters that are reserved for special purposes`, - value: [ - 'content', - 'context', // <- [🧠][🏍] Is parameter {context} good for anything? - 'knowledge', - 'examples', - 'modelName', - 'currentDate', + name: "RESERVED_PARAMETER_NAMES", + message: `The names of the parameters that are reserved for special purposes`, + value: [ + "content", + "context", // <- [🧠][🏍] Is parameter {context} good for anything? + "knowledge", + "examples", + "modelName", + "currentDate", - // <- TODO: list here all command names - // <- TODO: Add more like 'date', 'modelName',... - // <- TODO: Add [emoji] + instructions ACRY when adding new reserved parameter - ] as const, + // <- TODO: list here all command names + // <- TODO: Add more like 'date', 'modelName',... + // <- TODO: Add [emoji] + instructions ACRY when adding new reserved parameter + ] as const, }); /** diff --git a/src/conversion/archive/loadArchive.ts b/src/conversion/archive/loadArchive.ts index 139d090a7e..dda54f9c1e 100644 --- a/src/conversion/archive/loadArchive.ts +++ b/src/conversion/archive/loadArchive.ts @@ -1,9 +1,9 @@ -import JSZip from 'jszip'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { string_filename } from '../../types/typeAliases'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { FilesystemTools } from '../../execution/FilesystemTools'; -import { validatePipeline } from '../validation/validatePipeline'; +import JSZip from "jszip"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { FilesystemTools } from "../../execution/FilesystemTools"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { string_filename } from "../../types/typeAliases"; +import { validatePipeline } from "../validation/validatePipeline"; /** * Loads the books from the archive file with `.bookc` extension @@ -14,27 +14,32 @@ import { validatePipeline } from '../validation/validatePipeline'; * * @private utility of Prompbook */ -export async function loadArchive(filePath: string_filename, fs: FilesystemTools): Promise> { - if (!filePath.endsWith('.bookc')) { - throw new UnexpectedError(`Archive file must have '.bookc' extension`); - } +export async function loadArchive( + filePath: string_filename, + fs: FilesystemTools, +): Promise> { + if (!filePath.endsWith(".bookc")) { + throw new UnexpectedError(`Archive file must have '.bookc' extension`); + } - const data = await fs.readFile(filePath); - const archive = await JSZip.loadAsync(data); + const data = await fs.readFile(filePath); + const archive = await JSZip.loadAsync(data); - const indexFile = archive.file('index.book.json'); + const indexFile = archive.file("index.book.json"); - if (!indexFile) { - throw new UnexpectedError(`Archive does not contain 'index.book.json' file`); - } + if (!indexFile) { + throw new UnexpectedError( + `Archive does not contain 'index.book.json' file`, + ); + } - const collectionJson = JSON.parse(await indexFile.async('text')); + const collectionJson = JSON.parse(await indexFile.async("text")); - for (const pipeline of collectionJson) { - validatePipeline(pipeline); - } + for (const pipeline of collectionJson) { + validatePipeline(pipeline); + } - return collectionJson; + return collectionJson; } /** diff --git a/src/conversion/archive/saveArchive.ts b/src/conversion/archive/saveArchive.ts index 939015091d..259468cd63 100644 --- a/src/conversion/archive/saveArchive.ts +++ b/src/conversion/archive/saveArchive.ts @@ -1,10 +1,10 @@ -import JSZip from 'jszip'; -import { stringifyPipelineJson } from '../../utils/editable/utils/stringifyPipelineJson'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { string_filename } from '../../types/typeAliases'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { FilesystemTools } from '../../execution/FilesystemTools'; -import { validatePipeline } from '../validation/validatePipeline'; +import JSZip from "jszip"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { FilesystemTools } from "../../execution/FilesystemTools"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { string_filename } from "../../types/typeAliases"; +import { stringifyPipelineJson } from "../../utils/editable/utils/stringifyPipelineJson"; +import { validatePipeline } from "../validation/validatePipeline"; /** * Saves the given books into an archive file with `.bookc` extension @@ -16,26 +16,29 @@ import { validatePipeline } from '../validation/validatePipeline'; * @private utility of Prompbook */ export async function saveArchive( - filePath: string_filename, - collectionJson: ReadonlyArray, - fs: FilesystemTools, + filePath: string_filename, + collectionJson: ReadonlyArray, + fs: FilesystemTools, ): Promise { - if (!filePath.endsWith('.bookc')) { - throw new UnexpectedError(`Archive file must have '.bookc' extension`); - } + if (!filePath.endsWith(".bookc")) { + throw new UnexpectedError(`Archive file must have '.bookc' extension`); + } - // Note: We want to ensure that the generated JSONs are logically correct - for (const pipelineJson of collectionJson) { - validatePipeline(pipelineJson); - } + // Note: We want to ensure that the generated JSONs are logically correct + for (const pipelineJson of collectionJson) { + validatePipeline(pipelineJson); + } - const archive = new JSZip(); + const archive = new JSZip(); - const collectionJsonString = stringifyPipelineJson(collectionJson); + const collectionJsonString = stringifyPipelineJson(collectionJson); - archive.file('index.book.json', collectionJsonString); - const data = await archive.generateAsync({ type: 'nodebuffer', streamFiles: true }); - await fs.writeFile(filePath, data); + archive.file("index.book.json", collectionJsonString); + const data = await archive.generateAsync({ + type: "nodebuffer", + streamFiles: true, + }); + await fs.writeFile(filePath, data); } /** diff --git a/src/conversion/compilePipeline.test.ts b/src/conversion/compilePipeline.test.ts index 3bb2a16655..cbad7d8556 100644 --- a/src/conversion/compilePipeline.test.ts +++ b/src/conversion/compilePipeline.test.ts @@ -1,33 +1,41 @@ -import { describe, it } from '@jest/globals'; -import { readdirSync } from 'fs'; -import { join } from 'path'; -import { compilePipeline } from './compilePipeline'; -import { importPipelineWithoutPreparation } from './validation/_importPipeline'; +import { readdirSync } from "fs"; +import { join } from "path"; +import { describe, it } from "@jest/globals"; +import { compilePipeline } from "./compilePipeline"; +import { importPipelineWithoutPreparation } from "./validation/_importPipeline"; -describe('compilePipeline', () => { - const examplesDir = '../../examples/pipelines'; // <- TODO: [🚏] DRY, to config +describe("compilePipeline", () => { + const examplesDir = "../../examples/pipelines"; // <- TODO: [🚏] DRY, to config - const examples = readdirSync(join(__dirname, examplesDir), { withFileTypes: true, recursive: false }) - // <- Note: In production it is not good practice to use synchronous functions - // But this is only a test before the build, so it is okay - .filter((dirent) => dirent.isFile()) - .filter(({ name }) => name.endsWith('.book')); + const examples = readdirSync(join(__dirname, examplesDir), { + withFileTypes: true, + recursive: false, + }) + // <- Note: In production it is not good practice to use synchronous functions + // But this is only a test before the build, so it is okay + .filter((dirent) => dirent.isFile()) + .filter(({ name }) => name.endsWith(".book")); - for (const { name } of examples) { - it(`should compile ${name}`, async () => { - const pipelineFromMarkdownPromise = importPipelineWithoutPreparation(name as `${string}.book`) - .then((pipelineString) => compilePipeline(pipelineString)) - .then((pipeline) => ({ ...pipeline, title: undefined })); // <- Note: [0] Title is not compared because it can be changed in `preparePipeline` + for (const { name } of examples) { + it(`should compile ${name}`, async () => { + const pipelineFromMarkdownPromise = importPipelineWithoutPreparation( + name as `${string}.book`, + ) + .then((pipelineString) => compilePipeline(pipelineString)) + .then((pipeline) => ({ ...pipeline, title: undefined })); // <- Note: [0] Title is not compared because it can be changed in `preparePipeline` - const pipelineJson = { - ...(await importPipelineWithoutPreparation( - join(examplesDir, name).replace('.book', '.bookc') as `${string}.bookc`, - )), - title: undefined, - // <- Note: [0] - }; + const pipelineJson = { + ...(await importPipelineWithoutPreparation( + join(examplesDir, name).replace( + ".book", + ".bookc", + ) as `${string}.bookc`, + )), + title: undefined, + // <- Note: [0] + }; - await expect(pipelineFromMarkdownPromise).resolves.toEqual(pipelineJson); - }); - } + await expect(pipelineFromMarkdownPromise).resolves.toEqual(pipelineJson); + }); + } }); diff --git a/src/conversion/compilePipeline.ts b/src/conversion/compilePipeline.ts index a26278b526..b08c15027e 100644 --- a/src/conversion/compilePipeline.ts +++ b/src/conversion/compilePipeline.ts @@ -1,9 +1,9 @@ -import type { ExecutionTools } from '../execution/ExecutionTools'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import type { PipelineString } from '../pipeline/PipelineString'; -import type { PrepareAndScrapeOptions } from '../prepare/PrepareAndScrapeOptions'; -import { preparePipeline } from '../prepare/preparePipeline'; -import { parsePipeline } from './parsePipeline'; +import type { ExecutionTools } from "../execution/ExecutionTools"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import type { PipelineString } from "../pipeline/PipelineString"; +import type { PrepareAndScrapeOptions } from "../prepare/PrepareAndScrapeOptions"; +import { preparePipeline } from "../prepare/preparePipeline"; +import { parsePipeline } from "./parsePipeline"; /** * Compile pipeline from string (markdown) format to JSON format @@ -21,24 +21,24 @@ import { parsePipeline } from './parsePipeline'; * @public exported from `@promptbook/core` */ export async function compilePipeline( - pipelineString: PipelineString, - tools?: Pick, - options?: PrepareAndScrapeOptions, + pipelineString: PipelineString, + tools?: Pick, + options?: PrepareAndScrapeOptions, ): Promise { - let pipelineJson = parsePipeline(pipelineString); + let pipelineJson = parsePipeline(pipelineString); - if (tools !== undefined && tools.llm !== undefined) { - pipelineJson = await preparePipeline( - pipelineJson, - tools, - options || { - rootDirname: null, - }, - ); - } + if (tools !== undefined && tools.llm !== undefined) { + pipelineJson = await preparePipeline( + pipelineJson, + tools, + options || { + rootDirname: null, + }, + ); + } - // Note: No need to use `$exportJson` because `parsePipeline` and `preparePipeline` already do that - return pipelineJson; + // Note: No need to use `$exportJson` because `parsePipeline` and `preparePipeline` already do that + return pipelineJson; } /** diff --git a/src/conversion/compilePipelineOnRemoteServer.ts b/src/conversion/compilePipelineOnRemoteServer.ts index df324bf358..f26917c6b6 100644 --- a/src/conversion/compilePipelineOnRemoteServer.ts +++ b/src/conversion/compilePipelineOnRemoteServer.ts @@ -1,8 +1,8 @@ -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import type { PipelineString } from '../pipeline/PipelineString'; -import { preparePipelineOnRemoteServer } from '../prepare/preparePipelineOnRemoteServer'; -import type { RemoteClientOptions } from '../remote-server/types/RemoteClientOptions'; -import { parsePipeline } from './parsePipeline'; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import type { PipelineString } from "../pipeline/PipelineString"; +import { preparePipelineOnRemoteServer } from "../prepare/preparePipelineOnRemoteServer"; +import type { RemoteClientOptions } from "../remote-server/types/RemoteClientOptions"; +import { parsePipeline } from "./parsePipeline"; /** * Compile pipeline from string (markdown) format to JSON format @@ -19,15 +19,15 @@ import { parsePipeline } from './parsePipeline'; * @public exported from `@promptbook/remote-client` */ export async function compilePipelineOnRemoteServer( - pipelineString: PipelineString, - options: RemoteClientOptions, + pipelineString: PipelineString, + options: RemoteClientOptions, ): Promise { - let pipelineJson = parsePipeline(pipelineString); + let pipelineJson = parsePipeline(pipelineString); - pipelineJson = await preparePipelineOnRemoteServer(pipelineJson, options); + pipelineJson = await preparePipelineOnRemoteServer(pipelineJson, options); - // Note: No need to use `$exportJson` because `parsePipeline` and `preparePipeline` already do that - return pipelineJson; + // Note: No need to use `$exportJson` because `parsePipeline` and `preparePipeline` already do that + return pipelineJson; } /** diff --git a/src/conversion/parsePipeline.test.ts b/src/conversion/parsePipeline.test.ts index 618a76a4e6..582992fab2 100644 --- a/src/conversion/parsePipeline.test.ts +++ b/src/conversion/parsePipeline.test.ts @@ -1,33 +1,41 @@ -import { describe, expect, it } from '@jest/globals'; -import { readdirSync } from 'fs'; -import { join } from 'path'; -import { parsePipeline } from './parsePipeline'; -import { importPipelineWithoutPreparation } from './validation/_importPipeline'; +import { readdirSync } from "fs"; +import { join } from "path"; +import { describe, expect, it } from "@jest/globals"; +import { parsePipeline } from "./parsePipeline"; +import { importPipelineWithoutPreparation } from "./validation/_importPipeline"; -describe('parsePipeline', () => { - const examplesDir = '../../examples/pipelines'; // <- TODO: [🚏] DRY, to config +describe("parsePipeline", () => { + const examplesDir = "../../examples/pipelines"; // <- TODO: [🚏] DRY, to config - const examples = readdirSync(join(__dirname, examplesDir), { withFileTypes: true, recursive: false }) - // <- Note: In production it is not good practice to use synchronous functions - // But this is only a test before the build, so it is okay - .filter((dirent) => dirent.isFile()) - .filter(({ name }) => name.endsWith('.book')); + const examples = readdirSync(join(__dirname, examplesDir), { + withFileTypes: true, + recursive: false, + }) + // <- Note: In production it is not good practice to use synchronous functions + // But this is only a test before the build, so it is okay + .filter((dirent) => dirent.isFile()) + .filter(({ name }) => name.endsWith(".book")); - for (const { name } of examples) { - it(`should parse ${name}`, async () => { - const pipelineFromMarkdownPromise = importPipelineWithoutPreparation(name as `${string}.book`) - .then((pipelineString) => parsePipeline(pipelineString)) - .then((pipeline) => ({ ...pipeline, title: undefined })); // <- Note: [0] Title is not compared because it can be changed in `preparePipeline` + for (const { name } of examples) { + it(`should parse ${name}`, async () => { + const pipelineFromMarkdownPromise = importPipelineWithoutPreparation( + name as `${string}.book`, + ) + .then((pipelineString) => parsePipeline(pipelineString)) + .then((pipeline) => ({ ...pipeline, title: undefined })); // <- Note: [0] Title is not compared because it can be changed in `preparePipeline` - const pipelineJson = { - ...(await importPipelineWithoutPreparation( - join(examplesDir, name).replace('.book', '.bookc') as `${string}.bookc`, - )), - title: undefined, - // <- Note: [0] - }; + const pipelineJson = { + ...(await importPipelineWithoutPreparation( + join(examplesDir, name).replace( + ".book", + ".bookc", + ) as `${string}.bookc`, + )), + title: undefined, + // <- Note: [0] + }; - await expect(pipelineFromMarkdownPromise).resolves.toEqual(pipelineJson); - }); - } + await expect(pipelineFromMarkdownPromise).resolves.toEqual(pipelineJson); + }); + } }); diff --git a/src/conversion/parsePipeline.ts b/src/conversion/parsePipeline.ts index 18e22d4d9b..09bdef2d2b 100644 --- a/src/conversion/parsePipeline.ts +++ b/src/conversion/parsePipeline.ts @@ -1,42 +1,46 @@ -import { spaceTrim } from 'spacetrim'; -import type { Writable, WritableDeep } from 'type-fest'; -import { TODO_any } from '../_packages/types.index'; -import type { ParameterCommand } from '../commands/PARAMETER/ParameterCommand'; -import { sectionCommandParser } from '../commands/SECTION/sectionCommandParser'; -import { getParserForCommand } from '../commands/_common/getParserForCommand'; -import { parseCommand } from '../commands/_common/parseCommand'; +import { spaceTrim } from "spacetrim"; +import type { Writable, WritableDeep } from "type-fest"; +import type { TODO_any } from "../_packages/types.index"; +import type { ParameterCommand } from "../commands/PARAMETER/ParameterCommand"; +import { sectionCommandParser } from "../commands/SECTION/sectionCommandParser"; +import { getParserForCommand } from "../commands/_common/getParserForCommand"; +import { parseCommand } from "../commands/_common/parseCommand"; import type { - $PipelineJson, - $TaskJson, - CommandBase, - PipelineHeadCommandParser, - PipelineTaskCommandParser, -} from '../commands/_common/types/CommandParser'; -import { DEFAULT_BOOK_TITLE, DEFAULT_TASK_TITLE } from '../config'; -import { ORDER_OF_PIPELINE_JSON, RESERVED_PARAMETER_NAMES } from '../constants'; -import { ParseError } from '../errors/ParseError'; -import { UnexpectedError } from '../errors/UnexpectedError'; -import { HIGH_LEVEL_ABSTRACTIONS } from '../high-level-abstractions/index'; -import type { ParameterJson } from '../pipeline/PipelineJson/ParameterJson'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import type { ScriptTaskJson } from '../pipeline/PipelineJson/ScriptTaskJson'; -import type { TaskJson } from '../pipeline/PipelineJson/TaskJson'; -import type { PipelineString } from '../pipeline/PipelineString'; -import { validatePipelineString } from '../pipeline/validatePipelineString'; -import type { ScriptLanguage } from '../types/ScriptLanguage'; -import { SUPPORTED_SCRIPT_LANGUAGES } from '../types/ScriptLanguage'; -import type { number_integer, number_positive, string_name } from '../types/typeAliases'; -import { deflatePipeline } from '../utils/editable/edit-pipeline-string/deflatePipeline'; -import { extractAllListItemsFromMarkdown } from '../utils/markdown/extractAllListItemsFromMarkdown'; -import { extractOneBlockFromMarkdown } from '../utils/markdown/extractOneBlockFromMarkdown'; -import { flattenMarkdown } from '../utils/markdown/flattenMarkdown'; -import { parseMarkdownSection } from '../utils/markdown/parseMarkdownSection'; -import { removeMarkdownComments } from '../utils/markdown/removeMarkdownComments'; -import { splitMarkdownIntoSections } from '../utils/markdown/splitMarkdownIntoSections'; -import { titleToName } from '../utils/normalization/titleToName'; -import type { really_any } from '../utils/organization/really_any'; -import { exportJson } from '../utils/serialization/exportJson'; -import { extractParameterNamesFromTask } from './utils/extractParameterNamesFromTask'; + $PipelineJson, + $TaskJson, + CommandBase, + PipelineHeadCommandParser, + PipelineTaskCommandParser, +} from "../commands/_common/types/CommandParser"; +import { DEFAULT_BOOK_TITLE, DEFAULT_TASK_TITLE } from "../config"; +import { ORDER_OF_PIPELINE_JSON, RESERVED_PARAMETER_NAMES } from "../constants"; +import { ParseError } from "../errors/ParseError"; +import { UnexpectedError } from "../errors/UnexpectedError"; +import { HIGH_LEVEL_ABSTRACTIONS } from "../high-level-abstractions/index"; +import type { ParameterJson } from "../pipeline/PipelineJson/ParameterJson"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import type { ScriptTaskJson } from "../pipeline/PipelineJson/ScriptTaskJson"; +import type { TaskJson } from "../pipeline/PipelineJson/TaskJson"; +import type { PipelineString } from "../pipeline/PipelineString"; +import { validatePipelineString } from "../pipeline/validatePipelineString"; +import type { ScriptLanguage } from "../types/ScriptLanguage"; +import { SUPPORTED_SCRIPT_LANGUAGES } from "../types/ScriptLanguage"; +import type { + number_integer, + number_positive, + string_name, +} from "../types/typeAliases"; +import { deflatePipeline } from "../utils/editable/edit-pipeline-string/deflatePipeline"; +import { extractAllListItemsFromMarkdown } from "../utils/markdown/extractAllListItemsFromMarkdown"; +import { extractOneBlockFromMarkdown } from "../utils/markdown/extractOneBlockFromMarkdown"; +import { flattenMarkdown } from "../utils/markdown/flattenMarkdown"; +import { parseMarkdownSection } from "../utils/markdown/parseMarkdownSection"; +import { removeMarkdownComments } from "../utils/markdown/removeMarkdownComments"; +import { splitMarkdownIntoSections } from "../utils/markdown/splitMarkdownIntoSections"; +import { titleToName } from "../utils/normalization/titleToName"; +import type { really_any } from "../utils/organization/really_any"; +import { exportJson } from "../utils/serialization/exportJson"; +import { extractParameterNamesFromTask } from "./utils/extractParameterNamesFromTask"; /** * Compile pipeline from string (markdown) format to JSON format synchronously @@ -55,245 +59,252 @@ import { extractParameterNamesFromTask } from './utils/extractParameterNamesFrom * @public exported from `@promptbook/core` */ export function parsePipeline(pipelineString: PipelineString): PipelineJson { - const $pipelineJson: $PipelineJson = { - title: DEFAULT_BOOK_TITLE, - parameters: [], - tasks: [], - knowledgeSources: [], - knowledgePieces: [], - personas: [], - preparations: [], - sources: [ - { - type: 'BOOK', - path: null, - // <- TODO: !!6 Pass here path of the file - content: pipelineString, - }, - ], - }; - - function getPipelineIdentification() { - // Note: This is a 😐 implementation of [🚞] - const _: Array = []; - - if ($pipelineJson.sourceFile !== undefined) { - _.push(`File: ${$pipelineJson.sourceFile}`); - } - - if ($pipelineJson.pipelineUrl !== undefined) { - _.push(`Url: ${$pipelineJson.pipelineUrl}`); - } - - return _.join('\n'); - } - - // ============================================================= - // Note: 1️⃣ Parsing of the markdown into object - - // ============== - // Note: 1️⃣◽1️⃣ Remove #!shebang and comments - if (pipelineString.startsWith('#!')) { - const [shebangLine, ...restLines] = pipelineString.split('\n'); - - if (!(shebangLine || '').includes('ptbk')) { - throw new ParseError( - spaceTrim( - (block) => ` + const $pipelineJson: $PipelineJson = { + title: DEFAULT_BOOK_TITLE, + parameters: [], + tasks: [], + knowledgeSources: [], + knowledgePieces: [], + personas: [], + preparations: [], + sources: [ + { + type: "BOOK", + path: null, + // <- TODO: !!6 Pass here path of the file + content: pipelineString, + }, + ], + }; + + function getPipelineIdentification() { + // Note: This is a 😐 implementation of [🚞] + const _: Array = []; + + if ($pipelineJson.sourceFile !== undefined) { + _.push(`File: ${$pipelineJson.sourceFile}`); + } + + if ($pipelineJson.pipelineUrl !== undefined) { + _.push(`Url: ${$pipelineJson.pipelineUrl}`); + } + + return _.join("\n"); + } + + // ============================================================= + // Note: 1️⃣ Parsing of the markdown into object + + // ============== + // Note: 1️⃣◽1️⃣ Remove #!shebang and comments + if (pipelineString.startsWith("#!")) { + const [shebangLine, ...restLines] = pipelineString.split("\n"); + + if (!(shebangLine || "").includes("ptbk")) { + throw new ParseError( + spaceTrim( + (block) => ` It seems that you try to parse a book file which has non-standard shebang line for book files: Shebang line must contain 'ptbk' You have: - ${block(shebangLine || '(empty line)')} + ${block(shebangLine || "(empty line)")} It should look like this: #!/usr/bin/env ptbk ${block(getPipelineIdentification())} `, - ), - ); - } - pipelineString = validatePipelineString(restLines.join('\n')); - } - pipelineString = removeMarkdownComments(pipelineString); - pipelineString = spaceTrim(pipelineString) as PipelineString; - - // <- TODO: [😧] `spaceTrim` should preserve discriminated type *(or at lease `PipelineString`)* - pipelineString = deflatePipeline(pipelineString); - - // ============== - // Note: 1️⃣◽2️⃣ Parse the markdown - pipelineString = flattenMarkdown(pipelineString) /* <- Note: [🥞] */; - pipelineString = pipelineString.replaceAll( - /`\{(?[a-z0-9_]+)\}`/gi, - '{$}', - ) as PipelineString; - pipelineString = pipelineString.replaceAll( - /`->\s+\{(?[a-z0-9_]+)\}`/gi, - '-> {$}', - ) as PipelineString; - const [pipelineHead, ...pipelineSections] = - splitMarkdownIntoSections(pipelineString).map(parseMarkdownSection); /* <- Note: [🥞] */ - - // ============== - // Note: 1️⃣◽4️⃣ Check markdown structure - if (pipelineHead === undefined) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + ), + ); + } + pipelineString = validatePipelineString(restLines.join("\n")); + } + pipelineString = removeMarkdownComments(pipelineString); + pipelineString = spaceTrim(pipelineString) as PipelineString; + + // <- TODO: [😧] `spaceTrim` should preserve discriminated type *(or at lease `PipelineString`)* + pipelineString = deflatePipeline(pipelineString); + + // ============== + // Note: 1️⃣◽2️⃣ Parse the markdown + pipelineString = flattenMarkdown(pipelineString) /* <- Note: [🥞] */; + pipelineString = pipelineString.replaceAll( + /`\{(?[a-z0-9_]+)\}`/gi, + "{$}", + ) as PipelineString; + pipelineString = pipelineString.replaceAll( + /`->\s+\{(?[a-z0-9_]+)\}`/gi, + "-> {$}", + ) as PipelineString; + const [pipelineHead, ...pipelineSections] = + splitMarkdownIntoSections(pipelineString).map( + parseMarkdownSection, + ); /* <- Note: [🥞] */ + + // ============== + // Note: 1️⃣◽4️⃣ Check markdown structure + if (pipelineHead === undefined) { + throw new UnexpectedError( + spaceTrim( + (block) => ` Pipeline head is not defined ${block(getPipelineIdentification())} This should never happen, because the pipeline already flattened `, - ), // <- TODO: [🚞] - ); - } - if (pipelineHead.level !== 1) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + ), // <- TODO: [🚞] + ); + } + if (pipelineHead.level !== 1) { + throw new UnexpectedError( + spaceTrim( + (block) => ` Pipeline head is not h1 ${block(getPipelineIdentification())} This should never happen, because the pipeline already flattened `, - ), // <- TODO: [🚞] - ); - } - if (!pipelineSections.every((section) => section.level === 2)) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + ), // <- TODO: [🚞] + ); + } + if (!pipelineSections.every((section) => section.level === 2)) { + throw new UnexpectedError( + spaceTrim( + (block) => ` Not every pipeline section is h2 ${block(getPipelineIdentification())} This should never happen, because the pipeline already flattened `, - ), // <- TODO: [🚞] - ); - } - - // ============================================================= - ///Note: 2️⃣ Function for defining parameters - const defineParam = (parameterCommand: Omit) => { - const { parameterName, parameterDescription, isInput, isOutput } = parameterCommand; - - if (RESERVED_PARAMETER_NAMES.includes(parameterName as really_any)) { - throw new ParseError( - spaceTrim( - (block) => ` + ), // <- TODO: [🚞] + ); + } + + // ============================================================= + ///Note: 2️⃣ Function for defining parameters + const defineParam = (parameterCommand: Omit) => { + const { parameterName, parameterDescription, isInput, isOutput } = + parameterCommand; + + if (RESERVED_PARAMETER_NAMES.includes(parameterName as really_any)) { + throw new ParseError( + spaceTrim( + (block) => ` Parameter name {${parameterName}} is reserved and cannot be used as resulting parameter name ${block(getPipelineIdentification())} `, - ) /* <- TODO: [🚞] */, - ); - } - - const existingParameter = $pipelineJson.parameters.find( - (parameter: ParameterJson) => parameter.name === parameterName, - ); - if ( - existingParameter && - existingParameter.description && - existingParameter.description !== parameterDescription && - parameterDescription - ) { - throw new ParseError( - spaceTrim( - (block) => ` + ) /* <- TODO: [🚞] */, + ); + } + + const existingParameter = $pipelineJson.parameters.find( + (parameter: ParameterJson) => parameter.name === parameterName, + ); + if ( + existingParameter && + existingParameter.description && + existingParameter.description !== parameterDescription && + parameterDescription + ) { + throw new ParseError( + spaceTrim( + (block) => ` Parameter \`{${parameterName}}\` is defined multiple times with different description: ${block(getPipelineIdentification())} First definition: - ${block(existingParameter.description || '[undefined]')} + ${block(existingParameter.description || "[undefined]")} Second definition: - ${block(parameterDescription || '[undefined]')} + ${block(parameterDescription || "[undefined]")} `, // <- TODO: [🚞] - ), - ); - } - - if (existingParameter) { - if (parameterDescription) { - existingParameter.description = parameterDescription; - } - - existingParameter.isInput = existingParameter.isInput || isInput; - existingParameter.isOutput = existingParameter.isOutput || isOutput; - } else { - $pipelineJson.parameters.push( - { - name: parameterName, - description: parameterDescription || undefined, - isInput, - isOutput, - } as ParameterJson, - // <- Note: This type assertion is safe, only conflict is that in type definition `isInput` and `isOutput` cannot be both true - // but in this implementation it is possible, but it is not a problem it just does not make sense and its checked in [🆑] logic validaton - ); - } - }; - - // ============================================================= - // Note: 3️⃣ Process pipeline head - - $pipelineJson.title = pipelineHead.title; - - // TODO: [🎾][1] DRY description - let description: string | undefined = pipelineHead.content; - - // Note: Remove codeblocks - TODO: [🎾] Make util removeAllBlocksFromMarkdown (exported from `@promptbool/utils`) - description = description.split(/^```.*^```/gms).join(''); - description = description.split(/^>.*$/gm).join(''); - - //Note: Remove lists and return statement - TODO: [🎾] Make util (exported from `@promptbool/utils`) - description = description.split(/^(?:(?:-)|(?:\d\))|(?:`?->))\s+.*$/gm).join(''); - - description = spaceTrim(description); - - if (description === '') { - description = undefined; - } - $pipelineJson.description = description; - - const listItems = extractAllListItemsFromMarkdown(pipelineHead.content); - for (const listItem of listItems) { - // TODO: [🥥] Maybe move this logic to `$parseAndApplyPipelineHeadCommands` - const command = parseCommand(listItem, 'PIPELINE_HEAD'); - - const commandParser = getParserForCommand(command); - - if (commandParser.isUsedInPipelineHead !== true /* <- Note: [🦦][4] */) { - throw new ParseError( - spaceTrim( - (block) => ` + ), + ); + } + + if (existingParameter) { + if (parameterDescription) { + existingParameter.description = parameterDescription; + } + + existingParameter.isInput = existingParameter.isInput || isInput; + existingParameter.isOutput = existingParameter.isOutput || isOutput; + } else { + $pipelineJson.parameters.push( + { + name: parameterName, + description: parameterDescription || undefined, + isInput, + isOutput, + } as ParameterJson, + // <- Note: This type assertion is safe, only conflict is that in type definition `isInput` and `isOutput` cannot be both true + // but in this implementation it is possible, but it is not a problem it just does not make sense and its checked in [🆑] logic validaton + ); + } + }; + + // ============================================================= + // Note: 3️⃣ Process pipeline head + + $pipelineJson.title = pipelineHead.title; + + // TODO: [🎾][1] DRY description + let description: string | undefined = pipelineHead.content; + + // Note: Remove codeblocks - TODO: [🎾] Make util removeAllBlocksFromMarkdown (exported from `@promptbool/utils`) + description = description.split(/^```.*^```/gms).join(""); + description = description.split(/^>.*$/gm).join(""); + + //Note: Remove lists and return statement - TODO: [🎾] Make util (exported from `@promptbool/utils`) + description = description + .split(/^(?:(?:-)|(?:\d\))|(?:`?->))\s+.*$/gm) + .join(""); + + description = spaceTrim(description); + + if (description === "") { + description = undefined; + } + $pipelineJson.description = description; + + const listItems = extractAllListItemsFromMarkdown(pipelineHead.content); + for (const listItem of listItems) { + // TODO: [🥥] Maybe move this logic to `$parseAndApplyPipelineHeadCommands` + const command = parseCommand(listItem, "PIPELINE_HEAD"); + + const commandParser = getParserForCommand(command); + + if (commandParser.isUsedInPipelineHead !== true /* <- Note: [🦦][4] */) { + throw new ParseError( + spaceTrim( + (block) => ` Command \`${command.type}\` is not allowed in the head of the pipeline ONLY at the pipeline task ${block(getPipelineIdentification())} `, - ), - ); // <- TODO: [🚞] - } - - try { - (commandParser as PipelineHeadCommandParser).$applyToPipelineJson(command, $pipelineJson); - // <- Note: [🦦] Its strange that this assertion must be here, [🦦][4] should do this assertion implicitly - } catch (error) { - if (!(error instanceof ParseError)) { - throw error; - } - - throw new ParseError( - spaceTrim( - (block) => ` + ), + ); // <- TODO: [🚞] + } + + try { + ( + commandParser as PipelineHeadCommandParser + ).$applyToPipelineJson(command, $pipelineJson); + // <- Note: [🦦] Its strange that this assertion must be here, [🦦][4] should do this assertion implicitly + } catch (error) { + if (!(error instanceof ParseError)) { + throw error; + } + + throw new ParseError( + spaceTrim( + (block) => ` Command ${command.type} failed to apply to the pipeline The error: @@ -303,147 +314,157 @@ export function parsePipeline(pipelineString: PipelineString): PipelineJson { - ${listItem} Usage of ${command.type}: - ${block(commandParser.examples.map((example) => `- ${example}`).join('\n'))} + ${block(commandParser.examples.map((example) => `- ${example}`).join("\n"))} ${block(getPipelineIdentification())} `, - ), - ); // <- TODO: [🚞] - } - - if (command.type === 'PARAMETER') { - defineParam(command); - // <- Note: [🍣] - } - } - - // ============================================================= - // Note: 4️⃣ Prepare unique section names with indexes when needed - - const sectionCounts: Record< - string_name, - { count: number_integer & number_positive; currentIndex: number_integer & number_positive } - > = {}; - - for (const section of pipelineSections) { - const name = titleToName(section.title); - - if (sectionCounts[name] === undefined) { - sectionCounts[name] = { count: 0, currentIndex: 0 }; - } - - sectionCounts[name]!.count++; - } - - const getUniqueSectionName = (title: string) => { - const name = titleToName(title); - const count = sectionCounts[name]!; - - if (count.count === 1) { - return name; - } - - const nameWithSuffix = `${name}-${count.currentIndex}`; - count.currentIndex++; - - return nameWithSuffix; - }; - - // ============================================================= - // Note: 5️⃣ Process each section of the pipeline - for (const section of pipelineSections) { - // TODO: Parse section's description (the content out of the codeblock and lists) - - const listItems = extractAllListItemsFromMarkdown(section.content); - - const { language, content } = extractOneBlockFromMarkdown(section.content); - - // TODO: [🎾][1] DRY description - let description: string | undefined = section.content; - - // Note: Remove codeblocks - TODO: [🎾] - description = description.split(/^```.*^```/gms).join(''); - description = description.split(/^>.*$/gm).join(''); - - //Note: Remove lists and return statement - TODO: [🎾] - description = description.split(/^(?:(?:-)|(?:\d\))|(?:`?->))\s+.*$/gm).join(''); - - description = spaceTrim(description); - - if (description === '') { - description = undefined; - } - - const $taskJson: $TaskJson = { - isSectionTypeSet: false, - isTask: true, - taskType: undefined /* <- Note: [🍙] Putting here placeholder to keep `taskType` on top at final JSON */, - name: getUniqueSectionName(section.title || DEFAULT_TASK_TITLE), - title: section.title || DEFAULT_TASK_TITLE, - description, - content, - // <- TODO: [🍙] Some standard order of properties - }; - - const lastLine = section.content.split('\n').pop()!; - const resultingParameterNameMatch = /^->\s*\{(?[a-z0-9_]+)\}/im.exec(lastLine); - if ( - resultingParameterNameMatch && - resultingParameterNameMatch.groups !== undefined && - resultingParameterNameMatch.groups.resultingParamName !== undefined - ) { - $taskJson.resultingParameterName = resultingParameterNameMatch.groups.resultingParamName; - } - - // TODO: [🥥] Maybe move this logic to `$parseAndApplyPipelineTaskCommands` - const commands = listItems.map((listItem) => ({ - listItem, - command: parseCommand(listItem, 'PIPELINE_TASK'), - })); - - // Note: If block type is not set, set it to 'PROMPT_TASK' - if (commands.some(({ command }) => command.type === 'SECTION') === false) { - sectionCommandParser.$applyToTaskJson( - { type: 'SECTION', taskType: 'PROMPT_TASK' }, - $taskJson, - $pipelineJson, - ); - } - - // TODO [♓️] List commands and before apply order them to achieve order-agnostic commands - - for (const { listItem, command } of commands) { - const commandParser = getParserForCommand(command); - - if (commandParser.isUsedInPipelineTask !== true /* <- Note: [🦦][4] */) { - throw new ParseError( - spaceTrim( - (block) => ` + ), + ); // <- TODO: [🚞] + } + + if (command.type === "PARAMETER") { + defineParam(command); + // <- Note: [🍣] + } + } + + // ============================================================= + // Note: 4️⃣ Prepare unique section names with indexes when needed + + const sectionCounts: Record< + string_name, + { + count: number_integer & number_positive; + currentIndex: number_integer & number_positive; + } + > = {}; + + for (const section of pipelineSections) { + const name = titleToName(section.title); + + if (sectionCounts[name] === undefined) { + sectionCounts[name] = { count: 0, currentIndex: 0 }; + } + + sectionCounts[name]!.count++; + } + + const getUniqueSectionName = (title: string) => { + const name = titleToName(title); + const count = sectionCounts[name]!; + + if (count.count === 1) { + return name; + } + + const nameWithSuffix = `${name}-${count.currentIndex}`; + count.currentIndex++; + + return nameWithSuffix; + }; + + // ============================================================= + // Note: 5️⃣ Process each section of the pipeline + for (const section of pipelineSections) { + // TODO: Parse section's description (the content out of the codeblock and lists) + + const listItems = extractAllListItemsFromMarkdown(section.content); + + const { language, content } = extractOneBlockFromMarkdown(section.content); + + // TODO: [🎾][1] DRY description + let description: string | undefined = section.content; + + // Note: Remove codeblocks - TODO: [🎾] + description = description.split(/^```.*^```/gms).join(""); + description = description.split(/^>.*$/gm).join(""); + + //Note: Remove lists and return statement - TODO: [🎾] + description = description + .split(/^(?:(?:-)|(?:\d\))|(?:`?->))\s+.*$/gm) + .join(""); + + description = spaceTrim(description); + + if (description === "") { + description = undefined; + } + + const $taskJson: $TaskJson = { + isSectionTypeSet: false, + isTask: true, + taskType: + undefined /* <- Note: [🍙] Putting here placeholder to keep `taskType` on top at final JSON */, + name: getUniqueSectionName(section.title || DEFAULT_TASK_TITLE), + title: section.title || DEFAULT_TASK_TITLE, + description, + content, + // <- TODO: [🍙] Some standard order of properties + }; + + const lastLine = section.content.split("\n").pop()!; + const resultingParameterNameMatch = + /^->\s*\{(?[a-z0-9_]+)\}/im.exec(lastLine); + if ( + resultingParameterNameMatch && + resultingParameterNameMatch.groups !== undefined && + resultingParameterNameMatch.groups.resultingParamName !== undefined + ) { + $taskJson.resultingParameterName = + resultingParameterNameMatch.groups.resultingParamName; + } + + // TODO: [🥥] Maybe move this logic to `$parseAndApplyPipelineTaskCommands` + const commands = listItems.map((listItem) => ({ + listItem, + command: parseCommand(listItem, "PIPELINE_TASK"), + })); + + // Note: If block type is not set, set it to 'PROMPT_TASK' + if (commands.some(({ command }) => command.type === "SECTION") === false) { + sectionCommandParser.$applyToTaskJson( + { type: "SECTION", taskType: "PROMPT_TASK" }, + $taskJson, + $pipelineJson, + ); + } + + // TODO [♓️] List commands and before apply order them to achieve order-agnostic commands + + for (const { listItem, command } of commands) { + const commandParser = getParserForCommand(command); + + if (commandParser.isUsedInPipelineTask !== true /* <- Note: [🦦][4] */) { + throw new ParseError( + spaceTrim( + (block) => ` Command \`${ - command.type - }\` is not allowed in the task of the promptbook ONLY at the pipeline head + command.type + }\` is not allowed in the task of the promptbook ONLY at the pipeline head ${block(getPipelineIdentification())} `, - ), - ); // <- TODO: [🚞] - } - - try { - (commandParser as PipelineTaskCommandParser).$applyToTaskJson( - // <- Note: [🦦] Its strange that this assertion must be here, [🦦][4] should do this assertion implicitly - command, - $taskJson, - $pipelineJson, - ); - } catch (error) { - if (!(error instanceof ParseError)) { - throw error; - } - - throw new ParseError( - spaceTrim( - (block) => ` + ), + ); // <- TODO: [🚞] + } + + try { + ( + commandParser as PipelineTaskCommandParser + ).$applyToTaskJson( + // <- Note: [🦦] Its strange that this assertion must be here, [🦦][4] should do this assertion implicitly + command, + $taskJson, + $pipelineJson, + ); + } catch (error) { + if (!(error instanceof ParseError)) { + throw error; + } + + throw new ParseError( + spaceTrim( + (block) => ` Command \`${command.type}\` failed to apply to the task The error: @@ -461,72 +482,74 @@ export function parsePipeline(pipelineString: PipelineString): PipelineJson { - ${listItem} Usage of ${command.type}: - ${block(commandParser.examples.map((example) => `- ${example}`).join('\n'))} + ${block(commandParser.examples.map((example) => `- ${example}`).join("\n"))} ${block(getPipelineIdentification())} `, - ), - ); // <- TODO: [🚞] - } - - if (command.type === 'PARAMETER') { - defineParam(command); - // <- Note: [🍣] - } - } - - // TODO: [🍧] Should be done in SECTION command - if (($taskJson as WritableDeep).taskType === 'SCRIPT_TASK') { - if (!language) { - throw new ParseError( - spaceTrim( - (block) => ` + ), + ); // <- TODO: [🚞] + } + + if (command.type === "PARAMETER") { + defineParam(command); + // <- Note: [🍣] + } + } + + // TODO: [🍧] Should be done in SECTION command + if (($taskJson as WritableDeep).taskType === "SCRIPT_TASK") { + if (!language) { + throw new ParseError( + spaceTrim( + (block) => ` You must specify the language of the script in the \`SCRIPT\` task ${block(getPipelineIdentification())} `, - ), - // <- TODO: [🚞] - ); - } - - if (!SUPPORTED_SCRIPT_LANGUAGES.includes(language as ScriptLanguage)) { - throw new ParseError( - spaceTrim( - (block) => ` + ), + // <- TODO: [🚞] + ); + } + + if (!SUPPORTED_SCRIPT_LANGUAGES.includes(language as ScriptLanguage)) { + throw new ParseError( + spaceTrim( + (block) => ` Script language ${language} is not supported. Supported languages are: - ${block(SUPPORTED_SCRIPT_LANGUAGES.join(', '))} + ${block(SUPPORTED_SCRIPT_LANGUAGES.join(", "))} `, - // <- TODO: [🚞] - ), - ); - } - - ($taskJson as Partial<$TaskJson> as Writable).contentLanguage = language as ScriptLanguage; - } - - $taskJson.dependentParameterNames = Array.from( - extractParameterNamesFromTask( - $taskJson as TaskJson, - // <- TODO: [3] - ), - ); - - for (const parameterName of $taskJson.dependentParameterNames) { - // TODO: [🧠] This definition should be made first in the task - defineParam({ - parameterName, - parameterDescription: null, - isInput: false, - isOutput: false, - // <- Note: In this case null+false+false means that we do not know yet if it is input or output and we will set it later - }); - } - - /* + // <- TODO: [🚞] + ), + ); + } + + ( + $taskJson as Partial<$TaskJson> as Writable + ).contentLanguage = language as ScriptLanguage; + } + + $taskJson.dependentParameterNames = Array.from( + extractParameterNamesFromTask( + $taskJson as TaskJson, + // <- TODO: [3] + ), + ); + + for (const parameterName of $taskJson.dependentParameterNames) { + // TODO: [🧠] This definition should be made first in the task + defineParam({ + parameterName, + parameterDescription: null, + isInput: false, + isOutput: false, + // <- Note: In this case null+false+false means that we do not know yet if it is input or output and we will set it later + }); + } + + /* // TODO: [🍧] This should be checked in `MODEL` command + better error message if ($taskJson.taskType !== 'PROMPT_TASK' && $taskJson.modelRequirements !== undefined) { throw new UnexpectedError( @@ -545,90 +568,92 @@ export function parsePipeline(pipelineString: PipelineString): PipelineJson { } */ - if ($taskJson.isTask) { - delete ($taskJson as Partial<$TaskJson>).isSectionTypeSet; - delete ($taskJson as Partial<$TaskJson>).isTask; - - // TODO: [🍙] Maybe do reorder of `$taskJson` here - - $pipelineJson.tasks.push( - $taskJson as TaskJson, - // <- TODO: [3] Do not do `as TaskJson` BUT make 100% sure that nothing is missing - ); - } - } - - // ============================================================= - // Note: 6️⃣ Mark parameters as INPUT if not explicitly set - if ($pipelineJson.parameters.every((parameter) => !parameter.isInput)) { - for (const parameter of $pipelineJson.parameters) { - const isThisParameterResulting = $pipelineJson.tasks.some( - (task) => task.resultingParameterName === parameter.name, - ); - if (!isThisParameterResulting) { - parameter.isInput = true as TODO_any; - // <- TODO: [💔] Why this is making typescript error in vscode but not in cli - // > Type 'true' is not assignable to type 'false'.ts(2322) - // > (property) isInput: false - // > The parameter is input of the pipeline The parameter is NOT input of the pipeline - } - } - } - - // ============================================================= - // Note: 7️⃣ Mark all non-INPUT parameters as OUTPUT if any OUTPUT is not set - if ($pipelineJson.parameters.every((parameter) => !parameter.isOutput)) { - for (const parameter of $pipelineJson.parameters) { - if (!parameter.isInput) { - parameter.isOutput = true as TODO_any; - // <- TODO: [💔] - } - } - } - - // ============================================================= - // Note: 8️⃣ Cleanup of undefined values - $pipelineJson.tasks.forEach((tasks) => { - for (const [key, value] of Object.entries(tasks)) { - if (value === undefined) { - delete (tasks as really_any)[key]; - } - } - }); - $pipelineJson.parameters.forEach((parameter) => { - for (const [key, value] of Object.entries(parameter)) { - if (value === undefined) { - delete (parameter as really_any)[key]; - } - } - }); - - // ============================================================= - // Note: 9️⃣ Apply sync high-level abstractions - for (const highLevelAbstraction of HIGH_LEVEL_ABSTRACTIONS.filter(({ type }) => type === 'SYNC')) { - highLevelAbstraction.$applyToPipelineJson($pipelineJson); - } - - // ============================================================= - // Note: 🔟 Default formfactor - // Note: [🔆] If formfactor is still not set, set it to 'GENERIC' - if ($pipelineJson.formfactorName === undefined) { - $pipelineJson.formfactorName = 'GENERIC'; - } - - // ============================================================= - - return exportJson({ - name: 'pipelineJson', - message: `Result of \`parsePipeline\``, - order: ORDER_OF_PIPELINE_JSON, - value: { - formfactorName: 'GENERIC', - // <- Note: [🔆] Setting `formfactorName` is redundant to satisfy the typescript - - ...($pipelineJson as $PipelineJson), - }, - }); + if ($taskJson.isTask) { + delete ($taskJson as Partial<$TaskJson>).isSectionTypeSet; + delete ($taskJson as Partial<$TaskJson>).isTask; + + // TODO: [🍙] Maybe do reorder of `$taskJson` here + + $pipelineJson.tasks.push( + $taskJson as TaskJson, + // <- TODO: [3] Do not do `as TaskJson` BUT make 100% sure that nothing is missing + ); + } + } + + // ============================================================= + // Note: 6️⃣ Mark parameters as INPUT if not explicitly set + if ($pipelineJson.parameters.every((parameter) => !parameter.isInput)) { + for (const parameter of $pipelineJson.parameters) { + const isThisParameterResulting = $pipelineJson.tasks.some( + (task) => task.resultingParameterName === parameter.name, + ); + if (!isThisParameterResulting) { + parameter.isInput = true as TODO_any; + // <- TODO: [💔] Why this is making typescript error in vscode but not in cli + // > Type 'true' is not assignable to type 'false'.ts(2322) + // > (property) isInput: false + // > The parameter is input of the pipeline The parameter is NOT input of the pipeline + } + } + } + + // ============================================================= + // Note: 7️⃣ Mark all non-INPUT parameters as OUTPUT if any OUTPUT is not set + if ($pipelineJson.parameters.every((parameter) => !parameter.isOutput)) { + for (const parameter of $pipelineJson.parameters) { + if (!parameter.isInput) { + parameter.isOutput = true as TODO_any; + // <- TODO: [💔] + } + } + } + + // ============================================================= + // Note: 8️⃣ Cleanup of undefined values + $pipelineJson.tasks.forEach((tasks) => { + for (const [key, value] of Object.entries(tasks)) { + if (value === undefined) { + delete (tasks as really_any)[key]; + } + } + }); + $pipelineJson.parameters.forEach((parameter) => { + for (const [key, value] of Object.entries(parameter)) { + if (value === undefined) { + delete (parameter as really_any)[key]; + } + } + }); + + // ============================================================= + // Note: 9️⃣ Apply sync high-level abstractions + for (const highLevelAbstraction of HIGH_LEVEL_ABSTRACTIONS.filter( + ({ type }) => type === "SYNC", + )) { + highLevelAbstraction.$applyToPipelineJson($pipelineJson); + } + + // ============================================================= + // Note: 🔟 Default formfactor + // Note: [🔆] If formfactor is still not set, set it to 'GENERIC' + if ($pipelineJson.formfactorName === undefined) { + $pipelineJson.formfactorName = "GENERIC"; + } + + // ============================================================= + + return exportJson({ + name: "pipelineJson", + message: `Result of \`parsePipeline\``, + order: ORDER_OF_PIPELINE_JSON, + value: { + formfactorName: "GENERIC", + // <- Note: [🔆] Setting `formfactorName` is redundant to satisfy the typescript + + ...($pipelineJson as $PipelineJson), + }, + }); } /** diff --git a/src/conversion/pipelineJsonToString.ts b/src/conversion/pipelineJsonToString.ts index 2ccddd0b18..e0e00ff2cc 100644 --- a/src/conversion/pipelineJsonToString.ts +++ b/src/conversion/pipelineJsonToString.ts @@ -1,11 +1,11 @@ -import spaceTrim from 'spacetrim'; -import type { ParameterJson } from '../pipeline/PipelineJson/ParameterJson'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import type { PipelineString } from '../pipeline/PipelineString'; -import { validatePipelineString } from '../pipeline/validatePipelineString'; -import type { string_markdown } from '../types/typeAliases'; -import { prettifyMarkdown } from '../utils/markdown/prettifyMarkdown'; -import { capitalize } from '../utils/normalization/capitalize'; +import spaceTrim from "spacetrim"; +import type { ParameterJson } from "../pipeline/PipelineJson/ParameterJson"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import type { PipelineString } from "../pipeline/PipelineString"; +import { validatePipelineString } from "../pipeline/validatePipelineString"; +import type { string_markdown } from "../types/typeAliases"; +import { prettifyMarkdown } from "../utils/markdown/prettifyMarkdown"; +import { capitalize } from "../utils/normalization/capitalize"; /** * Converts promptbook in JSON format to string format @@ -15,157 +15,172 @@ import { capitalize } from '../utils/normalization/capitalize'; * @returns Promptbook in string format (.book.md) * @public exported from `@promptbook/core` */ -export function pipelineJsonToString(pipelineJson: PipelineJson): PipelineString { - const { title, pipelineUrl, bookVersion, description, parameters, tasks } = pipelineJson; - - let pipelineString: string_markdown = `# ${title}`; - - if (description) { - pipelineString += '\n\n'; - pipelineString += description; - } - - const commands: Array = []; - - if (pipelineUrl) { - commands.push(`PIPELINE URL ${pipelineUrl}`); - } - - if (bookVersion !== `undefined`) { - commands.push(`BOOK VERSION ${bookVersion}`); - } - - // TODO: [main] !!5 This increases size of the bundle and is probbably not necessary - pipelineString = prettifyMarkdown(pipelineString); - - for (const parameter of parameters.filter(({ isInput }) => isInput)) { - commands.push(`INPUT PARAMETER ${taskParameterJsonToString(parameter)}`); - } - - for (const parameter of parameters.filter(({ isOutput }) => isOutput)) { - commands.push(`OUTPUT PARAMETER ${taskParameterJsonToString(parameter)}`); - } - - pipelineString += '\n\n'; - pipelineString += commands.map((command) => `- ${command}`).join('\n'); - - for (const task of tasks) { - const { - /* Note: Not using:> name, */ - title, - description, - /* Note: dependentParameterNames, */ - jokerParameterNames: jokers, - taskType, - content, - postprocessingFunctionNames: postprocessing, - expectations, - format, - resultingParameterName, - } = task; - - pipelineString += '\n\n'; - pipelineString += `## ${title}`; - - if (description) { - pipelineString += '\n\n'; - pipelineString += description; - } - - const commands: Array = []; - let contentLanguage: 'markdown' | 'text' | 'javascript' | 'typescript' | 'python' | '' = 'text'; - - if (taskType === 'PROMPT_TASK') { - const { modelRequirements } = task; - const { modelName, modelVariant } = modelRequirements || {}; - - // Note: Do nothing, it is default - // commands.push(`PROMPT`); - - if (modelVariant) { - commands.push(`MODEL VARIANT ${capitalize(modelVariant)}`); - } - - if (modelName) { - commands.push(`MODEL NAME \`${modelName}\``); - } - } else if (taskType === 'SIMPLE_TASK') { - commands.push(`SIMPLE TEMPLATE`); - // Note: Nothing special here - } else if (taskType === 'SCRIPT_TASK') { - commands.push(`SCRIPT`); - if (task.contentLanguage) { - contentLanguage = task.contentLanguage; - } else { - contentLanguage = ''; - } - } else if (taskType === 'DIALOG_TASK') { - commands.push(`DIALOG`); - // Note: Nothing special here - } // <- }else if([🅱] - - if (jokers) { - for (const joker of jokers) { - commands.push(`JOKER {${joker}}`); - } - } /* not else */ - if (postprocessing) { - for (const postprocessingFunctionName of postprocessing) { - commands.push(`POSTPROCESSING \`${postprocessingFunctionName}\``); - } - } /* not else */ - if (expectations) { - for (const [unit, { min, max }] of Object.entries(expectations)) { - if (min === max) { - commands.push(`EXPECT EXACTLY ${min} ${capitalize(unit + (min! > 1 ? 's' : ''))}`); - } else { - if (min !== undefined) { - commands.push(`EXPECT MIN ${min} ${capitalize(unit + (min! > 1 ? 's' : ''))}`); - } /* not else */ - if (max !== undefined) { - commands.push(`EXPECT MAX ${max} ${capitalize(unit + (max! > 1 ? 's' : ''))}`); - } - } - } - } /* not else */ - if (format) { - if (format === 'JSON') { - // TODO: @deprecated remove - commands.push(`FORMAT JSON`); - } - } /* not else */ - - pipelineString += '\n\n'; - pipelineString += commands.map((command) => `- ${command}`).join('\n'); - - pipelineString += '\n\n'; - pipelineString += '```' + contentLanguage; - pipelineString += '\n'; - pipelineString += spaceTrim(content); - // <- TODO: [main] !!3 Escape - // <- TODO: [🧠] Some clear strategy how to spaceTrim the blocks - pipelineString += '\n'; - pipelineString += '```'; - - pipelineString += '\n\n'; - pipelineString += `\`-> {${resultingParameterName}}\``; // <- TODO: [main] !!3 If the parameter here has description, add it and use taskParameterJsonToString - } - - return validatePipelineString(pipelineString); +export function pipelineJsonToString( + pipelineJson: PipelineJson, +): PipelineString { + const { title, pipelineUrl, bookVersion, description, parameters, tasks } = + pipelineJson; + + let pipelineString: string_markdown = `# ${title}`; + + if (description) { + pipelineString += "\n\n"; + pipelineString += description; + } + + const commands: Array = []; + + if (pipelineUrl) { + commands.push(`PIPELINE URL ${pipelineUrl}`); + } + + if (bookVersion !== `undefined`) { + commands.push(`BOOK VERSION ${bookVersion}`); + } + + // TODO: [main] !!5 This increases size of the bundle and is probbably not necessary + pipelineString = prettifyMarkdown(pipelineString); + + for (const parameter of parameters.filter(({ isInput }) => isInput)) { + commands.push(`INPUT PARAMETER ${taskParameterJsonToString(parameter)}`); + } + + for (const parameter of parameters.filter(({ isOutput }) => isOutput)) { + commands.push(`OUTPUT PARAMETER ${taskParameterJsonToString(parameter)}`); + } + + pipelineString += "\n\n"; + pipelineString += commands.map((command) => `- ${command}`).join("\n"); + + for (const task of tasks) { + const { + /* Note: Not using:> name, */ + title, + description, + /* Note: dependentParameterNames, */ + jokerParameterNames: jokers, + taskType, + content, + postprocessingFunctionNames: postprocessing, + expectations, + format, + resultingParameterName, + } = task; + + pipelineString += "\n\n"; + pipelineString += `## ${title}`; + + if (description) { + pipelineString += "\n\n"; + pipelineString += description; + } + + const commands: Array = []; + let contentLanguage: + | "markdown" + | "text" + | "javascript" + | "typescript" + | "python" + | "" = "text"; + + if (taskType === "PROMPT_TASK") { + const { modelRequirements } = task; + const { modelName, modelVariant } = modelRequirements || {}; + + // Note: Do nothing, it is default + // commands.push(`PROMPT`); + + if (modelVariant) { + commands.push(`MODEL VARIANT ${capitalize(modelVariant)}`); + } + + if (modelName) { + commands.push(`MODEL NAME \`${modelName}\``); + } + } else if (taskType === "SIMPLE_TASK") { + commands.push(`SIMPLE TEMPLATE`); + // Note: Nothing special here + } else if (taskType === "SCRIPT_TASK") { + commands.push(`SCRIPT`); + if (task.contentLanguage) { + contentLanguage = task.contentLanguage; + } else { + contentLanguage = ""; + } + } else if (taskType === "DIALOG_TASK") { + commands.push(`DIALOG`); + // Note: Nothing special here + } // <- }else if([🅱] + + if (jokers) { + for (const joker of jokers) { + commands.push(`JOKER {${joker}}`); + } + } /* not else */ + if (postprocessing) { + for (const postprocessingFunctionName of postprocessing) { + commands.push(`POSTPROCESSING \`${postprocessingFunctionName}\``); + } + } /* not else */ + if (expectations) { + for (const [unit, { min, max }] of Object.entries(expectations)) { + if (min === max) { + commands.push( + `EXPECT EXACTLY ${min} ${capitalize(unit + (min! > 1 ? "s" : ""))}`, + ); + } else { + if (min !== undefined) { + commands.push( + `EXPECT MIN ${min} ${capitalize(unit + (min! > 1 ? "s" : ""))}`, + ); + } /* not else */ + if (max !== undefined) { + commands.push( + `EXPECT MAX ${max} ${capitalize(unit + (max! > 1 ? "s" : ""))}`, + ); + } + } + } + } /* not else */ + if (format) { + if (format === "JSON") { + // TODO: @deprecated remove + commands.push(`FORMAT JSON`); + } + } /* not else */ + + pipelineString += "\n\n"; + pipelineString += commands.map((command) => `- ${command}`).join("\n"); + + pipelineString += "\n\n"; + pipelineString += "```" + contentLanguage; + pipelineString += "\n"; + pipelineString += spaceTrim(content); + // <- TODO: [main] !!3 Escape + // <- TODO: [🧠] Some clear strategy how to spaceTrim the blocks + pipelineString += "\n"; + pipelineString += "```"; + + pipelineString += "\n\n"; + pipelineString += `\`-> {${resultingParameterName}}\``; // <- TODO: [main] !!3 If the parameter here has description, add it and use taskParameterJsonToString + } + + return validatePipelineString(pipelineString); } /** * @private internal utility of `pipelineJsonToString` */ function taskParameterJsonToString(taskParameterJson: ParameterJson): string { - const { name, description } = taskParameterJson; + const { name, description } = taskParameterJson; - let parameterString = `{${name}}`; + let parameterString = `{${name}}`; - if (description) { - parameterString = `${parameterString} ${description}`; - } - return parameterString; + if (description) { + parameterString = `${parameterString} ${description}`; + } + return parameterString; } /** diff --git a/src/conversion/prettify/PrettifyOptions.ts b/src/conversion/prettify/PrettifyOptions.ts index 48a9df6819..80d3da88a8 100644 --- a/src/conversion/prettify/PrettifyOptions.ts +++ b/src/conversion/prettify/PrettifyOptions.ts @@ -2,13 +2,13 @@ * Options for `prettifyPipelineString` function */ export type PrettifyOptions = { - /*** - * If true, adds Mermaid graph to the Promptbook string - */ - readonly isGraphAdded?: boolean; + /*** + * If true, adds Mermaid graph to the Promptbook string + */ + readonly isGraphAdded?: boolean; - /** - * If true, the string is prettifyed as markdown - */ - readonly isPrettifyed?: boolean; + /** + * If true, the string is prettifyed as markdown + */ + readonly isPrettifyed?: boolean; }; diff --git a/src/conversion/prettify/prettifyPipelineString.ts b/src/conversion/prettify/prettifyPipelineString.ts index 23da1178b2..ae8abfc9f2 100644 --- a/src/conversion/prettify/prettifyPipelineString.ts +++ b/src/conversion/prettify/prettifyPipelineString.ts @@ -1,10 +1,10 @@ -import { spaceTrim } from 'spacetrim'; -import type { PipelineString } from '../../pipeline/PipelineString'; -import { addAutoGeneratedSection } from '../../utils/markdown/addAutoGeneratedSection'; -import { prettifyMarkdown } from '../../utils/markdown/prettifyMarkdown'; -import { compilePipeline } from '../compilePipeline'; -import type { PrettifyOptions } from './PrettifyOptions'; -import { renderPromptbookMermaid } from './renderPipelineMermaidOptions'; +import { spaceTrim } from "spacetrim"; +import type { PipelineString } from "../../pipeline/PipelineString"; +import { addAutoGeneratedSection } from "../../utils/markdown/addAutoGeneratedSection"; +import { prettifyMarkdown } from "../../utils/markdown/prettifyMarkdown"; +import { compilePipeline } from "../compilePipeline"; +import type { PrettifyOptions } from "./PrettifyOptions"; +import { renderPromptbookMermaid } from "./renderPipelineMermaidOptions"; /** * Prettyfies Promptbook string and adds Mermaid graph @@ -12,41 +12,41 @@ import { renderPromptbookMermaid } from './renderPipelineMermaidOptions'; * @public exported from `@promptbook/core` */ export async function prettifyPipelineString( - pipelineString: PipelineString, - options: PrettifyOptions, + pipelineString: PipelineString, + options: PrettifyOptions, ): Promise { - const { isGraphAdded, isPrettifyed } = options; + const { isGraphAdded, isPrettifyed } = options; - if (isGraphAdded) { - const pipelineJson = await compilePipeline(pipelineString); + if (isGraphAdded) { + const pipelineJson = await compilePipeline(pipelineString); - const promptbookMermaid = renderPromptbookMermaid(pipelineJson, { - linkTask(task) { - return { href: `#${task.name}`, title: task.title }; - }, - }); + const promptbookMermaid = renderPromptbookMermaid(pipelineJson, { + linkTask(task) { + return { href: `#${task.name}`, title: task.title }; + }, + }); - const promptbookMermaidBlock = spaceTrim( - (block) => ` + const promptbookMermaidBlock = spaceTrim( + (block) => ` \`\`\`mermaid ${block(promptbookMermaid)} \`\`\` `, - ); + ); - pipelineString = addAutoGeneratedSection(pipelineString, { - sectionName: 'Graph', - sectionContent: promptbookMermaidBlock, - }) as PipelineString; - // <- TODO: [😧] `addAutoGeneratedSection` should preserve discriminated type *(or at lease `PipelineString`)* - } + pipelineString = addAutoGeneratedSection(pipelineString, { + sectionName: "Graph", + sectionContent: promptbookMermaidBlock, + }) as PipelineString; + // <- TODO: [😧] `addAutoGeneratedSection` should preserve discriminated type *(or at lease `PipelineString`)* + } - if (isPrettifyed) { - pipelineString = prettifyMarkdown(pipelineString) as PipelineString; - // <- TODO: [😧] `prettifyMarkdown` should preserve discriminated type *(or at lease `PipelineString`)* - } + if (isPrettifyed) { + pipelineString = prettifyMarkdown(pipelineString) as PipelineString; + // <- TODO: [😧] `prettifyMarkdown` should preserve discriminated type *(or at lease `PipelineString`)* + } - return pipelineString; + return pipelineString; } /** diff --git a/src/conversion/prettify/renderPipelineMermaidOptions.ts b/src/conversion/prettify/renderPipelineMermaidOptions.ts index 756e383d15..51880e1135 100644 --- a/src/conversion/prettify/renderPipelineMermaidOptions.ts +++ b/src/conversion/prettify/renderPipelineMermaidOptions.ts @@ -1,20 +1,20 @@ -import { spaceTrim } from 'spacetrim'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { TaskJson } from '../../pipeline/PipelineJson/TaskJson'; -import type { string_href } from '../../types/typeAliases'; -import type { string_name } from '../../types/typeAliases'; -import { normalizeTo_camelCase } from '../../utils/normalization/normalizeTo_camelCase'; -import { titleToName } from '../../utils/normalization/titleToName'; +import { spaceTrim } from "spacetrim"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { TaskJson } from "../../pipeline/PipelineJson/TaskJson"; +import type { string_href } from "../../types/typeAliases"; +import type { string_name } from "../../types/typeAliases"; +import { normalizeTo_camelCase } from "../../utils/normalization/normalizeTo_camelCase"; +import { titleToName } from "../../utils/normalization/titleToName"; /** * Addtional options for rendering Mermaid graph */ export type renderPipelineMermaidOptions = { - /** - * Callback for creating from task graph node - */ - linkTask?(task: TaskJson): { href: string_href; title: string } | null; + /** + * Callback for creating from task graph node + */ + linkTask?(task: TaskJson): { href: string_href; title: string } | null; }; /** @@ -24,32 +24,41 @@ export type renderPipelineMermaidOptions = { * * @public exported from `@promptbook/utils` */ -export function renderPromptbookMermaid(pipelineJson: PipelineJson, options?: renderPipelineMermaidOptions): string { - const { linkTask = () => null } = options || {}; - - const parameterNameToTaskName = (parameterName: string_name) => { - const parameter = pipelineJson.parameters.find((parameter) => parameter.name === parameterName); - - if (!parameter) { - throw new UnexpectedError(`Could not find {${parameterName}}`); - // <- TODO: !!6 This causes problems when {knowledge} and other reserved parameters are used - } - - if (parameter.isInput) { - return 'input'; - } - - const task = pipelineJson.tasks.find((task) => task.resultingParameterName === parameterName); - - if (!task) { - throw new Error(`Could not find task for {${parameterName}}`); - } - - return task.name || normalizeTo_camelCase('task-' + titleToName(task.title)); - }; - - const promptbookMermaid = spaceTrim( - (block) => ` +export function renderPromptbookMermaid( + pipelineJson: PipelineJson, + options?: renderPipelineMermaidOptions, +): string { + const { linkTask = () => null } = options || {}; + + const parameterNameToTaskName = (parameterName: string_name) => { + const parameter = pipelineJson.parameters.find( + (parameter) => parameter.name === parameterName, + ); + + if (!parameter) { + throw new UnexpectedError(`Could not find {${parameterName}}`); + // <- TODO: !!6 This causes problems when {knowledge} and other reserved parameters are used + } + + if (parameter.isInput) { + return "input"; + } + + const task = pipelineJson.tasks.find( + (task) => task.resultingParameterName === parameterName, + ); + + if (!task) { + throw new Error(`Could not find task for {${parameterName}}`); + } + + return ( + task.name || normalizeTo_camelCase("task-" + titleToName(task.title)) + ); + }; + + const promptbookMermaid = spaceTrim( + (block) => ` %% 🔮 Tip: Open this on GitHub or in the VSCode website to see the Mermaid graph visually @@ -60,47 +69,58 @@ export function renderPromptbookMermaid(pipelineJson: PipelineJson, options?: re input((Input)):::input ${block( - pipelineJson.tasks - .flatMap(({ title, dependentParameterNames, resultingParameterName }) => [ - `${parameterNameToTaskName(resultingParameterName)}("${title}")`, - ...dependentParameterNames.map( - (dependentParameterName) => - `${parameterNameToTaskName( - dependentParameterName, - )}--"{${dependentParameterName}}"-->${parameterNameToTaskName( - resultingParameterName, - )}`, - ), - ]) - .join('\n'), - )} + pipelineJson.tasks + .flatMap( + ({ + title, + dependentParameterNames, + resultingParameterName, + }) => [ + `${parameterNameToTaskName(resultingParameterName)}("${title}")`, + ...dependentParameterNames.map( + (dependentParameterName) => + `${parameterNameToTaskName( + dependentParameterName, + )}--"{${dependentParameterName}}"-->${parameterNameToTaskName( + resultingParameterName, + )}`, + ), + ], + ) + .join("\n"), + )} ${block( - pipelineJson.parameters - .filter(({ isOutput }) => isOutput) - .map(({ name }) => `${parameterNameToTaskName(name)}--"{${name}}"-->output`) - .join('\n'), - )} + pipelineJson.parameters + .filter(({ isOutput }) => isOutput) + .map( + ({ name }) => + `${parameterNameToTaskName(name)}--"{${name}}"-->output`, + ) + .join("\n"), + )} output((Output)):::output ${block( - pipelineJson.tasks - .map((task) => { - const link = linkTask(task); + pipelineJson.tasks + .map((task) => { + const link = linkTask(task); - if (link === null) { - return ''; - } + if (link === null) { + return ""; + } - const { href, title } = link; + const { href, title } = link; - const taskName = parameterNameToTaskName(task.resultingParameterName); + const taskName = parameterNameToTaskName( + task.resultingParameterName, + ); - return `click ${taskName} href "${href}" "${title}";`; - }) - .filter((line) => line !== '') - .join('\n'), - )} + return `click ${taskName} href "${href}" "${title}";`; + }) + .filter((line) => line !== "") + .join("\n"), + )} classDef input color: grey; classDef output color: grey; @@ -108,9 +128,9 @@ export function renderPromptbookMermaid(pipelineJson: PipelineJson, options?: re end; `, - ); + ); - return promptbookMermaid; + return promptbookMermaid; } /** diff --git a/src/conversion/utils/extractParameterNamesFromTask.test.ts b/src/conversion/utils/extractParameterNamesFromTask.test.ts index 2e361fe701..14f8baf894 100644 --- a/src/conversion/utils/extractParameterNamesFromTask.test.ts +++ b/src/conversion/utils/extractParameterNamesFromTask.test.ts @@ -1,27 +1,27 @@ -import { describe, expect, it } from '@jest/globals'; -import { extractParameterNamesFromTask } from './extractParameterNamesFromTask'; +import { describe, expect, it } from "@jest/globals"; +import { extractParameterNamesFromTask } from "./extractParameterNamesFromTask"; -describe('extractParameterNamesFromTask', () => { - it('should parse parameters from task', () => { - const task = { - title: 'name of {foo}', - description: 'description of {foo} and {bar}', - taskType: 'PROMPT_TASK', - content: 'hello {name}', - } as const; +describe("extractParameterNamesFromTask", () => { + it("should parse parameters from task", () => { + const task = { + title: "name of {foo}", + description: "description of {foo} and {bar}", + taskType: "PROMPT_TASK", + content: "hello {name}", + } as const; - expect(extractParameterNamesFromTask(task)).toContain('foo'); - expect(extractParameterNamesFromTask(task)).toContain('bar'); - expect(extractParameterNamesFromTask(task)).toContain('name'); - }); + expect(extractParameterNamesFromTask(task)).toContain("foo"); + expect(extractParameterNamesFromTask(task)).toContain("bar"); + expect(extractParameterNamesFromTask(task)).toContain("name"); + }); - it('should parse parameters from javascript script', () => { - expect( - extractParameterNamesFromTask({ - title: 'Script', - taskType: 'SCRIPT_TASK', - content: 'const greeting = hello;', - }), - ).toContain('hello'); - }); + it("should parse parameters from javascript script", () => { + expect( + extractParameterNamesFromTask({ + title: "Script", + taskType: "SCRIPT_TASK", + content: "const greeting = hello;", + }), + ).toContain("hello"); + }); }); diff --git a/src/conversion/utils/extractParameterNamesFromTask.ts b/src/conversion/utils/extractParameterNamesFromTask.ts index 298f7e2be7..3dc84947a0 100644 --- a/src/conversion/utils/extractParameterNamesFromTask.ts +++ b/src/conversion/utils/extractParameterNamesFromTask.ts @@ -1,8 +1,8 @@ -import type { ReadonlyDeep } from 'type-fest'; -import type { TaskJson } from '../../pipeline/PipelineJson/TaskJson'; -import type { string_parameter_name } from '../../types/typeAliases'; -import { extractParameterNames } from '../../utils/parameters/extractParameterNames'; -import { extractVariablesFromScript } from './extractVariablesFromScript'; +import type { ReadonlyDeep } from "type-fest"; +import type { TaskJson } from "../../pipeline/PipelineJson/TaskJson"; +import type { string_parameter_name } from "../../types/typeAliases"; +import { extractParameterNames } from "../../utils/parameters/extractParameterNames"; +import { extractVariablesFromScript } from "./extractVariablesFromScript"; /** * Parses the task and returns the set of all used parameters @@ -13,50 +13,64 @@ import { extractVariablesFromScript } from './extractVariablesFromScript'; * @public exported from `@promptbook/core` <- Note: [👖] This utility is so tightly interconnected with the Promptbook that it is not exported as util but in core */ export function extractParameterNamesFromTask( - task: ReadonlyDeep< - Pick< - TaskJson, - 'title' | 'description' | 'taskType' | 'content' | 'preparedContent' | 'jokerParameterNames' | 'foreach' - > - >, + task: ReadonlyDeep< + Pick< + TaskJson, + | "title" + | "description" + | "taskType" + | "content" + | "preparedContent" + | "jokerParameterNames" + | "foreach" + > + >, ): Set { - const { title, description, taskType, content, preparedContent, jokerParameterNames, foreach } = task; - const parameterNames = new Set(); + const { + title, + description, + taskType, + content, + preparedContent, + jokerParameterNames, + foreach, + } = task; + const parameterNames = new Set(); - for (const parameterName of [ - ...extractParameterNames(title), - ...extractParameterNames(description || ''), - ...extractParameterNames(content), - ...extractParameterNames(preparedContent || ''), - ]) { - parameterNames.add(parameterName); - } + for (const parameterName of [ + ...extractParameterNames(title), + ...extractParameterNames(description || ""), + ...extractParameterNames(content), + ...extractParameterNames(preparedContent || ""), + ]) { + parameterNames.add(parameterName); + } - if (taskType === 'SCRIPT_TASK') { - for (const parameterName of extractVariablesFromScript(content)) { - parameterNames.add(parameterName); - } - } + if (taskType === "SCRIPT_TASK") { + for (const parameterName of extractVariablesFromScript(content)) { + parameterNames.add(parameterName); + } + } - for (const jokerName of jokerParameterNames || []) { - parameterNames.add(jokerName); - } + for (const jokerName of jokerParameterNames || []) { + parameterNames.add(jokerName); + } - parameterNames.delete('content'); - // <- Note {websiteContent} is used in `preparedContent` + parameterNames.delete("content"); + // <- Note {websiteContent} is used in `preparedContent` - // Note: [🍭] Fixing dependent subparameterName from FOREACH command - if (foreach !== undefined) { - for (const subparameterName of foreach.inputSubparameterNames) { - if (parameterNames.has(subparameterName)) { - parameterNames.delete(subparameterName); - parameterNames.add(foreach.parameterName); - // <- TODO: [🚎] Warn/logic error when `subparameterName` not used - } - } - } + // Note: [🍭] Fixing dependent subparameterName from FOREACH command + if (foreach !== undefined) { + for (const subparameterName of foreach.inputSubparameterNames) { + if (parameterNames.has(subparameterName)) { + parameterNames.delete(subparameterName); + parameterNames.add(foreach.parameterName); + // <- TODO: [🚎] Warn/logic error when `subparameterName` not used + } + } + } - return parameterNames; + return parameterNames; } /** diff --git a/src/conversion/utils/extractVariablesFromScript.test.ts b/src/conversion/utils/extractVariablesFromScript.test.ts index c2dcc77b75..ba31134272 100644 --- a/src/conversion/utils/extractVariablesFromScript.test.ts +++ b/src/conversion/utils/extractVariablesFromScript.test.ts @@ -1,28 +1,42 @@ -import { describe, expect, it } from '@jest/globals'; -import { extractVariablesFromScript } from './extractVariablesFromScript'; +import { describe, expect, it } from "@jest/globals"; +import { extractVariablesFromScript } from "./extractVariablesFromScript"; //import { extractVariablesFromScript } from './extractVariablesFromScript'; -describe('extractVariablesFromScript', () => { - it('should work in supersimple case without any variables', () => { - expect([...extractVariablesFromScript('')]).toEqual([]); - expect([...extractVariablesFromScript('"Hello"')]).toEqual([]); - expect([...extractVariablesFromScript('const a = 1;')]).toEqual([]); - }); +describe("extractVariablesFromScript", () => { + it("should work in supersimple case without any variables", () => { + expect([...extractVariablesFromScript("")]).toEqual([]); + expect([...extractVariablesFromScript('"Hello"')]).toEqual([]); + expect([...extractVariablesFromScript("const a = 1;")]).toEqual([]); + }); - it('should parse one variable', () => { - expect([...extractVariablesFromScript('const a = name;')]).toEqual(['name']); - expect([...extractVariablesFromScript('console.log(name);')]).toEqual(['name']); - expect([...extractVariablesFromScript('const a = name; const b = name;')]).toEqual(['name']); - expect([...extractVariablesFromScript('const a = 1; const b = name; const c = name;')]).toEqual(['name']); - }); + it("should parse one variable", () => { + expect([...extractVariablesFromScript("const a = name;")]).toEqual([ + "name", + ]); + expect([...extractVariablesFromScript("console.log(name);")]).toEqual([ + "name", + ]); + expect([ + ...extractVariablesFromScript("const a = name; const b = name;"), + ]).toEqual(["name"]); + expect([ + ...extractVariablesFromScript( + "const a = 1; const b = name; const c = name;", + ), + ]).toEqual(["name"]); + }); - it('should NOT parse custom function', () => { - expect([...extractVariablesFromScript('foo();')]).toEqual([]); - expect([...extractVariablesFromScript('foo(name);')]).toEqual(['name']); - expect([...extractVariablesFromScript('console.log(name);')]).toEqual(['name']); - }); + it("should NOT parse custom function", () => { + expect([...extractVariablesFromScript("foo();")]).toEqual([]); + expect([...extractVariablesFromScript("foo(name);")]).toEqual(["name"]); + expect([...extractVariablesFromScript("console.log(name);")]).toEqual([ + "name", + ]); + }); - it('should parse multiple variables', () => { - expect([...extractVariablesFromScript('console.log(`${greeting} ${name}`);')]).toEqual(['greeting', 'name']); - }); + it("should parse multiple variables", () => { + expect([ + ...extractVariablesFromScript("console.log(`${greeting} ${name}`);"), + ]).toEqual(["greeting", "name"]); + }); }); diff --git a/src/conversion/utils/extractVariablesFromScript.ts b/src/conversion/utils/extractVariablesFromScript.ts index 2248c20de1..b3f31d436e 100644 --- a/src/conversion/utils/extractVariablesFromScript.ts +++ b/src/conversion/utils/extractVariablesFromScript.ts @@ -1,7 +1,7 @@ -import { spaceTrim } from 'spacetrim'; -import { ParseError } from '../../errors/ParseError'; -import type { string_javascript } from '../../types/typeAliases'; -import type { string_javascript_name } from '../../types/typeAliases'; +import { spaceTrim } from "spacetrim"; +import { ParseError } from "../../errors/ParseError"; +import type { string_javascript } from "../../types/typeAliases"; +import type { string_javascript_name } from "../../types/typeAliases"; /** * Parses the given script and returns the list of all used variables that are not defined in the script * @@ -10,22 +10,24 @@ import type { string_javascript_name } from '../../types/typeAliases'; * @throws {ParseError} if the script is invalid * @public exported from `@promptbook/utils` <- Note: [👖] This is usable elsewhere than in Promptbook, so keeping in utils */ -export function extractVariablesFromScript(script: string_javascript): Set { - const variables = new Set(); +export function extractVariablesFromScript( + script: string_javascript, +): Set { + const variables = new Set(); - const originalScript = script; - script = `(()=>{${script}})()`; + const originalScript = script; + script = `(()=>{${script}})()`; - try { - for (let i = 0; i < 100 /* <- TODO: This limit to configuration */; i++) - try { - eval(script); - } catch (error) { - if (!(error instanceof ReferenceError)) { - throw error; - } + try { + for (let i = 0; i < 100 /* <- TODO: This limit to configuration */; i++) + try { + eval(script); + } catch (error) { + if (!(error instanceof ReferenceError)) { + throw error; + } - /* + /* Note: Parsing the error 🌟 Most devices: [PipelineUrlError: thing is not defined] @@ -34,42 +36,42 @@ export function extractVariablesFromScript(script: string_javascript): Set'';` + script; - } else { - variables.add(variableName); - script = `const ${variableName} = '';` + script; - } - } - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } + if (script.includes(variableName + "(")) { + script = `const ${variableName} = ()=>'';` + script; + } else { + variables.add(variableName); + script = `const ${variableName} = '';` + script; + } + } + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } - throw new ParseError( - spaceTrim( - (block) => ` + throw new ParseError( + spaceTrim( + (block) => ` Can not extract variables from the script ${block((error as Error).stack || (error as Error).message)} Found variables: ${Array.from(variables) - .map((variableName, i) => `${i + 1}) ${variableName}`) - .join('\n')} + .map((variableName, i) => `${i + 1}) ${variableName}`) + .join("\n")} The script: @@ -78,12 +80,12 @@ export function extractVariablesFromScript(script: string_javascript): Set; -export async function importPipelineWithoutPreparation(path: `${string}.bookc`): Promise; -export async function importPipelineWithoutPreparation(path: string_filename): Promise { - const examplesDir = '../../../examples/pipelines'; // <- TODO: [🚏] DRY, to config - const filePath = join(__dirname, examplesDir, path); +export async function importPipelineWithoutPreparation( + path: `${string}.book`, +): Promise; +export async function importPipelineWithoutPreparation( + path: `${string}.bookc`, +): Promise; +export async function importPipelineWithoutPreparation( + path: string_filename, +): Promise { + const examplesDir = "../../../examples/pipelines"; // <- TODO: [🚏] DRY, to config + const filePath = join(__dirname, examplesDir, path); - if (path.endsWith('.bookc')) { - const pipelines = await loadArchive(filePath, $provideFilesystemForNode()); + if (path.endsWith(".bookc")) { + const pipelines = await loadArchive(filePath, $provideFilesystemForNode()); - if (pipelines.length !== 1) { - throw new Error('The archive must contain exactly one pipeline'); - } + if (pipelines.length !== 1) { + throw new Error("The archive must contain exactly one pipeline"); + } - let pipeline = pipelines[0]!; + let pipeline = pipelines[0]!; - pipeline = unpreparePipeline(pipeline); - return pipeline; - } else if (path.endsWith('.book')) { - const content = await readFile(filePath, 'utf-8'); + pipeline = unpreparePipeline(pipeline); + return pipeline; + } else if (path.endsWith(".book")) { + const content = await readFile(filePath, "utf-8"); - return validatePipelineString(content); - } else { - throw new Error('This should be used only for .book.md or .bookc files'); - } + return validatePipelineString(content); + } else { + throw new Error("This should be used only for .book.md or .bookc files"); + } } /** @@ -48,18 +54,23 @@ export async function importPipelineWithoutPreparation(path: string_filename): P * * @private internal function of tests */ -export async function importPipelineJson(path: `${string}.bookc`): Promise { - const examplesDir = '../../../examples/pipelines'; // <- TODO: [🚏] DRY, to config +export async function importPipelineJson( + path: `${string}.bookc`, +): Promise { + const examplesDir = "../../../examples/pipelines"; // <- TODO: [🚏] DRY, to config - const pipelines = await loadArchive(join(__dirname, examplesDir, path), $provideFilesystemForNode()); + const pipelines = await loadArchive( + join(__dirname, examplesDir, path), + $provideFilesystemForNode(), + ); - if (pipelines.length !== 1) { - throw new Error('The archive must contain exactly one pipeline'); - } + if (pipelines.length !== 1) { + throw new Error("The archive must contain exactly one pipeline"); + } - const pipeline = pipelines[0]!; + const pipeline = pipelines[0]!; - return pipeline; + return pipeline; } /* diff --git a/src/conversion/validation/pipelineStringToJson-parseErrors.test.ts b/src/conversion/validation/pipelineStringToJson-parseErrors.test.ts index ec98fabb5b..99141104d4 100644 --- a/src/conversion/validation/pipelineStringToJson-parseErrors.test.ts +++ b/src/conversion/validation/pipelineStringToJson-parseErrors.test.ts @@ -1,43 +1,49 @@ -import { describe, expect, it } from '@jest/globals'; -import { readdirSync } from 'fs'; -import { join } from 'path'; -import { ParseError } from '../../errors/ParseError'; -import { compilePipeline } from '../compilePipeline'; -import { importPipelineWithoutPreparation } from './_importPipeline'; -import { validatePipeline } from './validatePipeline'; +import { readdirSync } from "fs"; +import { join } from "path"; +import { describe, expect, it } from "@jest/globals"; +import { ParseError } from "../../errors/ParseError"; +import { compilePipeline } from "../compilePipeline"; +import { importPipelineWithoutPreparation } from "./_importPipeline"; +import { validatePipeline } from "./validatePipeline"; -describe('validatePipeline with parse errors', () => { - const examplesDir = '../../../examples/pipelines/'; // <- TODO: [🚏] DRY, to config - const examples = readdirSync(join(__dirname, examplesDir, 'errors/parse'), { - withFileTypes: true, - recursive: false, - }) - // <- Note: In production it is not good practice to use synchronous functions - // But this is only a test before the build, so it is okay - .filter((dirent) => dirent.isFile()) - .filter(({ name }) => name.endsWith('.book')); +describe("validatePipeline with parse errors", () => { + const examplesDir = "../../../examples/pipelines/"; // <- TODO: [🚏] DRY, to config + const examples = readdirSync(join(__dirname, examplesDir, "errors/parse"), { + withFileTypes: true, + recursive: false, + }) + // <- Note: In production it is not good practice to use synchronous functions + // But this is only a test before the build, so it is okay + .filter((dirent) => dirent.isFile()) + .filter(({ name }) => name.endsWith(".book")); - for (const { name } of examples) { - it(`should parse ${name} parse`, async () => { - await expect(async () => { - const pipelineString = await importPipelineWithoutPreparation( - ('errors/parse/' + name) as `${string}.book`, - ); - const pipelineJson = await compilePipeline(pipelineString); + for (const { name } of examples) { + it(`should parse ${name} parse`, async () => { + await expect(async () => { + const pipelineString = await importPipelineWithoutPreparation( + ("errors/parse/" + name) as `${string}.book`, + ); + const pipelineJson = await compilePipeline(pipelineString); - try { - validatePipeline(pipelineJson); - console.error('Pipeline should have ParseError error BUT it does not have any error:\n', name); - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } + try { + validatePipeline(pipelineJson); + console.error( + "Pipeline should have ParseError error BUT it does not have any error:\n", + name, + ); + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } - console.error(`Pipeline should have ParseError error BUT it has ${error.name}:\n`, name); - } - }).rejects.toThrowError(ParseError); - }); - } + console.error( + `Pipeline should have ParseError error BUT it has ${error.name}:\n`, + name, + ); + } + }).rejects.toThrowError(ParseError); + }); + } }); /** diff --git a/src/conversion/validation/validatePipeline-logicErrors.test.ts b/src/conversion/validation/validatePipeline-logicErrors.test.ts index 3d31359527..e772da8e13 100644 --- a/src/conversion/validation/validatePipeline-logicErrors.test.ts +++ b/src/conversion/validation/validatePipeline-logicErrors.test.ts @@ -1,35 +1,38 @@ -import { describe, expect, it } from '@jest/globals'; -import { readdirSync } from 'fs'; -import { join } from 'path'; -import { PipelineLogicError } from '../../errors/PipelineLogicError'; -import { compilePipeline } from '../compilePipeline'; -import { importPipelineWithoutPreparation } from './_importPipeline'; -import { validatePipeline } from './validatePipeline'; +import { readdirSync } from "fs"; +import { join } from "path"; +import { describe, expect, it } from "@jest/globals"; +import { PipelineLogicError } from "../../errors/PipelineLogicError"; +import { compilePipeline } from "../compilePipeline"; +import { importPipelineWithoutPreparation } from "./_importPipeline"; +import { validatePipeline } from "./validatePipeline"; -describe('validatePipeline with logic errors', () => { - const examplesDir = '../../../examples/pipelines/'; // <- TODO: [🚏] DRY, to config - const examples = readdirSync(join(__dirname, examplesDir, 'errors/logic'), { - withFileTypes: true, - recursive: false, - }) - // <- Note: In production it is not good practice to use synchronous functions - // But this is only a test before the build, so it is okay - .filter((dirent) => dirent.isFile()) - .filter(({ name }) => name.endsWith('.book')); +describe("validatePipeline with logic errors", () => { + const examplesDir = "../../../examples/pipelines/"; // <- TODO: [🚏] DRY, to config + const examples = readdirSync(join(__dirname, examplesDir, "errors/logic"), { + withFileTypes: true, + recursive: false, + }) + // <- Note: In production it is not good practice to use synchronous functions + // But this is only a test before the build, so it is okay + .filter((dirent) => dirent.isFile()) + .filter(({ name }) => name.endsWith(".book")); - for (const { name } of examples) { - it(`should validate ${name} logic`, () => { - expect(async () => { - const pipelineString = await importPipelineWithoutPreparation( - ('errors/logic/' + name) as `${string}.book`, - ); - const pipelineJson = await compilePipeline(pipelineString); - validatePipeline(pipelineJson); + for (const { name } of examples) { + it(`should validate ${name} logic`, () => { + expect(async () => { + const pipelineString = await importPipelineWithoutPreparation( + ("errors/logic/" + name) as `${string}.book`, + ); + const pipelineJson = await compilePipeline(pipelineString); + validatePipeline(pipelineJson); - console.error('Pipeline should have logic error BUT it does not:\n', name); - }).rejects.toThrowError(PipelineLogicError); - }); - } + console.error( + "Pipeline should have logic error BUT it does not:\n", + name, + ); + }).rejects.toThrowError(PipelineLogicError); + }); + } }); /** diff --git a/src/conversion/validation/validatePipeline.test.ts b/src/conversion/validation/validatePipeline.test.ts index c73c674ebc..e3c207d6f1 100644 --- a/src/conversion/validation/validatePipeline.test.ts +++ b/src/conversion/validation/validatePipeline.test.ts @@ -1,48 +1,53 @@ -import { describe, expect, it } from '@jest/globals'; -import { readdirSync } from 'fs'; -import { join } from 'path'; -import { spaceTrim } from 'spacetrim'; -import { compilePipeline } from '../compilePipeline'; -import { importPipelineWithoutPreparation } from './_importPipeline'; -import { validatePipeline } from './validatePipeline'; +import { readdirSync } from "fs"; +import { join } from "path"; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { compilePipeline } from "../compilePipeline"; +import { importPipelineWithoutPreparation } from "./_importPipeline"; +import { validatePipeline } from "./validatePipeline"; -describe('validatePipeline with valid examples', () => { - const examplesDir = '../../../examples/pipelines'; // <- TODO: [🚏] DRY, to config - const examples = readdirSync(join(__dirname, examplesDir), { withFileTypes: true, recursive: false }) - // <- Note: In production it is not good practice to use synchronous functions - // But this is only a test before the build, so it is okay - .filter((dirent) => dirent.isFile()) - .filter(({ name }) => name.endsWith('.book')); +describe("validatePipeline with valid examples", () => { + const examplesDir = "../../../examples/pipelines"; // <- TODO: [🚏] DRY, to config + const examples = readdirSync(join(__dirname, examplesDir), { + withFileTypes: true, + recursive: false, + }) + // <- Note: In production it is not good practice to use synchronous functions + // But this is only a test before the build, so it is okay + .filter((dirent) => dirent.isFile()) + .filter(({ name }) => name.endsWith(".book")); - for (const { name } of examples) { - it(`should validate ${name} syntax, parsing and logic`, () => { - expect( - (async () => { - try { - const pipelineString = await importPipelineWithoutPreparation(name as `${string}.book`); - const pipelineJson = await compilePipeline(pipelineString); - validatePipeline(pipelineJson); - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } + for (const { name } of examples) { + it(`should validate ${name} syntax, parsing and logic`, () => { + expect( + (async () => { + try { + const pipelineString = await importPipelineWithoutPreparation( + name as `${string}.book`, + ); + const pipelineJson = await compilePipeline(pipelineString); + validatePipeline(pipelineJson); + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } - throw new Error( - spaceTrim( - (block) => ` + throw new Error( + spaceTrim( + (block) => ` - Error in ${join(__dirname, examplesDir, name).split('\\').join('/')}: + Error in ${join(__dirname, examplesDir, name).split("\\").join("/")}: ${block((error as Error).message)} `, - ), - ); - } - })(), - ).resolves.not.toThrowError(); - }); - } + ), + ); + } + })(), + ).resolves.not.toThrowError(); + }); + } }); /** diff --git a/src/conversion/validation/validatePipeline.ts b/src/conversion/validation/validatePipeline.ts index 4c06ebff6d..6f99c987c2 100644 --- a/src/conversion/validation/validatePipeline.ts +++ b/src/conversion/validation/validatePipeline.ts @@ -1,17 +1,17 @@ -import { spaceTrim } from 'spacetrim'; -import { IS_PIPELINE_LOGIC_VALIDATED } from '../../config'; -import { LOOP_LIMIT } from '../../config'; -import { RESERVED_PARAMETER_NAMES } from '../../constants'; -import { ParseError } from '../../errors/ParseError'; -import { PipelineLogicError } from '../../errors/PipelineLogicError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { ParameterJson } from '../../pipeline/PipelineJson/ParameterJson'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { TaskJson } from '../../pipeline/PipelineJson/TaskJson'; -import type { string_name } from '../../types/typeAliases'; -import type { string_reserved_parameter_name } from '../../types/typeAliases'; -import { isValidPromptbookVersion } from '../../utils/validators/semanticVersion/isValidPromptbookVersion'; -import { isValidPipelineUrl } from '../../utils/validators/url/isValidPipelineUrl'; +import { spaceTrim } from "spacetrim"; +import { IS_PIPELINE_LOGIC_VALIDATED } from "../../config"; +import { LOOP_LIMIT } from "../../config"; +import { RESERVED_PARAMETER_NAMES } from "../../constants"; +import { ParseError } from "../../errors/ParseError"; +import { PipelineLogicError } from "../../errors/PipelineLogicError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { ParameterJson } from "../../pipeline/PipelineJson/ParameterJson"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { TaskJson } from "../../pipeline/PipelineJson/TaskJson"; +import type { string_name } from "../../types/typeAliases"; +import type { string_reserved_parameter_name } from "../../types/typeAliases"; +import { isValidPromptbookVersion } from "../../utils/validators/semanticVersion/isValidPromptbookVersion"; +import { isValidPipelineUrl } from "../../utils/validators/url/isValidPipelineUrl"; /** * Validates PipelineJson if it is logically valid @@ -29,116 +29,122 @@ import { isValidPipelineUrl } from '../../utils/validators/url/isValidPipelineUr * @public exported from `@promptbook/core` */ export function validatePipeline(pipeline: PipelineJson): PipelineJson { - if (IS_PIPELINE_LOGIC_VALIDATED) { - validatePipeline_InnerFunction(pipeline); - } else { - try { - validatePipeline_InnerFunction(pipeline); - } catch (error) { - if (!(error instanceof PipelineLogicError)) { - throw error; - } - - console.error( - spaceTrim( - (block) => ` + if (IS_PIPELINE_LOGIC_VALIDATED) { + validatePipeline_InnerFunction(pipeline); + } else { + try { + validatePipeline_InnerFunction(pipeline); + } catch (error) { + if (!(error instanceof PipelineLogicError)) { + throw error; + } + + console.error( + spaceTrim( + (block) => ` Pipeline is not valid but logic errors are temporarily disabled via \`IS_PIPELINE_LOGIC_VALIDATED\` ${block((error as PipelineLogicError).message)} `, - ), - ); - } - } + ), + ); + } + } - return pipeline; + return pipeline; } /** * @private internal function for `validatePipeline` */ export function validatePipeline_InnerFunction(pipeline: PipelineJson): void { - // TODO: [🧠] Maybe test if promptbook is a promise and make specific error case for that - - const pipelineIdentification = (() => { - // Note: This is a 😐 implementation of [🚞] - const _: Array = []; - - if (pipeline.sourceFile !== undefined) { - _.push(`File: ${pipeline.sourceFile}`); - } - - if (pipeline.pipelineUrl !== undefined) { - _.push(`Url: ${pipeline.pipelineUrl}`); - } - - return _.join('\n'); - })(); - - if (pipeline.pipelineUrl !== undefined && !isValidPipelineUrl(pipeline.pipelineUrl)) { - // <- Note: [🚲] - throw new PipelineLogicError( - spaceTrim( - (block) => ` + // TODO: [🧠] Maybe test if promptbook is a promise and make specific error case for that + + const pipelineIdentification = (() => { + // Note: This is a 😐 implementation of [🚞] + const _: Array = []; + + if (pipeline.sourceFile !== undefined) { + _.push(`File: ${pipeline.sourceFile}`); + } + + if (pipeline.pipelineUrl !== undefined) { + _.push(`Url: ${pipeline.pipelineUrl}`); + } + + return _.join("\n"); + })(); + + if ( + pipeline.pipelineUrl !== undefined && + !isValidPipelineUrl(pipeline.pipelineUrl) + ) { + // <- Note: [🚲] + throw new PipelineLogicError( + spaceTrim( + (block) => ` Invalid promptbook URL "${pipeline.pipelineUrl}" ${block(pipelineIdentification)} `, - ), - // <- TODO: [🐠] - // <- TODO: [🚞] - ); - } - - if (pipeline.bookVersion !== undefined && !isValidPromptbookVersion(pipeline.bookVersion)) { - // <- Note: [🚲] - throw new PipelineLogicError( - spaceTrim( - (block) => ` + ), + // <- TODO: [🐠] + // <- TODO: [🚞] + ); + } + + if ( + pipeline.bookVersion !== undefined && + !isValidPromptbookVersion(pipeline.bookVersion) + ) { + // <- Note: [🚲] + throw new PipelineLogicError( + spaceTrim( + (block) => ` Invalid Promptbook Version "${pipeline.bookVersion}" ${block(pipelineIdentification)} `, - ), - // <- TODO: [🚞] - ); - } - - // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking - if (!Array.isArray(pipeline.parameters)) { - // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError - throw new ParseError( - spaceTrim( - (block) => ` + ), + // <- TODO: [🚞] + ); + } + + // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking + if (!Array.isArray(pipeline.parameters)) { + // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError + throw new ParseError( + spaceTrim( + (block) => ` Pipeline is valid JSON but with wrong structure \`PipelineJson.parameters\` expected to be an array, but got ${typeof pipeline.parameters} ${block(pipelineIdentification)} `, - ), - // <- TODO: [🚞] - ); - } - - // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking - if (!Array.isArray(pipeline.tasks)) { - // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError - throw new ParseError( - spaceTrim( - (block) => ` + ), + // <- TODO: [🚞] + ); + } + + // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking + if (!Array.isArray(pipeline.tasks)) { + // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError + throw new ParseError( + spaceTrim( + (block) => ` Pipeline is valid JSON but with wrong structure \`PipelineJson.tasks\` expected to be an array, but got ${typeof pipeline.tasks} ${block(pipelineIdentification)} `, - ), - // <- TODO: [🚞] - ); - } + ), + // <- TODO: [🚞] + ); + } - /* + /* TODO: [🧠][🅾] Should be empty pipeline valid or not // Note: Check that pipeline has some tasks if (pipeline.tasks.length === 0) { @@ -154,50 +160,57 @@ export function validatePipeline_InnerFunction(pipeline: PipelineJson): void { } */ - // Note: Check each parameter individually - for (const parameter of pipeline.parameters) { - if (parameter.isInput && parameter.isOutput) { - throw new PipelineLogicError( - spaceTrim( - (block) => ` + // Note: Check each parameter individually + for (const parameter of pipeline.parameters) { + if (parameter.isInput && parameter.isOutput) { + throw new PipelineLogicError( + spaceTrim( + (block) => ` Parameter \`{${(parameter as ParameterJson).name}}\` can not be both input and output ${block(pipelineIdentification)} `, - ), - // <- Note: [🆑] - // <- TODO: [🚞] - ); - } - - // Note: Testing that parameter is either intermediate or output BUT not created and unused - if ( - !parameter.isInput && - !parameter.isOutput && - !pipeline.tasks.some((task) => task.dependentParameterNames.includes(parameter.name)) - ) { - throw new PipelineLogicError( - spaceTrim( - (block) => ` + ), + // <- Note: [🆑] + // <- TODO: [🚞] + ); + } + + // Note: Testing that parameter is either intermediate or output BUT not created and unused + if ( + !parameter.isInput && + !parameter.isOutput && + !pipeline.tasks.some((task) => + task.dependentParameterNames.includes(parameter.name), + ) + ) { + throw new PipelineLogicError( + spaceTrim( + (block) => ` Parameter \`{${parameter.name}}\` is created but not used You can declare {${parameter.name}} as output parameter by adding in the header: - - OUTPUT PARAMETER \`{${parameter.name}}\` ${parameter.description || ''} + - OUTPUT PARAMETER \`{${parameter.name}}\` ${parameter.description || ""} ${block(pipelineIdentification)} `, - ), - // <- TODO: [🚞] - ); - } - - // Note: Testing that parameter is either input or result of some task - if (!parameter.isInput && !pipeline.tasks.some((task) => task.resultingParameterName === parameter.name)) { - throw new PipelineLogicError( - spaceTrim( - (block) => ` + ), + // <- TODO: [🚞] + ); + } + + // Note: Testing that parameter is either input or result of some task + if ( + !parameter.isInput && + !pipeline.tasks.some( + (task) => task.resultingParameterName === parameter.name, + ) + ) { + throw new PipelineLogicError( + spaceTrim( + (block) => ` Parameter \`{${parameter.name}}\` is declared but not defined You can do one of these: @@ -206,166 +219,174 @@ export function validatePipeline_InnerFunction(pipeline: PipelineJson): void { ${block(pipelineIdentification)} `, - ), - // <- TODO: [🚞] - ); - } - } - - // Note: All input parameters are defined - so that they can be used as result of some task - const definedParameters: Set = new Set( - pipeline.parameters.filter(({ isInput }) => isInput).map(({ name }) => name), - ); - - // Note: Checking each task individually - for (const task of pipeline.tasks) { - if (definedParameters.has(task.resultingParameterName)) { - throw new PipelineLogicError( - spaceTrim( - (block) => ` + ), + // <- TODO: [🚞] + ); + } + } + + // Note: All input parameters are defined - so that they can be used as result of some task + const definedParameters: Set = new Set( + pipeline.parameters + .filter(({ isInput }) => isInput) + .map(({ name }) => name), + ); + + // Note: Checking each task individually + for (const task of pipeline.tasks) { + if (definedParameters.has(task.resultingParameterName)) { + throw new PipelineLogicError( + spaceTrim( + (block) => ` Parameter \`{${task.resultingParameterName}}\` is defined multiple times ${block(pipelineIdentification)} `, - ), - // <- TODO: [🚞] - ); - } - - if (RESERVED_PARAMETER_NAMES.includes(task.resultingParameterName as string_reserved_parameter_name)) { - throw new PipelineLogicError( - spaceTrim( - (block) => ` + ), + // <- TODO: [🚞] + ); + } + + if ( + RESERVED_PARAMETER_NAMES.includes( + task.resultingParameterName as string_reserved_parameter_name, + ) + ) { + throw new PipelineLogicError( + spaceTrim( + (block) => ` Parameter name {${task.resultingParameterName}} is reserved, please use different name ${block(pipelineIdentification)} `, - ), - - // <- TODO: [🚞] - ); - } - - definedParameters.add(task.resultingParameterName); - - if (task.jokerParameterNames && task.jokerParameterNames.length > 0) { - if ( - !task.format && - !task.expectations /* <- TODO: Require at least 1 -> min <- expectation to use jokers */ - ) { - throw new PipelineLogicError( - spaceTrim( - (block) => ` + ), + + // <- TODO: [🚞] + ); + } + + definedParameters.add(task.resultingParameterName); + + if (task.jokerParameterNames && task.jokerParameterNames.length > 0) { + if ( + !task.format && + !task.expectations /* <- TODO: Require at least 1 -> min <- expectation to use jokers */ + ) { + throw new PipelineLogicError( + spaceTrim( + (block) => ` Joker parameters are used for {${ - task.resultingParameterName - }} but no expectations are defined + task.resultingParameterName + }} but no expectations are defined ${block(pipelineIdentification)} `, - ), - // <- TODO: [🚞] - ); - } - - for (const joker of task.jokerParameterNames) { - if (!task.dependentParameterNames.includes(joker)) { - throw new PipelineLogicError( - spaceTrim( - (block) => ` + ), + // <- TODO: [🚞] + ); + } + + for (const joker of task.jokerParameterNames) { + if (!task.dependentParameterNames.includes(joker)) { + throw new PipelineLogicError( + spaceTrim( + (block) => ` Parameter \`{${joker}}\` is used for {${ - task.resultingParameterName - }} as joker but not in \`dependentParameterNames\` + task.resultingParameterName + }} as joker but not in \`dependentParameterNames\` ${block(pipelineIdentification)} `, - ), - // <- TODO: [🚞] - ); - } - } - } - - if (task.expectations) { - for (const [unit, { min, max }] of Object.entries(task.expectations)) { - if (min !== undefined && max !== undefined && min > max) { - throw new PipelineLogicError( - spaceTrim( - (block) => ` + ), + // <- TODO: [🚞] + ); + } + } + } + + if (task.expectations) { + for (const [unit, { min, max }] of Object.entries(task.expectations)) { + if (min !== undefined && max !== undefined && min > max) { + throw new PipelineLogicError( + spaceTrim( + (block) => ` Min expectation (=${min}) of ${unit} is higher than max expectation (=${max}) ${block(pipelineIdentification)} `, - ), - // <- TODO: [🚞] - ); - } - - if (min !== undefined && min < 0) { - throw new PipelineLogicError( - spaceTrim( - (block) => ` + ), + // <- TODO: [🚞] + ); + } + + if (min !== undefined && min < 0) { + throw new PipelineLogicError( + spaceTrim( + (block) => ` Min expectation of ${unit} must be zero or positive ${block(pipelineIdentification)} `, - ), - // <- TODO: [🚞] - ); - } - - if (max !== undefined && max <= 0) { - throw new PipelineLogicError( - spaceTrim( - (block) => ` + ), + // <- TODO: [🚞] + ); + } + + if (max !== undefined && max <= 0) { + throw new PipelineLogicError( + spaceTrim( + (block) => ` Max expectation of ${unit} must be positive ${block(pipelineIdentification)} `, - ), - // <- TODO: [🚞] - ); - } - } - } - } - - // Note: Detect circular dependencies - let resovedParameters: ReadonlyArray = pipeline.parameters - .filter(({ isInput }) => isInput) - .map(({ name }) => name); - - // Note: All reserved parameters are resolved - for (const reservedParameterName of RESERVED_PARAMETER_NAMES) { - resovedParameters = [...resovedParameters, reservedParameterName]; - } - - let unresovedTasks: ReadonlyArray = [...pipeline.tasks]; - - let loopLimit = LOOP_LIMIT; - while (unresovedTasks.length > 0) { - if (loopLimit-- < 0) { - // Note: Really UnexpectedError not LimitReachedError - this should not happen and be caught below - throw new UnexpectedError( - spaceTrim( - (block) => ` + ), + // <- TODO: [🚞] + ); + } + } + } + } + + // Note: Detect circular dependencies + let resovedParameters: ReadonlyArray = pipeline.parameters + .filter(({ isInput }) => isInput) + .map(({ name }) => name); + + // Note: All reserved parameters are resolved + for (const reservedParameterName of RESERVED_PARAMETER_NAMES) { + resovedParameters = [...resovedParameters, reservedParameterName]; + } + + let unresovedTasks: ReadonlyArray = [...pipeline.tasks]; + + let loopLimit = LOOP_LIMIT; + while (unresovedTasks.length > 0) { + if (loopLimit-- < 0) { + // Note: Really UnexpectedError not LimitReachedError - this should not happen and be caught below + throw new UnexpectedError( + spaceTrim( + (block) => ` Loop limit reached during detection of circular dependencies in \`validatePipeline\` ${block(pipelineIdentification)} `, - ), - // <- TODO: [🚞] - ); - } - - const currentlyResovedTasks = unresovedTasks.filter((task) => - task.dependentParameterNames.every((name) => resovedParameters.includes(name)), - ); - - if (currentlyResovedTasks.length === 0) { - throw new PipelineLogicError( - // TODO: [🐎] DRY - spaceTrim( - (block) => ` + ), + // <- TODO: [🚞] + ); + } + + const currentlyResovedTasks = unresovedTasks.filter((task) => + task.dependentParameterNames.every((name) => + resovedParameters.includes(name), + ), + ); + + if (currentlyResovedTasks.length === 0) { + throw new PipelineLogicError( + // TODO: [🐎] DRY + spaceTrim( + (block) => ` Can not resolve some parameters: Either you are using a parameter that is not defined, or there are some circular dependencies. @@ -374,56 +395,70 @@ export function validatePipeline_InnerFunction(pipeline: PipelineJson): void { **Can not resolve:** ${block( - unresovedTasks - .map( - ({ resultingParameterName, dependentParameterNames }) => - `- Parameter \`{${resultingParameterName}}\` which depends on ${dependentParameterNames - .map((dependentParameterName) => `\`{${dependentParameterName}}\``) - .join(' and ')}`, - ) - .join('\n'), - )} + unresovedTasks + .map( + ({ + resultingParameterName, + dependentParameterNames, + }) => + `- Parameter \`{${resultingParameterName}}\` which depends on ${dependentParameterNames + .map( + (dependentParameterName) => + `\`{${dependentParameterName}}\``, + ) + .join(" and ")}`, + ) + .join("\n"), + )} **Resolved:** ${block( - resovedParameters - .filter( - (name) => - !RESERVED_PARAMETER_NAMES.includes(name as string_reserved_parameter_name), - ) - .map((name) => `- Parameter \`{${name}}\``) - .join('\n'), - )} + resovedParameters + .filter( + (name) => + !RESERVED_PARAMETER_NAMES.includes( + name as string_reserved_parameter_name, + ), + ) + .map((name) => `- Parameter \`{${name}}\``) + .join("\n"), + )} **Reserved (which are available):** ${block( - resovedParameters - .filter((name) => - RESERVED_PARAMETER_NAMES.includes(name as string_reserved_parameter_name), - ) - .map((name) => `- Parameter \`{${name}}\``) - .join('\n'), - )} + resovedParameters + .filter((name) => + RESERVED_PARAMETER_NAMES.includes( + name as string_reserved_parameter_name, + ), + ) + .map((name) => `- Parameter \`{${name}}\``) + .join("\n"), + )} `, - // <- TODO: [🚞] - ), - ); - } - - resovedParameters = [ - ...resovedParameters, - ...currentlyResovedTasks.map(({ resultingParameterName }) => resultingParameterName), - ]; - - unresovedTasks = unresovedTasks.filter((task) => !currentlyResovedTasks.includes(task)); - } - - // Note: Check that formfactor is corresponding to the pipeline interface - // TODO: !!6 Implement this - // pipeline.formfactorName + // <- TODO: [🚞] + ), + ); + } + + resovedParameters = [ + ...resovedParameters, + ...currentlyResovedTasks.map( + ({ resultingParameterName }) => resultingParameterName, + ), + ]; + + unresovedTasks = unresovedTasks.filter( + (task) => !currentlyResovedTasks.includes(task), + ); + } + + // Note: Check that formfactor is corresponding to the pipeline interface + // TODO: !!6 Implement this + // pipeline.formfactorName } /** diff --git a/src/dialogs/callback/CallbackInterfaceTools.ts b/src/dialogs/callback/CallbackInterfaceTools.ts index 34a67b4ee2..b4098c16ee 100644 --- a/src/dialogs/callback/CallbackInterfaceTools.ts +++ b/src/dialogs/callback/CallbackInterfaceTools.ts @@ -1,7 +1,7 @@ -import { spaceTrim } from 'spacetrim'; -import type { UserInterfaceTools } from '../../execution/UserInterfaceTools'; -import type { UserInterfaceToolsPromptDialogOptions } from '../../execution/UserInterfaceTools'; -import type { CallbackInterfaceToolsOptions } from './CallbackInterfaceToolsOptions'; +import { spaceTrim } from "spacetrim"; +import type { UserInterfaceTools } from "../../execution/UserInterfaceTools"; +import type { UserInterfaceToolsPromptDialogOptions } from "../../execution/UserInterfaceTools"; +import type { CallbackInterfaceToolsOptions } from "./CallbackInterfaceToolsOptions"; /** * Delagates the user interaction to a async callback function @@ -10,25 +10,29 @@ import type { CallbackInterfaceToolsOptions } from './CallbackInterfaceToolsOpti * @public exported from `@promptbook/core` */ export class CallbackInterfaceTools implements UserInterfaceTools { - public constructor(protected readonly options: CallbackInterfaceToolsOptions) {} + public constructor( + protected readonly options: CallbackInterfaceToolsOptions, + ) {} - /** - * Trigger the custom callback function - */ - public async promptDialog(options: UserInterfaceToolsPromptDialogOptions): Promise { - const answer = await this.options.callback(options); + /** + * Trigger the custom callback function + */ + public async promptDialog( + options: UserInterfaceToolsPromptDialogOptions, + ): Promise { + const answer = await this.options.callback(options); - if (this.options.isVerbose) { - console.info( - spaceTrim( - (block) => ` + if (this.options.isVerbose) { + console.info( + spaceTrim( + (block) => ` 📖 ${block(options.promptTitle)} 👤 ${block(answer)} `, - ), - ); - } + ), + ); + } - return answer; - } + return answer; + } } diff --git a/src/dialogs/callback/CallbackInterfaceToolsOptions.ts b/src/dialogs/callback/CallbackInterfaceToolsOptions.ts index df19cb8ff6..115590d38b 100644 --- a/src/dialogs/callback/CallbackInterfaceToolsOptions.ts +++ b/src/dialogs/callback/CallbackInterfaceToolsOptions.ts @@ -1,6 +1,6 @@ -import type { Promisable } from 'type-fest'; -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; -import type { UserInterfaceToolsPromptDialogOptions } from '../../execution/UserInterfaceTools'; +import type { Promisable } from "type-fest"; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; +import type { UserInterfaceToolsPromptDialogOptions } from "../../execution/UserInterfaceTools"; /** * Options for `CallbackInterfaceTools` @@ -8,8 +8,8 @@ import type { UserInterfaceToolsPromptDialogOptions } from '../../execution/User * @public exported from `@promptbook/core` */ export type CallbackInterfaceToolsOptions = CommonToolsOptions & { - /** - * The callback function to be called when promptDialog is called - */ - callback(prompt: UserInterfaceToolsPromptDialogOptions): Promisable; + /** + * The callback function to be called when promptDialog is called + */ + callback(prompt: UserInterfaceToolsPromptDialogOptions): Promisable; }; diff --git a/src/dialogs/simple-prompt/SimplePromptInterfaceTools.ts b/src/dialogs/simple-prompt/SimplePromptInterfaceTools.ts index 6340c0936b..a62190d48a 100644 --- a/src/dialogs/simple-prompt/SimplePromptInterfaceTools.ts +++ b/src/dialogs/simple-prompt/SimplePromptInterfaceTools.ts @@ -1,8 +1,8 @@ -import { spaceTrim } from 'spacetrim'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; -import type { UserInterfaceTools } from '../../execution/UserInterfaceTools'; -import type { UserInterfaceToolsPromptDialogOptions } from '../../execution/UserInterfaceTools'; +import { spaceTrim } from "spacetrim"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; +import type { UserInterfaceTools } from "../../execution/UserInterfaceTools"; +import type { UserInterfaceToolsPromptDialogOptions } from "../../execution/UserInterfaceTools"; /** * Wrapper around `window.prompt` synchronous function that interacts with the user via browser prompt @@ -13,39 +13,41 @@ import type { UserInterfaceToolsPromptDialogOptions } from '../../execution/User * @public exported from `@promptbook/browser` */ export class SimplePromptInterfaceTools implements UserInterfaceTools { - public constructor(protected readonly options: CommonToolsOptions = {}) {} + public constructor(protected readonly options: CommonToolsOptions = {}) {} - /** - * Trigger window.prompt dialog - */ - public async promptDialog(options: UserInterfaceToolsPromptDialogOptions): Promise { - const answer = window.prompt( - spaceTrim( - (block) => ` + /** + * Trigger window.prompt dialog + */ + public async promptDialog( + options: UserInterfaceToolsPromptDialogOptions, + ): Promise { + const answer = window.prompt( + spaceTrim( + (block) => ` ${block(options.promptTitle)} ${block(options.promptMessage)} `, - ), - ); + ), + ); - if (this.options.isVerbose) { - console.info( - spaceTrim( - (block) => ` + if (this.options.isVerbose) { + console.info( + spaceTrim( + (block) => ` 📖 ${block(options.promptTitle)} - 👤 ${block(answer || '🚫 User cancelled prompt')} + 👤 ${block(answer || "🚫 User cancelled prompt")} `, - ), - ); - } + ), + ); + } - if (answer === null) { - throw new PipelineExecutionError('User cancelled prompt'); - } + if (answer === null) { + throw new PipelineExecutionError("User cancelled prompt"); + } - return answer; - } + return answer; + } } /** diff --git a/src/dialogs/user-interface-execution-tools.test.ts b/src/dialogs/user-interface-execution-tools.test.ts index 932dd1d858..4d6c9e79f4 100644 --- a/src/dialogs/user-interface-execution-tools.test.ts +++ b/src/dialogs/user-interface-execution-tools.test.ts @@ -1,45 +1,50 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { compilePipeline } from '../conversion/compilePipeline'; -import { createPipelineExecutor } from '../execution/createPipelineExecutor/00-createPipelineExecutor'; -import { MockedEchoLlmExecutionTools } from '../llm-providers/mocked/MockedEchoLlmExecutionTools'; -import type { PipelineString } from '../pipeline/PipelineString'; -import { CallbackInterfaceTools } from './callback/CallbackInterfaceTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { compilePipeline } from "../conversion/compilePipeline"; +import { createPipelineExecutor } from "../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { MockedEchoLlmExecutionTools } from "../llm-providers/mocked/MockedEchoLlmExecutionTools"; +import type { PipelineString } from "../pipeline/PipelineString"; +import { CallbackInterfaceTools } from "./callback/CallbackInterfaceTools"; -describe('createPipelineExecutor + executing user interface prompts in promptbook', () => { - it('should work when every INPUT PARAMETER defined', async () => { - const pipelineExecutor = await getPipelineExecutor(); +describe("createPipelineExecutor + executing user interface prompts in promptbook", () => { + it("should work when every INPUT PARAMETER defined", async () => { + const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({ thing: 'apple' }).asPromise()).resolves.toMatchObject({ - outputParameters: { - favoriteThing: 'Answer to question "Thing: What is your favorite apple to buy?" is not apple but Pear.', - }, - }); - expect(pipelineExecutor({ thing: 'a cup of coffee' }).asPromise()).resolves.toMatchObject({ - outputParameters: { - favoriteThing: - 'Answer to question "Thing: What is your favorite a cup of coffee to buy?" is not a cup of coffee but Pear.', - }, - }); - }); + expect( + pipelineExecutor({ thing: "apple" }).asPromise(), + ).resolves.toMatchObject({ + outputParameters: { + favoriteThing: + 'Answer to question "Thing: What is your favorite apple to buy?" is not apple but Pear.', + }, + }); + expect( + pipelineExecutor({ thing: "a cup of coffee" }).asPromise(), + ).resolves.toMatchObject({ + outputParameters: { + favoriteThing: + 'Answer to question "Thing: What is your favorite a cup of coffee to buy?" is not a cup of coffee but Pear.', + }, + }); + }); - it('should fail when some INPUT PARAMETER is missing', async () => { - const pipelineExecutor = await getPipelineExecutor(); + it("should fail when some INPUT PARAMETER is missing", async () => { + const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({}).asPromise()).resolves.toMatchObject({ - isSuccessful: false, - errors: [/Parameter `{thing}` is required as an input parameter/i], - }); + expect(pipelineExecutor({}).asPromise()).resolves.toMatchObject({ + isSuccessful: false, + errors: [/Parameter `{thing}` is required as an input parameter/i], + }); - expect(() => pipelineExecutor({}).asPromise()).rejects.toThrowError( - /Parameter `\{thing\}` is required as an input parameter/i, - ); - }); + expect(() => pipelineExecutor({}).asPromise()).rejects.toThrowError( + /Parameter `\{thing\}` is required as an input parameter/i, + ); + }); }); async function getPipelineExecutor() { - const pipeline = await compilePipeline( - spaceTrim(` + const pipeline = await compilePipeline( + spaceTrim(` # Example prompt Show how to use prompt dialog @@ -60,21 +65,21 @@ async function getPipelineExecutor() { -> {favoriteThing} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - ); - const pipelineExecutor = createPipelineExecutor({ - pipeline, - tools: { - llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), - script: [], - userInterface: new CallbackInterfaceTools({ - isVerbose: true, - async callback({ promptTitle, promptMessage, defaultValue }) { - return `Answer to question "${promptTitle}: ${promptMessage}" is not ${defaultValue} but Pear.`; - }, - }), - }, - }); + // <- TODO: [📼] Use`book\`` string literal notation + ); + const pipelineExecutor = createPipelineExecutor({ + pipeline, + tools: { + llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), + script: [], + userInterface: new CallbackInterfaceTools({ + isVerbose: true, + async callback({ promptTitle, promptMessage, defaultValue }) { + return `Answer to question "${promptTitle}: ${promptMessage}" is not ${defaultValue} but Pear.`; + }, + }), + }, + }); - return pipelineExecutor; + return pipelineExecutor; } diff --git a/src/errors/0-BoilerplateError.ts b/src/errors/0-BoilerplateError.ts index 4ee90d849b..d9bca294de 100644 --- a/src/errors/0-BoilerplateError.ts +++ b/src/errors/0-BoilerplateError.ts @@ -4,11 +4,11 @@ * @public exported from `@promptbook/core` */ export class BoilerplateError extends Error { - public readonly name = 'BoilerplateError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, BoilerplateError.prototype); - } + public readonly name = "BoilerplateError"; + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, BoilerplateError.prototype); + } } /** diff --git a/src/errors/0-index.ts b/src/errors/0-index.ts index e0463e33cc..1cf6ca1562 100644 --- a/src/errors/0-index.ts +++ b/src/errors/0-index.ts @@ -1,18 +1,18 @@ -import { CsvFormatError } from '../formats/csv/CsvFormatError'; -import { AbstractFormatError } from './AbstractFormatError'; -import { CollectionError } from './CollectionError'; -import { EnvironmentMismatchError } from './EnvironmentMismatchError'; -import { ExpectError } from './ExpectError'; -import { KnowledgeScrapeError } from './KnowledgeScrapeError'; -import { LimitReachedError } from './LimitReachedError'; -import { MissingToolsError } from './MissingToolsError'; -import { NotFoundError } from './NotFoundError'; -import { NotYetImplementedError } from './NotYetImplementedError'; -import { ParseError } from './ParseError'; -import { PipelineExecutionError } from './PipelineExecutionError'; -import { PipelineLogicError } from './PipelineLogicError'; -import { PipelineUrlError } from './PipelineUrlError'; -import { UnexpectedError } from './UnexpectedError'; +import { CsvFormatError } from "../formats/csv/CsvFormatError"; +import { AbstractFormatError } from "./AbstractFormatError"; +import { CollectionError } from "./CollectionError"; +import { EnvironmentMismatchError } from "./EnvironmentMismatchError"; +import { ExpectError } from "./ExpectError"; +import { KnowledgeScrapeError } from "./KnowledgeScrapeError"; +import { LimitReachedError } from "./LimitReachedError"; +import { MissingToolsError } from "./MissingToolsError"; +import { NotFoundError } from "./NotFoundError"; +import { NotYetImplementedError } from "./NotYetImplementedError"; +import { ParseError } from "./ParseError"; +import { PipelineExecutionError } from "./PipelineExecutionError"; +import { PipelineLogicError } from "./PipelineLogicError"; +import { PipelineUrlError } from "./PipelineUrlError"; +import { UnexpectedError } from "./UnexpectedError"; /** * Index of all custom errors @@ -20,22 +20,22 @@ import { UnexpectedError } from './UnexpectedError'; * @public exported from `@promptbook/core` */ export const PROMPTBOOK_ERRORS = { - AbstractFormatError, - CsvFormatError, - CollectionError, - EnvironmentMismatchError, - ExpectError, - KnowledgeScrapeError, - LimitReachedError, - MissingToolsError, - NotFoundError, - NotYetImplementedError, - ParseError, - PipelineExecutionError, - PipelineLogicError, - PipelineUrlError, - UnexpectedError, - // TODO: [🪑]> VersionMismatchError, + AbstractFormatError, + CsvFormatError, + CollectionError, + EnvironmentMismatchError, + ExpectError, + KnowledgeScrapeError, + LimitReachedError, + MissingToolsError, + NotFoundError, + NotYetImplementedError, + ParseError, + PipelineExecutionError, + PipelineLogicError, + PipelineUrlError, + UnexpectedError, + // TODO: [🪑]> VersionMismatchError, } as const; /** @@ -44,15 +44,15 @@ export const PROMPTBOOK_ERRORS = { * @private for internal usage */ export const COMMON_JAVASCRIPT_ERRORS = { - Error, - EvalError, - RangeError, - ReferenceError, - SyntaxError, - TypeError, - URIError, - AggregateError, - /* + Error, + EvalError, + RangeError, + ReferenceError, + SyntaxError, + TypeError, + URIError, + AggregateError, + /* Note: Not widely supported > InternalError, > ModuleError, @@ -68,8 +68,8 @@ export const COMMON_JAVASCRIPT_ERRORS = { * @private for internal usage */ export const ALL_ERRORS = { - ...PROMPTBOOK_ERRORS, - ...COMMON_JAVASCRIPT_ERRORS, + ...PROMPTBOOK_ERRORS, + ...COMMON_JAVASCRIPT_ERRORS, } as const; /** diff --git a/src/errors/AbstractFormatError.ts b/src/errors/AbstractFormatError.ts index fed515d539..0b6ecebbf3 100644 --- a/src/errors/AbstractFormatError.ts +++ b/src/errors/AbstractFormatError.ts @@ -7,11 +7,11 @@ * @public exported from `@promptbook/core` */ export class AbstractFormatError extends Error { - // Note: To allow instanceof do not put here error `name` - // public readonly name = 'AbstractFormatError'; + // Note: To allow instanceof do not put here error `name` + // public readonly name = 'AbstractFormatError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, AbstractFormatError.prototype); - } + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, AbstractFormatError.prototype); + } } diff --git a/src/errors/CollectionError.ts b/src/errors/CollectionError.ts index b759fb26fd..dd1b752888 100644 --- a/src/errors/CollectionError.ts +++ b/src/errors/CollectionError.ts @@ -4,9 +4,9 @@ * @public exported from `@promptbook/core` */ export class CollectionError extends Error { - public readonly name = 'CollectionError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, CollectionError.prototype); - } + public readonly name = "CollectionError"; + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, CollectionError.prototype); + } } diff --git a/src/errors/EnvironmentMismatchError.ts b/src/errors/EnvironmentMismatchError.ts index ec677f9524..e33cda0970 100644 --- a/src/errors/EnvironmentMismatchError.ts +++ b/src/errors/EnvironmentMismatchError.ts @@ -1,12 +1,12 @@ /** * This error type indicates that you try to use a feature that is not available in the current environment - * + * * @public exported from `@promptbook/core` */ export class EnvironmentMismatchError extends Error { - public readonly name = 'EnvironmentMismatchError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, EnvironmentMismatchError.prototype); - } + public readonly name = "EnvironmentMismatchError"; + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, EnvironmentMismatchError.prototype); + } } diff --git a/src/errors/ExpectError.ts b/src/errors/ExpectError.ts index 35af9a6487..5585585367 100644 --- a/src/errors/ExpectError.ts +++ b/src/errors/ExpectError.ts @@ -7,9 +7,9 @@ * Note: This is a kindof subtype of PipelineExecutionError */ export class ExpectError extends Error { - public readonly name = 'ExpectError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, ExpectError.prototype); - } + public readonly name = "ExpectError"; + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, ExpectError.prototype); + } } diff --git a/src/errors/KnowledgeScrapeError.ts b/src/errors/KnowledgeScrapeError.ts index 4cbe0d0495..a8b3a3701b 100644 --- a/src/errors/KnowledgeScrapeError.ts +++ b/src/errors/KnowledgeScrapeError.ts @@ -4,9 +4,9 @@ * @public exported from `@promptbook/core` */ export class KnowledgeScrapeError extends Error { - public readonly name = 'KnowledgeScrapeError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, KnowledgeScrapeError.prototype); - } + public readonly name = "KnowledgeScrapeError"; + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, KnowledgeScrapeError.prototype); + } } diff --git a/src/errors/LimitReachedError.ts b/src/errors/LimitReachedError.ts index 2c683aeed9..3ccc878d23 100644 --- a/src/errors/LimitReachedError.ts +++ b/src/errors/LimitReachedError.ts @@ -1,12 +1,12 @@ /** * This error type indicates that some limit was reached - * + * * @public exported from `@promptbook/core` */ export class LimitReachedError extends Error { - public readonly name = 'LimitReachedError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, LimitReachedError.prototype); - } + public readonly name = "LimitReachedError"; + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, LimitReachedError.prototype); + } } diff --git a/src/errors/MissingToolsError.ts b/src/errors/MissingToolsError.ts index e09aa1839b..575be7bbb5 100644 --- a/src/errors/MissingToolsError.ts +++ b/src/errors/MissingToolsError.ts @@ -1,4 +1,4 @@ -import { spaceTrim } from 'spacetrim'; +import { spaceTrim } from "spacetrim"; /** * This error type indicates that some tools are missing for pipeline execution or preparation @@ -6,18 +6,18 @@ import { spaceTrim } from 'spacetrim'; * @public exported from `@promptbook/core` */ export class MissingToolsError extends Error { - public readonly name = 'MissingToolsError'; - public constructor(message: string) { - super( - spaceTrim( - (block) => ` + public readonly name = "MissingToolsError"; + public constructor(message: string) { + super( + spaceTrim( + (block) => ` ${block(message)} Note: You have probbably forgot to provide some tools for pipeline execution or preparation `, - ), - ); - Object.setPrototypeOf(this, MissingToolsError.prototype); - } + ), + ); + Object.setPrototypeOf(this, MissingToolsError.prototype); + } } diff --git a/src/errors/NotFoundError.ts b/src/errors/NotFoundError.ts index 31e44badf6..7327d62eb6 100644 --- a/src/errors/NotFoundError.ts +++ b/src/errors/NotFoundError.ts @@ -1,12 +1,12 @@ /** * This error indicates that promptbook not found in the collection - * + * * @public exported from `@promptbook/core` */ export class NotFoundError extends Error { - public readonly name = 'NotFoundError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, NotFoundError.prototype); - } + public readonly name = "NotFoundError"; + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, NotFoundError.prototype); + } } diff --git a/src/errors/NotYetImplementedError.ts b/src/errors/NotYetImplementedError.ts index f6e78c000c..4d8a83c091 100644 --- a/src/errors/NotYetImplementedError.ts +++ b/src/errors/NotYetImplementedError.ts @@ -1,4 +1,4 @@ -import { spaceTrim } from 'spacetrim'; +import { spaceTrim } from "spacetrim"; /** * This error type indicates that some part of the code is not implemented yet @@ -6,11 +6,11 @@ import { spaceTrim } from 'spacetrim'; * @public exported from `@promptbook/core` */ export class NotYetImplementedError extends Error { - public readonly name = 'NotYetImplementedError'; - public constructor(message: string) { - super( - spaceTrim( - (block) => ` + public readonly name = "NotYetImplementedError"; + public constructor(message: string) { + super( + spaceTrim( + (block) => ` ${block(message)} Note: This feature is not implemented yet but it will be soon. @@ -21,8 +21,8 @@ export class NotYetImplementedError extends Error { Or contact us on pavol@ptbk.io `, - ), - ); - Object.setPrototypeOf(this, NotYetImplementedError.prototype); - } + ), + ); + Object.setPrototypeOf(this, NotYetImplementedError.prototype); + } } diff --git a/src/errors/ParseError.ts b/src/errors/ParseError.ts index d765a92463..63b7a18bd7 100644 --- a/src/errors/ParseError.ts +++ b/src/errors/ParseError.ts @@ -4,11 +4,11 @@ * @public exported from `@promptbook/core` */ export class ParseError extends Error { - public readonly name = 'ParseError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, ParseError.prototype); - } + public readonly name = "ParseError"; + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, ParseError.prototype); + } } /** diff --git a/src/errors/PipelineExecutionError.ts b/src/errors/PipelineExecutionError.ts index e94c9ea825..9c1a75cb76 100644 --- a/src/errors/PipelineExecutionError.ts +++ b/src/errors/PipelineExecutionError.ts @@ -1,12 +1,12 @@ /** * This error indicates errors during the execution of the pipeline - * + * * @public exported from `@promptbook/core` */ export class PipelineExecutionError extends Error { - public readonly name = 'PipelineExecutionError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, PipelineExecutionError.prototype); - } + public readonly name = "PipelineExecutionError"; + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, PipelineExecutionError.prototype); + } } diff --git a/src/errors/PipelineLogicError.ts b/src/errors/PipelineLogicError.ts index 34c0a76ad7..5fd19213eb 100644 --- a/src/errors/PipelineLogicError.ts +++ b/src/errors/PipelineLogicError.ts @@ -4,9 +4,9 @@ * @public exported from `@promptbook/core` */ export class PipelineLogicError extends Error { - public readonly name = 'PipelineLogicError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, PipelineLogicError.prototype); - } + public readonly name = "PipelineLogicError"; + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, PipelineLogicError.prototype); + } } diff --git a/src/errors/PipelineUrlError.ts b/src/errors/PipelineUrlError.ts index ed852f6aff..a9a337c98a 100644 --- a/src/errors/PipelineUrlError.ts +++ b/src/errors/PipelineUrlError.ts @@ -4,9 +4,9 @@ * @public exported from `@promptbook/core` */ export class PipelineUrlError extends Error { - public readonly name = 'PipelineUrlError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, PipelineUrlError.prototype); - } + public readonly name = "PipelineUrlError"; + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, PipelineUrlError.prototype); + } } diff --git a/src/errors/UnexpectedError.ts b/src/errors/UnexpectedError.ts index 7de054b753..baff6038c5 100644 --- a/src/errors/UnexpectedError.ts +++ b/src/errors/UnexpectedError.ts @@ -1,6 +1,6 @@ -import { spaceTrim } from 'spacetrim'; -import { ADMIN_EMAIL } from '../config'; -import { getErrorReportUrl } from './utils/getErrorReportUrl'; +import { spaceTrim } from "spacetrim"; +import { ADMIN_EMAIL } from "../config"; +import { getErrorReportUrl } from "./utils/getErrorReportUrl"; /** * This error type indicates that the error should not happen and its last check before crashing with some other error @@ -8,11 +8,11 @@ import { getErrorReportUrl } from './utils/getErrorReportUrl'; * @public exported from `@promptbook/core` */ export class UnexpectedError extends Error { - public readonly name = 'UnexpectedError'; - public constructor(message: string) { - super( - spaceTrim( - (block) => ` + public readonly name = "UnexpectedError"; + public constructor(message: string) { + super( + spaceTrim( + (block) => ` ${block(message)} Note: This error should not happen. @@ -24,8 +24,8 @@ export class UnexpectedError extends Error { Or contact us on ${ADMIN_EMAIL} `, - ), - ); - Object.setPrototypeOf(this, UnexpectedError.prototype); - } + ), + ); + Object.setPrototypeOf(this, UnexpectedError.prototype); + } } diff --git a/src/errors/utils/ErrorJson.ts b/src/errors/utils/ErrorJson.ts index c21a9dd581..38d1d691c3 100644 --- a/src/errors/utils/ErrorJson.ts +++ b/src/errors/utils/ErrorJson.ts @@ -1,4 +1,4 @@ -import type { ALL_ERRORS } from '../0-index'; // <- TODO: [🤛] This should be this automatically repared to type import, but its not +import type { ALL_ERRORS } from "../0-index"; // <- TODO: [🤛] This should be this automatically repared to type import, but its not /** * Represents a serialized error or custom Promptbook error @@ -6,18 +6,18 @@ import type { ALL_ERRORS } from '../0-index'; // <- TODO: [🤛] This should be * Note: [🚉] This is fully serializable as JSON */ export type ErrorJson = { - /** - * The type of the error - */ - readonly name: keyof typeof ALL_ERRORS; + /** + * The type of the error + */ + readonly name: keyof typeof ALL_ERRORS; - /** - * The message of the error - */ - readonly message: string; + /** + * The message of the error + */ + readonly message: string; - /** - * The stack trace of the error - */ - readonly stack?: string; + /** + * The stack trace of the error + */ + readonly stack?: string; }; diff --git a/src/errors/utils/deserializeError.test.ts b/src/errors/utils/deserializeError.test.ts index 1e426ce7bb..6d4cf9203d 100644 --- a/src/errors/utils/deserializeError.test.ts +++ b/src/errors/utils/deserializeError.test.ts @@ -1,16 +1,18 @@ -import { describe, expect, it } from '@jest/globals'; -import { CollectionError } from '../CollectionError'; -import { deserializeError } from './deserializeError'; +import { describe, expect, it } from "@jest/globals"; +import { CollectionError } from "../CollectionError"; +import { deserializeError } from "./deserializeError"; -describe('how `deserializeError` works', () => { - it('should deserialize vanilla Error', () => - expect(deserializeError({ name: 'Error', message: 'Foo' })).toEqual(new Error('Foo'))); +describe("how `deserializeError` works", () => { + it("should deserialize vanilla Error", () => + expect(deserializeError({ name: "Error", message: "Foo" })).toEqual( + new Error("Foo"), + )); - it('should deserialize `CollectionError`', () => - expect( - deserializeError({ - name: 'CollectionError', - message: 'Bar', - }), - ).toEqual(new CollectionError('Bar'))); + it("should deserialize `CollectionError`", () => + expect( + deserializeError({ + name: "CollectionError", + message: "Bar", + }), + ).toEqual(new CollectionError("Bar"))); }); diff --git a/src/errors/utils/deserializeError.ts b/src/errors/utils/deserializeError.ts index d5bd699397..00fe6aaba2 100644 --- a/src/errors/utils/deserializeError.ts +++ b/src/errors/utils/deserializeError.ts @@ -1,6 +1,6 @@ -import spaceTrim from 'spacetrim'; -import { ALL_ERRORS } from '../0-index'; -import type { ErrorJson } from './ErrorJson'; +import spaceTrim from "spacetrim"; +import { ALL_ERRORS } from "../0-index"; +import type { ErrorJson } from "./ErrorJson"; /** * Deserializes the error object @@ -8,25 +8,25 @@ import type { ErrorJson } from './ErrorJson'; * @public exported from `@promptbook/utils` */ export function deserializeError(error: ErrorJson): Error { - const { name, stack } = error; - let { message } = error; - let ErrorClass = ALL_ERRORS[error.name as keyof typeof ALL_ERRORS]; + const { name, stack } = error; + let { message } = error; + let ErrorClass = ALL_ERRORS[error.name as keyof typeof ALL_ERRORS]; - if (ErrorClass === undefined) { - ErrorClass = Error; - message = `${name}: ${message}`; - } + if (ErrorClass === undefined) { + ErrorClass = Error; + message = `${name}: ${message}`; + } - if (stack !== undefined && stack !== '') { - message = spaceTrim( - (block) => ` + if (stack !== undefined && stack !== "") { + message = spaceTrim( + (block) => ` ${block(message)} Original stack trace: - ${block(stack || '')} + ${block(stack || "")} `, - ); - } + ); + } - return new ErrorClass(message); + return new ErrorClass(message); } diff --git a/src/errors/utils/getErrorReportUrl.ts b/src/errors/utils/getErrorReportUrl.ts index 6bd687a89d..91aabaec0b 100644 --- a/src/errors/utils/getErrorReportUrl.ts +++ b/src/errors/utils/getErrorReportUrl.ts @@ -1,8 +1,8 @@ -import spaceTrim from 'spacetrim'; -import { ADMIN_GITHUB_NAME } from '../../config'; -import { NAME } from '../../config'; -import { BOOK_LANGUAGE_VERSION } from '../../version'; -import { PROMPTBOOK_ENGINE_VERSION } from '../../version'; +import spaceTrim from "spacetrim"; +import { ADMIN_GITHUB_NAME } from "../../config"; +import { NAME } from "../../config"; +import { BOOK_LANGUAGE_VERSION } from "../../version"; +import { PROMPTBOOK_ENGINE_VERSION } from "../../version"; /** * Make error report URL for the given error @@ -10,16 +10,16 @@ import { PROMPTBOOK_ENGINE_VERSION } from '../../version'; * @private private within the repository */ export function getErrorReportUrl(error: Error): URL { - const report = { - title: `🐜 Error report from ${NAME}`, - body: spaceTrim( - (block) => ` + const report = { + title: `🐜 Error report from ${NAME}`, + body: spaceTrim( + (block) => ` - \`${error.name || 'Error'}\` has occurred in the [${NAME}], please look into it @${ADMIN_GITHUB_NAME}. + \`${error.name || "Error"}\` has occurred in the [${NAME}], please look into it @${ADMIN_GITHUB_NAME}. \`\`\` - ${block(error.message || '(no error message)')} + ${block(error.message || "(no error message)")} \`\`\` @@ -35,19 +35,21 @@ export function getErrorReportUrl(error: Error): URL { ## Stack trace: \`\`\`stacktrace - ${block(error.stack || '(empty)')} + ${block(error.stack || "(empty)")} \`\`\` `, // <- TODO: Add all registrations - llm providers and scrapers - ), - }; - - const reportUrl = new URL(`https://github.com/webgptorg/promptbook/issues/new`); - reportUrl.searchParams.set('labels', 'bug'); - reportUrl.searchParams.set('assignees', ADMIN_GITHUB_NAME); - reportUrl.searchParams.set('title', report.title); - reportUrl.searchParams.set('body', report.body); - - return reportUrl; + ), + }; + + const reportUrl = new URL( + `https://github.com/webgptorg/promptbook/issues/new`, + ); + reportUrl.searchParams.set("labels", "bug"); + reportUrl.searchParams.set("assignees", ADMIN_GITHUB_NAME); + reportUrl.searchParams.set("title", report.title); + reportUrl.searchParams.set("body", report.body); + + return reportUrl; } diff --git a/src/errors/utils/serializeError.test.ts b/src/errors/utils/serializeError.test.ts index 533537eeac..3e76d2a8a2 100644 --- a/src/errors/utils/serializeError.test.ts +++ b/src/errors/utils/serializeError.test.ts @@ -1,15 +1,19 @@ -import { describe, expect, it } from '@jest/globals'; -import { CollectionError } from '../CollectionError'; -import { serializeError } from './serializeError'; +import { describe, expect, it } from "@jest/globals"; +import { CollectionError } from "../CollectionError"; +import { serializeError } from "./serializeError"; -describe('how `serializeError` works', () => { - it('should serialize vanilla Error', () => - expect(serializeError(new Error('Foo'))).toEqual({ name: 'Error', message: 'Foo', stack: expect.any(String) })); +describe("how `serializeError` works", () => { + it("should serialize vanilla Error", () => + expect(serializeError(new Error("Foo"))).toEqual({ + name: "Error", + message: "Foo", + stack: expect.any(String), + })); - it('should deserialize `CollectionError`', () => - expect(serializeError(new CollectionError('Bar'))).toEqual({ - name: 'CollectionError', - message: 'Bar', - stack: expect.any(String), - })); + it("should deserialize `CollectionError`", () => + expect(serializeError(new CollectionError("Bar"))).toEqual({ + name: "CollectionError", + message: "Bar", + stack: expect.any(String), + })); }); diff --git a/src/errors/utils/serializeError.ts b/src/errors/utils/serializeError.ts index 5cc81fc0f6..d18841438c 100644 --- a/src/errors/utils/serializeError.ts +++ b/src/errors/utils/serializeError.ts @@ -1,6 +1,6 @@ -import spaceTrim from 'spacetrim'; -import { ALL_ERRORS } from '../0-index'; -import type { ErrorJson } from './ErrorJson'; +import spaceTrim from "spacetrim"; +import { ALL_ERRORS } from "../0-index"; +import type { ErrorJson } from "./ErrorJson"; /** * Serializes an error into a [🚉] JSON-serializable object @@ -9,27 +9,25 @@ import type { ErrorJson } from './ErrorJson'; */ export function serializeError(error: Error): ErrorJson { - const { name, message, stack } = error; + const { name, message, stack } = error; - if ( - !Object.keys(ALL_ERRORS).includes(name) - ) { - console.error( - spaceTrim( - (block) => ` + if (!Object.keys(ALL_ERRORS).includes(name)) { + console.error( + spaceTrim( + (block) => ` Cannot serialize error with name "${name}" ${block(stack || message)} `, - ), - ); - } + ), + ); + } - return { - name: name as ErrorJson['name'], - message, - stack, - }; + return { + name: name as ErrorJson["name"], + message, + stack, + }; } diff --git a/src/executables/$provideExecutablesForNode.ts b/src/executables/$provideExecutablesForNode.ts index cf71107242..b19c55e7cc 100644 --- a/src/executables/$provideExecutablesForNode.ts +++ b/src/executables/$provideExecutablesForNode.ts @@ -1,33 +1,40 @@ -import { DEFAULT_IS_AUTO_INSTALLED } from '../config'; -import { DEFAULT_IS_VERBOSE } from '../config'; -import { EnvironmentMismatchError } from '../errors/EnvironmentMismatchError'; -import type { Executables } from '../execution/Executables'; -import type { PrepareAndScrapeOptions } from '../prepare/PrepareAndScrapeOptions'; -import { $isRunningInNode } from '../utils/environment/$isRunningInNode'; -import { TODO_USE } from '../utils/organization/TODO_USE'; -import { locateLibreoffice } from './apps/locateLibreoffice'; -import { locatePandoc } from './apps/locatePandoc'; +import { DEFAULT_IS_AUTO_INSTALLED } from "../config"; +import { DEFAULT_IS_VERBOSE } from "../config"; +import { EnvironmentMismatchError } from "../errors/EnvironmentMismatchError"; +import type { Executables } from "../execution/Executables"; +import type { PrepareAndScrapeOptions } from "../prepare/PrepareAndScrapeOptions"; +import { $isRunningInNode } from "../utils/environment/$isRunningInNode"; +import { TODO_USE } from "../utils/organization/TODO_USE"; +import { locateLibreoffice } from "./apps/locateLibreoffice"; +import { locatePandoc } from "./apps/locatePandoc"; /** * @@@ * * @public exported from `@promptbook/node` */ -export async function $provideExecutablesForNode(options?: PrepareAndScrapeOptions): Promise { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment'); - } +export async function $provideExecutablesForNode( + options?: PrepareAndScrapeOptions, +): Promise { + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Function `$getScrapersForNode` works only in Node.js environment", + ); + } - const { isAutoInstalled = DEFAULT_IS_AUTO_INSTALLED, isVerbose = DEFAULT_IS_VERBOSE } = options || {}; + const { + isAutoInstalled = DEFAULT_IS_AUTO_INSTALLED, + isVerbose = DEFAULT_IS_VERBOSE, + } = options || {}; - TODO_USE(isAutoInstalled); // <- TODO: [🔱][🧠] Auto-install the executables - TODO_USE(isVerbose); + TODO_USE(isAutoInstalled); // <- TODO: [🔱][🧠] Auto-install the executables + TODO_USE(isVerbose); - return { - pandocPath: (await locatePandoc()) || undefined, - libreOfficePath: (await locateLibreoffice()) || undefined, - // <- TODO: [🧠] `null` vs `undefined` - }; + return { + pandocPath: (await locatePandoc()) || undefined, + libreOfficePath: (await locateLibreoffice()) || undefined, + // <- TODO: [🧠] `null` vs `undefined` + }; } /** diff --git a/src/executables/apps/locateLibreoffice.test.ts b/src/executables/apps/locateLibreoffice.test.ts index d81100d439..8e36b0cba2 100644 --- a/src/executables/apps/locateLibreoffice.test.ts +++ b/src/executables/apps/locateLibreoffice.test.ts @@ -1,8 +1,8 @@ -import { locateLibreoffice } from './locateLibreoffice'; +import { locateLibreoffice } from "./locateLibreoffice"; -describe('locating the LibreOffice', () => { - it('should locate LibreOffice', async () => { - await expect(locateLibreoffice()).resolves.toMatch(/office/i); - expect.assertions(1); - }); +describe("locating the LibreOffice", () => { + it("should locate LibreOffice", async () => { + await expect(locateLibreoffice()).resolves.toMatch(/office/i); + expect.assertions(1); + }); }); diff --git a/src/executables/apps/locateLibreoffice.ts b/src/executables/apps/locateLibreoffice.ts index 148e12c5f7..938fb38e75 100644 --- a/src/executables/apps/locateLibreoffice.ts +++ b/src/executables/apps/locateLibreoffice.ts @@ -1,5 +1,5 @@ -import type { string_executable_path } from '../../types/typeAliases'; -import { locateApp } from '../locateApp'; +import type { string_executable_path } from "../../types/typeAliases"; +import { locateApp } from "../locateApp"; /** * @@@ @@ -7,12 +7,12 @@ import { locateApp } from '../locateApp'; * @private within the repository */ export function locateLibreoffice(): Promise { - return locateApp({ - appName: 'Libreoffice', - linuxWhich: 'libreoffice', - windowsSuffix: '\\LibreOffice\\program\\soffice.exe', - macOsName: 'LibreOffice', - }); + return locateApp({ + appName: "Libreoffice", + linuxWhich: "libreoffice", + windowsSuffix: "\\LibreOffice\\program\\soffice.exe", + macOsName: "LibreOffice", + }); } /** diff --git a/src/executables/apps/locatePandoc.test.ts b/src/executables/apps/locatePandoc.test.ts index 2380b875cf..862c56cdb0 100644 --- a/src/executables/apps/locatePandoc.test.ts +++ b/src/executables/apps/locatePandoc.test.ts @@ -1,8 +1,8 @@ -import { locatePandoc } from './locatePandoc'; +import { locatePandoc } from "./locatePandoc"; -describe('locating the Pandoc - The universal markup converter', () => { - it('should locate Pandoc', async () => { - await expect(locatePandoc()).resolves.toMatch(/pandoc/i); - expect.assertions(1); - }); +describe("locating the Pandoc - The universal markup converter", () => { + it("should locate Pandoc", async () => { + await expect(locatePandoc()).resolves.toMatch(/pandoc/i); + expect.assertions(1); + }); }); diff --git a/src/executables/apps/locatePandoc.ts b/src/executables/apps/locatePandoc.ts index 4f9103f1b9..274d5974b1 100644 --- a/src/executables/apps/locatePandoc.ts +++ b/src/executables/apps/locatePandoc.ts @@ -1,5 +1,5 @@ -import type { string_executable_path } from '../../types/typeAliases'; -import { locateApp } from '../locateApp'; +import type { string_executable_path } from "../../types/typeAliases"; +import { locateApp } from "../locateApp"; /** * @@@ @@ -7,12 +7,12 @@ import { locateApp } from '../locateApp'; * @private within the repository */ export function locatePandoc(): Promise { - return locateApp({ - appName: 'Pandoc', - linuxWhich: 'pandoc', - windowsSuffix: '\\Pandoc\\pandoc.exe', - macOsName: 'Pandoc', - }); + return locateApp({ + appName: "Pandoc", + linuxWhich: "pandoc", + windowsSuffix: "\\Pandoc\\pandoc.exe", + macOsName: "Pandoc", + }); } /** diff --git a/src/executables/locateApp.test.ts b/src/executables/locateApp.test.ts index cd6345aa94..20ceb2ca29 100644 --- a/src/executables/locateApp.test.ts +++ b/src/executables/locateApp.test.ts @@ -1,15 +1,15 @@ -import { locateApp } from './locateApp'; +import { locateApp } from "./locateApp"; -describe('locating the app', () => { - it('should locate Chrome', async () => { - await expect( - locateApp({ - appName: 'Chrome', - linuxWhich: 'google-chrome', - windowsSuffix: '\\Google\\Chrome\\Application\\chrome.exe', - macOsName: 'Google Chrome', - }), - ).resolves.toMatch(/chrome/i); - expect.assertions(1); - }); +describe("locating the app", () => { + it("should locate Chrome", async () => { + await expect( + locateApp({ + appName: "Chrome", + linuxWhich: "google-chrome", + windowsSuffix: "\\Google\\Chrome\\Application\\chrome.exe", + macOsName: "Google Chrome", + }), + ).resolves.toMatch(/chrome/i); + expect.assertions(1); + }); }); diff --git a/src/executables/locateApp.ts b/src/executables/locateApp.ts index 6a510b19c0..3d09c0f766 100644 --- a/src/executables/locateApp.ts +++ b/src/executables/locateApp.ts @@ -1,34 +1,34 @@ -import type { RequireAtLeastOne } from 'type-fest'; -import { EnvironmentMismatchError } from '../errors/EnvironmentMismatchError'; -import type { string_executable_path } from '../types/typeAliases'; -import { $isRunningInNode } from '../utils/environment/$isRunningInNode'; -import { locateAppOnLinux } from './platforms/locateAppOnLinux'; -import { locateAppOnMacOs } from './platforms/locateAppOnMacOs'; -import { locateAppOnWindows } from './platforms/locateAppOnWindows'; +import type { RequireAtLeastOne } from "type-fest"; +import { EnvironmentMismatchError } from "../errors/EnvironmentMismatchError"; +import type { string_executable_path } from "../types/typeAliases"; +import { $isRunningInNode } from "../utils/environment/$isRunningInNode"; +import { locateAppOnLinux } from "./platforms/locateAppOnLinux"; +import { locateAppOnMacOs } from "./platforms/locateAppOnMacOs"; +import { locateAppOnWindows } from "./platforms/locateAppOnWindows"; /** * Options for locating any application */ export interface LocateAppOptions { - /** - * Name of the application - */ - appName: string; + /** + * Name of the application + */ + appName: string; - /** - * Name of the executable on Linux - */ - linuxWhich?: string; + /** + * Name of the executable on Linux + */ + linuxWhich?: string; - /** - * Path suffix on Windows - */ - windowsSuffix?: string; + /** + * Path suffix on Windows + */ + windowsSuffix?: string; - /** - * Name of the application on macOS - */ - macOsName?: string; + /** + * Name of the application on macOS + */ + macOsName?: string; } /** @@ -37,33 +37,38 @@ export interface LocateAppOptions { * @private within the repository */ export function locateApp( - options: RequireAtLeastOne, + options: RequireAtLeastOne< + LocateAppOptions, + "linuxWhich" | "windowsSuffix" | "macOsName" + >, ): Promise { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError('Locating apps works only in Node.js environment'); - } + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Locating apps works only in Node.js environment", + ); + } - const { appName, linuxWhich, windowsSuffix, macOsName } = options; + const { appName, linuxWhich, windowsSuffix, macOsName } = options; - if (process.platform === 'win32') { - if (windowsSuffix) { - return locateAppOnWindows({ appName, windowsSuffix }); - } else { - throw new Error(`${appName} is not available on Windows.`); - } - } else if (process.platform === 'darwin') { - if (macOsName) { - return locateAppOnMacOs({ macOsName }); - } else { - throw new Error(`${appName} is not available on macOS.`); - } - } else { - if (linuxWhich) { - return locateAppOnLinux({ linuxWhich }); - } else { - throw new Error(`${appName} is not available on Linux.`); - } - } + if (process.platform === "win32") { + if (windowsSuffix) { + return locateAppOnWindows({ appName, windowsSuffix }); + } else { + throw new Error(`${appName} is not available on Windows.`); + } + } else if (process.platform === "darwin") { + if (macOsName) { + return locateAppOnMacOs({ macOsName }); + } else { + throw new Error(`${appName} is not available on macOS.`); + } + } else { + if (linuxWhich) { + return locateAppOnLinux({ linuxWhich }); + } else { + throw new Error(`${appName} is not available on Linux.`); + } + } } /** diff --git a/src/executables/platforms/locateAppOnLinux.ts b/src/executables/platforms/locateAppOnLinux.ts index 2d4acd1b52..ce24cb256f 100644 --- a/src/executables/platforms/locateAppOnLinux.ts +++ b/src/executables/platforms/locateAppOnLinux.ts @@ -1,6 +1,6 @@ -import { $execCommand } from '../../utils/execCommand/$execCommand'; -import type { string_executable_path } from '../../types/typeAliases'; -import type { LocateAppOptions } from '../locateApp'; +import type { string_executable_path } from "../../types/typeAliases"; +import { $execCommand } from "../../utils/execCommand/$execCommand"; +import type { LocateAppOptions } from "../locateApp"; /** * @@@ @@ -8,19 +8,25 @@ import type { LocateAppOptions } from '../locateApp'; * @private within the repository */ export async function locateAppOnLinux({ - linuxWhich, -}: Pick, 'linuxWhich'>): Promise { - try { - const result = await $execCommand({ crashOnError: true, command: `which ${linuxWhich}` }); + linuxWhich, +}: Pick< + Required, + "linuxWhich" +>): Promise { + try { + const result = await $execCommand({ + crashOnError: true, + command: `which ${linuxWhich}`, + }); - return result.trim(); - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } + return result.trim(); + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } - return null; - } + return null; + } } /** diff --git a/src/executables/platforms/locateAppOnMacOs.ts b/src/executables/platforms/locateAppOnMacOs.ts index 3049b79b79..065b36ba97 100644 --- a/src/executables/platforms/locateAppOnMacOs.ts +++ b/src/executables/platforms/locateAppOnMacOs.ts @@ -1,13 +1,13 @@ -import { $execCommand } from '../../utils/execCommand/$execCommand'; -import { $provideFilesystemForNode } from '../../scrapers/_common/register/$provideFilesystemForNode'; -import type { string_executable_path } from '../../types/typeAliases'; -import { isExecutable } from '../../utils/files/isExecutable'; -import type { LocateAppOptions } from '../locateApp'; +import { $provideFilesystemForNode } from "../../scrapers/_common/register/$provideFilesystemForNode"; +import type { string_executable_path } from "../../types/typeAliases"; +import { $execCommand } from "../../utils/execCommand/$execCommand"; +import { isExecutable } from "../../utils/files/isExecutable"; +import type { LocateAppOptions } from "../locateApp"; // Note: Module `userhome` has no types available, so it is imported using `require` // @see https://stackoverflow.com/questions/37000981/how-to-import-node-module-in-typescript-without-type-definitions // eslint-disable-next-line @typescript-eslint/no-var-requires -const userhome = require('userhome'); +const userhome = require("userhome"); /** * @@@ @@ -15,32 +15,35 @@ const userhome = require('userhome'); * @private within the repository */ export async function locateAppOnMacOs({ - macOsName, -}: Pick, 'macOsName'>): Promise { - try { - const toExec = `/Contents/MacOS/${macOsName}`; - const regPath = `/Applications/${macOsName}.app` + toExec; - const altPath = userhome(regPath.slice(1)); + macOsName, +}: Pick< + Required, + "macOsName" +>): Promise { + try { + const toExec = `/Contents/MacOS/${macOsName}`; + const regPath = `/Applications/${macOsName}.app` + toExec; + const altPath = userhome(regPath.slice(1)); - if (await isExecutable(regPath, $provideFilesystemForNode())) { - return regPath; - } else if (await isExecutable(altPath, $provideFilesystemForNode())) { - return altPath; - } + if (await isExecutable(regPath, $provideFilesystemForNode())) { + return regPath; + } else if (await isExecutable(altPath, $provideFilesystemForNode())) { + return altPath; + } - const result = await $execCommand({ - crashOnError: true, - command: `mdfind 'kMDItemDisplayName == "${macOsName}" && kMDItemKind == Application'`, - }); + const result = await $execCommand({ + crashOnError: true, + command: `mdfind 'kMDItemDisplayName == "${macOsName}" && kMDItemKind == Application'`, + }); - return result.trim() + toExec; - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } + return result.trim() + toExec; + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } - return null; - } + return null; + } } /** diff --git a/src/executables/platforms/locateAppOnWindows.ts b/src/executables/platforms/locateAppOnWindows.ts index b4aeec44a0..f924cc8f94 100644 --- a/src/executables/platforms/locateAppOnWindows.ts +++ b/src/executables/platforms/locateAppOnWindows.ts @@ -1,8 +1,8 @@ -import { join } from 'path'; -import { $provideFilesystemForNode } from '../../scrapers/_common/register/$provideFilesystemForNode'; -import type { string_executable_path } from '../../types/typeAliases'; -import { isExecutable } from '../../utils/files/isExecutable'; -import type { LocateAppOptions } from '../locateApp'; +import { join } from "path"; +import { $provideFilesystemForNode } from "../../scrapers/_common/register/$provideFilesystemForNode"; +import type { string_executable_path } from "../../types/typeAliases"; +import { isExecutable } from "../../utils/files/isExecutable"; +import type { LocateAppOptions } from "../locateApp"; /** * @@@ @@ -10,33 +10,36 @@ import type { LocateAppOptions } from '../locateApp'; * @private within the repository */ export async function locateAppOnWindows({ - appName, - windowsSuffix, -}: Pick, 'appName' | 'windowsSuffix'>): Promise { - try { - const prefixes = [ - process.env.LOCALAPPDATA, - join(process.env.LOCALAPPDATA || '', 'Programs'), - process.env.PROGRAMFILES, - process.env['PROGRAMFILES(X86)'], - ]; + appName, + windowsSuffix, +}: Pick< + Required, + "appName" | "windowsSuffix" +>): Promise { + try { + const prefixes = [ + process.env.LOCALAPPDATA, + join(process.env.LOCALAPPDATA || "", "Programs"), + process.env.PROGRAMFILES, + process.env["PROGRAMFILES(X86)"], + ]; - for (const prefix of prefixes) { - const path = prefix + windowsSuffix; + for (const prefix of prefixes) { + const path = prefix + windowsSuffix; - if (await isExecutable(path, $provideFilesystemForNode())) { - return path; - } - } + if (await isExecutable(path, $provideFilesystemForNode())) { + return path; + } + } - throw new Error(`Can not locate app ${appName} on Windows.`); - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } + throw new Error(`Can not locate app ${appName} on Windows.`); + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } - return null; - } + return null; + } } /** diff --git a/src/execution/AbstractTaskResult.ts b/src/execution/AbstractTaskResult.ts index 1129a9bedd..b23279b8e3 100644 --- a/src/execution/AbstractTaskResult.ts +++ b/src/execution/AbstractTaskResult.ts @@ -1,5 +1,5 @@ -import { ReadonlyDeep } from 'type-fest'; -import type { ErrorJson } from '../errors/utils/ErrorJson'; +import type { ReadonlyDeep } from "type-fest"; +import type { ErrorJson } from "../errors/utils/ErrorJson"; /** * @@@ @@ -7,20 +7,20 @@ import type { ErrorJson } from '../errors/utils/ErrorJson'; * Note: [🚉] This is fully serializable as JSON */ export type AbstractTaskResult = { - /** - * Whether the execution was successful, details are available in `executionReport` - */ - readonly isSuccessful: boolean; + /** + * Whether the execution was successful, details are available in `executionReport` + */ + readonly isSuccessful: boolean; - /** - * Errors that occured during the execution, details are available in `executionReport` - */ - readonly errors: ReadonlyDeep>; + /** + * Errors that occured during the execution, details are available in `executionReport` + */ + readonly errors: ReadonlyDeep>; - /** - * Warnings that occured during the execution, details are available in `executionReport` - */ - readonly warnings: ReadonlyDeep>; + /** + * Warnings that occured during the execution, details are available in `executionReport` + */ + readonly warnings: ReadonlyDeep>; }; /** diff --git a/src/execution/AvailableModel.ts b/src/execution/AvailableModel.ts index f5430d5c39..b620250508 100644 --- a/src/execution/AvailableModel.ts +++ b/src/execution/AvailableModel.ts @@ -1,41 +1,41 @@ -import type { ModelVariant } from '../types/ModelVariant'; -import type { string_model_description } from '../types/typeAliases'; -import type { string_model_name } from '../types/typeAliases'; -import type { string_title } from '../types/typeAliases'; +import type { ModelVariant } from "../types/ModelVariant"; +import type { string_model_description } from "../types/typeAliases"; +import type { string_model_name } from "../types/typeAliases"; +import type { string_title } from "../types/typeAliases"; /** * Represents a model that can be used for prompt execution */ export type AvailableModel = { - /** - * The model title, when not provided, the `modelName` should be used - * - * @example "GPT o1" - */ - readonly modelTitle?: string_title; + /** + * The model title, when not provided, the `modelName` should be used + * + * @example "GPT o1" + */ + readonly modelTitle?: string_title; - /** - * The model name aviailable - * - * @example "o1" - */ - readonly modelName: string_model_name; + /** + * The model name aviailable + * + * @example "o1" + */ + readonly modelName: string_model_name; - /** - * Variant of the model - * - * @example "CHAT" - */ - readonly modelVariant: ModelVariant; + /** + * Variant of the model + * + * @example "CHAT" + */ + readonly modelVariant: ModelVariant; - /** - * Unstructured description of the model - * - * This will be used to pick the best available model for each task - * - * @example "Model with 1 billion parameters and advanced reasoning capabilities" - */ - readonly modelDescription?: string_model_description; + /** + * Unstructured description of the model + * + * This will be used to pick the best available model for each task + * + * @example "Model with 1 billion parameters and advanced reasoning capabilities" + */ + readonly modelDescription?: string_model_description; }; /** diff --git a/src/execution/CommonToolsOptions.ts b/src/execution/CommonToolsOptions.ts index ee7cad30e0..9273a3c3b2 100644 --- a/src/execution/CommonToolsOptions.ts +++ b/src/execution/CommonToolsOptions.ts @@ -1,4 +1,4 @@ -import type { string_user_id } from '../types/typeAliases'; +import type { string_user_id } from "../types/typeAliases"; /** * @@@ @@ -6,18 +6,18 @@ import type { string_user_id } from '../types/typeAliases'; * Note: Keep it public to allow people to make their own execution tools */ export type CommonToolsOptions = { - /** - * A unique identifier representing your end-user - * - * Note: For example it can help to detect abuse - * For example for OpenAi @see https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids - */ - readonly userId?: string_user_id; + /** + * A unique identifier representing your end-user + * + * Note: For example it can help to detect abuse + * For example for OpenAi @see https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids + */ + readonly userId?: string_user_id; - /** - * If true, the internal executions will be logged - */ - readonly isVerbose?: boolean; + /** + * If true, the internal executions will be logged + */ + readonly isVerbose?: boolean; }; /** diff --git a/src/execution/Executables.ts b/src/execution/Executables.ts index 4dee351d9a..beaafbbf42 100644 --- a/src/execution/Executables.ts +++ b/src/execution/Executables.ts @@ -1,20 +1,20 @@ -import type { string_executable_path } from '../types/typeAliases'; +import type { string_executable_path } from "../types/typeAliases"; /** * Paths to the external programs executables */ export type Executables = { - /** - * Path to the `pandoc` executable - * - * @example 'C:/Users/me/AppData/Local/Pandoc/pandoc.exe' - */ - pandocPath?: string_executable_path; + /** + * Path to the `pandoc` executable + * + * @example 'C:/Users/me/AppData/Local/Pandoc/pandoc.exe' + */ + pandocPath?: string_executable_path; - /** - * Path to the LibreOffice executable - * - * @example 'C:/Program Files/LibreOffice/program/swriter.exe' - */ - libreOfficePath?: string_executable_path; + /** + * Path to the LibreOffice executable + * + * @example 'C:/Program Files/LibreOffice/program/swriter.exe' + */ + libreOfficePath?: string_executable_path; }; diff --git a/src/execution/ExecutionTask.ts b/src/execution/ExecutionTask.ts index 4431e35082..b9d0127e79 100644 --- a/src/execution/ExecutionTask.ts +++ b/src/execution/ExecutionTask.ts @@ -1,32 +1,32 @@ -import type { Observable } from 'rxjs'; -import { BehaviorSubject } from 'rxjs'; -import { PartialDeep } from 'type-fest'; -import type { task_id } from '../types/typeAliases'; -import type { string_SCREAMING_CASE } from '../utils/normalization/normalizeTo_SCREAMING_CASE'; -import type { TODO_remove_as } from '../utils/organization/TODO_remove_as'; -import type { really_any } from '../utils/organization/really_any'; -import { $randomToken } from '../utils/random/$randomToken'; -import type { AbstractTaskResult } from './AbstractTaskResult'; -import type { PipelineExecutorResult } from './PipelineExecutorResult'; -import { assertsTaskSuccessful } from './assertsTaskSuccessful'; +import type { Observable } from "rxjs"; +import { BehaviorSubject } from "rxjs"; +import type { PartialDeep } from "type-fest"; +import type { task_id } from "../types/typeAliases"; +import type { string_SCREAMING_CASE } from "../utils/normalization/normalizeTo_SCREAMING_CASE"; +import type { TODO_remove_as } from "../utils/organization/TODO_remove_as"; +import type { really_any } from "../utils/organization/really_any"; +import { $randomToken } from "../utils/random/$randomToken"; +import type { AbstractTaskResult } from "./AbstractTaskResult"; +import type { PipelineExecutorResult } from "./PipelineExecutorResult"; +import { assertsTaskSuccessful } from "./assertsTaskSuccessful"; /** * Options for creating a new task */ type CreateTaskOptions = { - /** - * The type of task to create - */ - readonly taskType: AbstractTask['taskType']; - - /** - * Callback that processes the task and updates the ongoing result - * @param ongoingResult The partial result of the task processing - * @returns The final task result - */ - taskProcessCallback( - updateOngoingResult: (newOngoingResult: PartialDeep) => void, - ): Promise; + /** + * The type of task to create + */ + readonly taskType: AbstractTask["taskType"]; + + /** + * Callback that processes the task and updates the ongoing result + * @param ongoingResult The partial result of the task processing + * @returns The final task result + */ + taskProcessCallback( + updateOngoingResult: (newOngoingResult: PartialDeep) => void, + ): Promise; }; /** @@ -35,68 +35,72 @@ type CreateTaskOptions = { * @private internal helper function */ export function createTask( - options: CreateTaskOptions, + options: CreateTaskOptions, ): AbstractTask { - const { taskType, taskProcessCallback } = options; - - const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken( - 8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts */, - )}`; - - const partialResultSubject = new BehaviorSubject>({} as PartialDeep); - - const finalResultPromise = /* not await */ taskProcessCallback((newOngoingResult: PartialDeep) => { - partialResultSubject.next(newOngoingResult); - }); - - finalResultPromise - .catch((error) => { - partialResultSubject.error(error); - }) - .then((value) => { - if (value) { - try { - assertsTaskSuccessful(value); - partialResultSubject.next(value as really_any); - } catch (error) { - partialResultSubject.error(error); - } - } - - partialResultSubject.complete(); - }); - - async function asPromise(options?: { readonly isCrashedOnError?: boolean }) { - const { isCrashedOnError = true } = options || {}; - - const finalResult = await finalResultPromise; - - if (isCrashedOnError) { - assertsTaskSuccessful(finalResult); - } - - return finalResult; - } - - return { - taskType, - taskId, - asPromise, - asObservable() { - return partialResultSubject.asObservable(); - }, - get currentValue() { - return partialResultSubject.value; - }, - } as TODO_remove_as>; + const { taskType, taskProcessCallback } = options; + + const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken( + 8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts */, + )}`; + + const partialResultSubject = new BehaviorSubject>( + {} as PartialDeep, + ); + + const finalResultPromise = /* not await */ taskProcessCallback( + (newOngoingResult: PartialDeep) => { + partialResultSubject.next(newOngoingResult); + }, + ); + + finalResultPromise + .catch((error) => { + partialResultSubject.error(error); + }) + .then((value) => { + if (value) { + try { + assertsTaskSuccessful(value); + partialResultSubject.next(value as really_any); + } catch (error) { + partialResultSubject.error(error); + } + } + + partialResultSubject.complete(); + }); + + async function asPromise(options?: { readonly isCrashedOnError?: boolean }) { + const { isCrashedOnError = true } = options || {}; + + const finalResult = await finalResultPromise; + + if (isCrashedOnError) { + assertsTaskSuccessful(finalResult); + } + + return finalResult; + } + + return { + taskType, + taskId, + asPromise, + asObservable() { + return partialResultSubject.asObservable(); + }, + get currentValue() { + return partialResultSubject.value; + }, + } as TODO_remove_as>; } /** * Represents a task that executes a pipeline */ export type ExecutionTask = AbstractTask & { - readonly taskType: 'EXECUTION'; - readonly taskId: `exec-${task_id}`; // <- Note: This is an exception to use shortcuts + readonly taskType: "EXECUTION"; + readonly taskId: `exec-${task_id}`; // <- Note: This is an exception to use shortcuts }; /** @@ -104,40 +108,42 @@ export type ExecutionTask = AbstractTask & { * @deprecated TODO: [🐚] Currently unused - use */ export type PreparationTask = AbstractTask & { - readonly taskType: 'PREPARATION'; - readonly taskId: `prep-${task_id}`; // <- Note: This is an exception to use shortcuts + readonly taskType: "PREPARATION"; + readonly taskId: `prep-${task_id}`; // <- Note: This is an exception to use shortcuts }; /** * Base interface for all task types */ export type AbstractTask = { - /** - * Type of the task - */ - readonly taskType: string_SCREAMING_CASE; - - /** - * Unique identifier for the task - */ - readonly taskId: task_id; - - /** - * Gets a promise that resolves with the task result - */ - asPromise(options?: { readonly isCrashedOnError?: boolean }): Promise; - - /** - * Gets an observable stream of partial task results - */ - asObservable(): Observable>; - - /** - * Gets just the current value which is mutated during the task processing - */ - currentValue: PartialDeep; - - // <- TODO: asMutableObject(): PartialDeep; + /** + * Type of the task + */ + readonly taskType: string_SCREAMING_CASE; + + /** + * Unique identifier for the task + */ + readonly taskId: task_id; + + /** + * Gets a promise that resolves with the task result + */ + asPromise(options?: { + readonly isCrashedOnError?: boolean; + }): Promise; + + /** + * Gets an observable stream of partial task results + */ + asObservable(): Observable>; + + /** + * Gets just the current value which is mutated during the task processing + */ + currentValue: PartialDeep; + + // <- TODO: asMutableObject(): PartialDeep; }; export type Task = ExecutionTask | PreparationTask; diff --git a/src/execution/ExecutionTools.ts b/src/execution/ExecutionTools.ts index fc42eaa787..5f8b0e0797 100644 --- a/src/execution/ExecutionTools.ts +++ b/src/execution/ExecutionTools.ts @@ -1,11 +1,11 @@ -import type { Scraper } from '../scrapers/_common/Scraper'; -import type { Arrayable } from '../types/Arrayable'; -import type { Executables } from './Executables'; -import type { FilesystemTools } from './FilesystemTools'; -import type { LlmExecutionTools } from './LlmExecutionTools'; -import type { PromptbookFetch } from './PromptbookFetch'; -import type { ScriptExecutionTools } from './ScriptExecutionTools'; -import type { UserInterfaceTools } from './UserInterfaceTools'; +import type { Scraper } from "../scrapers/_common/Scraper"; +import type { Arrayable } from "../types/Arrayable"; +import type { Executables } from "./Executables"; +import type { FilesystemTools } from "./FilesystemTools"; +import type { LlmExecutionTools } from "./LlmExecutionTools"; +import type { PromptbookFetch } from "./PromptbookFetch"; +import type { ScriptExecutionTools } from "./ScriptExecutionTools"; +import type { UserInterfaceTools } from "./UserInterfaceTools"; /** * All the tools needed to execute pipelines. @@ -13,64 +13,64 @@ import type { UserInterfaceTools } from './UserInterfaceTools'; * @see https://github.com/webgptorg/promptbook#execution-tools */ export type ExecutionTools = { - /** - * Tools for executing prompts in large language models - * - * Tip: Use `$provideLlmToolsFromEnv` to use all available LLM providers you configured - * - * @default [] - If not provided, no LLM execution will be possible - it does not make sense in most cases - */ - readonly llm?: Arrayable; + /** + * Tools for executing prompts in large language models + * + * Tip: Use `$provideLlmToolsFromEnv` to use all available LLM providers you configured + * + * @default [] - If not provided, no LLM execution will be possible - it does not make sense in most cases + */ + readonly llm?: Arrayable; - /** - * Tools for filesystem operations - * - * Tip: Use `$provideFilesystemForNode` to use filesystem in Node.js - * - * @default undefined - If not provided, no filesystem operations will be possible - */ - readonly fs?: FilesystemTools; + /** + * Tools for filesystem operations + * + * Tip: Use `$provideFilesystemForNode` to use filesystem in Node.js + * + * @default undefined - If not provided, no filesystem operations will be possible + */ + readonly fs?: FilesystemTools; - /** - * Fetch function for fetching resources - * - * @default `fetch` - If not provided, the built-in `fetch' function is used with a lightweight error handling wrapper. - */ - readonly fetch?: PromptbookFetch; + /** + * Fetch function for fetching resources + * + * @default `fetch` - If not provided, the built-in `fetch' function is used with a lightweight error handling wrapper. + */ + readonly fetch?: PromptbookFetch; - /** - * Scrapers for extracting knowledge from external sources - * - * Tip: Use `$provideExecutablesForNode` - * - * @default [] - If not provided, no external knowledge extraction will be possible - */ - readonly executables?: Executables; + /** + * Scrapers for extracting knowledge from external sources + * + * Tip: Use `$provideExecutablesForNode` + * + * @default [] - If not provided, no external knowledge extraction will be possible + */ + readonly executables?: Executables; - /** - * Scrapers for extracting knowledge from external sources - * - * Tip: Use `$provideScrapersForNode` OR `$provideScrapersForBrowser` to use all available scrapers - * - * @default [] - If not provided, no external knowledge extraction will be possible - */ - readonly scrapers?: Arrayable; + /** + * Scrapers for extracting knowledge from external sources + * + * Tip: Use `$provideScrapersForNode` OR `$provideScrapersForBrowser` to use all available scrapers + * + * @default [] - If not provided, no external knowledge extraction will be possible + */ + readonly scrapers?: Arrayable; - /** - * Tools for executing scripts - * - * Note: You can pass multiple ScriptExecutionTools, they will be tried one by one until one of them supports the script - * If none of them supports the script, an error is thrown - * Tip: Use here `new JavascriptExecutionTools` - * - * @default [] - If not provided, no script execution will be possible - */ - readonly script?: Arrayable; + /** + * Tools for executing scripts + * + * Note: You can pass multiple ScriptExecutionTools, they will be tried one by one until one of them supports the script + * If none of them supports the script, an error is thrown + * Tip: Use here `new JavascriptExecutionTools` + * + * @default [] - If not provided, no script execution will be possible + */ + readonly script?: Arrayable; - /** - * Tools for interacting with the user - * - * Note: When undefined, the user interface is disabled and promptbook which requires user interaction will fail - */ - readonly userInterface?: UserInterfaceTools; + /** + * Tools for interacting with the user + * + * Note: When undefined, the user interface is disabled and promptbook which requires user interaction will fail + */ + readonly userInterface?: UserInterfaceTools; }; diff --git a/src/execution/FilesystemTools.ts b/src/execution/FilesystemTools.ts index 6aca9b123d..56e6b9b559 100644 --- a/src/execution/FilesystemTools.ts +++ b/src/execution/FilesystemTools.ts @@ -1,11 +1,17 @@ -import type fs from 'fs/promises'; +import type fs from "fs/promises"; /** * Container for all the tools needed to manipulate with filesystem */ export type FilesystemTools = Pick< - typeof fs, - 'access' | 'constants' | 'readFile' | 'writeFile' | 'stat' | 'readdir' | 'mkdir' + typeof fs, + | "access" + | "constants" + | "readFile" + | "writeFile" + | "stat" + | "readdir" + | "mkdir" >; /** diff --git a/src/execution/LlmExecutionTools.ts b/src/execution/LlmExecutionTools.ts index b479761e6b..b25ac6576b 100644 --- a/src/execution/LlmExecutionTools.ts +++ b/src/execution/LlmExecutionTools.ts @@ -1,12 +1,12 @@ -import type { Promisable } from 'type-fest'; -import type { Prompt } from '../types/Prompt'; -import type { string_markdown } from '../types/typeAliases'; -import type { string_markdown_text } from '../types/typeAliases'; -import type { string_title } from '../types/typeAliases'; -import type { AvailableModel } from './AvailableModel'; -import type { ChatPromptResult } from './PromptResult'; -import type { CompletionPromptResult } from './PromptResult'; -import type { EmbeddingPromptResult } from './PromptResult'; +import type { Promisable } from "type-fest"; +import type { Prompt } from "../types/Prompt"; +import type { string_markdown } from "../types/typeAliases"; +import type { string_markdown_text } from "../types/typeAliases"; +import type { string_title } from "../types/typeAliases"; +import type { AvailableModel } from "./AvailableModel"; +import type { ChatPromptResult } from "./PromptResult"; +import type { CompletionPromptResult } from "./PromptResult"; +import type { EmbeddingPromptResult } from "./PromptResult"; /** * Container for all the tools needed to execute prompts to large language models like GPT-4 @@ -16,47 +16,47 @@ import type { EmbeddingPromptResult } from './PromptResult'; * @see https://github.com/webgptorg/promptbook#llm-execution-tools */ export type LlmExecutionTools = { - /** - * Title of the model provider - * - * @example "OpenAI" - */ - readonly title: string_title & string_markdown_text; + /** + * Title of the model provider + * + * @example "OpenAI" + */ + readonly title: string_title & string_markdown_text; - /** - * Description of the provider - * - * @example "Use all models from OpenAI" - */ - readonly description?: string_markdown; + /** + * Description of the provider + * + * @example "Use all models from OpenAI" + */ + readonly description?: string_markdown; - /** - * Check comfiguration - * - * @returns nothing if configuration is correct - * @throws {Error} if configuration is incorrect - */ - checkConfiguration(): Promisable; + /** + * Check comfiguration + * + * @returns nothing if configuration is correct + * @throws {Error} if configuration is incorrect + */ + checkConfiguration(): Promisable; - /** - * List all available models that can be used - */ - listModels(): Promisable>; + /** + * List all available models that can be used + */ + listModels(): Promisable>; - /** - * Calls a chat model - */ - callChatModel?(prompt: Prompt): Promise; + /** + * Calls a chat model + */ + callChatModel?(prompt: Prompt): Promise; - /** - * Calls a completion model - */ - callCompletionModel?(prompt: Prompt): Promise; + /** + * Calls a completion model + */ + callCompletionModel?(prompt: Prompt): Promise; - /** - * Calls an embedding model - */ - callEmbeddingModel?(prompt: Prompt): Promise; + /** + * Calls an embedding model + */ + callEmbeddingModel?(prompt: Prompt): Promise; }; /** diff --git a/src/execution/LlmExecutionToolsConstructor.ts b/src/execution/LlmExecutionToolsConstructor.ts index 8b167caeda..85117f0bfd 100644 --- a/src/execution/LlmExecutionToolsConstructor.ts +++ b/src/execution/LlmExecutionToolsConstructor.ts @@ -1,11 +1,12 @@ -import type { Registered } from '../utils/$Register'; -import type { TODO_any } from '../utils/organization/TODO_any'; -import type { LlmExecutionTools } from './LlmExecutionTools'; +import type { Registered } from "../utils/$Register"; +import type { TODO_any } from "../utils/organization/TODO_any"; +import type { LlmExecutionTools } from "./LlmExecutionTools"; /** * @@@ */ -export type LlmExecutionToolsConstructor = Registered & ((options: TODO_any) => LlmExecutionTools); +export type LlmExecutionToolsConstructor = Registered & + ((options: TODO_any) => LlmExecutionTools); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/execution/PipelineExecutor.ts b/src/execution/PipelineExecutor.ts index 33707d783c..87b4e9a9a3 100644 --- a/src/execution/PipelineExecutor.ts +++ b/src/execution/PipelineExecutor.ts @@ -1,5 +1,5 @@ -import type { InputParameters } from '../types/typeAliases'; -import type { ExecutionTask } from './ExecutionTask'; +import type { InputParameters } from "../types/typeAliases"; +import type { ExecutionTask } from "./ExecutionTask"; /** * Executor is a simple async function that takes INPUT PARAMETERs and returns result parameters _(along with all intermediate parameters and INPUT PARAMETERs = it extends input object)_. @@ -9,9 +9,9 @@ import type { ExecutionTask } from './ExecutionTask'; * * @see https://github.com/webgptorg/promptbook#executor */ -export type PipelineExecutor = { - (inputParameters: InputParameters): ExecutionTask; -}; +export type PipelineExecutor = ( + inputParameters: InputParameters, +) => ExecutionTask; /** * TODO: [🧠] Should this file be in /execution or /types folder? diff --git a/src/execution/PipelineExecutorResult.ts b/src/execution/PipelineExecutorResult.ts index c5e831eeae..5966817362 100644 --- a/src/execution/PipelineExecutorResult.ts +++ b/src/execution/PipelineExecutorResult.ts @@ -1,9 +1,9 @@ -import { ReadonlyDeep } from 'type-fest'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import type { Parameters } from '../types/typeAliases'; -import type { AbstractTaskResult } from './AbstractTaskResult'; -import type { ExecutionReportJson } from './execution-report/ExecutionReportJson'; -import type { PromptResultUsage } from './PromptResultUsage'; +import type { ReadonlyDeep } from "type-fest"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import type { Parameters } from "../types/typeAliases"; +import type { AbstractTaskResult } from "./AbstractTaskResult"; +import type { PromptResultUsage } from "./PromptResultUsage"; +import type { ExecutionReportJson } from "./execution-report/ExecutionReportJson"; /** * @@@ @@ -11,30 +11,30 @@ import type { PromptResultUsage } from './PromptResultUsage'; * Note: [🚉] This is fully serializable as JSON */ export type PipelineExecutorResult = AbstractTaskResult & { - /** - * Result parameters of the execution - * - * Note: If the execution was not successful, there are only some of the result parameters - */ - readonly outputParameters: Readonly; + /** + * Result parameters of the execution + * + * Note: If the execution was not successful, there are only some of the result parameters + */ + readonly outputParameters: Readonly; - /** - * Added usage of whole execution, detailed usage is available in `executionReport` - */ - readonly usage: ReadonlyDeep; + /** + * Added usage of whole execution, detailed usage is available in `executionReport` + */ + readonly usage: ReadonlyDeep; - /** - * The report of the execution with all details - */ - readonly executionReport: ReadonlyDeep; + /** + * The report of the execution with all details + */ + readonly executionReport: ReadonlyDeep; - /** - * The prepared pipeline that was used for the execution - * - * Note: If you called `createPipelineExecutor` with fully prepared pipeline, this is the same object as this pipeline - * If you passed not fully prepared pipeline, this is same pipeline but fully prepared - */ - readonly preparedPipeline: ReadonlyDeep; + /** + * The prepared pipeline that was used for the execution + * + * Note: If you called `createPipelineExecutor` with fully prepared pipeline, this is the same object as this pipeline + * If you passed not fully prepared pipeline, this is same pipeline but fully prepared + */ + readonly preparedPipeline: ReadonlyDeep; }; /** diff --git a/src/execution/PromptResult.ts b/src/execution/PromptResult.ts index 18f1905c42..e3873ea0ba 100644 --- a/src/execution/PromptResult.ts +++ b/src/execution/PromptResult.ts @@ -1,9 +1,9 @@ -import type { string_date_iso8601 } from '../types/typeAliases'; -import type { string_model_name } from '../types/typeAliases'; -import type { string_prompt } from '../types/typeAliases'; -import type { TODO_object } from '../utils/organization/TODO_object'; -import type { EmbeddingVector } from './EmbeddingVector'; -import type { PromptResultUsage } from './PromptResultUsage'; +import type { string_date_iso8601 } from "../types/typeAliases"; +import type { string_model_name } from "../types/typeAliases"; +import type { string_prompt } from "../types/typeAliases"; +import type { TODO_object } from "../utils/organization/TODO_object"; +import type { EmbeddingVector } from "./EmbeddingVector"; +import type { PromptResultUsage } from "./PromptResultUsage"; /** * Prompt result is the simplest concept of execution. @@ -11,7 +11,10 @@ import type { PromptResultUsage } from './PromptResultUsage'; * * @see https://github.com/webgptorg/promptbook#prompt-result */ -export type PromptResult = CompletionPromptResult | ChatPromptResult | EmbeddingPromptResult /* <- [🤖] */; +export type PromptResult = + | CompletionPromptResult + | ChatPromptResult + | EmbeddingPromptResult /* <- [🤖] */; /** * Completion prompt result @@ -27,7 +30,7 @@ export type CompletionPromptResult = CommonPromptResult; * Note: [🚉] This is fully serializable as JSON */ export type ChatPromptResult = CommonPromptResult & { - // TODO: [🗯][🧠] Figure out way how to pass thread / previous messages + // TODO: [🗯][🧠] Figure out way how to pass thread / previous messages }; /** @@ -35,11 +38,11 @@ export type ChatPromptResult = CommonPromptResult & { * * Note: [🚉] This is fully serializable as JSON */ -export type EmbeddingPromptResult = Omit & { - /** - * The response from the model - */ - content: EmbeddingVector; +export type EmbeddingPromptResult = Omit & { + /** + * The response from the model + */ + content: EmbeddingVector; }; // <- Note: [🤖] Add new model variant here @@ -52,64 +55,64 @@ export type EmbeddingPromptResult = Omit & { * @private just abstract the common properties of the prompt results */ export type CommonPromptResult = { - // TODO: [🗯] Unique messageId + threadId + remoteId - // TODO: [🗯] prompt - - /** - * Exact text response from the model - */ - readonly content: string; - - /** - * Name of the model used to generate the response - */ - readonly modelName: string_model_name; - - /** - * Timing - */ - readonly timing: { - /** - * Start of the execution - */ - readonly start: string_date_iso8601; - - /** - * First token generated - */ - readonly firstToken?: string_date_iso8601; - - /** - * End of the execution - */ - readonly complete: string_date_iso8601; - }; - - /** - * Usage of the prompt execution - */ - readonly usage: PromptResultUsage; - - /** - * Exact text of the prompt (with all replacements) - * - * Note: This contains redundant information - */ - readonly rawPromptContent: string_prompt; - - /** - * Raw request to the model - * - * Note: This contains redundant information - */ - readonly rawRequest: TODO_object | null; - - /** - * Raw response from the model - * - * Note: This contains redundant information - */ - readonly rawResponse: TODO_object; + // TODO: [🗯] Unique messageId + threadId + remoteId + // TODO: [🗯] prompt + + /** + * Exact text response from the model + */ + readonly content: string; + + /** + * Name of the model used to generate the response + */ + readonly modelName: string_model_name; + + /** + * Timing + */ + readonly timing: { + /** + * Start of the execution + */ + readonly start: string_date_iso8601; + + /** + * First token generated + */ + readonly firstToken?: string_date_iso8601; + + /** + * End of the execution + */ + readonly complete: string_date_iso8601; + }; + + /** + * Usage of the prompt execution + */ + readonly usage: PromptResultUsage; + + /** + * Exact text of the prompt (with all replacements) + * + * Note: This contains redundant information + */ + readonly rawPromptContent: string_prompt; + + /** + * Raw request to the model + * + * Note: This contains redundant information + */ + readonly rawRequest: TODO_object | null; + + /** + * Raw response from the model + * + * Note: This contains redundant information + */ + readonly rawResponse: TODO_object; }; /** diff --git a/src/execution/PromptResultUsage.ts b/src/execution/PromptResultUsage.ts index 0e321b4fb8..f6c35bcd0f 100644 --- a/src/execution/PromptResultUsage.ts +++ b/src/execution/PromptResultUsage.ts @@ -1,6 +1,6 @@ -import type { KebabCase } from 'type-fest'; -import type { ExpectationUnit } from '../pipeline/PipelineJson/Expectations'; -import type { UncertainNumber } from './UncertainNumber'; +import type { KebabCase } from "type-fest"; +import type { ExpectationUnit } from "../pipeline/PipelineJson/Expectations"; +import type { UncertainNumber } from "./UncertainNumber"; /** * Usage statistics for one or many prompt results @@ -8,22 +8,22 @@ import type { UncertainNumber } from './UncertainNumber'; * Note: [🚉] This is fully serializable as JSON */ export type PromptResultUsage = { - /** - * Cost of the execution in USD - * - * Note: If the cost is unknown, the value 0 and isUncertain is true - */ - readonly price: UncertainNumber; + /** + * Cost of the execution in USD + * + * Note: If the cost is unknown, the value 0 and isUncertain is true + */ + readonly price: UncertainNumber; - /** - * Number of whatever used in the input aka. `prompt_tokens` - */ - readonly input: PromptResultUsageCounts; + /** + * Number of whatever used in the input aka. `prompt_tokens` + */ + readonly input: PromptResultUsageCounts; - /** - * Number of tokens used in the output aka. `completion_tokens` - */ - readonly output: PromptResultUsageCounts; + /** + * Number of tokens used in the output aka. `completion_tokens` + */ + readonly output: PromptResultUsageCounts; }; /** @@ -31,7 +31,10 @@ export type PromptResultUsage = { * * Note: [🚉] This is fully serializable as JSON */ -export type PromptResultUsageCounts = Record<`${KebabCase<'TOKENS' | ExpectationUnit>}Count`, UncertainNumber>; +export type PromptResultUsageCounts = Record< + `${KebabCase<"TOKENS" | ExpectationUnit>}Count`, + UncertainNumber +>; /** * TODO: [🍙] Make some standard order of json properties diff --git a/src/execution/PromptbookFetch.test-type.ts b/src/execution/PromptbookFetch.test-type.ts index e6c860fe1c..deeb365a92 100644 --- a/src/execution/PromptbookFetch.test-type.ts +++ b/src/execution/PromptbookFetch.test-type.ts @@ -1,5 +1,5 @@ import { keepUnused } from "../utils/organization/keepUnused"; -import { PromptbookFetch } from "./PromptbookFetch"; +import type { PromptbookFetch } from "./PromptbookFetch"; // Note: Checking that `fetch` is implementing `PromptbookFetch` let compatibleFetch: PromptbookFetch | null; diff --git a/src/execution/PromptbookFetch.ts b/src/execution/PromptbookFetch.ts index 90b1265188..3365fd7580 100644 --- a/src/execution/PromptbookFetch.ts +++ b/src/execution/PromptbookFetch.ts @@ -1,4 +1,4 @@ -import type { string_url } from '../types/typeAliases'; +import type { string_url } from "../types/typeAliases"; /** * Fetch function used in Promptbook engine @@ -10,4 +10,7 @@ import type { string_url } from '../types/typeAliases'; * - Fetching knowledge sources * - Callbacks from remote server ([👩🏾‍🤝‍🧑🏾] Not yet implemented) */ -export type PromptbookFetch = (url: string_url, init?: RequestInit) => Promise; +export type PromptbookFetch = ( + url: string_url, + init?: RequestInit, +) => Promise; diff --git a/src/execution/ScriptExecutionTools.ts b/src/execution/ScriptExecutionTools.ts index fcbf8a812d..de64a458d5 100644 --- a/src/execution/ScriptExecutionTools.ts +++ b/src/execution/ScriptExecutionTools.ts @@ -1,6 +1,6 @@ -import type { ScriptLanguage } from '../types/ScriptLanguage'; -import type { Parameters } from '../types/typeAliases'; -import type { string_script } from '../types/typeAliases'; +import type { ScriptLanguage } from "../types/ScriptLanguage"; +import type { Parameters } from "../types/typeAliases"; +import type { string_script } from "../types/typeAliases"; /** * Represents all the tools needed to execute `SCRIPT` tasks @@ -8,34 +8,34 @@ import type { string_script } from '../types/typeAliases'; * @see https://github.com/webgptorg/promptbook#script-execution-tools */ export type ScriptExecutionTools = { - execute(options: ScriptExecutionToolsExecuteOptions): Promise; + execute(options: ScriptExecutionToolsExecuteOptions): Promise; }; /** * Input for the script execution */ export type ScriptExecutionToolsExecuteOptions = { - /** - * Language of the script - */ - readonly scriptLanguage: ScriptLanguage; + /** + * Language of the script + */ + readonly scriptLanguage: ScriptLanguage; - /** - * Parameters for the script - * Theese parameters are passed to the script as variables - * For example: { "name": "John" } => const name = "John"; - */ - readonly parameters: Parameters; + /** + * Parameters for the script + * Theese parameters are passed to the script as variables + * For example: { "name": "John" } => const name = "John"; + */ + readonly parameters: Parameters; - /** - * The content of the script to execute - * - It can be a single statement - * - It can be multiple statements separated by semicolon and return - * - It can be a function (but you need to call it) - * - It can be IIFE (immediately invoked function expression) - * - It can use the parameters as variables and functions from global scope - */ - readonly script: string_script; + /** + * The content of the script to execute + * - It can be a single statement + * - It can be multiple statements separated by semicolon and return + * - It can be a function (but you need to call it) + * - It can be IIFE (immediately invoked function expression) + * - It can use the parameters as variables and functions from global scope + */ + readonly script: string_script; }; /** diff --git a/src/execution/UncertainNumber.ts b/src/execution/UncertainNumber.ts index 105ded9da4..3d27b22dcf 100644 --- a/src/execution/UncertainNumber.ts +++ b/src/execution/UncertainNumber.ts @@ -1,5 +1,5 @@ -import type { number_positive } from '../types/typeAliases'; -import type { number_usd } from '../types/typeAliases'; +import type { number_positive } from "../types/typeAliases"; +import type { number_usd } from "../types/typeAliases"; /** * Number which can be uncertain @@ -9,13 +9,13 @@ import type { number_usd } from '../types/typeAliases'; * Note: Not using NaN or null because it looses the value which is better to be uncertain then not to be at all */ export type UncertainNumber = { - /** - * The numeric value - */ - readonly value: number_usd & (number_positive | 0); + /** + * The numeric value + */ + readonly value: number_usd & (number_positive | 0); - /** - * Is the value uncertain - */ - readonly isUncertain?: true; + /** + * Is the value uncertain + */ + readonly isUncertain?: true; }; diff --git a/src/execution/UserInterfaceTools.ts b/src/execution/UserInterfaceTools.ts index a45f21a2d5..9869251376 100644 --- a/src/execution/UserInterfaceTools.ts +++ b/src/execution/UserInterfaceTools.ts @@ -1,5 +1,5 @@ -import type { number_integer } from '../types/typeAliases'; -import type { number_positive } from '../types/typeAliases'; +import type { number_integer } from "../types/typeAliases"; +import type { number_positive } from "../types/typeAliases"; /** * Represents all the tools needed to interact with the user. @@ -7,46 +7,46 @@ import type { number_positive } from '../types/typeAliases'; * @see https://github.com/webgptorg/promptbook#user-interface-tools */ export type UserInterfaceTools = { - /** - * Asks the user to answer a free-text (multi-line) question - * - * @param options the question to ask - * @returns the answer from the user - */ - promptDialog(options: UserInterfaceToolsPromptDialogOptions): Promise; + /** + * Asks the user to answer a free-text (multi-line) question + * + * @param options the question to ask + * @returns the answer from the user + */ + promptDialog(options: UserInterfaceToolsPromptDialogOptions): Promise; }; export type UserInterfaceToolsPromptDialogOptions = { - /** - * Prompt title - * - * Note: This is not a prompt to language model but a prompt to the user - * @example "Your name" - */ - readonly promptTitle: string; + /** + * Prompt title + * + * Note: This is not a prompt to language model but a prompt to the user + * @example "Your name" + */ + readonly promptTitle: string; - /** - * Prompt message - * - * Note: This is not a prompt to language model but a prompt to the user - * @example "Please enter your name, including your last name, title, etc." - */ - readonly promptMessage: string; + /** + * Prompt message + * + * Note: This is not a prompt to language model but a prompt to the user + * @example "Please enter your name, including your last name, title, etc." + */ + readonly promptMessage: string; - /** - * Default value for the input/textarea - */ - readonly defaultValue: string | null; + /** + * Default value for the input/textarea + */ + readonly defaultValue: string | null; - /** - * Placeholder for the input/textarea - */ - readonly placeholder?: string; + /** + * Placeholder for the input/textarea + */ + readonly placeholder?: string; - /** - * Priority of the prompt - * - * Note: This would be reflected for example into the UI z-index of the prompt modal - */ - readonly priority: number_integer & number_positive; + /** + * Priority of the prompt + * + * Note: This would be reflected for example into the UI z-index of the prompt modal + */ + readonly priority: number_integer & number_positive; }; diff --git a/src/execution/assertsTaskSuccessful.ts b/src/execution/assertsTaskSuccessful.ts index b3653cef76..d65dc15c2c 100644 --- a/src/execution/assertsTaskSuccessful.ts +++ b/src/execution/assertsTaskSuccessful.ts @@ -1,7 +1,7 @@ -import { spaceTrim } from 'spacetrim'; -import { PipelineExecutionError } from '../errors/PipelineExecutionError'; -import { deserializeError } from '../errors/utils/deserializeError'; -import type { PipelineExecutorResult } from './PipelineExecutorResult'; +import { spaceTrim } from "spacetrim"; +import { PipelineExecutionError } from "../errors/PipelineExecutionError"; +import { deserializeError } from "../errors/utils/deserializeError"; +import type { PipelineExecutorResult } from "./PipelineExecutorResult"; /** * Asserts that the execution of a Promptbook is successful @@ -13,43 +13,48 @@ import type { PipelineExecutorResult } from './PipelineExecutorResult'; * @private internal helper function of `asPromise` method of `ExecutionTask` */ export function assertsTaskSuccessful( - executionResult: Pick, + executionResult: Pick< + PipelineExecutorResult, + "isSuccessful" | "errors" | "warnings" + >, ): void { - const { isSuccessful, errors, warnings } = executionResult; + const { isSuccessful, errors, warnings } = executionResult; - for (const warning of warnings) { - console.warn(warning.message); - } + for (const warning of warnings) { + console.warn(warning.message); + } - if (isSuccessful === true) { - return; - } - if (errors.length === 0) { - throw new PipelineExecutionError(`Promptbook Execution failed because of unknown reason`); - } else if (errors.length === 1) { - throw deserializeError(errors[0]!); - } else { - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + if (isSuccessful === true) { + return; + } + if (errors.length === 0) { + throw new PipelineExecutionError( + `Promptbook Execution failed because of unknown reason`, + ); + } else if (errors.length === 1) { + throw deserializeError(errors[0]!); + } else { + throw new PipelineExecutionError( + spaceTrim( + (block) => ` Multiple errors occurred during Promptbook execution ${block( - errors - .map(({ name, stack, message }, index) => - spaceTrim( - (block) => ` + errors + .map(({ name, stack, message }, index) => + spaceTrim( + (block) => ` ${name} ${index + 1}: ${block(stack || message)} `, - ), - ) - .join('\n'), - )} + ), + ) + .join("\n"), + )} `, - ), - ); - } + ), + ); + } } /** diff --git a/src/execution/createPipelineExecutor/$OngoingTaskResult.ts b/src/execution/createPipelineExecutor/$OngoingTaskResult.ts index 22d134c7ff..b0b3df2b3f 100644 --- a/src/execution/createPipelineExecutor/$OngoingTaskResult.ts +++ b/src/execution/createPipelineExecutor/$OngoingTaskResult.ts @@ -1,9 +1,9 @@ -import { ExpectError } from '../../errors/ExpectError'; -import type { Prompt } from '../../types/Prompt'; -import type { ChatPromptResult } from '../PromptResult'; -import type { CompletionPromptResult } from '../PromptResult'; -import type { EmbeddingPromptResult } from '../PromptResult'; -import type { PromptResult } from '../PromptResult'; +import type { ExpectError } from "../../errors/ExpectError"; +import type { Prompt } from "../../types/Prompt"; +import type { ChatPromptResult } from "../PromptResult"; +import type { CompletionPromptResult } from "../PromptResult"; +import type { EmbeddingPromptResult } from "../PromptResult"; +import type { PromptResult } from "../PromptResult"; /** * @@@ @@ -11,44 +11,44 @@ import type { PromptResult } from '../PromptResult'; * @private internal utility of `createPipelineExecutor` */ export type $OngoingTaskResult = { - /** - * @@@ - */ - $prompt?: Prompt; - - /** - * @@@ - */ - $chatResult?: ChatPromptResult; - - /** - * @@@ - */ - $completionResult?: CompletionPromptResult; - - /** - * @@@ - */ - $embeddingResult?: EmbeddingPromptResult; - // <- Note: [🤖] - - /** - * @@@ - */ - $result: PromptResult | null; - - /** - * @@@ - */ - $resultString: string | null; - - /** - * @@@ - */ - $expectError: ExpectError | null; - - /** - * @@@ - */ - $scriptPipelineExecutionErrors: Array; + /** + * @@@ + */ + $prompt?: Prompt; + + /** + * @@@ + */ + $chatResult?: ChatPromptResult; + + /** + * @@@ + */ + $completionResult?: CompletionPromptResult; + + /** + * @@@ + */ + $embeddingResult?: EmbeddingPromptResult; + // <- Note: [🤖] + + /** + * @@@ + */ + $result: PromptResult | null; + + /** + * @@@ + */ + $resultString: string | null; + + /** + * @@@ + */ + $expectError: ExpectError | null; + + /** + * @@@ + */ + $scriptPipelineExecutionErrors: Array; }; diff --git a/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.ts b/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.ts index b6147bb315..8a55853fb9 100644 --- a/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.ts +++ b/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.ts @@ -1,49 +1,49 @@ -import type { CsvSettings } from '../../formats/csv/CsvSettings'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import type { ExecutionTools } from '../ExecutionTools'; +import type { CsvSettings } from "../../formats/csv/CsvSettings"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import type { ExecutionTools } from "../ExecutionTools"; /** * Options for `createPipelineExecutor` */ export type CreatePipelineExecutorOptions = PrepareAndScrapeOptions & { - /** - * The pipeline to be executed - */ - readonly pipeline: PipelineJson; + /** + * The pipeline to be executed + */ + readonly pipeline: PipelineJson; - /** - * The execution tools to be used during the execution of the pipeline - */ - readonly tools: ExecutionTools; + /** + * The execution tools to be used during the execution of the pipeline + */ + readonly tools: ExecutionTools; - /** - * When executor does not satisfy expectations it will be retried this amount of times - * - * @default MAX_EXECUTION_ATTEMPTS - */ - readonly maxExecutionAttempts?: number; - // <- TODO: [💿] Maybe move to `PrepareAndScrapeOptions` + also use in preparation + /** + * When executor does not satisfy expectations it will be retried this amount of times + * + * @default MAX_EXECUTION_ATTEMPTS + */ + readonly maxExecutionAttempts?: number; + // <- TODO: [💿] Maybe move to `PrepareAndScrapeOptions` + also use in preparation - /** - * Settings for CSV format - * - * @default DEFAULT_CSV_SETTINGS - */ - readonly csvSettings?: CsvSettings; - // <- TODO: [💿] Maybe move to `PrepareAndScrapeOptions` + also use in preparation - // <- TODO: [🧎][💩] Move to better place or make more univeral + /** + * Settings for CSV format + * + * @default DEFAULT_CSV_SETTINGS + */ + readonly csvSettings?: CsvSettings; + // <- TODO: [💿] Maybe move to `PrepareAndScrapeOptions` + also use in preparation + // <- TODO: [🧎][💩] Move to better place or make more univeral - /** - * If you pass fully prepared pipeline, this does not matter - * - * Otherwise: - * If false or not set, warning is shown when pipeline is not prepared - * If true, warning is suppressed - * - * @default false - */ - readonly isNotPreparedWarningSupressed?: boolean; + /** + * If you pass fully prepared pipeline, this does not matter + * + * Otherwise: + * If false or not set, warning is shown when pipeline is not prepared + * If true, warning is suppressed + * + * @default false + */ + readonly isNotPreparedWarningSupressed?: boolean; }; /** diff --git a/src/execution/createPipelineExecutor/00-createPipelineExecutor.ts b/src/execution/createPipelineExecutor/00-createPipelineExecutor.ts index d041a84b27..e92c6bd1ae 100644 --- a/src/execution/createPipelineExecutor/00-createPipelineExecutor.ts +++ b/src/execution/createPipelineExecutor/00-createPipelineExecutor.ts @@ -1,23 +1,23 @@ -import { spaceTrim } from 'spacetrim'; -import type { PartialDeep, Promisable, ReadonlyDeep } from 'type-fest'; -import { DEFAULT_CSV_SETTINGS } from '../../config'; -import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from '../../config'; -import { DEFAULT_IS_AUTO_INSTALLED } from '../../config'; -import { DEFAULT_IS_VERBOSE } from '../../config'; -import { DEFAULT_MAX_EXECUTION_ATTEMPTS } from '../../config'; -import { DEFAULT_MAX_PARALLEL_COUNT } from '../../config'; -import { DEFAULT_SCRAPE_CACHE_DIRNAME } from '../../config'; -import { validatePipeline } from '../../conversion/validation/validatePipeline'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import { isPipelinePrepared } from '../../prepare/isPipelinePrepared'; - -import type { InputParameters } from '../../types/typeAliases'; -import type { ExecutionTask } from '../ExecutionTask'; -import { createTask } from '../ExecutionTask'; -import type { PipelineExecutor } from '../PipelineExecutor'; -import type { PipelineExecutorResult } from '../PipelineExecutorResult'; -import type { CreatePipelineExecutorOptions } from './00-CreatePipelineExecutorOptions'; -import { executePipeline } from './10-executePipeline'; +import { spaceTrim } from "spacetrim"; +import type { PartialDeep, Promisable, ReadonlyDeep } from "type-fest"; +import { DEFAULT_CSV_SETTINGS } from "../../config"; +import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from "../../config"; +import { DEFAULT_IS_AUTO_INSTALLED } from "../../config"; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import { DEFAULT_MAX_EXECUTION_ATTEMPTS } from "../../config"; +import { DEFAULT_MAX_PARALLEL_COUNT } from "../../config"; +import { DEFAULT_SCRAPE_CACHE_DIRNAME } from "../../config"; +import { validatePipeline } from "../../conversion/validation/validatePipeline"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import { isPipelinePrepared } from "../../prepare/isPipelinePrepared"; + +import type { InputParameters } from "../../types/typeAliases"; +import type { ExecutionTask } from "../ExecutionTask"; +import { createTask } from "../ExecutionTask"; +import type { PipelineExecutor } from "../PipelineExecutor"; +import type { PipelineExecutorResult } from "../PipelineExecutorResult"; +import type { CreatePipelineExecutorOptions } from "./00-CreatePipelineExecutorOptions"; +import { executePipeline } from "./10-executePipeline"; /** * Creates executor function from pipeline and execution tools. @@ -26,47 +26,49 @@ import { executePipeline } from './10-executePipeline'; * @throws {PipelineLogicError} on logical error in the pipeline * @public exported from `@promptbook/core` */ -export function createPipelineExecutor(options: CreatePipelineExecutorOptions): PipelineExecutor { - const { - pipeline, - tools, - maxExecutionAttempts = DEFAULT_MAX_EXECUTION_ATTEMPTS, - maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, - csvSettings = DEFAULT_CSV_SETTINGS, - isVerbose = DEFAULT_IS_VERBOSE, - isNotPreparedWarningSupressed = false, - cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, - intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, - isAutoInstalled = DEFAULT_IS_AUTO_INSTALLED, - - rootDirname = null, - } = options; - - validatePipeline(pipeline); - - const pipelineIdentification = (() => { - // Note: This is a 😐 implementation of [🚞] - const _: Array = []; - - if (pipeline.sourceFile !== undefined) { - _.push(`File: ${pipeline.sourceFile}`); - } - - if (pipeline.pipelineUrl !== undefined) { - _.push(`Url: ${pipeline.pipelineUrl}`); - } - - return _.join('\n'); - })(); - - let preparedPipeline: ReadonlyDeep; - - if (isPipelinePrepared(pipeline)) { - preparedPipeline = pipeline; - } else if (isNotPreparedWarningSupressed !== true) { - console.warn( - spaceTrim( - (block) => ` +export function createPipelineExecutor( + options: CreatePipelineExecutorOptions, +): PipelineExecutor { + const { + pipeline, + tools, + maxExecutionAttempts = DEFAULT_MAX_EXECUTION_ATTEMPTS, + maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, + csvSettings = DEFAULT_CSV_SETTINGS, + isVerbose = DEFAULT_IS_VERBOSE, + isNotPreparedWarningSupressed = false, + cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, + intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, + isAutoInstalled = DEFAULT_IS_AUTO_INSTALLED, + + rootDirname = null, + } = options; + + validatePipeline(pipeline); + + const pipelineIdentification = (() => { + // Note: This is a 😐 implementation of [🚞] + const _: Array = []; + + if (pipeline.sourceFile !== undefined) { + _.push(`File: ${pipeline.sourceFile}`); + } + + if (pipeline.pipelineUrl !== undefined) { + _.push(`Url: ${pipeline.pipelineUrl}`); + } + + return _.join("\n"); + })(); + + let preparedPipeline: ReadonlyDeep; + + if (isPipelinePrepared(pipeline)) { + preparedPipeline = pipeline; + } else if (isNotPreparedWarningSupressed !== true) { + console.warn( + spaceTrim( + (block) => ` Pipeline is not prepared ${block(pipelineIdentification)} @@ -76,55 +78,66 @@ export function createPipelineExecutor(options: CreatePipelineExecutorOptions): @see more at https://ptbk.io/prepare-pipeline `, - ), - ); - } - - let runCount = 0; - - const pipelineExecutorWithCallback = async ( - inputParameters: InputParameters, - onProgress?: (newOngoingResult: PartialDeep) => Promisable, - ): Promise => { - runCount++; - - return /* not await */ executePipeline({ - pipeline, - preparedPipeline, - setPreparedPipeline: (newPreparedPipeline) => { - preparedPipeline = newPreparedPipeline; - }, - inputParameters, - tools, - onProgress, - pipelineIdentification: spaceTrim( - (block) => ` + ), + ); + } + + let runCount = 0; + + const pipelineExecutorWithCallback = async ( + inputParameters: InputParameters, + onProgress?: ( + newOngoingResult: PartialDeep, + ) => Promisable, + ): Promise => { + runCount++; + + return /* not await */ executePipeline({ + pipeline, + preparedPipeline, + setPreparedPipeline: (newPreparedPipeline) => { + preparedPipeline = newPreparedPipeline; + }, + inputParameters, + tools, + onProgress, + pipelineIdentification: spaceTrim( + (block) => ` ${block(pipelineIdentification)} - ${runCount === 1 ? '' : `Run #${runCount}`} + ${runCount === 1 ? "" : `Run #${runCount}`} `, - ), - maxExecutionAttempts, - maxParallelCount, - csvSettings, - isVerbose, - isNotPreparedWarningSupressed, - rootDirname, - cacheDirname, - intermediateFilesStrategy, - isAutoInstalled, - }); - }; - - const pipelineExecutor: PipelineExecutor = (inputParameters: InputParameters): ExecutionTask => - createTask({ - taskType: 'EXECUTION', - taskProcessCallback(updateOngoingResult: (newOngoingResult: PartialDeep) => void) { - return pipelineExecutorWithCallback(inputParameters, async (newOngoingResult) => { - updateOngoingResult(newOngoingResult); - }); - }, - }) as ExecutionTask; - // <- TODO: Make types such as there is no need to do `as` for `createTask` - - return pipelineExecutor; + ), + maxExecutionAttempts, + maxParallelCount, + csvSettings, + isVerbose, + isNotPreparedWarningSupressed, + rootDirname, + cacheDirname, + intermediateFilesStrategy, + isAutoInstalled, + }); + }; + + const pipelineExecutor: PipelineExecutor = ( + inputParameters: InputParameters, + ): ExecutionTask => + createTask({ + taskType: "EXECUTION", + taskProcessCallback( + updateOngoingResult: ( + newOngoingResult: PartialDeep, + ) => void, + ) { + return pipelineExecutorWithCallback( + inputParameters, + async (newOngoingResult) => { + updateOngoingResult(newOngoingResult); + }, + ); + }, + }) as ExecutionTask; + // <- TODO: Make types such as there is no need to do `as` for `createTask` + + return pipelineExecutor; } diff --git a/src/execution/createPipelineExecutor/10-executePipeline.ts b/src/execution/createPipelineExecutor/10-executePipeline.ts index 1185cc168c..d4c57a3536 100644 --- a/src/execution/createPipelineExecutor/10-executePipeline.ts +++ b/src/execution/createPipelineExecutor/10-executePipeline.ts @@ -1,29 +1,34 @@ -import { spaceTrim } from 'spacetrim'; -import type { PartialDeep, Promisable, ReadonlyDeep, WritableDeep } from 'type-fest'; -import { forTime } from 'waitasecond'; -import { IMMEDIATE_TIME } from '../../config'; -import { LOOP_LIMIT } from '../../config'; -import { RESERVED_PARAMETER_NAMES } from '../../constants'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import { serializeError } from '../../errors/utils/serializeError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { TaskJson } from '../../pipeline/PipelineJson/TaskJson'; -import { preparePipeline } from '../../prepare/preparePipeline'; -import type { InputParameters } from '../../types/typeAliases'; -import type { Parameters } from '../../types/typeAliases'; -import type { string_name } from '../../types/typeAliases'; -import type { string_reserved_parameter_name } from '../../types/typeAliases'; -import { valueToString } from '../../utils/parameters/valueToString'; -import { exportJson } from '../../utils/serialization/exportJson'; -import { PROMPTBOOK_ENGINE_VERSION } from '../../version'; -import type { ExecutionReportJson } from '../execution-report/ExecutionReportJson'; -import type { PipelineExecutorResult } from '../PipelineExecutorResult'; -import { addUsage } from '../utils/addUsage'; -import { ZERO_USAGE } from '../utils/usage-constants'; -import type { CreatePipelineExecutorOptions } from './00-CreatePipelineExecutorOptions'; -import { executeTask } from './20-executeTask'; -import { filterJustOutputParameters } from './filterJustOutputParameters'; +import { spaceTrim } from "spacetrim"; +import type { + PartialDeep, + Promisable, + ReadonlyDeep, + WritableDeep, +} from "type-fest"; +import { forTime } from "waitasecond"; +import { IMMEDIATE_TIME } from "../../config"; +import { LOOP_LIMIT } from "../../config"; +import { RESERVED_PARAMETER_NAMES } from "../../constants"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import { serializeError } from "../../errors/utils/serializeError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { TaskJson } from "../../pipeline/PipelineJson/TaskJson"; +import { preparePipeline } from "../../prepare/preparePipeline"; +import type { InputParameters } from "../../types/typeAliases"; +import type { Parameters } from "../../types/typeAliases"; +import type { string_name } from "../../types/typeAliases"; +import type { string_reserved_parameter_name } from "../../types/typeAliases"; +import { valueToString } from "../../utils/parameters/valueToString"; +import { exportJson } from "../../utils/serialization/exportJson"; +import { PROMPTBOOK_ENGINE_VERSION } from "../../version"; +import type { PipelineExecutorResult } from "../PipelineExecutorResult"; +import type { ExecutionReportJson } from "../execution-report/ExecutionReportJson"; +import { addUsage } from "../utils/addUsage"; +import { ZERO_USAGE } from "../utils/usage-constants"; +import type { CreatePipelineExecutorOptions } from "./00-CreatePipelineExecutorOptions"; +import { executeTask } from "./20-executeTask"; +import { filterJustOutputParameters } from "./filterJustOutputParameters"; /** * @@@ @@ -31,35 +36,39 @@ import { filterJustOutputParameters } from './filterJustOutputParameters'; * @private internal type of `executePipeline` */ type ExecutePipelineOptions = Required & { - /** - * @@@ - */ - readonly inputParameters: Readonly; - - /** - * @@@ - */ - onProgress?(newOngoingResult: PartialDeep): Promisable; - - /** - * @@@ - */ - readonly pipeline: PipelineJson; - - /** - * @@@ - */ - readonly preparedPipeline: ReadonlyDeep; - - /** - * @@@ - */ - readonly setPreparedPipeline: (preparedPipeline: ReadonlyDeep) => void; - - /** - * @@@ - */ - readonly pipelineIdentification: string; + /** + * @@@ + */ + readonly inputParameters: Readonly; + + /** + * @@@ + */ + onProgress?( + newOngoingResult: PartialDeep, + ): Promisable; + + /** + * @@@ + */ + readonly pipeline: PipelineJson; + + /** + * @@@ + */ + readonly preparedPipeline: ReadonlyDeep; + + /** + * @@@ + */ + readonly setPreparedPipeline: ( + preparedPipeline: ReadonlyDeep, + ) => void; + + /** + * @@@ + */ + readonly pipelineIdentification: string; }; /** @@ -69,348 +78,388 @@ type ExecutePipelineOptions = Required & { * * @private internal utility of `createPipelineExecutor` */ -export async function executePipeline(options: ExecutePipelineOptions): Promise { - const { - inputParameters, - tools, - onProgress, - pipeline, - setPreparedPipeline, - pipelineIdentification, - maxParallelCount, - rootDirname, - isVerbose, - } = options; - let { preparedPipeline } = options; - - if (preparedPipeline === undefined) { - preparedPipeline = await preparePipeline(pipeline, tools, { - rootDirname, - isVerbose, - maxParallelCount, - }); - setPreparedPipeline(preparedPipeline); - } - - const errors: Array = []; - const warnings: Array = []; - - const executionReport: WritableDeep = { - pipelineUrl: preparedPipeline.pipelineUrl, - title: preparedPipeline.title, - promptbookUsedVersion: PROMPTBOOK_ENGINE_VERSION, - promptbookRequestedVersion: preparedPipeline.bookVersion, - description: preparedPipeline.description, - promptExecutions: [], - }; - - /** - * Note: This is a flag to prevent `onProgress` call after the pipeline execution is finished - */ - let isReturned = false; - - // Note: Check that all input input parameters are defined - for (const parameter of preparedPipeline.parameters.filter(({ isInput }) => isInput)) { - if (inputParameters[parameter.name] === undefined) { - isReturned = true; - - if (onProgress !== undefined) { - // Note: Wait a short time to prevent race conditions - await forTime(IMMEDIATE_TIME); - } - - return exportJson({ - name: `executionReport`, - message: `Unuccessful PipelineExecutorResult (with missing parameter {${parameter.name}}) PipelineExecutorResult`, - order: [], - value: { - isSuccessful: false, - errors: [ - new PipelineExecutionError( - `Parameter \`{${parameter.name}}\` is required as an input parameter`, - ), - ...errors, - ].map(serializeError), - warnings: [], - executionReport, - outputParameters: {}, - usage: ZERO_USAGE, - preparedPipeline, - }, - }) satisfies PipelineExecutorResult; - } - } - - // Note: Check that no extra input parameters are passed - for (const parameterName of Object.keys(inputParameters)) { - const parameter = preparedPipeline.parameters.find(({ name }) => name === parameterName); - - if (parameter === undefined) { - warnings.push( - new PipelineExecutionError( - spaceTrim( - (block) => ` +export async function executePipeline( + options: ExecutePipelineOptions, +): Promise { + const { + inputParameters, + tools, + onProgress, + pipeline, + setPreparedPipeline, + pipelineIdentification, + maxParallelCount, + rootDirname, + isVerbose, + } = options; + let { preparedPipeline } = options; + + if (preparedPipeline === undefined) { + preparedPipeline = await preparePipeline(pipeline, tools, { + rootDirname, + isVerbose, + maxParallelCount, + }); + setPreparedPipeline(preparedPipeline); + } + + const errors: Array = []; + const warnings: Array = + []; + + const executionReport: WritableDeep = { + pipelineUrl: preparedPipeline.pipelineUrl, + title: preparedPipeline.title, + promptbookUsedVersion: PROMPTBOOK_ENGINE_VERSION, + promptbookRequestedVersion: preparedPipeline.bookVersion, + description: preparedPipeline.description, + promptExecutions: [], + }; + + /** + * Note: This is a flag to prevent `onProgress` call after the pipeline execution is finished + */ + let isReturned = false; + + // Note: Check that all input input parameters are defined + for (const parameter of preparedPipeline.parameters.filter( + ({ isInput }) => isInput, + )) { + if (inputParameters[parameter.name] === undefined) { + isReturned = true; + + if (onProgress !== undefined) { + // Note: Wait a short time to prevent race conditions + await forTime(IMMEDIATE_TIME); + } + + return exportJson({ + name: `executionReport`, + message: `Unuccessful PipelineExecutorResult (with missing parameter {${parameter.name}}) PipelineExecutorResult`, + order: [], + value: { + isSuccessful: false, + errors: [ + new PipelineExecutionError( + `Parameter \`{${parameter.name}}\` is required as an input parameter`, + ), + ...errors, + ].map(serializeError), + warnings: [], + executionReport, + outputParameters: {}, + usage: ZERO_USAGE, + preparedPipeline, + }, + }) satisfies PipelineExecutorResult; + } + } + + // Note: Check that no extra input parameters are passed + for (const parameterName of Object.keys(inputParameters)) { + const parameter = preparedPipeline.parameters.find( + ({ name }) => name === parameterName, + ); + + if (parameter === undefined) { + warnings.push( + new PipelineExecutionError( + spaceTrim( + (block) => ` Extra parameter {${parameterName}} is being passed which is not part of the pipeline. ${block(pipelineIdentification)} `, - ), - ), - ); - } else if (parameter.isInput === false) { - isReturned = true; - - if (onProgress !== undefined) { - // Note: Wait a short time to prevent race conditions - await forTime(IMMEDIATE_TIME); - } - - // TODO: [🧠] This should be also non-critical error - return exportJson({ - name: 'pipelineExecutorResult', - message: spaceTrim( - (block) => ` + ), + ), + ); + } else if (parameter.isInput === false) { + isReturned = true; + + if (onProgress !== undefined) { + // Note: Wait a short time to prevent race conditions + await forTime(IMMEDIATE_TIME); + } + + // TODO: [🧠] This should be also non-critical error + return exportJson({ + name: "pipelineExecutorResult", + message: spaceTrim( + (block) => ` Unuccessful PipelineExecutorResult (with extra parameter {${ - parameter.name - }}) PipelineExecutorResult + parameter.name + }}) PipelineExecutorResult ${block(pipelineIdentification)} `, - ), - order: [], - value: { - isSuccessful: false, - errors: [ - new PipelineExecutionError( - spaceTrim( - (block) => ` + ), + order: [], + value: { + isSuccessful: false, + errors: [ + new PipelineExecutionError( + spaceTrim( + (block) => ` Parameter \`{${parameter.name}}\` is passed as input parameter but it is not input ${block(pipelineIdentification)} `, - ), - ), - ...errors, - ].map(serializeError), - warnings: warnings.map(serializeError), - executionReport, - outputParameters: {}, - usage: ZERO_USAGE, - preparedPipeline, - }, - }) satisfies PipelineExecutorResult; - } - } - - let parametersToPass: Parameters = Object.fromEntries( - Object.entries(inputParameters).map(([key, value]) => [key, valueToString(value)]), - ); - - try { - let resovedParameterNames: ReadonlyArray = preparedPipeline.parameters - .filter(({ isInput }) => isInput) - .map(({ name }) => name); - let unresovedTasks: ReadonlyArray> = [...preparedPipeline.tasks]; - let resolving: Array> = []; - - let loopLimit = LOOP_LIMIT; - while (unresovedTasks.length > 0) { - if (loopLimit-- < 0) { - // Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline - throw new UnexpectedError( - spaceTrim( - (block) => ` + ), + ), + ...errors, + ].map(serializeError), + warnings: warnings.map(serializeError), + executionReport, + outputParameters: {}, + usage: ZERO_USAGE, + preparedPipeline, + }, + }) satisfies PipelineExecutorResult; + } + } + + let parametersToPass: Parameters = Object.fromEntries( + Object.entries(inputParameters).map(([key, value]) => [ + key, + valueToString(value), + ]), + ); + + try { + let resovedParameterNames: ReadonlyArray = + preparedPipeline.parameters + .filter(({ isInput }) => isInput) + .map(({ name }) => name); + let unresovedTasks: ReadonlyArray> = [ + ...preparedPipeline.tasks, + ]; + let resolving: Array> = []; + + let loopLimit = LOOP_LIMIT; + while (unresovedTasks.length > 0) { + if (loopLimit-- < 0) { + // Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline + throw new UnexpectedError( + spaceTrim( + (block) => ` Loop limit reached during resolving parameters pipeline execution ${block(pipelineIdentification)} `, - ), - ); - } - - const currentTask = unresovedTasks.find((task) => - task.dependentParameterNames.every((name) => - [...resovedParameterNames, ...RESERVED_PARAMETER_NAMES].includes(name), - ), - ); - - if (!currentTask && resolving.length === 0) { - throw new UnexpectedError( - // TODO: [🐎] DRY - spaceTrim( - (block) => ` + ), + ); + } + + const currentTask = unresovedTasks.find((task) => + task.dependentParameterNames.every((name) => + [...resovedParameterNames, ...RESERVED_PARAMETER_NAMES].includes( + name, + ), + ), + ); + + if (!currentTask && resolving.length === 0) { + throw new UnexpectedError( + // TODO: [🐎] DRY + spaceTrim( + (block) => ` Can not resolve some parameters: ${block(pipelineIdentification)} **Can not resolve:** ${block( - unresovedTasks - .map( - ({ resultingParameterName, dependentParameterNames }) => - `- Parameter \`{${resultingParameterName}}\` which depends on ${dependentParameterNames - .map((dependentParameterName) => `\`{${dependentParameterName}}\``) - .join(' and ')}`, - ) - .join('\n'), - )} + unresovedTasks + .map( + ({ + resultingParameterName, + dependentParameterNames, + }) => + `- Parameter \`{${resultingParameterName}}\` which depends on ${dependentParameterNames + .map( + (dependentParameterName) => + `\`{${dependentParameterName}}\``, + ) + .join(" and ")}`, + ) + .join("\n"), + )} **Resolved:** ${block( - resovedParameterNames - .filter( - (name) => - !RESERVED_PARAMETER_NAMES.includes(name as string_reserved_parameter_name), - ) - .map((name) => `- Parameter \`{${name}}\``) - .join('\n'), - )} + resovedParameterNames + .filter( + (name) => + !RESERVED_PARAMETER_NAMES.includes( + name as string_reserved_parameter_name, + ), + ) + .map((name) => `- Parameter \`{${name}}\``) + .join("\n"), + )} **Reserved (which are available):** ${block( - resovedParameterNames - .filter((name) => - RESERVED_PARAMETER_NAMES.includes(name as string_reserved_parameter_name), - ) - .map((name) => `- Parameter \`{${name}}\``) - .join('\n'), - )} + resovedParameterNames + .filter((name) => + RESERVED_PARAMETER_NAMES.includes( + name as string_reserved_parameter_name, + ), + ) + .map((name) => `- Parameter \`{${name}}\``) + .join("\n"), + )} *Note: This should be catched in \`validatePipeline\`* `, - ), - ); - } else if (!currentTask) { - /* [🤹‍♂️] */ await Promise.race(resolving); - } else { - unresovedTasks = unresovedTasks.filter((task) => task !== currentTask); - - const work = /* [🤹‍♂️] not await */ executeTask({ - ...options, - currentTask, - preparedPipeline, - parametersToPass, - tools, - onProgress(newOngoingResult: PartialDeep) { - if (isReturned) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + ), + ); + } else if (!currentTask) { + /* [🤹‍♂️] */ await Promise.race(resolving); + } else { + unresovedTasks = unresovedTasks.filter((task) => task !== currentTask); + + const work = /* [🤹‍♂️] not await */ executeTask({ + ...options, + currentTask, + preparedPipeline, + parametersToPass, + tools, + onProgress(newOngoingResult: PartialDeep) { + if (isReturned) { + throw new UnexpectedError( + spaceTrim( + (block) => ` Can not call \`onProgress\` after pipeline execution is finished ${block(pipelineIdentification)} ${block( - JSON.stringify(newOngoingResult, null, 4) - .split('\n') - .map((line) => `> ${line}`) - .join('\n'), - )} + JSON.stringify( + newOngoingResult, + null, + 4, + ) + .split("\n") + .map((line) => `> ${line}`) + .join("\n"), + )} `, - ), - ); - } - - if (onProgress) { - onProgress(newOngoingResult); - } - }, - $executionReport: executionReport, - pipelineIdentification: spaceTrim( - (block) => ` + ), + ); + } + + if (onProgress) { + onProgress(newOngoingResult); + } + }, + $executionReport: executionReport, + pipelineIdentification: spaceTrim( + (block) => ` ${block(pipelineIdentification)} Task name: ${currentTask.name} Task title: ${currentTask.title} `, - ), - }) - .then((newParametersToPass) => { - parametersToPass = { ...newParametersToPass, ...parametersToPass }; - resovedParameterNames = [...resovedParameterNames, currentTask.resultingParameterName]; - }) - .then(() => { - resolving = resolving.filter((w) => w !== work); - }); - // <- Note: Errors are catched here [3] - // TODO: BUT if in multiple tasks are errors, only the first one is catched so maybe we should catch errors here and save them to errors array here - - resolving.push(work); - } - } - - await Promise.all(resolving); - } catch (error /* <- Note: [3] */) { - if (!(error instanceof Error)) { - throw error; - } - - // Note: No need to rethrow UnexpectedError - // if (error instanceof UnexpectedError) { - - // Note: Count usage, [🧠] Maybe put to separate function executionReportJsonToUsage + DRY [🤹‍♂️] - const usage = addUsage(...executionReport.promptExecutions.map(({ result }) => result?.usage || ZERO_USAGE)); - - // Note: Making this on separate line before `return` to grab errors [4] - const outputParameters = filterJustOutputParameters({ - preparedPipeline, - parametersToPass, - $warnings: warnings, - pipelineIdentification, - }); - - isReturned = true; - - if (onProgress !== undefined) { - // Note: Wait a short time to prevent race conditions - await forTime(IMMEDIATE_TIME); - } - - return exportJson({ - name: 'pipelineExecutorResult', - message: `Unuccessful PipelineExecutorResult (with misc errors) PipelineExecutorResult`, - order: [], - value: { - isSuccessful: false, - errors: [error, ...errors].map(serializeError), - warnings: warnings.map(serializeError), - usage, - executionReport, - outputParameters, - preparedPipeline, - }, - }) satisfies PipelineExecutorResult; - } - - // Note: Count usage, [🧠] Maybe put to separate function executionReportJsonToUsage + DRY [🤹‍♂️] - const usage = addUsage(...executionReport.promptExecutions.map(({ result }) => result?.usage || ZERO_USAGE)); - - // Note: Making this on separate line before `return` to grab errors [4] - const outputParameters = filterJustOutputParameters({ - preparedPipeline, - parametersToPass, - $warnings: warnings, - pipelineIdentification, - }); - - isReturned = true; - - if (onProgress !== undefined) { - // Note: Wait a short time to prevent race conditions - await forTime(IMMEDIATE_TIME); - } - - return exportJson({ - name: 'pipelineExecutorResult', - message: `Successful PipelineExecutorResult`, - order: [], - value: { - isSuccessful: true, - errors: errors.map(serializeError), - warnings: warnings.map(serializeError), - usage, - executionReport, - outputParameters, - preparedPipeline, - }, - }) satisfies PipelineExecutorResult; + ), + }) + .then((newParametersToPass) => { + parametersToPass = { ...newParametersToPass, ...parametersToPass }; + resovedParameterNames = [ + ...resovedParameterNames, + currentTask.resultingParameterName, + ]; + }) + .then(() => { + resolving = resolving.filter((w) => w !== work); + }); + // <- Note: Errors are catched here [3] + // TODO: BUT if in multiple tasks are errors, only the first one is catched so maybe we should catch errors here and save them to errors array here + + resolving.push(work); + } + } + + await Promise.all(resolving); + } catch (error /* <- Note: [3] */) { + if (!(error instanceof Error)) { + throw error; + } + + // Note: No need to rethrow UnexpectedError + // if (error instanceof UnexpectedError) { + + // Note: Count usage, [🧠] Maybe put to separate function executionReportJsonToUsage + DRY [🤹‍♂️] + const usage = addUsage( + ...executionReport.promptExecutions.map( + ({ result }) => result?.usage || ZERO_USAGE, + ), + ); + + // Note: Making this on separate line before `return` to grab errors [4] + const outputParameters = filterJustOutputParameters({ + preparedPipeline, + parametersToPass, + $warnings: warnings, + pipelineIdentification, + }); + + isReturned = true; + + if (onProgress !== undefined) { + // Note: Wait a short time to prevent race conditions + await forTime(IMMEDIATE_TIME); + } + + return exportJson({ + name: "pipelineExecutorResult", + message: `Unuccessful PipelineExecutorResult (with misc errors) PipelineExecutorResult`, + order: [], + value: { + isSuccessful: false, + errors: [error, ...errors].map(serializeError), + warnings: warnings.map(serializeError), + usage, + executionReport, + outputParameters, + preparedPipeline, + }, + }) satisfies PipelineExecutorResult; + } + + // Note: Count usage, [🧠] Maybe put to separate function executionReportJsonToUsage + DRY [🤹‍♂️] + const usage = addUsage( + ...executionReport.promptExecutions.map( + ({ result }) => result?.usage || ZERO_USAGE, + ), + ); + + // Note: Making this on separate line before `return` to grab errors [4] + const outputParameters = filterJustOutputParameters({ + preparedPipeline, + parametersToPass, + $warnings: warnings, + pipelineIdentification, + }); + + isReturned = true; + + if (onProgress !== undefined) { + // Note: Wait a short time to prevent race conditions + await forTime(IMMEDIATE_TIME); + } + + return exportJson({ + name: "pipelineExecutorResult", + message: `Successful PipelineExecutorResult`, + order: [], + value: { + isSuccessful: true, + errors: errors.map(serializeError), + warnings: warnings.map(serializeError), + usage, + executionReport, + outputParameters, + preparedPipeline, + }, + }) satisfies PipelineExecutorResult; } diff --git a/src/execution/createPipelineExecutor/20-executeTask.ts b/src/execution/createPipelineExecutor/20-executeTask.ts index a18e5eab6d..6990927847 100644 --- a/src/execution/createPipelineExecutor/20-executeTask.ts +++ b/src/execution/createPipelineExecutor/20-executeTask.ts @@ -1,17 +1,22 @@ -import { spaceTrim } from 'spacetrim'; -import type { PartialDeep, Promisable, ReadonlyDeep, WritableDeep } from 'type-fest'; -import { extractParameterNamesFromTask } from '../../conversion/utils/extractParameterNamesFromTask'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { TaskJson } from '../../pipeline/PipelineJson/TaskJson'; -import type { Parameters } from '../../types/typeAliases'; -import { difference } from '../../utils/sets/difference'; -import { union } from '../../utils/sets/union'; -import type { ExecutionReportJson } from '../execution-report/ExecutionReportJson'; -import type { PipelineExecutorResult } from '../PipelineExecutorResult'; -import type { CreatePipelineExecutorOptions } from './00-CreatePipelineExecutorOptions'; -import { executeFormatSubvalues } from './30-executeFormatSubvalues'; -import { getReservedParametersForTask } from './getReservedParametersForTask'; +import { spaceTrim } from "spacetrim"; +import type { + PartialDeep, + Promisable, + ReadonlyDeep, + WritableDeep, +} from "type-fest"; +import { extractParameterNamesFromTask } from "../../conversion/utils/extractParameterNamesFromTask"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { TaskJson } from "../../pipeline/PipelineJson/TaskJson"; +import type { Parameters } from "../../types/typeAliases"; +import { difference } from "../../utils/sets/difference"; +import { union } from "../../utils/sets/union"; +import type { PipelineExecutorResult } from "../PipelineExecutorResult"; +import type { ExecutionReportJson } from "../execution-report/ExecutionReportJson"; +import type { CreatePipelineExecutorOptions } from "./00-CreatePipelineExecutorOptions"; +import { executeFormatSubvalues } from "./30-executeFormatSubvalues"; +import { getReservedParametersForTask } from "./getReservedParametersForTask"; /** * @@@ @@ -19,35 +24,37 @@ import { getReservedParametersForTask } from './getReservedParametersForTask'; * @private internal type of `executeTask` */ type executeSingleTaskOptions = Required & { - /** - * @@@ - */ - readonly currentTask: ReadonlyDeep; - - /** - * @@@ - */ - readonly preparedPipeline: ReadonlyDeep; - - /** - * @@@ - */ - readonly parametersToPass: Readonly; - - /** - * @@@ - */ - readonly onProgress: (newOngoingResult: PartialDeep) => Promisable; - - /** - * @@@ - */ - readonly $executionReport: WritableDeep; - - /** - * @@@ - */ - readonly pipelineIdentification: string; + /** + * @@@ + */ + readonly currentTask: ReadonlyDeep; + + /** + * @@@ + */ + readonly preparedPipeline: ReadonlyDeep; + + /** + * @@@ + */ + readonly parametersToPass: Readonly; + + /** + * @@@ + */ + readonly onProgress: ( + newOngoingResult: PartialDeep, + ) => Promisable; + + /** + * @@@ + */ + readonly $executionReport: WritableDeep; + + /** + * @@@ + */ + readonly pipelineIdentification: string; }; /** @@ -55,97 +62,111 @@ type executeSingleTaskOptions = Required & { * * @private internal utility of `createPipelineExecutor` */ -export async function executeTask(options: executeSingleTaskOptions): Promise> { - const { - currentTask, - preparedPipeline, - parametersToPass, - tools, - onProgress, - $executionReport, - pipelineIdentification, - maxExecutionAttempts, - maxParallelCount, - csvSettings, - isVerbose, - rootDirname, - cacheDirname, - intermediateFilesStrategy, - isAutoInstalled, - isNotPreparedWarningSupressed, - } = options; - - const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask); - - await onProgress({ - outputParameters: { - [currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here? - }, - }); - - // Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck - const usedParameterNames = extractParameterNamesFromTask(currentTask); - const dependentParameterNames = new Set(currentTask.dependentParameterNames); - // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters` - if ( - union( - difference(usedParameterNames, dependentParameterNames), - difference(dependentParameterNames, usedParameterNames), - // <- TODO: [💯] - ).size !== 0 - ) { - throw new UnexpectedError( - spaceTrim( - (block) => ` +export async function executeTask( + options: executeSingleTaskOptions, +): Promise> { + const { + currentTask, + preparedPipeline, + parametersToPass, + tools, + onProgress, + $executionReport, + pipelineIdentification, + maxExecutionAttempts, + maxParallelCount, + csvSettings, + isVerbose, + rootDirname, + cacheDirname, + intermediateFilesStrategy, + isAutoInstalled, + isNotPreparedWarningSupressed, + } = options; + + const priority = + preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask); + + await onProgress({ + outputParameters: { + [currentTask.resultingParameterName]: "", // <- TODO: [🧠] What is the best value here? + }, + }); + + // Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck + const usedParameterNames = extractParameterNamesFromTask(currentTask); + const dependentParameterNames = new Set(currentTask.dependentParameterNames); + // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters` + if ( + union( + difference(usedParameterNames, dependentParameterNames), + difference(dependentParameterNames, usedParameterNames), + // <- TODO: [💯] + ).size !== 0 + ) { + throw new UnexpectedError( + spaceTrim( + (block) => ` Dependent parameters are not consistent with used parameters: Dependent parameters: ${Array.from(dependentParameterNames) - .map((name) => `{${name}}`) - .join(', ')} + .map((name) => `{${name}}`) + .join(", ")} Used parameters: ${Array.from(usedParameterNames) - .map((name) => `{${name}}`) - .join(', ')} + .map((name) => `{${name}}`) + .join(", ")} ${block(pipelineIdentification)} `, - ), - ); - } - - const definedParameters: Parameters = Object.freeze({ - ...(await getReservedParametersForTask({ - preparedPipeline, - task: currentTask, - pipelineIdentification, - })), - ...parametersToPass, - }); - - const definedParameterNames = new Set(Object.keys(definedParameters)); - const parameters: Parameters = {}; - - // Note: [2] Check that all used parameters are defined and removing unused parameters for this task - // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters` - for (const parameterName of Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))) { - // Situation: Parameter is defined and used - if (definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) { - parameters[parameterName] = definedParameters[parameterName]!; - } - // Situation: Parameter is defined but NOT used - else if (definedParameterNames.has(parameterName) && !usedParameterNames.has(parameterName)) { - // Do not pass this parameter to prompt - } - // Situation: Parameter is NOT defined BUT used - else if (!definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) { - // Houston, we have a problem - // Note: Checking part is also done in `validatePipeline`, but it’s good to doublecheck - throw new UnexpectedError( - spaceTrim( - (block) => ` + ), + ); + } + + const definedParameters: Parameters = Object.freeze({ + ...(await getReservedParametersForTask({ + preparedPipeline, + task: currentTask, + pipelineIdentification, + })), + ...parametersToPass, + }); + + const definedParameterNames = new Set(Object.keys(definedParameters)); + const parameters: Parameters = {}; + + // Note: [2] Check that all used parameters are defined and removing unused parameters for this task + // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters` + for (const parameterName of Array.from( + union(definedParameterNames, usedParameterNames, dependentParameterNames), + )) { + // Situation: Parameter is defined and used + if ( + definedParameterNames.has(parameterName) && + usedParameterNames.has(parameterName) + ) { + parameters[parameterName] = definedParameters[parameterName]!; + } + // Situation: Parameter is defined but NOT used + else if ( + definedParameterNames.has(parameterName) && + !usedParameterNames.has(parameterName) + ) { + // Do not pass this parameter to prompt + } + // Situation: Parameter is NOT defined BUT used + else if ( + !definedParameterNames.has(parameterName) && + usedParameterNames.has(parameterName) + ) { + // Houston, we have a problem + // Note: Checking part is also done in `validatePipeline`, but it’s good to doublecheck + throw new UnexpectedError( + spaceTrim( + (block) => ` Parameter \`{${parameterName}}\` is NOT defined BUT used in task "${currentTask.title || currentTask.name}" @@ -154,53 +175,58 @@ export async function executeTask(options: executeSingleTaskOptions): Promise { - const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options; - - if (task.foreach === undefined) { - return /* not await */ executeAttempts(options); - } - - if (jokerParameterNames.length !== 0) { - throw new UnexpectedError( - spaceTrim( - (block) => ` +export async function executeFormatSubvalues( + options: ExecuteFormatCellsOptions, +): Promise { + const { + task, + jokerParameterNames, + parameters, + priority, + csvSettings, + pipelineIdentification, + } = options; + + if (task.foreach === undefined) { + return /* not await */ executeAttempts(options); + } + + if (jokerParameterNames.length !== 0) { + throw new UnexpectedError( + spaceTrim( + (block) => ` JOKER parameters are not supported together with FOREACH command [🧞‍♀️] This should be prevented in \`validatePipeline\` ${block(pipelineIdentification)} `, - ), - ); - } - - const parameterValue = parameters[task.foreach.parameterName] || ''; - - const formatDefinition = FORMAT_DEFINITIONS.find( - (formatDefinition) => - [formatDefinition.formatName, ...(formatDefinition.aliases || [])].includes(task.foreach!.formatName), - // <- Note: All names here are already normalized - ); - - if (formatDefinition === undefined) { - throw new UnexpectedError( - // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError) - spaceTrim( - (block) => ` + ), + ); + } + + const parameterValue = parameters[task.foreach.parameterName] || ""; + + const formatDefinition = FORMAT_DEFINITIONS.find( + (formatDefinition) => + [ + formatDefinition.formatName, + ...(formatDefinition.aliases || []), + ].includes(task.foreach!.formatName), + // <- Note: All names here are already normalized + ); + + if (formatDefinition === undefined) { + throw new UnexpectedError( + // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError) + spaceTrim( + (block) => ` Unsupported format "${task.foreach!.formatName}" Available formats: ${block( - FORMAT_DEFINITIONS.map((formatDefinition) => formatDefinition.formatName) - .map((formatName) => `- ${formatName}`) - .join('\n'), - )} + FORMAT_DEFINITIONS.map( + (formatDefinition) => formatDefinition.formatName, + ) + .map((formatName) => `- ${formatName}`) + .join("\n"), + )} [⛷] This should never happen because format name should be validated during parsing ${block(pipelineIdentification)} `, - ), - ); - } - - const subvalueDefinition = formatDefinition.subvalueDefinitions.find( - (subvalueDefinition) => - [subvalueDefinition.subvalueName, ...(subvalueDefinition.aliases || [])].includes( - task.foreach!.subformatName, - ), - // <- Note: All names here are already normalized - ); - - if (subvalueDefinition === undefined) { - throw new UnexpectedError( - // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError) - spaceTrim( - (block) => ` + ), + ); + } + + const subvalueDefinition = formatDefinition.subvalueDefinitions.find( + (subvalueDefinition) => + [ + subvalueDefinition.subvalueName, + ...(subvalueDefinition.aliases || []), + ].includes(task.foreach!.subformatName), + // <- Note: All names here are already normalized + ); + + if (subvalueDefinition === undefined) { + throw new UnexpectedError( + // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError) + spaceTrim( + (block) => ` Unsupported subformat name "${task.foreach!.subformatName}" for format "${ - task.foreach!.formatName - }" + task.foreach!.formatName + }" Available subformat names for format "${formatDefinition.formatName}": ${block( - formatDefinition.subvalueDefinitions - .map((subvalueDefinition) => subvalueDefinition.subvalueName) - .map((subvalueName) => `- ${subvalueName}`) - .join('\n'), - )} + formatDefinition.subvalueDefinitions + .map( + (subvalueDefinition) => + subvalueDefinition.subvalueName, + ) + .map((subvalueName) => `- ${subvalueName}`) + .join("\n"), + )} [⛷] This should never happen because subformat name should be validated during parsing ${block(pipelineIdentification)} `, - ), - ); - } - - let formatSettings: TODO_any; - - if (formatDefinition.formatName === 'CSV') { - formatSettings = csvSettings; - // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text - } - - const resultString = await subvalueDefinition.mapValues( - parameterValue, - task.foreach.outputSubparameterName, - formatSettings, - async (subparameters, index) => { - let mappedParameters: Record; - - // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions - // TODO: When done [🐚] Report progress also for each subvalue here - - try { - mappedParameters = mapAvailableToExpectedParameters({ - expectedParameters: Object.fromEntries( - task.foreach!.inputSubparameterNames.map((subparameterName) => [subparameterName, null]), - ), - availableParameters: subparameters, - }); - } catch (error) { - if (!(error instanceof PipelineExecutionError)) { - throw error; - } - - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + ), + ); + } + + let formatSettings: TODO_any; + + if (formatDefinition.formatName === "CSV") { + formatSettings = csvSettings; + // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text + } + + const resultString = await subvalueDefinition.mapValues( + parameterValue, + task.foreach.outputSubparameterName, + formatSettings, + async (subparameters, index) => { + let mappedParameters: Record< + string_parameter_name, + string_parameter_value + >; + + // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions + // TODO: When done [🐚] Report progress also for each subvalue here + + try { + mappedParameters = mapAvailableToExpectedParameters({ + expectedParameters: Object.fromEntries( + task.foreach!.inputSubparameterNames.map((subparameterName) => [ + subparameterName, + null, + ]), + ), + availableParameters: subparameters, + }); + } catch (error) { + if (!(error instanceof PipelineExecutionError)) { + throw error; + } + + throw new PipelineExecutionError( + spaceTrim( + (block) => ` ${(error as PipelineExecutionError).message} This is error in FOREACH command @@ -145,33 +169,33 @@ export async function executeFormatSubvalues(options: ExecuteFormatCellsOptions) ${block(pipelineIdentification)} Subparameter index: ${index} `, - ), - ); - } - - const allSubparameters = { - ...parameters, - ...mappedParameters, - }; - - // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed - Object.freeze(allSubparameters); - - const subresultString = await executeAttempts({ - ...options, - priority: priority + index, - parameters: allSubparameters, - pipelineIdentification: spaceTrim( - (block) => ` + ), + ); + } + + const allSubparameters = { + ...parameters, + ...mappedParameters, + }; + + // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed + Object.freeze(allSubparameters); + + const subresultString = await executeAttempts({ + ...options, + priority: priority + index, + parameters: allSubparameters, + pipelineIdentification: spaceTrim( + (block) => ` ${block(pipelineIdentification)} Subparameter index: ${index} `, - ), - }); + ), + }); - return subresultString; - }, - ); + return subresultString; + }, + ); - return resultString; + return resultString; } diff --git a/src/execution/createPipelineExecutor/40-executeAttempts.ts b/src/execution/createPipelineExecutor/40-executeAttempts.ts index c68db385bd..ab71a81dd9 100644 --- a/src/execution/createPipelineExecutor/40-executeAttempts.ts +++ b/src/execution/createPipelineExecutor/40-executeAttempts.ts @@ -1,32 +1,32 @@ -import { spaceTrim } from 'spacetrim'; -import type { ReadonlyDeep, WritableDeep } from 'type-fest'; -import { ExpectError } from '../../errors/ExpectError'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import { serializeError } from '../../errors/utils/serializeError'; -import { isValidJsonString } from '../../formats/json/utils/isValidJsonString'; -import { joinLlmExecutionTools } from '../../llm-providers/multiple/joinLlmExecutionTools'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { TaskJson } from '../../pipeline/PipelineJson/TaskJson'; -import { extractJsonBlock } from '../../postprocessing/utils/extractJsonBlock'; -import type { ModelRequirements } from '../../types/ModelRequirements'; -import type { ChatPrompt } from '../../types/Prompt'; -import type { CompletionPrompt } from '../../types/Prompt'; -import type { Prompt } from '../../types/Prompt'; -import type { Parameters } from '../../types/typeAliases'; -import type { string_parameter_name } from '../../types/typeAliases'; -import { arrayableToArray } from '../../utils/arrayableToArray'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import type { really_any } from '../../utils/organization/really_any'; -import type { TODO_any } from '../../utils/organization/TODO_any'; -import type { TODO_string } from '../../utils/organization/TODO_string'; -import { templateParameters } from '../../utils/parameters/templateParameters'; -import { $deepFreeze } from '../../utils/serialization/$deepFreeze'; -import type { ExecutionReportJson } from '../execution-report/ExecutionReportJson'; -import { checkExpectations } from '../utils/checkExpectations'; -import type { $OngoingTaskResult } from './$OngoingTaskResult'; -import type { CreatePipelineExecutorOptions } from './00-CreatePipelineExecutorOptions'; +import { spaceTrim } from "spacetrim"; +import type { ReadonlyDeep, WritableDeep } from "type-fest"; +import { ExpectError } from "../../errors/ExpectError"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import { serializeError } from "../../errors/utils/serializeError"; +import { isValidJsonString } from "../../formats/json/utils/isValidJsonString"; +import { joinLlmExecutionTools } from "../../llm-providers/multiple/joinLlmExecutionTools"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { TaskJson } from "../../pipeline/PipelineJson/TaskJson"; +import { extractJsonBlock } from "../../postprocessing/utils/extractJsonBlock"; +import type { ModelRequirements } from "../../types/ModelRequirements"; +import type { ChatPrompt } from "../../types/Prompt"; +import type { CompletionPrompt } from "../../types/Prompt"; +import type { Prompt } from "../../types/Prompt"; +import type { Parameters } from "../../types/typeAliases"; +import type { string_parameter_name } from "../../types/typeAliases"; +import { arrayableToArray } from "../../utils/arrayableToArray"; +import type { TODO_any } from "../../utils/organization/TODO_any"; +import type { TODO_string } from "../../utils/organization/TODO_string"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import type { really_any } from "../../utils/organization/really_any"; +import { templateParameters } from "../../utils/parameters/templateParameters"; +import { $deepFreeze } from "../../utils/serialization/$deepFreeze"; +import type { ExecutionReportJson } from "../execution-report/ExecutionReportJson"; +import { checkExpectations } from "../utils/checkExpectations"; +import type { $OngoingTaskResult } from "./$OngoingTaskResult"; +import type { CreatePipelineExecutorOptions } from "./00-CreatePipelineExecutorOptions"; keepTypeImported(); @@ -35,55 +35,57 @@ keepTypeImported(); * * @private internal type of `executeAttempts` */ -export type ExecuteAttemptsOptions = Required> & { - /** - * @@@ - */ - readonly jokerParameterNames: Readonly>; - - /** - * @@@ - */ - readonly priority: number; - - /** - * @@@ - * - * Note: [💂] There are two distinct variabiles - * 1) `maxExecutionAttempts` - the amount of attempts LLM model - * 2) `maxAttempts` - the amount of attempts for any task - LLM, SCRIPT, DIALOG, etc. - */ - readonly maxAttempts: number; - - /** - * @@@ - */ - readonly preparedContent: TODO_string; - - /** - * @@@ - */ - readonly parameters: Readonly; - - /** - * @@@ - */ - readonly task: ReadonlyDeep; - - /** - * @@@ - */ - readonly preparedPipeline: ReadonlyDeep; - - /** - * @@@ - */ - readonly $executionReport: WritableDeep; - - /** - * @@@ - */ - readonly pipelineIdentification: string; +export type ExecuteAttemptsOptions = Required< + Omit +> & { + /** + * @@@ + */ + readonly jokerParameterNames: Readonly>; + + /** + * @@@ + */ + readonly priority: number; + + /** + * @@@ + * + * Note: [💂] There are two distinct variabiles + * 1) `maxExecutionAttempts` - the amount of attempts LLM model + * 2) `maxAttempts` - the amount of attempts for any task - LLM, SCRIPT, DIALOG, etc. + */ + readonly maxAttempts: number; + + /** + * @@@ + */ + readonly preparedContent: TODO_string; + + /** + * @@@ + */ + readonly parameters: Readonly; + + /** + * @@@ + */ + readonly task: ReadonlyDeep; + + /** + * @@@ + */ + readonly preparedPipeline: ReadonlyDeep; + + /** + * @@@ + */ + readonly $executionReport: WritableDeep; + + /** + * @@@ + */ + readonly pipelineIdentification: string; }; /** @@ -91,129 +93,149 @@ export type ExecuteAttemptsOptions = Required { - const { - jokerParameterNames, - priority, - maxAttempts, // <- Note: [💂] - preparedContent, - parameters, - task, - preparedPipeline, - tools, - $executionReport, - pipelineIdentification, - maxExecutionAttempts, - } = options; - - const $ongoingTaskResult: $OngoingTaskResult = { - $result: null, - $resultString: null, - $expectError: null, - $scriptPipelineExecutionErrors: [], - }; - - // TODO: [🚐] Make arrayable LLMs -> single LLM DRY - const _llms = arrayableToArray(tools.llm); - const llmTools = _llms.length === 1 ? _llms[0]! : joinLlmExecutionTools(..._llms); - - attempts: for (let attempt = -jokerParameterNames.length; attempt < maxAttempts; attempt++) { - const isJokerAttempt = attempt < 0; - const jokerParameterName = jokerParameterNames[jokerParameterNames.length + attempt]; - - // TODO: [🧠][🍭] JOKERS, EXPECTATIONS, POSTPROCESSING and FOREACH - if (isJokerAttempt && !jokerParameterName) { - throw new UnexpectedError( - spaceTrim( - (block) => ` +export async function executeAttempts( + options: ExecuteAttemptsOptions, +): Promise { + const { + jokerParameterNames, + priority, + maxAttempts, // <- Note: [💂] + preparedContent, + parameters, + task, + preparedPipeline, + tools, + $executionReport, + pipelineIdentification, + maxExecutionAttempts, + } = options; + + const $ongoingTaskResult: $OngoingTaskResult = { + $result: null, + $resultString: null, + $expectError: null, + $scriptPipelineExecutionErrors: [], + }; + + // TODO: [🚐] Make arrayable LLMs -> single LLM DRY + const _llms = arrayableToArray(tools.llm); + const llmTools = + _llms.length === 1 ? _llms[0]! : joinLlmExecutionTools(..._llms); + + for ( + let attempt = -jokerParameterNames.length; + attempt < maxAttempts; + attempt++ + ) { + const isJokerAttempt = attempt < 0; + const jokerParameterName = + jokerParameterNames[jokerParameterNames.length + attempt]; + + // TODO: [🧠][🍭] JOKERS, EXPECTATIONS, POSTPROCESSING and FOREACH + if (isJokerAttempt && !jokerParameterName) { + throw new UnexpectedError( + spaceTrim( + (block) => ` Joker not found in attempt ${attempt} ${block(pipelineIdentification)} `, - ), - ); - } - - $ongoingTaskResult.$result = null; - $ongoingTaskResult.$resultString = null; - $ongoingTaskResult.$expectError = null; - - if (isJokerAttempt) { - if (parameters[jokerParameterName!] === undefined) { - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + ), + ); + } + + $ongoingTaskResult.$result = null; + $ongoingTaskResult.$resultString = null; + $ongoingTaskResult.$expectError = null; + + if (isJokerAttempt) { + if (parameters[jokerParameterName!] === undefined) { + throw new PipelineExecutionError( + spaceTrim( + (block) => ` Joker parameter {${jokerParameterName}} not defined ${block(pipelineIdentification)} `, - ), - ); - // <- TODO: This is maybe `PipelineLogicError` which should be detected in `validatePipeline` and here just thrown as `UnexpectedError` - } else { - $ongoingTaskResult.$resultString = parameters[jokerParameterName!]!; - } - } - - try { - if (!isJokerAttempt) { - taskType: switch (task.taskType) { - case 'SIMPLE_TASK': - $ongoingTaskResult.$resultString = templateParameters(preparedContent, parameters); - break taskType; - - case 'PROMPT_TASK': - { - const modelRequirements = { - modelVariant: 'CHAT', - ...(preparedPipeline.defaultModelRequirements || {}), - ...(task.modelRequirements || {}), - } satisfies ModelRequirements; /* <- Note: [🤛] */ - - $ongoingTaskResult.$prompt = { - title: task.title, - pipelineUrl: `${ - preparedPipeline.pipelineUrl - ? preparedPipeline.pipelineUrl - : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */ - }#${ - task.name - // <- TODO: Here should be maybe also subformat index to distinguish between same task with different subformat values - }`, - parameters, - content: preparedContent, // <- Note: For LLM execution, parameters are replaced in the content - modelRequirements, - expectations: { - ...(preparedPipeline.personas.find(({ name }) => name === task.personaName) || {}), - ...task.expectations, - }, - format: task.format, - postprocessingFunctionNames: task.postprocessingFunctionNames, - } as Prompt; // <- TODO: Not very good type guard - - variant: switch (modelRequirements.modelVariant) { - case 'CHAT': - $ongoingTaskResult.$chatResult = await llmTools.callChatModel!( - // <- TODO: [🧁] Check that `callChatModel` is defined - $deepFreeze($ongoingTaskResult.$prompt) as ChatPrompt, - ); - // TODO: [🍬] Destroy chatThread - $ongoingTaskResult.$result = $ongoingTaskResult.$chatResult; - $ongoingTaskResult.$resultString = $ongoingTaskResult.$chatResult.content; - break variant; - case 'COMPLETION': - $ongoingTaskResult.$completionResult = await llmTools.callCompletionModel!( - // <- TODO: [🧁] Check that `callCompletionModel` is defined - $deepFreeze($ongoingTaskResult.$prompt) as CompletionPrompt, - ); - $ongoingTaskResult.$result = $ongoingTaskResult.$completionResult; - $ongoingTaskResult.$resultString = $ongoingTaskResult.$completionResult.content; - break variant; - - case 'EMBEDDING': - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + ), + ); + // <- TODO: This is maybe `PipelineLogicError` which should be detected in `validatePipeline` and here just thrown as `UnexpectedError` + } else { + $ongoingTaskResult.$resultString = parameters[jokerParameterName!]!; + } + } + + try { + if (!isJokerAttempt) { + switch (task.taskType) { + case "SIMPLE_TASK": + $ongoingTaskResult.$resultString = templateParameters( + preparedContent, + parameters, + ); + break; + + case "PROMPT_TASK": + { + const modelRequirements = { + modelVariant: "CHAT", + ...(preparedPipeline.defaultModelRequirements || {}), + ...(task.modelRequirements || {}), + } satisfies ModelRequirements; /* <- Note: [🤛] */ + + $ongoingTaskResult.$prompt = { + title: task.title, + pipelineUrl: `${ + preparedPipeline.pipelineUrl + ? preparedPipeline.pipelineUrl + : "anonymous" /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */ + }#${ + task.name + // <- TODO: Here should be maybe also subformat index to distinguish between same task with different subformat values + }`, + parameters, + content: preparedContent, // <- Note: For LLM execution, parameters are replaced in the content + modelRequirements, + expectations: { + ...(preparedPipeline.personas.find( + ({ name }) => name === task.personaName, + ) || {}), + ...task.expectations, + }, + format: task.format, + postprocessingFunctionNames: task.postprocessingFunctionNames, + } as Prompt; // <- TODO: Not very good type guard + + switch (modelRequirements.modelVariant) { + case "CHAT": + $ongoingTaskResult.$chatResult = + await llmTools.callChatModel!( + // <- TODO: [🧁] Check that `callChatModel` is defined + $deepFreeze($ongoingTaskResult.$prompt) as ChatPrompt, + ); + // TODO: [🍬] Destroy chatThread + $ongoingTaskResult.$result = $ongoingTaskResult.$chatResult; + $ongoingTaskResult.$resultString = + $ongoingTaskResult.$chatResult.content; + break; + case "COMPLETION": + $ongoingTaskResult.$completionResult = + await llmTools.callCompletionModel!( + // <- TODO: [🧁] Check that `callCompletionModel` is defined + $deepFreeze( + $ongoingTaskResult.$prompt, + ) as CompletionPrompt, + ); + $ongoingTaskResult.$result = + $ongoingTaskResult.$completionResult; + $ongoingTaskResult.$resultString = + $ongoingTaskResult.$completionResult.content; + break; + + case "EMBEDDING": + throw new PipelineExecutionError( + spaceTrim( + (block) => ` Embedding model can not be used in pipeline This should be catched during parsing @@ -221,266 +243,286 @@ export async function executeAttempts(options: ExecuteAttemptsOptions): Promise< ${block(pipelineIdentification)} `, - ), - ); - break variant; + ), + ); + break; - // <- case [🤖]: + // <- case [🤖]: - default: - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + default: + throw new PipelineExecutionError( + spaceTrim( + (block) => ` Unknown model variant "${ - (task as really_any).modelRequirements.modelVariant - }" + (task as really_any) + .modelRequirements + .modelVariant + }" ${block(pipelineIdentification)} `, - ), - ); - } - } - break; - - case 'SCRIPT_TASK': - if (arrayableToArray(tools.script).length === 0) { - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + ), + ); + } + } + break; + + case "SCRIPT_TASK": + if (arrayableToArray(tools.script).length === 0) { + throw new PipelineExecutionError( + spaceTrim( + (block) => ` No script execution tools are available ${block(pipelineIdentification)} `, - ), - ); - } - if (!task.contentLanguage) { - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + ), + ); + } + if (!task.contentLanguage) { + throw new PipelineExecutionError( + spaceTrim( + (block) => ` Script language is not defined for SCRIPT TASK "${task.name}" ${block(pipelineIdentification)} `, - ), - ); - } - - // TODO: DRY [☯] - scripts: for (const scriptTools of arrayableToArray(tools.script)) { - try { - $ongoingTaskResult.$resultString = await scriptTools.execute( - $deepFreeze({ - scriptLanguage: task.contentLanguage, - script: preparedContent, // <- Note: For Script execution, parameters are used as variables - parameters, - }), - ); - - break scripts; - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - if (error instanceof UnexpectedError) { - throw error; - } - - $ongoingTaskResult.$scriptPipelineExecutionErrors.push(error); - } - } - - if ($ongoingTaskResult.$resultString !== null) { - break taskType; - } - - if ($ongoingTaskResult.$scriptPipelineExecutionErrors.length === 1) { - throw $ongoingTaskResult.$scriptPipelineExecutionErrors[0]; - } else { - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + ), + ); + } + + // TODO: DRY [☯] + for (const scriptTools of arrayableToArray(tools.script)) { + try { + $ongoingTaskResult.$resultString = await scriptTools.execute( + $deepFreeze({ + scriptLanguage: task.contentLanguage, + script: preparedContent, // <- Note: For Script execution, parameters are used as variables + parameters, + }), + ); + + break; + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + if (error instanceof UnexpectedError) { + throw error; + } + + $ongoingTaskResult.$scriptPipelineExecutionErrors.push(error); + } + } + + if ($ongoingTaskResult.$resultString !== null) { + break; + } + + if ( + $ongoingTaskResult.$scriptPipelineExecutionErrors.length === 1 + ) { + throw $ongoingTaskResult.$scriptPipelineExecutionErrors[0]; + } else { + throw new PipelineExecutionError( + spaceTrim( + (block) => ` Script execution failed ${ - $ongoingTaskResult.$scriptPipelineExecutionErrors.length - }x + $ongoingTaskResult + .$scriptPipelineExecutionErrors + .length + }x ${block(pipelineIdentification)} ${block( - $ongoingTaskResult.$scriptPipelineExecutionErrors - .map((error) => '- ' + error.message) - .join('\n\n'), - )} + $ongoingTaskResult.$scriptPipelineExecutionErrors + .map( + (error) => "- " + error.message, + ) + .join("\n\n"), + )} `, - ), - ); - } - - // Note: This line is unreachable because of the break taskType above - break taskType; - - case 'DIALOG_TASK': - if (tools.userInterface === undefined) { - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + ), + ); + } + + // Note: This line is unreachable because of the break taskType above + break; + + case "DIALOG_TASK": + if (tools.userInterface === undefined) { + throw new PipelineExecutionError( + spaceTrim( + (block) => ` User interface tools are not available ${block(pipelineIdentification)} `, - ), - ); - } - - // TODO: [🌹] When making next attempt for `DIALOG TASK`, preserve the previous user input - $ongoingTaskResult.$resultString = await tools.userInterface.promptDialog( - $deepFreeze({ - promptTitle: task.title, - promptMessage: templateParameters(task.description || '', parameters), - defaultValue: templateParameters(preparedContent, parameters), - - // TODO: [🧠] Figure out how to define placeholder in .book.md file - placeholder: undefined, - priority, - }), - ); - break taskType; - - // <- case: [🅱] - - default: - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + ), + ); + } + + // TODO: [🌹] When making next attempt for `DIALOG TASK`, preserve the previous user input + $ongoingTaskResult.$resultString = + await tools.userInterface.promptDialog( + $deepFreeze({ + promptTitle: task.title, + promptMessage: templateParameters( + task.description || "", + parameters, + ), + defaultValue: templateParameters(preparedContent, parameters), + + // TODO: [🧠] Figure out how to define placeholder in .book.md file + placeholder: undefined, + priority, + }), + ); + break; + + // <- case: [🅱] + + default: + throw new PipelineExecutionError( + spaceTrim( + (block) => ` Unknown execution type "${(task as TODO_any).taskType}" ${block(pipelineIdentification)} `, - ), - ); - } - } - - if (!isJokerAttempt && task.postprocessingFunctionNames) { - for (const functionName of task.postprocessingFunctionNames) { - let postprocessingError = null; - - scripts: for (const scriptTools of arrayableToArray(tools.script)) { - try { - $ongoingTaskResult.$resultString = await scriptTools.execute({ - scriptLanguage: `javascript` /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */, - script: `${functionName}(resultString)`, - parameters: { - resultString: $ongoingTaskResult.$resultString || '', - // Note: No ...parametersForTask, because working with result only - }, - }); - - postprocessingError = null; - break scripts; - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - if (error instanceof UnexpectedError) { - throw error; - } - - postprocessingError = error; - $ongoingTaskResult.$scriptPipelineExecutionErrors.push(error); - } - } - - if (postprocessingError) { - throw postprocessingError; - } - } - } - - // TODO: [💝] Unite object for expecting amount and format - if (task.format) { - if (task.format === 'JSON') { - if (!isValidJsonString($ongoingTaskResult.$resultString || '')) { - // TODO: [🏢] Do more universally via `FormatDefinition` - - try { - $ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || ''); - } catch (error) { - keepUnused( - error, - // <- Note: This error is not important - // ONLY imporant thing is the information that `resultString` not contain valid JSON block - ); - - throw new ExpectError( - spaceTrim( - (block) => ` + ), + ); + } + } + + if (!isJokerAttempt && task.postprocessingFunctionNames) { + for (const functionName of task.postprocessingFunctionNames) { + let postprocessingError = null; + + for (const scriptTools of arrayableToArray(tools.script)) { + try { + $ongoingTaskResult.$resultString = await scriptTools.execute({ + scriptLanguage: `javascript` /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */, + script: `${functionName}(resultString)`, + parameters: { + resultString: $ongoingTaskResult.$resultString || "", + // Note: No ...parametersForTask, because working with result only + }, + }); + + postprocessingError = null; + break; + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + if (error instanceof UnexpectedError) { + throw error; + } + + postprocessingError = error; + $ongoingTaskResult.$scriptPipelineExecutionErrors.push(error); + } + } + + if (postprocessingError) { + throw postprocessingError; + } + } + } + + // TODO: [💝] Unite object for expecting amount and format + if (task.format) { + if (task.format === "JSON") { + if (!isValidJsonString($ongoingTaskResult.$resultString || "")) { + // TODO: [🏢] Do more universally via `FormatDefinition` + + try { + $ongoingTaskResult.$resultString = extractJsonBlock( + $ongoingTaskResult.$resultString || "", + ); + } catch (error) { + keepUnused( + error, + // <- Note: This error is not important + // ONLY imporant thing is the information that `resultString` not contain valid JSON block + ); + + throw new ExpectError( + spaceTrim( + (block) => ` Expected valid JSON string ${block( - /*<- Note: No need for `pipelineIdentification`, it will be catched and added later */ '', - )} + /*<- Note: No need for `pipelineIdentification`, it will be catched and added later */ "", + )} `, - ), - ); - } - } - } else { - throw new UnexpectedError( - spaceTrim( - (block) => ` + ), + ); + } + } + } else { + throw new UnexpectedError( + spaceTrim( + (block) => ` Unknown format "${task.format}" ${block(pipelineIdentification)} `, - ), - ); - } - } - - // TODO: [💝] Unite object for expecting amount and format - if (task.expectations) { - checkExpectations(task.expectations, $ongoingTaskResult.$resultString || ''); - } - - break attempts; - } catch (error) { - if (!(error instanceof ExpectError)) { - throw error; - } - - $ongoingTaskResult.$expectError = error; - } finally { - if ( - !isJokerAttempt && - task.taskType === 'PROMPT_TASK' && - $ongoingTaskResult.$prompt! - // <- Note: [2] When some expected parameter is not defined, error will occur in templateParameters - // In that case we don’t want to make a report about it because it’s not a llm execution error - ) { - // TODO: [🧠] Maybe put other taskTypes into report - $executionReport.promptExecutions.push({ - prompt: { - ...$ongoingTaskResult.$prompt, - // <- TODO: [🧠] How to pick everyhing except `pipelineUrl` - } as really_any, - result: $ongoingTaskResult.$result || undefined, - error: - $ongoingTaskResult.$expectError === null - ? undefined - : serializeError($ongoingTaskResult.$expectError), - }); - } - } - - if ($ongoingTaskResult.$expectError !== null && attempt === maxAttempts - 1) { - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + ), + ); + } + } + + // TODO: [💝] Unite object for expecting amount and format + if (task.expectations) { + checkExpectations( + task.expectations, + $ongoingTaskResult.$resultString || "", + ); + } + + break; + } catch (error) { + if (!(error instanceof ExpectError)) { + throw error; + } + + $ongoingTaskResult.$expectError = error; + } finally { + if ( + !isJokerAttempt && + task.taskType === "PROMPT_TASK" && + $ongoingTaskResult.$prompt! + // <- Note: [2] When some expected parameter is not defined, error will occur in templateParameters + // In that case we don’t want to make a report about it because it’s not a llm execution error + ) { + // TODO: [🧠] Maybe put other taskTypes into report + $executionReport.promptExecutions.push({ + prompt: { + ...$ongoingTaskResult.$prompt, + // <- TODO: [🧠] How to pick everyhing except `pipelineUrl` + } as really_any, + result: $ongoingTaskResult.$result || undefined, + error: + $ongoingTaskResult.$expectError === null + ? undefined + : serializeError($ongoingTaskResult.$expectError), + }); + } + } + + if ( + $ongoingTaskResult.$expectError !== null && + attempt === maxAttempts - 1 + ) { + throw new PipelineExecutionError( + spaceTrim( + (block) => ` LLM execution failed ${maxExecutionAttempts}x ${block(pipelineIdentification)} @@ -488,49 +530,49 @@ export async function executeAttempts(options: ExecuteAttemptsOptions): Promise< --- The Prompt: ${block( - ($ongoingTaskResult.$prompt?.content || '') - .split('\n') - .map((line) => `> ${line}`) - .join('\n'), - )} + ($ongoingTaskResult.$prompt?.content || "") + .split("\n") + .map((line) => `> ${line}`) + .join("\n"), + )} - Last error ${$ongoingTaskResult.$expectError?.name || ''}: + Last error ${$ongoingTaskResult.$expectError?.name || ""}: ${block( - ($ongoingTaskResult.$expectError?.message || '') - .split('\n') - .map((line) => `> ${line}`) - .join('\n'), - )} + ($ongoingTaskResult.$expectError?.message || "") + .split("\n") + .map((line) => `> ${line}`) + .join("\n"), + )} Last result: ${block( - $ongoingTaskResult.$resultString === null - ? 'null' - : $ongoingTaskResult.$resultString - .split('\n') - .map((line) => `> ${line}`) - .join('\n'), - )} + $ongoingTaskResult.$resultString === null + ? "null" + : $ongoingTaskResult.$resultString + .split("\n") + .map((line) => `> ${line}`) + .join("\n"), + )} --- `, - ), - ); - } - } - - if ($ongoingTaskResult.$resultString === null) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + ), + ); + } + } + + if ($ongoingTaskResult.$resultString === null) { + throw new UnexpectedError( + spaceTrim( + (block) => ` Something went wrong and prompt result is null ${block(pipelineIdentification)} `, - ), - ); - } + ), + ); + } - return $ongoingTaskResult.$resultString; + return $ongoingTaskResult.$resultString; } /** diff --git a/src/execution/createPipelineExecutor/filterJustOutputParameters.ts b/src/execution/createPipelineExecutor/filterJustOutputParameters.ts index bc20324093..8951c5ca69 100644 --- a/src/execution/createPipelineExecutor/filterJustOutputParameters.ts +++ b/src/execution/createPipelineExecutor/filterJustOutputParameters.ts @@ -1,8 +1,8 @@ -import { spaceTrim } from 'spacetrim'; -import type { ReadonlyDeep } from 'type-fest'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { Parameters } from '../../types/typeAliases'; +import { spaceTrim } from "spacetrim"; +import type { ReadonlyDeep } from "type-fest"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { Parameters } from "../../types/typeAliases"; /** * @@@ @@ -10,25 +10,25 @@ import type { Parameters } from '../../types/typeAliases'; * @private internal type of `createPipelineExecutor` */ type FilterJustOutputParametersOptions = { - /** - * @@@ - */ - readonly preparedPipeline: ReadonlyDeep; + /** + * @@@ + */ + readonly preparedPipeline: ReadonlyDeep; - /** - * @@@ - */ - readonly parametersToPass: Readonly; + /** + * @@@ + */ + readonly parametersToPass: Readonly; - /** - * @@@ - */ - readonly $warnings: PipelineExecutionError[]; + /** + * @@@ + */ + readonly $warnings: PipelineExecutionError[]; - /** - * @@@ - */ - readonly pipelineIdentification: string; + /** + * @@@ + */ + readonly pipelineIdentification: string; }; /** @@ -36,34 +36,43 @@ type FilterJustOutputParametersOptions = { * * @private internal utility of `createPipelineExecutor` */ -export function filterJustOutputParameters(options: FilterJustOutputParametersOptions): Parameters { - const { preparedPipeline, parametersToPass, $warnings, pipelineIdentification } = options; +export function filterJustOutputParameters( + options: FilterJustOutputParametersOptions, +): Parameters { + const { + preparedPipeline, + parametersToPass, + $warnings, + pipelineIdentification, + } = options; - const outputParameters: Parameters = {}; + const outputParameters: Parameters = {}; - // Note: Filter ONLY output parameters - // TODO: [👩🏾‍🤝‍👩🏻] Maybe use here `mapAvailableToExpectedParameters` - for (const parameter of preparedPipeline.parameters.filter(({ isOutput }) => isOutput)) { - if (parametersToPass[parameter.name] === undefined) { - // [4] - $warnings.push( - new PipelineExecutionError( - spaceTrim( - (block) => ` + // Note: Filter ONLY output parameters + // TODO: [👩🏾‍🤝‍👩🏻] Maybe use here `mapAvailableToExpectedParameters` + for (const parameter of preparedPipeline.parameters.filter( + ({ isOutput }) => isOutput, + )) { + if (parametersToPass[parameter.name] === undefined) { + // [4] + $warnings.push( + new PipelineExecutionError( + spaceTrim( + (block) => ` Parameter \`{${ - parameter.name - }}\` should be an output parameter, but it was not generated during pipeline execution + parameter.name + }}\` should be an output parameter, but it was not generated during pipeline execution ${block(pipelineIdentification)} `, - ), - ), - // <- TODO: This should be maybe `UnexpectedError` because it should be catched during `validatePipeline` - ); - continue; - } - outputParameters[parameter.name] = parametersToPass[parameter.name] || ''; - } + ), + ), + // <- TODO: This should be maybe `UnexpectedError` because it should be catched during `validatePipeline` + ); + continue; + } + outputParameters[parameter.name] = parametersToPass[parameter.name] || ""; + } - return outputParameters; + return outputParameters; } diff --git a/src/execution/createPipelineExecutor/getContextForTask.ts b/src/execution/createPipelineExecutor/getContextForTask.ts index 271768e606..c9d407f56f 100644 --- a/src/execution/createPipelineExecutor/getContextForTask.ts +++ b/src/execution/createPipelineExecutor/getContextForTask.ts @@ -1,9 +1,9 @@ -import type { ReadonlyDeep } from 'type-fest'; -import { RESERVED_PARAMETER_MISSING_VALUE } from '../../constants'; -import type { TaskJson } from '../../pipeline/PipelineJson/TaskJson'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_parameter_value } from '../../types/typeAliases'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; +import type { ReadonlyDeep } from "type-fest"; +import { RESERVED_PARAMETER_MISSING_VALUE } from "../../constants"; +import type { TaskJson } from "../../pipeline/PipelineJson/TaskJson"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_parameter_value } from "../../types/typeAliases"; +import { TODO_USE } from "../../utils/organization/TODO_USE"; /** * @@@ @@ -11,8 +11,8 @@ import { TODO_USE } from '../../utils/organization/TODO_USE'; * @private internal utility of `createPipelineExecutor` */ export async function getContextForTask( - task: ReadonlyDeep, + task: ReadonlyDeep, ): Promise { - TODO_USE(task); - return RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [🏍] Implement */; + TODO_USE(task); + return RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [🏍] Implement */; } diff --git a/src/execution/createPipelineExecutor/getExamplesForTask.ts b/src/execution/createPipelineExecutor/getExamplesForTask.ts index 2053b3212e..ff73ff667d 100644 --- a/src/execution/createPipelineExecutor/getExamplesForTask.ts +++ b/src/execution/createPipelineExecutor/getExamplesForTask.ts @@ -1,9 +1,9 @@ -import type { ReadonlyDeep } from 'type-fest'; -import { RESERVED_PARAMETER_MISSING_VALUE } from '../../constants'; -import type { TaskJson } from '../../pipeline/PipelineJson/TaskJson'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_parameter_value } from '../../types/typeAliases'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; +import type { ReadonlyDeep } from "type-fest"; +import { RESERVED_PARAMETER_MISSING_VALUE } from "../../constants"; +import type { TaskJson } from "../../pipeline/PipelineJson/TaskJson"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_parameter_value } from "../../types/typeAliases"; +import { TODO_USE } from "../../utils/organization/TODO_USE"; /** * @@@ @@ -11,10 +11,10 @@ import { TODO_USE } from '../../utils/organization/TODO_USE'; * @private internal utility of `createPipelineExecutor` */ export async function getExamplesForTask( - task: ReadonlyDeep, + task: ReadonlyDeep, ): Promise { - // TODO: [♨][💩] Implement Better - use real index and keyword search + // TODO: [♨][💩] Implement Better - use real index and keyword search - TODO_USE(task); - return RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [♨] Implement */; + TODO_USE(task); + return RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [♨] Implement */; } diff --git a/src/execution/createPipelineExecutor/getKnowledgeForTask.ts b/src/execution/createPipelineExecutor/getKnowledgeForTask.ts index 415a90dc56..df72b2a4a9 100644 --- a/src/execution/createPipelineExecutor/getKnowledgeForTask.ts +++ b/src/execution/createPipelineExecutor/getKnowledgeForTask.ts @@ -1,9 +1,9 @@ -import type { ReadonlyDeep } from 'type-fest'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { TaskJson } from '../../pipeline/PipelineJson/TaskJson'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_parameter_value } from '../../types/typeAliases'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; +import type { ReadonlyDeep } from "type-fest"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { TaskJson } from "../../pipeline/PipelineJson/TaskJson"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_parameter_value } from "../../types/typeAliases"; +import { TODO_USE } from "../../utils/organization/TODO_USE"; /** * @@@ @@ -11,15 +11,15 @@ import { TODO_USE } from '../../utils/organization/TODO_USE'; * @private internal type of `getKnowledgeFoTask` */ type GetKnowledgeForTaskOptions = { - /** - * @@@ - */ - readonly preparedPipeline: ReadonlyDeep; + /** + * @@@ + */ + readonly preparedPipeline: ReadonlyDeep; - /** - * @@@ - */ - readonly task: ReadonlyDeep; + /** + * @@@ + */ + readonly task: ReadonlyDeep; }; /** @@ -28,13 +28,15 @@ type GetKnowledgeForTaskOptions = { * @private internal utility of `createPipelineExecutor` */ export async function getKnowledgeForTask( - options: GetKnowledgeForTaskOptions, + options: GetKnowledgeForTaskOptions, ): Promise { - const { preparedPipeline, task } = options; + const { preparedPipeline, task } = options; - // TODO: [♨][💩] Implement Better - use real index and keyword search from `task` and {examples} + // TODO: [♨][💩] Implement Better - use real index and keyword search from `task` and {examples} - TODO_USE(task); - return preparedPipeline.knowledgePieces.map(({ content }) => `- ${content}`).join('\n'); - // <- TODO: [🧠] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed + TODO_USE(task); + return preparedPipeline.knowledgePieces + .map(({ content }) => `- ${content}`) + .join("\n"); + // <- TODO: [🧠] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed } diff --git a/src/execution/createPipelineExecutor/getReservedParametersForTask.ts b/src/execution/createPipelineExecutor/getReservedParametersForTask.ts index bd8599b928..6f477fe090 100644 --- a/src/execution/createPipelineExecutor/getReservedParametersForTask.ts +++ b/src/execution/createPipelineExecutor/getReservedParametersForTask.ts @@ -1,15 +1,15 @@ -import { spaceTrim } from 'spacetrim'; -import type { ReadonlyDeep } from 'type-fest'; -import { RESERVED_PARAMETER_MISSING_VALUE } from '../../constants'; -import { RESERVED_PARAMETER_NAMES } from '../../constants'; -import { RESERVED_PARAMETER_RESTRICTED } from '../../constants'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { TaskJson } from '../../pipeline/PipelineJson/TaskJson'; -import type { ReservedParameters } from '../../types/typeAliases'; -import { getContextForTask } from './getContextForTask'; -import { getExamplesForTask } from './getExamplesForTask'; -import { getKnowledgeForTask } from './getKnowledgeForTask'; +import { spaceTrim } from "spacetrim"; +import type { ReadonlyDeep } from "type-fest"; +import { RESERVED_PARAMETER_MISSING_VALUE } from "../../constants"; +import { RESERVED_PARAMETER_NAMES } from "../../constants"; +import { RESERVED_PARAMETER_RESTRICTED } from "../../constants"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { TaskJson } from "../../pipeline/PipelineJson/TaskJson"; +import type { ReservedParameters } from "../../types/typeAliases"; +import { getContextForTask } from "./getContextForTask"; +import { getExamplesForTask } from "./getExamplesForTask"; +import { getKnowledgeForTask } from "./getKnowledgeForTask"; /** * @@@ @@ -17,20 +17,20 @@ import { getKnowledgeForTask } from './getKnowledgeForTask'; * @private internal type of `getReservedParametersForTask` */ type GetReservedParametersForTaskOptions = { - /** - * @@@ - */ - readonly preparedPipeline: ReadonlyDeep; + /** + * @@@ + */ + readonly preparedPipeline: ReadonlyDeep; - /** - * @@@ - */ - readonly task: ReadonlyDeep; + /** + * @@@ + */ + readonly task: ReadonlyDeep; - /** - * @@@ - */ - readonly pipelineIdentification: string; + /** + * @@@ + */ + readonly pipelineIdentification: string; }; /** @@ -39,39 +39,39 @@ type GetReservedParametersForTaskOptions = { * @private internal utility of `createPipelineExecutor` */ export async function getReservedParametersForTask( - options: GetReservedParametersForTaskOptions, + options: GetReservedParametersForTaskOptions, ): Promise> { - const { preparedPipeline, task, pipelineIdentification } = options; + const { preparedPipeline, task, pipelineIdentification } = options; - const context = await getContextForTask(task); // <- [🏍] - const knowledge = await getKnowledgeForTask({ preparedPipeline, task }); - const examples = await getExamplesForTask(task); - const currentDate = new Date().toISOString(); // <- TODO: [🧠][💩] Better - const modelName = RESERVED_PARAMETER_MISSING_VALUE; + const context = await getContextForTask(task); // <- [🏍] + const knowledge = await getKnowledgeForTask({ preparedPipeline, task }); + const examples = await getExamplesForTask(task); + const currentDate = new Date().toISOString(); // <- TODO: [🧠][💩] Better + const modelName = RESERVED_PARAMETER_MISSING_VALUE; - const reservedParameters: ReservedParameters = { - content: RESERVED_PARAMETER_RESTRICTED, - context, // <- [🏍] - knowledge, - examples, - currentDate, - modelName, - }; + const reservedParameters: ReservedParameters = { + content: RESERVED_PARAMETER_RESTRICTED, + context, // <- [🏍] + knowledge, + examples, + currentDate, + modelName, + }; - // Note: Doublecheck that ALL reserved parameters are defined: - for (const parameterName of RESERVED_PARAMETER_NAMES) { - if (reservedParameters[parameterName] === undefined) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + // Note: Doublecheck that ALL reserved parameters are defined: + for (const parameterName of RESERVED_PARAMETER_NAMES) { + if (reservedParameters[parameterName] === undefined) { + throw new UnexpectedError( + spaceTrim( + (block) => ` Reserved parameter {${parameterName}} is not defined ${block(pipelineIdentification)} `, - ), - ); - } - } + ), + ); + } + } - return reservedParameters; + return reservedParameters; } diff --git a/src/execution/embeddingVectorToString.ts b/src/execution/embeddingVectorToString.ts index 70bfb0904a..36ad312c89 100644 --- a/src/execution/embeddingVectorToString.ts +++ b/src/execution/embeddingVectorToString.ts @@ -1,14 +1,17 @@ -import type { EmbeddingVector } from './EmbeddingVector'; +import type { EmbeddingVector } from "./EmbeddingVector"; /** * Pretty print an embedding vector for logging * * @public exported from `@promptbook/core` */ -export function embeddingVectorToString(embeddingVector: Readonly) { - const vectorLength = embeddingVector.reduce((acc, val) => acc + val ** 2, 0) ** 0.5; +export function embeddingVectorToString( + embeddingVector: Readonly, +) { + const vectorLength = + embeddingVector.reduce((acc, val) => acc + val ** 2, 0) ** 0.5; - return `[EmbeddingVector; ${embeddingVector.length} dimensions; length: ${vectorLength.toFixed( - 2, - )}; ${embeddingVector.slice(0, 3).join(', ')}...]`; + return `[EmbeddingVector; ${embeddingVector.length} dimensions; length: ${vectorLength.toFixed( + 2, + )}; ${embeddingVector.slice(0, 3).join(", ")}...]`; } diff --git a/src/execution/execution-report/ExecutionPromptReportJson.ts b/src/execution/execution-report/ExecutionPromptReportJson.ts index df4719b818..a7d6afa361 100644 --- a/src/execution/execution-report/ExecutionPromptReportJson.ts +++ b/src/execution/execution-report/ExecutionPromptReportJson.ts @@ -1,6 +1,6 @@ -import type { ErrorJson } from '../../errors/utils/ErrorJson'; -import type { Prompt } from '../../types/Prompt'; -import type { PromptResult } from '../PromptResult'; +import type { ErrorJson } from "../../errors/utils/ErrorJson"; +import type { Prompt } from "../../types/Prompt"; +import type { PromptResult } from "../PromptResult"; /** * Report of single prompt execution @@ -8,20 +8,20 @@ import type { PromptResult } from '../PromptResult'; * Note: [🚉] This is fully serializable as JSON */ export type ExecutionPromptReportJson = { - /** - * The prompt wich was executed - */ - readonly prompt: Omit; + /** + * The prompt wich was executed + */ + readonly prompt: Omit; - /** - * Result of the prompt execution (if not failed during LLM execution) - */ - readonly result?: PromptResult; + /** + * Result of the prompt execution (if not failed during LLM execution) + */ + readonly result?: PromptResult; - /** - * The error which occured during LLM execution or during postprocessing or expectation checking - * - * Note: It makes sense to have both error and result defined, for example when the result not pass expectations - */ - readonly error?: ErrorJson; + /** + * The error which occured during LLM execution or during postprocessing or expectation checking + * + * Note: It makes sense to have both error and result defined, for example when the result not pass expectations + */ + readonly error?: ErrorJson; }; diff --git a/src/execution/execution-report/ExecutionReportJson.ts b/src/execution/execution-report/ExecutionReportJson.ts index 131862cb2b..36957d3595 100644 --- a/src/execution/execution-report/ExecutionReportJson.ts +++ b/src/execution/execution-report/ExecutionReportJson.ts @@ -1,7 +1,7 @@ -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_pipeline_url } from '../../types/typeAliases'; -import type { string_semantic_version } from '../../types/typeAliases'; -import type { ExecutionPromptReportJson } from './ExecutionPromptReportJson'; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_pipeline_url } from "../../types/typeAliases"; +import type { string_semantic_version } from "../../types/typeAliases"; +import type { ExecutionPromptReportJson } from "./ExecutionPromptReportJson"; /** * ExecutionReport is result of executing one promptbook @@ -15,40 +15,40 @@ import type { ExecutionPromptReportJson } from './ExecutionPromptReportJson'; * @see https://github.com/webgptorg/promptbook#execution-report */ export type ExecutionReportJson = { - /* + /* TODO: [💼] > readonly type: 'REPORT'; + make type test for this */ - /** - * Unique identifier of the pipeline from promptbook which was executed - */ - readonly pipelineUrl?: string_pipeline_url; - - /** - * Title of from promptbook which was executed - */ - readonly title?: string; - - /** - * Version from promptbook which was executed - */ - readonly promptbookUsedVersion: string_semantic_version; - - /** - * Version from promptbook which was requested by promptbook - */ - readonly promptbookRequestedVersion?: string_semantic_version; - - /** - * Description of the promptbook which was executed - */ - readonly description?: string_markdown_text; - - /** - * Sequence of tasks in order which were executed - */ - readonly promptExecutions: ReadonlyArray; + /** + * Unique identifier of the pipeline from promptbook which was executed + */ + readonly pipelineUrl?: string_pipeline_url; + + /** + * Title of from promptbook which was executed + */ + readonly title?: string; + + /** + * Version from promptbook which was executed + */ + readonly promptbookUsedVersion: string_semantic_version; + + /** + * Version from promptbook which was requested by promptbook + */ + readonly promptbookRequestedVersion?: string_semantic_version; + + /** + * Description of the promptbook which was executed + */ + readonly description?: string_markdown_text; + + /** + * Sequence of tasks in order which were executed + */ + readonly promptExecutions: ReadonlyArray; }; diff --git a/src/execution/execution-report/ExecutionReportString.ts b/src/execution/execution-report/ExecutionReportString.ts index 75bc10e532..5cbc4477fc 100644 --- a/src/execution/execution-report/ExecutionReportString.ts +++ b/src/execution/execution-report/ExecutionReportString.ts @@ -9,7 +9,7 @@ * @see https://github.com/webgptorg/promptbook#execution-report */ export type ExecutionReportString = string & { - readonly _type: 'ExecutionReportString' /* <- TODO: [🏟] What is the best shape of the additional object in branded types */; + readonly _type: "ExecutionReportString" /* <- TODO: [🏟] What is the best shape of the additional object in branded types */; }; /** diff --git a/src/execution/execution-report/ExecutionReportStringOptions.ts b/src/execution/execution-report/ExecutionReportStringOptions.ts index 2d03b8d790..a4fcff27fc 100644 --- a/src/execution/execution-report/ExecutionReportStringOptions.ts +++ b/src/execution/execution-report/ExecutionReportStringOptions.ts @@ -1,4 +1,4 @@ -import type { number_percent } from '../../types/typeAliases'; +import type { number_percent } from "../../types/typeAliases"; /** * Options for `executionReportJsonToString` @@ -6,14 +6,14 @@ import type { number_percent } from '../../types/typeAliases'; * @public exported from `@promptbook/core` */ export type ExecutionReportStringOptions = { - /** - * The tax rate to be applied, expressed as a percentage from 0 to 1 (=100%) or even more - */ - readonly taxRate: number_percent; - /** - * The width of the charts in the report - */ - readonly chartsWidth: number; + /** + * The tax rate to be applied, expressed as a percentage from 0 to 1 (=100%) or even more + */ + readonly taxRate: number_percent; + /** + * The width of the charts in the report + */ + readonly chartsWidth: number; }; /** @@ -22,6 +22,6 @@ export type ExecutionReportStringOptions = { * @public exported from `@promptbook/core` */ export const ExecutionReportStringOptionsDefaults = { - taxRate: 0, - chartsWidth: 36, + taxRate: 0, + chartsWidth: 36, } satisfies ExecutionReportStringOptions; diff --git a/src/execution/execution-report/countWorkingDuration.test.ts b/src/execution/execution-report/countWorkingDuration.test.ts index 9bd423997f..1c7fb2b74c 100644 --- a/src/execution/execution-report/countWorkingDuration.test.ts +++ b/src/execution/execution-report/countWorkingDuration.test.ts @@ -1,50 +1,52 @@ -import { describe, expect, it } from '@jest/globals'; -import { countWorkingDuration } from './countWorkingDuration'; +import { describe, expect, it } from "@jest/globals"; +import { countWorkingDuration } from "./countWorkingDuration"; -describe('how countWorkingDuration works', () => { - it('should work with single item', () => { - expect(countWorkingDuration([{ title: 'Task 1', from: 0, to: 1 }])).toBe(1); - expect(countWorkingDuration([{ title: 'Task 1', from: 0, to: 2 }])).toBe(2); - expect(countWorkingDuration([{ title: 'Task 1', from: -1, to: 1 }])).toBe(2); - }); +describe("how countWorkingDuration works", () => { + it("should work with single item", () => { + expect(countWorkingDuration([{ title: "Task 1", from: 0, to: 1 }])).toBe(1); + expect(countWorkingDuration([{ title: "Task 1", from: 0, to: 2 }])).toBe(2); + expect(countWorkingDuration([{ title: "Task 1", from: -1, to: 1 }])).toBe( + 2, + ); + }); - it('should work with full coverage', () => { - expect( - countWorkingDuration([ - { title: 'Task 1', from: 0, to: 10 }, - { title: 'Task 2', from: 4, to: 6 }, - { title: 'Task 3', from: 3, to: 9 }, - { title: 'Task 4', from: 6, to: 11 }, - { title: 'Task 5', from: 11, to: 12 }, - ]), - ).toBe(12); - }); + it("should work with full coverage", () => { + expect( + countWorkingDuration([ + { title: "Task 1", from: 0, to: 10 }, + { title: "Task 2", from: 4, to: 6 }, + { title: "Task 3", from: 3, to: 9 }, + { title: "Task 4", from: 6, to: 11 }, + { title: "Task 5", from: 11, to: 12 }, + ]), + ).toBe(12); + }); - it('should work with sparse coverage', () => { - expect( - countWorkingDuration([ - { title: 'Task 1', from: 0, to: 1 }, - { title: 'Task 2', from: 2, to: 3 }, - ]), - ).toBe(2); - expect( - countWorkingDuration([ - { title: 'Task 1', from: 0, to: 1 }, - { title: 'Task 2', from: 3, to: 4 }, - ]), - ).toBe(2); - }); + it("should work with sparse coverage", () => { + expect( + countWorkingDuration([ + { title: "Task 1", from: 0, to: 1 }, + { title: "Task 2", from: 2, to: 3 }, + ]), + ).toBe(2); + expect( + countWorkingDuration([ + { title: "Task 1", from: 0, to: 1 }, + { title: "Task 2", from: 3, to: 4 }, + ]), + ).toBe(2); + }); - it('should work in advanced case', () => { - expect( - countWorkingDuration([ - { title: 'Task 1', from: 0, to: 1 }, - { title: 'Task 2', from: -2.2, to: 3 }, - { title: 'Task 3', from: 3, to: 4 }, - { title: 'Task 4', from: 3, to: 6.4 }, - { title: 'Task 5', from: 5, to: 7 }, - { title: 'Task 5', from: 85, to: 100 }, - ]), - ).toBeCloseTo(24.2); - }); + it("should work in advanced case", () => { + expect( + countWorkingDuration([ + { title: "Task 1", from: 0, to: 1 }, + { title: "Task 2", from: -2.2, to: 3 }, + { title: "Task 3", from: 3, to: 4 }, + { title: "Task 4", from: 3, to: 6.4 }, + { title: "Task 5", from: 5, to: 7 }, + { title: "Task 5", from: 85, to: 100 }, + ]), + ).toBeCloseTo(24.2); + }); }); diff --git a/src/execution/execution-report/countWorkingDuration.ts b/src/execution/execution-report/countWorkingDuration.ts index 0fc2bcb366..6971597c76 100644 --- a/src/execution/execution-report/countWorkingDuration.ts +++ b/src/execution/execution-report/countWorkingDuration.ts @@ -1,4 +1,4 @@ -import type { FromtoItems } from '../../utils/FromtoItems'; +import type { FromtoItems } from "../../utils/FromtoItems"; /** * Count the duration of working time @@ -6,18 +6,22 @@ import type { FromtoItems } from '../../utils/FromtoItems'; * @private within the repository */ export function countWorkingDuration(items: FromtoItems): number { - const steps = Array.from(new Set(items.flatMap((item) => [item.from, item.to]))); - steps.sort((a, b) => a - b); - const intervals = steps.map((step, index) => [step, steps[index + 1] || 0] as const).slice(0, -1); + const steps = Array.from( + new Set(items.flatMap((item) => [item.from, item.to])), + ); + steps.sort((a, b) => a - b); + const intervals = steps + .map((step, index) => [step, steps[index + 1] || 0] as const) + .slice(0, -1); - let duration = 0; + let duration = 0; - for (const interval of intervals) { - const [from, to] = interval; - if (items.some((item) => item.from < to && item.to > from)) { - duration += to - from; - } - } + for (const interval of intervals) { + const [from, to] = interval; + if (items.some((item) => item.from < to && item.to > from)) { + duration += to - from; + } + } - return duration; + return duration; } diff --git a/src/execution/execution-report/executionReportJsonToString.ts b/src/execution/execution-report/executionReportJsonToString.ts index 2b755e7a46..3bb9b21850 100644 --- a/src/execution/execution-report/executionReportJsonToString.ts +++ b/src/execution/execution-report/executionReportJsonToString.ts @@ -1,21 +1,21 @@ -import moment from 'moment'; -import { spaceTrim } from 'spacetrim'; -import type { ReadonlyDeep } from 'type-fest'; -import { MOMENT_ARG_THRESHOLDS } from '../../config'; -import type { number_usd } from '../../types/typeAliases'; -import type { FromtoItems } from '../../utils/FromtoItems'; -import { createMarkdownChart } from '../../utils/markdown/createMarkdownChart'; -import { escapeMarkdownBlock } from '../../utils/markdown/escapeMarkdownBlock'; -import { prettifyMarkdown } from '../../utils/markdown/prettifyMarkdown'; -import { normalizeToKebabCase } from '../../utils/normalization/normalize-to-kebab-case'; -import { just } from '../../utils/organization/just'; -import { numberToString } from '../../utils/parameters/numberToString'; -import { embeddingVectorToString } from '../embeddingVectorToString'; -import type { ExecutionReportJson } from './ExecutionReportJson'; -import type { ExecutionReportString } from './ExecutionReportString'; -import type { ExecutionReportStringOptions } from './ExecutionReportStringOptions'; -import { ExecutionReportStringOptionsDefaults } from './ExecutionReportStringOptions'; -import { countWorkingDuration } from './countWorkingDuration'; +import moment from "moment"; +import { spaceTrim } from "spacetrim"; +import type { ReadonlyDeep } from "type-fest"; +import { MOMENT_ARG_THRESHOLDS } from "../../config"; +import type { number_usd } from "../../types/typeAliases"; +import type { FromtoItems } from "../../utils/FromtoItems"; +import { createMarkdownChart } from "../../utils/markdown/createMarkdownChart"; +import { escapeMarkdownBlock } from "../../utils/markdown/escapeMarkdownBlock"; +import { prettifyMarkdown } from "../../utils/markdown/prettifyMarkdown"; +import { normalizeToKebabCase } from "../../utils/normalization/normalize-to-kebab-case"; +import { just } from "../../utils/organization/just"; +import { numberToString } from "../../utils/parameters/numberToString"; +import { embeddingVectorToString } from "../embeddingVectorToString"; +import type { ExecutionReportJson } from "./ExecutionReportJson"; +import type { ExecutionReportString } from "./ExecutionReportString"; +import type { ExecutionReportStringOptions } from "./ExecutionReportStringOptions"; +import { ExecutionReportStringOptionsDefaults } from "./ExecutionReportStringOptions"; +import { countWorkingDuration } from "./countWorkingDuration"; /** * Converts execution report from JSON to string format @@ -23,212 +23,246 @@ import { countWorkingDuration } from './countWorkingDuration'; * @public exported from `@promptbook/core` */ export function executionReportJsonToString( - executionReportJson: ReadonlyDeep, - options?: Partial, + executionReportJson: ReadonlyDeep, + options?: Partial, ): ExecutionReportString { - const { taxRate, chartsWidth } = { ...ExecutionReportStringOptionsDefaults, ...(options || {}) }; + const { taxRate, chartsWidth } = { + ...ExecutionReportStringOptionsDefaults, + ...(options || {}), + }; - let executionReportString = spaceTrim( - (block) => ` - # ${executionReportJson.title || 'Execution report'} + let executionReportString = spaceTrim( + (block) => ` + # ${executionReportJson.title || "Execution report"} - ${block(executionReportJson.description || '')} + ${block(executionReportJson.description || "")} `, - ); - - const headerList: Array = []; - - if (executionReportJson.pipelineUrl) { - headerList.push(`PIPELINE URL ${executionReportJson.pipelineUrl}`); - } - - headerList.push( - `PROMPTBOOK VERSION ${executionReportJson.promptbookUsedVersion}` + - (!executionReportJson.promptbookRequestedVersion - ? '' - : ` *(requested ${executionReportJson.promptbookRequestedVersion})*`), - ); - - if (executionReportJson.promptExecutions.length !== 0) { - // TODO: What if startedAt OR/AND completedAt is not defined? - const startedAt = moment( - Math.min( - ...executionReportJson.promptExecutions - .filter((promptExecution) => promptExecution.result?.timing?.start) - .map((promptExecution) => moment(promptExecution.result!.timing.start).valueOf()), - ), - ); - const completedAt = moment( - Math.max( - ...executionReportJson.promptExecutions - .filter((promptExecution) => promptExecution.result?.timing?.complete) - .map((promptExecution) => moment(promptExecution.result!.timing.complete).valueOf()), - ), - ); - - const timingItems: FromtoItems = executionReportJson.promptExecutions.map((promptExecution) => ({ - title: promptExecution.prompt.title, - from: moment(promptExecution.result?.timing?.start).valueOf() / 1000, - to: moment(promptExecution.result?.timing?.complete).valueOf() / 1000, - })); - - const costItems: FromtoItems = executionReportJson.promptExecutions - .filter((promptExecution) => typeof promptExecution.result?.usage?.price === 'number') - .map((promptExecution) => ({ - title: promptExecution.prompt.title, - from: 0, - to: - (promptExecution.result?.usage?.price.value || 0) /* <- TODO: look at uncertain numbers */ * - (1 + taxRate), - })); - - const duration = moment.duration(completedAt.diff(startedAt)); - const llmDuration = moment.duration(countWorkingDuration(timingItems) * 1000); - - const executionsWithKnownCost = executionReportJson.promptExecutions.filter( - (promptExecution) => (promptExecution.result?.usage?.price || 'UNKNOWN') !== 'UNKNOWN', - ); - const cost: number_usd = executionsWithKnownCost.reduce( - (cost, promptExecution) => - cost + (promptExecution.result!.usage.price.value /* <- Look at uncertain number */ || 0), - 0, - ); - - headerList.push(`STARTED AT ${moment(startedAt).format(`YYYY-MM-DD HH:mm:ss`)}`); - headerList.push(`COMPLETED AT ${moment(completedAt).format(`YYYY-MM-DD HH:mm:ss`)}`); - headerList.push(`TOTAL DURATION ${duration.humanize(MOMENT_ARG_THRESHOLDS)}`); - headerList.push(`TOTAL LLM DURATION ${llmDuration.humanize(MOMENT_ARG_THRESHOLDS)}`); - - headerList.push( - `TOTAL COST $${numberToString(cost * (1 + taxRate))}` + - (executionsWithKnownCost.length === executionReportJson.promptExecutions.length - ? '' - : ` *(Some cost is unknown)*`) + - (taxRate !== 0 ? ` *(with tax ${taxRate * 100}%)*` : ''), - ); - - executionReportString += '\n\n' + headerList.map((header) => `- ${header}`).join('\n'); - - executionReportString += - '\n\n' + - '## 🗃 Index' + - '\n\n' + - executionReportJson.promptExecutions - .map((promptExecution) => { - // TODO: [💩] Make some better system to convert headings to links - let hash = normalizeToKebabCase(promptExecution.prompt.title); - if (/^\s*\p{Extended_Pictographic}/u.test(promptExecution.prompt.title)) { - hash = '-' + hash; - } - - // TODO: Make working hash link for the task in md + pdf - - return `- [${promptExecution.prompt.title}](#${hash})`; - }) - .join('\n'); - - executionReportString += - '\n\n' + - '## ⌚ Time chart' + - '\n\n' + - createMarkdownChart({ - nameHeader: 'Task', - valueHeader: 'Timeline', - items: timingItems, - width: chartsWidth, - unitName: 'seconds', - }); - - executionReportString += - '\n\n' + - '## 💸 Cost chart' + - '\n\n' + - createMarkdownChart({ - nameHeader: 'Task', - valueHeader: 'Cost', - items: costItems, - width: chartsWidth, - unitName: 'USD', - }); - } else { - headerList.push(`TOTAL COST $0 *(Nothing executed)*`); - } - - for (const promptExecution of executionReportJson.promptExecutions) { - executionReportString += '\n\n\n\n' + `## ${promptExecution.prompt.title}`; - - const taskList: Array = []; - - // TODO: What if startedAt OR/AND completedAt is not defined? - const startedAt = moment(promptExecution.result?.timing?.start); - const completedAt = moment(promptExecution.result?.timing?.complete); - const duration = moment.duration(completedAt.diff(startedAt)); - - // Not need here: - // > taskList.push(`STARTED AT ${moment(startedAt).calendar()}`); - taskList.push(`DURATION ${duration.humanize(MOMENT_ARG_THRESHOLDS)}`); - - if (typeof promptExecution.result?.usage?.price === 'number') { - taskList.push( - `COST $${numberToString(promptExecution.result.usage.price * (1 + taxRate))}` + - (taxRate !== 0 ? ` *(with tax ${taxRate * 100}%)*` : ''), - ); - } else { - taskList.push(`COST UNKNOWN`); - } - - executionReportString += '\n\n' + taskList.map((header) => `- ${header}`).join('\n'); - - /* + ); + + const headerList: Array = []; + + if (executionReportJson.pipelineUrl) { + headerList.push(`PIPELINE URL ${executionReportJson.pipelineUrl}`); + } + + headerList.push( + `PROMPTBOOK VERSION ${executionReportJson.promptbookUsedVersion}` + + (!executionReportJson.promptbookRequestedVersion + ? "" + : ` *(requested ${executionReportJson.promptbookRequestedVersion})*`), + ); + + if (executionReportJson.promptExecutions.length !== 0) { + // TODO: What if startedAt OR/AND completedAt is not defined? + const startedAt = moment( + Math.min( + ...executionReportJson.promptExecutions + .filter((promptExecution) => promptExecution.result?.timing?.start) + .map((promptExecution) => + moment(promptExecution.result!.timing.start).valueOf(), + ), + ), + ); + const completedAt = moment( + Math.max( + ...executionReportJson.promptExecutions + .filter((promptExecution) => promptExecution.result?.timing?.complete) + .map((promptExecution) => + moment(promptExecution.result!.timing.complete).valueOf(), + ), + ), + ); + + const timingItems: FromtoItems = executionReportJson.promptExecutions.map( + (promptExecution) => ({ + title: promptExecution.prompt.title, + from: moment(promptExecution.result?.timing?.start).valueOf() / 1000, + to: moment(promptExecution.result?.timing?.complete).valueOf() / 1000, + }), + ); + + const costItems: FromtoItems = executionReportJson.promptExecutions + .filter( + (promptExecution) => + typeof promptExecution.result?.usage?.price === "number", + ) + .map((promptExecution) => ({ + title: promptExecution.prompt.title, + from: 0, + to: + (promptExecution.result?.usage?.price.value || + 0) /* <- TODO: look at uncertain numbers */ * + (1 + taxRate), + })); + + const duration = moment.duration(completedAt.diff(startedAt)); + const llmDuration = moment.duration( + countWorkingDuration(timingItems) * 1000, + ); + + const executionsWithKnownCost = executionReportJson.promptExecutions.filter( + (promptExecution) => + (promptExecution.result?.usage?.price || "UNKNOWN") !== "UNKNOWN", + ); + const cost: number_usd = executionsWithKnownCost.reduce( + (cost, promptExecution) => + cost + + (promptExecution.result!.usage.price + .value /* <- Look at uncertain number */ || 0), + 0, + ); + + headerList.push( + `STARTED AT ${moment(startedAt).format(`YYYY-MM-DD HH:mm:ss`)}`, + ); + headerList.push( + `COMPLETED AT ${moment(completedAt).format(`YYYY-MM-DD HH:mm:ss`)}`, + ); + headerList.push( + `TOTAL DURATION ${duration.humanize(MOMENT_ARG_THRESHOLDS)}`, + ); + headerList.push( + `TOTAL LLM DURATION ${llmDuration.humanize(MOMENT_ARG_THRESHOLDS)}`, + ); + + headerList.push( + `TOTAL COST $${numberToString(cost * (1 + taxRate))}` + + (executionsWithKnownCost.length === + executionReportJson.promptExecutions.length + ? "" + : ` *(Some cost is unknown)*`) + + (taxRate !== 0 ? ` *(with tax ${taxRate * 100}%)*` : ""), + ); + + executionReportString += + "\n\n" + headerList.map((header) => `- ${header}`).join("\n"); + + executionReportString += + "\n\n" + + "## 🗃 Index" + + "\n\n" + + executionReportJson.promptExecutions + .map((promptExecution) => { + // TODO: [💩] Make some better system to convert headings to links + let hash = normalizeToKebabCase(promptExecution.prompt.title); + if ( + /^\s*\p{Extended_Pictographic}/u.test(promptExecution.prompt.title) + ) { + hash = "-" + hash; + } + + // TODO: Make working hash link for the task in md + pdf + + return `- [${promptExecution.prompt.title}](#${hash})`; + }) + .join("\n"); + + executionReportString += + "\n\n" + + "## ⌚ Time chart" + + "\n\n" + + createMarkdownChart({ + nameHeader: "Task", + valueHeader: "Timeline", + items: timingItems, + width: chartsWidth, + unitName: "seconds", + }); + + executionReportString += + "\n\n" + + "## 💸 Cost chart" + + "\n\n" + + createMarkdownChart({ + nameHeader: "Task", + valueHeader: "Cost", + items: costItems, + width: chartsWidth, + unitName: "USD", + }); + } else { + headerList.push(`TOTAL COST $0 *(Nothing executed)*`); + } + + for (const promptExecution of executionReportJson.promptExecutions) { + executionReportString += "\n\n\n\n" + `## ${promptExecution.prompt.title}`; + + const taskList: Array = []; + + // TODO: What if startedAt OR/AND completedAt is not defined? + const startedAt = moment(promptExecution.result?.timing?.start); + const completedAt = moment(promptExecution.result?.timing?.complete); + const duration = moment.duration(completedAt.diff(startedAt)); + + // Not need here: + // > taskList.push(`STARTED AT ${moment(startedAt).calendar()}`); + taskList.push(`DURATION ${duration.humanize(MOMENT_ARG_THRESHOLDS)}`); + + if (typeof promptExecution.result?.usage?.price === "number") { + taskList.push( + `COST $${numberToString(promptExecution.result.usage.price * (1 + taxRate))}` + + (taxRate !== 0 ? ` *(with tax ${taxRate * 100}%)*` : ""), + ); + } else { + taskList.push(`COST UNKNOWN`); + } + + executionReportString += + "\n\n" + taskList.map((header) => `- ${header}`).join("\n"); + + /* - MODEL VARIANT ${promptExecution.prompt.modelRequirements.modelVariant} - MODEL NAME \`${promptExecution.result?.model}\` (requested \`${ promptExecution.prompt.modelRequirements.modelName */ - if (just(true)) { - executionReportString += - '\n\n\n\n' + - spaceTrim( - (block) => ` + if (just(true)) { + executionReportString += + "\n\n\n\n" + + spaceTrim( + (block) => ` ### Prompt \`\`\` ${block( - escapeMarkdownBlock( - promptExecution.result?.rawPromptContent || promptExecution.prompt.content, - ), - )} + escapeMarkdownBlock( + promptExecution.result?.rawPromptContent || + promptExecution.prompt.content, + ), + )} \`\`\` `, - ); - } + ); + } - if (promptExecution.result && promptExecution.result.content) { - executionReportString += '\n\n\n\n' + '### Result' + '\n\n'; + if (promptExecution.result && promptExecution.result.content) { + executionReportString += "\n\n\n\n" + "### Result" + "\n\n"; - if (promptExecution.result === undefined) { - executionReportString += '*No result*'; - } else if (typeof promptExecution.result.content === 'string') { - executionReportString += spaceTrim( - (block) => ` + if (promptExecution.result === undefined) { + executionReportString += "*No result*"; + } else if (typeof promptExecution.result.content === "string") { + executionReportString += spaceTrim( + (block) => ` \`\`\` ${block(escapeMarkdownBlock(promptExecution.result!.content as string))} \`\`\` `, - ); - } else { - executionReportString += embeddingVectorToString(promptExecution.result.content); - } - } - - if (promptExecution.error && promptExecution.error.message) { - executionReportString += - '\n\n\n\n' + - spaceTrim( - (block) => ` + ); + } else { + executionReportString += embeddingVectorToString( + promptExecution.result.content, + ); + } + } + + if (promptExecution.error && promptExecution.error.message) { + executionReportString += + "\n\n\n\n" + + spaceTrim( + (block) => ` ### Error @@ -237,12 +271,12 @@ export function executionReportJsonToString( \`\`\` `, - ); - } - } + ); + } + } - executionReportString = prettifyMarkdown(executionReportString); - return executionReportString as ExecutionReportString; + executionReportString = prettifyMarkdown(executionReportString); + return executionReportString as ExecutionReportString; } /** diff --git a/src/execution/translation/automatic-translate/automatic-translators/AutomaticTranslator.ts b/src/execution/translation/automatic-translate/automatic-translators/AutomaticTranslator.ts index 1d081ceeb0..b3e36e68c6 100644 --- a/src/execution/translation/automatic-translate/automatic-translators/AutomaticTranslator.ts +++ b/src/execution/translation/automatic-translate/automatic-translators/AutomaticTranslator.ts @@ -1,8 +1,8 @@ -import type { Promisable } from 'type-fest'; +import type { Promisable } from "type-fest"; /** * @private still in development [🏳] */ export type AutomaticTranslator = { - translate(message: string): Promisable; + translate(message: string): Promisable; }; diff --git a/src/execution/translation/automatic-translate/automatic-translators/DebugAutomaticTranslator.ts b/src/execution/translation/automatic-translate/automatic-translators/DebugAutomaticTranslator.ts index 573212f855..d59e098b80 100644 --- a/src/execution/translation/automatic-translate/automatic-translators/DebugAutomaticTranslator.ts +++ b/src/execution/translation/automatic-translate/automatic-translators/DebugAutomaticTranslator.ts @@ -1,5 +1,5 @@ -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import type { AutomaticTranslator } from './AutomaticTranslator'; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import type { AutomaticTranslator } from "./AutomaticTranslator"; /** * This will wrap an automatic translator and log each translation into the console @@ -7,14 +7,16 @@ import type { AutomaticTranslator } from './AutomaticTranslator'; * @private still in development [🏳] */ export class DebugAutomaticTranslator implements AutomaticTranslator { - public constructor(private readonly automaticTranslator: AutomaticTranslator) {} + public constructor( + private readonly automaticTranslator: AutomaticTranslator, + ) {} - public async translate(message: string): Promise { - const messageTranslated = await this.automaticTranslator.translate(message); + public async translate(message: string): Promise { + const messageTranslated = await this.automaticTranslator.translate(message); - // TODO: Write by "" only if needed - console.log(colors.green(`"${message}" → "${messageTranslated}"`)); + // TODO: Write by "" only if needed + console.log(colors.green(`"${message}" → "${messageTranslated}"`)); - return messageTranslated; - } + return messageTranslated; + } } diff --git a/src/execution/translation/automatic-translate/automatic-translators/FakeAutomaticTranslator.ts b/src/execution/translation/automatic-translate/automatic-translators/FakeAutomaticTranslator.ts index 1d511fe16f..496483f4c9 100644 --- a/src/execution/translation/automatic-translate/automatic-translators/FakeAutomaticTranslator.ts +++ b/src/execution/translation/automatic-translate/automatic-translators/FakeAutomaticTranslator.ts @@ -1,12 +1,12 @@ -import type { AutomaticTranslator } from './AutomaticTranslator'; +import type { AutomaticTranslator } from "./AutomaticTranslator"; /** * @private still in development [🏳] */ export class FakeAutomaticTranslator implements AutomaticTranslator { - public constructor() {} + public constructor() {} - public translate(message: string): string { - return message; - } + public translate(message: string): string { + return message; + } } diff --git a/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.ts b/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.ts index a1114fb540..75ae1398a5 100644 --- a/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.ts +++ b/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.ts @@ -1,18 +1,18 @@ -import FormData from 'form-data'; -import fetch from 'node-fetch'; /* <- TODO: [🌿] Use the Node native fetch */ -import { spaceTrim } from 'spacetrim'; -import { PipelineExecutionError } from '../../../../errors/PipelineExecutionError'; -import type { AutomaticTranslator } from './AutomaticTranslator'; -import type { TranslatorOptions } from './TranslatorOptions'; +import FormData from "form-data"; +import fetch from "node-fetch"; /* <- TODO: [🌿] Use the Node native fetch */ +import { spaceTrim } from "spacetrim"; +import { PipelineExecutionError } from "../../../../errors/PipelineExecutionError"; +import type { AutomaticTranslator } from "./AutomaticTranslator"; +import type { TranslatorOptions } from "./TranslatorOptions"; /** * @@@ */ type LindatAutomaticTranslatorOptions = TranslatorOptions & { - /** - * @@@ - */ - readonly apiUrl?: URL; + /** + * @@@ + */ + readonly apiUrl?: URL; }; /** @@ -21,32 +21,35 @@ type LindatAutomaticTranslatorOptions = TranslatorOptions & { * @private still in development [🏳] */ export class LindatAutomaticTranslator implements AutomaticTranslator { - public constructor(protected readonly options: LindatAutomaticTranslatorOptions) {} - public async translate(message: string): Promise { - const formData = new FormData(); - formData.append('input_text', message); - formData.append('src', this.options.from); - formData.append('tgt', this.options.to); + public constructor( + protected readonly options: LindatAutomaticTranslatorOptions, + ) {} + public async translate(message: string): Promise { + const formData = new FormData(); + formData.append("input_text", message); + formData.append("src", this.options.from); + formData.append("tgt", this.options.to); - const response = await fetch( // <- TODO: [🏳] Probbably pass the fetching function - this.options.apiUrl || '!!', + const response = await fetch( + // <- TODO: [🏳] Probbably pass the fetching function + this.options.apiUrl || "!!", - { - method: 'POST', - body: formData, - }, - ); + { + method: "POST", + body: formData, + }, + ); - if (response.status === 200) { - const translation = await response.text(); - return spaceTrim(translation); - } else { - const json = await response.json(); - if (json.message) { - throw new PipelineExecutionError(json.message); - } else { - throw new PipelineExecutionError( - spaceTrim(` + if (response.status === 200) { + const translation = await response.text(); + return spaceTrim(translation); + } else { + const json = await response.json(); + if (json.message) { + throw new PipelineExecutionError(json.message); + } else { + throw new PipelineExecutionError( + spaceTrim(` Lindat: Unknown error From: ${this.options.from} To: ${this.options.to} @@ -55,8 +58,8 @@ export class LindatAutomaticTranslator implements AutomaticTranslator { Response: ${JSON.stringify(json)} `), - ); - } - } - } + ); + } + } + } } diff --git a/src/execution/translation/automatic-translate/automatic-translators/TranslatorOptions.ts b/src/execution/translation/automatic-translate/automatic-translators/TranslatorOptions.ts index 878e5fb3e7..4f3a7959ec 100644 --- a/src/execution/translation/automatic-translate/automatic-translators/TranslatorOptions.ts +++ b/src/execution/translation/automatic-translate/automatic-translators/TranslatorOptions.ts @@ -2,6 +2,6 @@ * @private still in development [🏳] */ export type TranslatorOptions = { - readonly from?: string; - readonly to: string; + readonly from?: string; + readonly to: string; }; diff --git a/src/execution/translation/automatic-translate/automatic-translators/utils/extractMultiplicatedOccurrence.test.ts b/src/execution/translation/automatic-translate/automatic-translators/utils/extractMultiplicatedOccurrence.test.ts index 9881212e16..da6a7394cb 100644 --- a/src/execution/translation/automatic-translate/automatic-translators/utils/extractMultiplicatedOccurrence.test.ts +++ b/src/execution/translation/automatic-translate/automatic-translators/utils/extractMultiplicatedOccurrence.test.ts @@ -1,12 +1,12 @@ -import { extractMultiplicatedOccurrence } from './extractMultiplicatedOccurrence'; +import { extractMultiplicatedOccurrence } from "./extractMultiplicatedOccurrence"; -describe('how extractMultiplicatedOccurrence works', () => { - it('extracts correct pattern', () => { - expect(extractMultiplicatedOccurrence('hellohellohello')).toEqual('hello'); - expect( - extractMultiplicatedOccurrence( - 'Шарна дошка негайно для використанняШарна дошка негайно для використання...Допоміжна дошка негайно для використання...Úplné výsledky se nepodařilo načístZkusit znovuOpakování…Opakování… ...(Upraveno)Obnovit originálSharna doshka nehayno dlya vykorystannyaZobrazit víceZobrazit méněШарна дошка негайно для використанняclearVymazat textZrušitOdeslatVáš příspěvek poslouží ke zlepšení kvality překladu a může být zobrazen dalším uživatelům (aniž by byla uvedena vaše totožnost). Další informacevolume_upHlasový výstup není v tomto prohlížeči podporovánNačítání…stopvolume_upPoslech výslovnosticontent_copyKopírovat překladOhodnoťte tento překladOhodnoťte tento překladJste s tímto překladem spokojeni?Dobrý překladŠpatný překladNavrhnout úpravuVaše zpětná vazba bude použita k vylepšení službyshareSdílet překladSdílení překladuE-mailTwitter', - ), - ).toEqual('Шарна дошка негайно для використання'); - }); +describe("how extractMultiplicatedOccurrence works", () => { + it("extracts correct pattern", () => { + expect(extractMultiplicatedOccurrence("hellohellohello")).toEqual("hello"); + expect( + extractMultiplicatedOccurrence( + "Шарна дошка негайно для використанняШарна дошка негайно для використання...Допоміжна дошка негайно для використання...Úplné výsledky se nepodařilo načístZkusit znovuOpakování…Opakování… ...(Upraveno)Obnovit originálSharna doshka nehayno dlya vykorystannyaZobrazit víceZobrazit méněШарна дошка негайно для використанняclearVymazat textZrušitOdeslatVáš příspěvek poslouží ke zlepšení kvality překladu a může být zobrazen dalším uživatelům (aniž by byla uvedena vaše totožnost). Další informacevolume_upHlasový výstup není v tomto prohlížeči podporovánNačítání…stopvolume_upPoslech výslovnosticontent_copyKopírovat překladOhodnoťte tento překladOhodnoťte tento překladJste s tímto překladem spokojeni?Dobrý překladŠpatný překladNavrhnout úpravuVaše zpětná vazba bude použita k vylepšení službyshareSdílet překladSdílení překladuE-mailTwitter", + ), + ).toEqual("Шарна дошка негайно для використання"); + }); }); diff --git a/src/execution/translation/automatic-translate/automatic-translators/utils/extractMultiplicatedOccurrence.ts b/src/execution/translation/automatic-translate/automatic-translators/utils/extractMultiplicatedOccurrence.ts index 08c6031a47..215fa620f4 100644 --- a/src/execution/translation/automatic-translate/automatic-translators/utils/extractMultiplicatedOccurrence.ts +++ b/src/execution/translation/automatic-translate/automatic-translators/utils/extractMultiplicatedOccurrence.ts @@ -1,4 +1,4 @@ -import { PipelineExecutionError } from '../../../../../errors/PipelineExecutionError'; +import { PipelineExecutionError } from "../../../../../errors/PipelineExecutionError"; /** * @@ -8,11 +8,16 @@ import { PipelineExecutionError } from '../../../../../errors/PipelineExecutionE * @private still in development [🏳] */ export function extractMultiplicatedOccurrence(message: string): string { - for (let subLength = 1; subLength < message.length / 2; subLength++) { - if (message.substring(subLength * 0, subLength * 1) === message.substring(subLength * 1, subLength * 2)) { - return message.substring(subLength * 0, subLength * 1); - } - } + for (let subLength = 1; subLength < message.length / 2; subLength++) { + if ( + message.substring(subLength * 0, subLength * 1) === + message.substring(subLength * 1, subLength * 2) + ) { + return message.substring(subLength * 0, subLength * 1); + } + } - throw new PipelineExecutionError(`Cannot extract multiplicated occurrence from "${message}"`); + throw new PipelineExecutionError( + `Cannot extract multiplicated occurrence from "${message}"`, + ); } diff --git a/src/execution/translation/automatic-translate/translateMessages.ts b/src/execution/translation/automatic-translate/translateMessages.ts index 373a7aac2e..a42c5d998c 100644 --- a/src/execution/translation/automatic-translate/translateMessages.ts +++ b/src/execution/translation/automatic-translate/translateMessages.ts @@ -1,52 +1,56 @@ -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { readFile, writeFile } from 'fs'; -import glob from 'glob-promise'; -import JSON5 from 'json5'; -import { join } from 'path'; -import { spaceTrim } from 'spacetrim'; -import { promisify } from 'util'; -import { PipelineExecutionError } from '../../../errors/PipelineExecutionError'; -import type { AutomaticTranslator } from './automatic-translators/AutomaticTranslator'; -import type { TranslatorOptions } from './automatic-translators/TranslatorOptions'; +import { readFile, writeFile } from "fs"; +import { join } from "path"; +import { promisify } from "util"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import glob from "glob-promise"; +import JSON5 from "json5"; +import { spaceTrim } from "spacetrim"; +import { PipelineExecutionError } from "../../../errors/PipelineExecutionError"; +import type { AutomaticTranslator } from "./automatic-translators/AutomaticTranslator"; +import type { TranslatorOptions } from "./automatic-translators/TranslatorOptions"; /** * @private still in development [🏳] */ export async function translateMessages({ - automaticTranslator, - from, - to, + automaticTranslator, + from, + to, }: { automaticTranslator: AutomaticTranslator } & TranslatorOptions) { - for (const filename of await glob(join(__dirname, '../../translations/', from || 'en', '/**/*.json5'))) { - // <- TODO: [😶] - const fileData = JSON5.parse(await promisify(readFile)(filename, 'utf-8')); + for (const filename of await glob( + join(__dirname, "../../translations/", from || "en", "/**/*.json5"), + )) { + // <- TODO: [😶] + const fileData = JSON5.parse(await promisify(readFile)(filename, "utf-8")); - for (const row of fileData) { - if (row.language !== from) { - throw new PipelineExecutionError( - spaceTrim(` + for (const row of fileData) { + if (row.language !== from) { + throw new PipelineExecutionError( + spaceTrim(` Language ${row.language} is not ${from} Check the file: ${filename} `), - ); - } + ); + } - const source = spaceTrim(row.message); - const translated = spaceTrim(await automaticTranslator.translate(row.message)); - console.info(colors.gray(`${source} ▶️ ${translated}`)); - row.message = translated; - row.language = to; - row.isAutomaticTranslation = true; - } + const source = spaceTrim(row.message); + const translated = spaceTrim( + await automaticTranslator.translate(row.message), + ); + console.info(colors.gray(`${source} ▶️ ${translated}`)); + row.message = translated; + row.language = to; + row.isAutomaticTranslation = true; + } - // TODO: [🏳] Use here `FileCacheStorage` - await promisify(writeFile)( - filename.split(`/${from}/`).join(`/${to}/`), - JSON5.stringify(fileData, null, 4) + '\n', - 'utf-8', - ); - } + // TODO: [🏳] Use here `FileCacheStorage` + await promisify(writeFile)( + filename.split(`/${from}/`).join(`/${to}/`), + JSON5.stringify(fileData, null, 4) + "\n", + "utf-8", + ); + } } /** diff --git a/src/execution/utils/$provideExecutionToolsForNode.ts b/src/execution/utils/$provideExecutionToolsForNode.ts index a25c06efc0..4ccc0fb3f0 100644 --- a/src/execution/utils/$provideExecutionToolsForNode.ts +++ b/src/execution/utils/$provideExecutionToolsForNode.ts @@ -1,12 +1,12 @@ -import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError'; -import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode'; -import { $provideLlmToolsFromEnv } from '../../llm-providers/_common/register/$provideLlmToolsFromEnv'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { $provideFilesystemForNode } from '../../scrapers/_common/register/$provideFilesystemForNode'; -import { $provideScrapersForNode } from '../../scrapers/_common/register/$provideScrapersForNode'; -import { JavascriptExecutionTools } from '../../scripting/javascript/JavascriptExecutionTools'; -import { $isRunningInNode } from '../../utils/environment/$isRunningInNode'; -import type { ExecutionTools } from '../ExecutionTools'; +import { EnvironmentMismatchError } from "../../errors/EnvironmentMismatchError"; +import { $provideExecutablesForNode } from "../../executables/$provideExecutablesForNode"; +import { $provideLlmToolsFromEnv } from "../../llm-providers/_common/register/$provideLlmToolsFromEnv"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { $provideFilesystemForNode } from "../../scrapers/_common/register/$provideFilesystemForNode"; +import { $provideScrapersForNode } from "../../scrapers/_common/register/$provideScrapersForNode"; +import { JavascriptExecutionTools } from "../../scripting/javascript/JavascriptExecutionTools"; +import { $isRunningInNode } from "../../utils/environment/$isRunningInNode"; +import type { ExecutionTools } from "../ExecutionTools"; /** * Note: There is unfortunately no equivalent for this function in the browser environment @@ -15,24 +15,28 @@ import type { ExecutionTools } from '../ExecutionTools'; * * @public exported from `@promptbook/node` */ -export async function $provideExecutionToolsForNode(options?: PrepareAndScrapeOptions): Promise { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError('Function `$getExecutionToolsForNode` works only in Node.js environment'); - } +export async function $provideExecutionToolsForNode( + options?: PrepareAndScrapeOptions, +): Promise { + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Function `$getExecutionToolsForNode` works only in Node.js environment", + ); + } - const fs = $provideFilesystemForNode(); - const llm = await $provideLlmToolsFromEnv(options); - const executables = await $provideExecutablesForNode(options); + const fs = $provideFilesystemForNode(); + const llm = await $provideLlmToolsFromEnv(options); + const executables = await $provideExecutablesForNode(options); - const tools = { - llm, - fs, - executables, - scrapers: await $provideScrapersForNode({ fs, llm, executables }, options), - script: [new JavascriptExecutionTools(options)], - } satisfies ExecutionTools; + const tools = { + llm, + fs, + executables, + scrapers: await $provideScrapersForNode({ fs, llm, executables }, options), + script: [new JavascriptExecutionTools(options)], + } satisfies ExecutionTools; - return tools; + return tools; } /** diff --git a/src/execution/utils/addUsage.test.ts b/src/execution/utils/addUsage.test.ts index 907b2efadb..eac7d13a2b 100644 --- a/src/execution/utils/addUsage.test.ts +++ b/src/execution/utils/addUsage.test.ts @@ -1,239 +1,243 @@ -import { describe, expect, it } from '@jest/globals'; -import { addUsage } from './addUsage'; -import { ZERO_USAGE } from './usage-constants'; +import { describe, expect, it } from "@jest/globals"; +import { addUsage } from "./addUsage"; +import { ZERO_USAGE } from "./usage-constants"; -describe('how addUsage works', () => { - it('should create void usage with 0 items', () => - expect(addUsage()).toEqual({ - price: { value: 0 }, - input: { - tokensCount: { value: 0 }, - charactersCount: { value: 0 }, - wordsCount: { value: 0 }, - sentencesCount: { value: 0 }, - linesCount: { value: 0 }, - paragraphsCount: { value: 0 }, - pagesCount: { value: 0 }, - }, - output: { - tokensCount: { value: 0 }, - charactersCount: { value: 0 }, - wordsCount: { value: 0 }, - sentencesCount: { value: 0 }, - linesCount: { value: 0 }, - paragraphsCount: { value: 0 }, - pagesCount: { value: 0 }, - }, - })); +describe("how addUsage works", () => { + it("should create void usage with 0 items", () => + expect(addUsage()).toEqual({ + price: { value: 0 }, + input: { + tokensCount: { value: 0 }, + charactersCount: { value: 0 }, + wordsCount: { value: 0 }, + sentencesCount: { value: 0 }, + linesCount: { value: 0 }, + paragraphsCount: { value: 0 }, + pagesCount: { value: 0 }, + }, + output: { + tokensCount: { value: 0 }, + charactersCount: { value: 0 }, + wordsCount: { value: 0 }, + sentencesCount: { value: 0 }, + linesCount: { value: 0 }, + paragraphsCount: { value: 0 }, + pagesCount: { value: 0 }, + }, + })); - it('should add multiple ZERO_USAGE and still get ZERO_USAGE', () => { - expect(addUsage()).toEqual(ZERO_USAGE); - expect(addUsage(ZERO_USAGE)).toEqual(ZERO_USAGE); - expect(addUsage(ZERO_USAGE, ZERO_USAGE)).toEqual(ZERO_USAGE); - expect(addUsage(ZERO_USAGE, ZERO_USAGE, ZERO_USAGE)).toEqual(ZERO_USAGE); - expect(addUsage(ZERO_USAGE, ZERO_USAGE, ZERO_USAGE, ZERO_USAGE)).toEqual(ZERO_USAGE); - expect(addUsage(ZERO_USAGE, ZERO_USAGE, addUsage(ZERO_USAGE, ZERO_USAGE))).toEqual(ZERO_USAGE); - }); + it("should add multiple ZERO_USAGE and still get ZERO_USAGE", () => { + expect(addUsage()).toEqual(ZERO_USAGE); + expect(addUsage(ZERO_USAGE)).toEqual(ZERO_USAGE); + expect(addUsage(ZERO_USAGE, ZERO_USAGE)).toEqual(ZERO_USAGE); + expect(addUsage(ZERO_USAGE, ZERO_USAGE, ZERO_USAGE)).toEqual(ZERO_USAGE); + expect(addUsage(ZERO_USAGE, ZERO_USAGE, ZERO_USAGE, ZERO_USAGE)).toEqual( + ZERO_USAGE, + ); + expect( + addUsage(ZERO_USAGE, ZERO_USAGE, addUsage(ZERO_USAGE, ZERO_USAGE)), + ).toEqual(ZERO_USAGE); + }); - it('should preserve 1 item', () => - expect( - addUsage({ - price: { value: 1 }, - input: { - tokensCount: { value: 20 }, - charactersCount: { value: 1 }, - wordsCount: { value: 2 }, - sentencesCount: { value: 3 }, - linesCount: { value: 4 }, - paragraphsCount: { value: 5, isUncertain: true }, - pagesCount: { value: 6, isUncertain: true }, - }, - output: { - tokensCount: { value: 20 }, - charactersCount: { value: 1 }, - wordsCount: { value: 2 }, - sentencesCount: { value: 3 }, - linesCount: { value: 4 }, - paragraphsCount: { value: 5, isUncertain: true }, - pagesCount: { value: 6, isUncertain: true }, - }, - }), - ).toEqual({ - price: { value: 1 }, - input: { - tokensCount: { value: 20 }, - charactersCount: { value: 1 }, - wordsCount: { value: 2 }, - sentencesCount: { value: 3 }, - linesCount: { value: 4 }, - paragraphsCount: { value: 5, isUncertain: true }, - pagesCount: { value: 6, isUncertain: true }, - }, - output: { - tokensCount: { value: 20 }, - charactersCount: { value: 1 }, - wordsCount: { value: 2 }, - sentencesCount: { value: 3 }, - linesCount: { value: 4 }, - paragraphsCount: { value: 5, isUncertain: true }, - pagesCount: { value: 6, isUncertain: true }, - }, - })); + it("should preserve 1 item", () => + expect( + addUsage({ + price: { value: 1 }, + input: { + tokensCount: { value: 20 }, + charactersCount: { value: 1 }, + wordsCount: { value: 2 }, + sentencesCount: { value: 3 }, + linesCount: { value: 4 }, + paragraphsCount: { value: 5, isUncertain: true }, + pagesCount: { value: 6, isUncertain: true }, + }, + output: { + tokensCount: { value: 20 }, + charactersCount: { value: 1 }, + wordsCount: { value: 2 }, + sentencesCount: { value: 3 }, + linesCount: { value: 4 }, + paragraphsCount: { value: 5, isUncertain: true }, + pagesCount: { value: 6, isUncertain: true }, + }, + }), + ).toEqual({ + price: { value: 1 }, + input: { + tokensCount: { value: 20 }, + charactersCount: { value: 1 }, + wordsCount: { value: 2 }, + sentencesCount: { value: 3 }, + linesCount: { value: 4 }, + paragraphsCount: { value: 5, isUncertain: true }, + pagesCount: { value: 6, isUncertain: true }, + }, + output: { + tokensCount: { value: 20 }, + charactersCount: { value: 1 }, + wordsCount: { value: 2 }, + sentencesCount: { value: 3 }, + linesCount: { value: 4 }, + paragraphsCount: { value: 5, isUncertain: true }, + pagesCount: { value: 6, isUncertain: true }, + }, + })); - it('should add 2 items', () => - expect( - addUsage( - { - price: { value: 1 }, - input: { - tokensCount: { value: 10 }, - charactersCount: { value: 5 }, - wordsCount: { value: 7 }, - sentencesCount: { value: 3 }, - linesCount: { value: 2 }, - paragraphsCount: { value: 1 }, - pagesCount: { value: 0 }, - }, - output: { - tokensCount: { value: 10 }, - charactersCount: { value: 5 }, - wordsCount: { value: 7 }, - sentencesCount: { value: 3 }, - linesCount: { value: 2 }, - paragraphsCount: { value: 1 }, - pagesCount: { value: 0 }, - }, - }, - { - price: { value: 2 }, - input: { - tokensCount: { value: 15 }, - charactersCount: { value: 10 }, - wordsCount: { value: 5 }, - sentencesCount: { value: 2 }, - linesCount: { value: 3 }, - paragraphsCount: { value: 2 }, - pagesCount: { value: 1 }, - }, - output: { - tokensCount: { value: 15 }, - charactersCount: { value: 10 }, - wordsCount: { value: 5 }, - sentencesCount: { value: 2 }, - linesCount: { value: 3 }, - paragraphsCount: { value: 2 }, - pagesCount: { value: 1 }, - }, - }, - ), - ).toEqual({ - price: { value: 3 }, - input: { - tokensCount: { value: 25 }, - charactersCount: { value: 15 }, - wordsCount: { value: 12 }, - sentencesCount: { value: 5 }, - linesCount: { value: 5 }, - paragraphsCount: { value: 3 }, - pagesCount: { value: 1 }, - }, - output: { - tokensCount: { value: 25 }, - charactersCount: { value: 15 }, - wordsCount: { value: 12 }, - sentencesCount: { value: 5 }, - linesCount: { value: 5 }, - paragraphsCount: { value: 3 }, - pagesCount: { value: 1 }, - }, - })); + it("should add 2 items", () => + expect( + addUsage( + { + price: { value: 1 }, + input: { + tokensCount: { value: 10 }, + charactersCount: { value: 5 }, + wordsCount: { value: 7 }, + sentencesCount: { value: 3 }, + linesCount: { value: 2 }, + paragraphsCount: { value: 1 }, + pagesCount: { value: 0 }, + }, + output: { + tokensCount: { value: 10 }, + charactersCount: { value: 5 }, + wordsCount: { value: 7 }, + sentencesCount: { value: 3 }, + linesCount: { value: 2 }, + paragraphsCount: { value: 1 }, + pagesCount: { value: 0 }, + }, + }, + { + price: { value: 2 }, + input: { + tokensCount: { value: 15 }, + charactersCount: { value: 10 }, + wordsCount: { value: 5 }, + sentencesCount: { value: 2 }, + linesCount: { value: 3 }, + paragraphsCount: { value: 2 }, + pagesCount: { value: 1 }, + }, + output: { + tokensCount: { value: 15 }, + charactersCount: { value: 10 }, + wordsCount: { value: 5 }, + sentencesCount: { value: 2 }, + linesCount: { value: 3 }, + paragraphsCount: { value: 2 }, + pagesCount: { value: 1 }, + }, + }, + ), + ).toEqual({ + price: { value: 3 }, + input: { + tokensCount: { value: 25 }, + charactersCount: { value: 15 }, + wordsCount: { value: 12 }, + sentencesCount: { value: 5 }, + linesCount: { value: 5 }, + paragraphsCount: { value: 3 }, + pagesCount: { value: 1 }, + }, + output: { + tokensCount: { value: 25 }, + charactersCount: { value: 15 }, + wordsCount: { value: 12 }, + sentencesCount: { value: 5 }, + linesCount: { value: 5 }, + paragraphsCount: { value: 3 }, + pagesCount: { value: 1 }, + }, + })); - it('should add 3 items', () => - expect( - addUsage( - { - price: { value: 1 }, - input: { - tokensCount: { value: 10 }, - charactersCount: { value: 5 }, - wordsCount: { value: 7 }, - sentencesCount: { value: 3 }, - linesCount: { value: 2 }, - paragraphsCount: { value: 1 }, - pagesCount: { value: 0 }, - }, - output: { - tokensCount: { value: 10 }, - charactersCount: { value: 5 }, - wordsCount: { value: 7 }, - sentencesCount: { value: 3 }, - linesCount: { value: 2 }, - paragraphsCount: { value: 1 }, - pagesCount: { value: 0 }, - }, - }, - { - price: { value: 2 }, - input: { - tokensCount: { value: 15 }, - charactersCount: { value: 10 }, - wordsCount: { value: 5 }, - sentencesCount: { value: 2 }, - linesCount: { value: 3 }, - paragraphsCount: { value: 2 }, - pagesCount: { value: 1 }, - }, - output: { - tokensCount: { value: 15 }, - charactersCount: { value: 10 }, - wordsCount: { value: 5 }, - sentencesCount: { value: 2 }, - linesCount: { value: 3 }, - paragraphsCount: { value: 2 }, - pagesCount: { value: 1 }, - }, - }, - { - price: { value: 3 }, - input: { - tokensCount: { value: 5 }, - charactersCount: { value: 2 }, - wordsCount: { value: 3 }, - sentencesCount: { value: 1 }, - linesCount: { value: 1 }, - paragraphsCount: { value: 0 }, - pagesCount: { value: 0 }, - }, - output: { - tokensCount: { value: 5 }, - charactersCount: { value: 2 }, - wordsCount: { value: 3 }, - sentencesCount: { value: 1 }, - linesCount: { value: 1 }, - paragraphsCount: { value: 0 }, - pagesCount: { value: 0 }, - }, - }, - ), - ).toEqual({ - price: { value: 6 }, - input: { - tokensCount: { value: 30 }, - charactersCount: { value: 17 }, - wordsCount: { value: 15 }, - sentencesCount: { value: 6 }, - linesCount: { value: 6 }, - paragraphsCount: { value: 3 }, - pagesCount: { value: 1 }, - }, - output: { - tokensCount: { value: 30 }, - charactersCount: { value: 17 }, - wordsCount: { value: 15 }, - sentencesCount: { value: 6 }, - linesCount: { value: 6 }, - paragraphsCount: { value: 3 }, - pagesCount: { value: 1 }, - }, - })); + it("should add 3 items", () => + expect( + addUsage( + { + price: { value: 1 }, + input: { + tokensCount: { value: 10 }, + charactersCount: { value: 5 }, + wordsCount: { value: 7 }, + sentencesCount: { value: 3 }, + linesCount: { value: 2 }, + paragraphsCount: { value: 1 }, + pagesCount: { value: 0 }, + }, + output: { + tokensCount: { value: 10 }, + charactersCount: { value: 5 }, + wordsCount: { value: 7 }, + sentencesCount: { value: 3 }, + linesCount: { value: 2 }, + paragraphsCount: { value: 1 }, + pagesCount: { value: 0 }, + }, + }, + { + price: { value: 2 }, + input: { + tokensCount: { value: 15 }, + charactersCount: { value: 10 }, + wordsCount: { value: 5 }, + sentencesCount: { value: 2 }, + linesCount: { value: 3 }, + paragraphsCount: { value: 2 }, + pagesCount: { value: 1 }, + }, + output: { + tokensCount: { value: 15 }, + charactersCount: { value: 10 }, + wordsCount: { value: 5 }, + sentencesCount: { value: 2 }, + linesCount: { value: 3 }, + paragraphsCount: { value: 2 }, + pagesCount: { value: 1 }, + }, + }, + { + price: { value: 3 }, + input: { + tokensCount: { value: 5 }, + charactersCount: { value: 2 }, + wordsCount: { value: 3 }, + sentencesCount: { value: 1 }, + linesCount: { value: 1 }, + paragraphsCount: { value: 0 }, + pagesCount: { value: 0 }, + }, + output: { + tokensCount: { value: 5 }, + charactersCount: { value: 2 }, + wordsCount: { value: 3 }, + sentencesCount: { value: 1 }, + linesCount: { value: 1 }, + paragraphsCount: { value: 0 }, + pagesCount: { value: 0 }, + }, + }, + ), + ).toEqual({ + price: { value: 6 }, + input: { + tokensCount: { value: 30 }, + charactersCount: { value: 17 }, + wordsCount: { value: 15 }, + sentencesCount: { value: 6 }, + linesCount: { value: 6 }, + paragraphsCount: { value: 3 }, + pagesCount: { value: 1 }, + }, + output: { + tokensCount: { value: 30 }, + charactersCount: { value: 17 }, + wordsCount: { value: 15 }, + sentencesCount: { value: 6 }, + linesCount: { value: 6 }, + paragraphsCount: { value: 3 }, + pagesCount: { value: 1 }, + }, + })); }); diff --git a/src/execution/utils/addUsage.ts b/src/execution/utils/addUsage.ts index 037fd12575..bfc9a68b2e 100644 --- a/src/execution/utils/addUsage.ts +++ b/src/execution/utils/addUsage.ts @@ -1,7 +1,7 @@ -import type { WritableDeep } from 'type-fest'; -import { deepClone } from '../../utils/serialization/deepClone'; -import type { PromptResultUsage } from '../PromptResultUsage'; -import { ZERO_USAGE } from './usage-constants'; +import type { WritableDeep } from "type-fest"; +import { deepClone } from "../../utils/serialization/deepClone"; +import type { PromptResultUsage } from "../PromptResultUsage"; +import { ZERO_USAGE } from "./usage-constants"; /** * Function `addUsage` will add multiple usages into one @@ -10,44 +10,49 @@ import { ZERO_USAGE } from './usage-constants'; * * @public exported from `@promptbook/core` */ -export function addUsage(...usageItems: ReadonlyArray): PromptResultUsage { - return usageItems.reduce((acc: WritableDeep, item) => { - acc.price.value += item.price?.value || 0; +export function addUsage( + ...usageItems: ReadonlyArray +): PromptResultUsage { + return usageItems.reduce( + (acc: WritableDeep, item) => { + acc.price.value += item.price?.value || 0; - for (const key of Object.keys(acc.input)) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - if (item.input[key]) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - acc.input[key].value += item.input[key].value || 0; - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - if (item.input[key].isUncertain) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - acc.input[key].isUncertain = true; - } - } - } + for (const key of Object.keys(acc.input)) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + if (item.input[key]) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + acc.input[key].value += item.input[key].value || 0; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + if (item.input[key].isUncertain) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + acc.input[key].isUncertain = true; + } + } + } - for (const key of Object.keys(acc.output)) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - if (item.output[key]) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - acc.output[key].value += item.output[key].value || 0; - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - if (item.output[key].isUncertain) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - acc.output[key].isUncertain = true; - } - } - } + for (const key of Object.keys(acc.output)) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + if (item.output[key]) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + acc.output[key].value += item.output[key].value || 0; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + if (item.output[key].isUncertain) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + acc.output[key].isUncertain = true; + } + } + } - return acc; - }, deepClone(ZERO_USAGE)); + return acc; + }, + deepClone(ZERO_USAGE), + ); } diff --git a/src/execution/utils/checkExpectations.test.ts b/src/execution/utils/checkExpectations.test.ts index 3cc63e5d2e..8596df2962 100644 --- a/src/execution/utils/checkExpectations.test.ts +++ b/src/execution/utils/checkExpectations.test.ts @@ -1,55 +1,67 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { checkExpectations } from './checkExpectations'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { checkExpectations } from "./checkExpectations"; -describe('how checkExpectations works', () => { - it('should pass the expectations', () => { - expect(() => checkExpectations({}, ``)).not.toThrow(); - expect(() => checkExpectations({ words: { min: 1, max: 5 } }, `Foo bar`)).not.toThrow(); - expect(() => checkExpectations({ words: { min: 1, max: 5 } }, `Foo bar baz`)).not.toThrow(); - expect(() => checkExpectations({ sentences: { min: 1, max: 1 } }, `Foo bar baz qux`)).not.toThrow(); +describe("how checkExpectations works", () => { + it("should pass the expectations", () => { + expect(() => checkExpectations({}, ``)).not.toThrow(); + expect(() => + checkExpectations({ words: { min: 1, max: 5 } }, `Foo bar`), + ).not.toThrow(); + expect(() => + checkExpectations({ words: { min: 1, max: 5 } }, `Foo bar baz`), + ).not.toThrow(); + expect(() => + checkExpectations({ sentences: { min: 1, max: 1 } }, `Foo bar baz qux`), + ).not.toThrow(); - expect(() => - checkExpectations( - { - paragraphs: { min: 1, max: 1 }, - }, - spaceTrim(` + expect(() => + checkExpectations( + { + paragraphs: { min: 1, max: 1 }, + }, + spaceTrim(` Foo Bar Baz `), - ), - ).not.toThrow(); - }); + ), + ).not.toThrow(); + }); - it('should fail the expectations', () => { - expect(() => checkExpectations({ characters: { min: 1, max: 5 } }, ``)).toThrowError( - /Expected at least 1 characters but got 0/, - ); - expect(() => checkExpectations({ words: { min: 1, max: 5 } }, `Foo bar baz brr grr hoo`)).toThrowError( - /Expected at most 5 words but got 6/, - ); - expect(() => checkExpectations({ words: { min: 1, max: 5 } }, ``)).toThrowError( - /Expected at least 1 words but got 0/, - ); - expect(() => checkExpectations({ sentences: { min: 1, max: 1 } }, `Foo bar baz qux. And foo.`)).toThrowError( - /Expected at most 1 sentences but got 2/, - ); + it("should fail the expectations", () => { + expect(() => + checkExpectations({ characters: { min: 1, max: 5 } }, ``), + ).toThrowError(/Expected at least 1 characters but got 0/); + expect(() => + checkExpectations( + { words: { min: 1, max: 5 } }, + `Foo bar baz brr grr hoo`, + ), + ).toThrowError(/Expected at most 5 words but got 6/); + expect(() => + checkExpectations({ words: { min: 1, max: 5 } }, ``), + ).toThrowError(/Expected at least 1 words but got 0/); + expect(() => + checkExpectations( + { sentences: { min: 1, max: 1 } }, + `Foo bar baz qux. And foo.`, + ), + ).toThrowError(/Expected at most 1 sentences but got 2/); - expect(() => - checkExpectations( - { - paragraphs: { min: 1, max: 1 }, - }, - spaceTrim(` + expect(() => + checkExpectations( + { + paragraphs: { min: 1, max: 1 }, + }, + spaceTrim(` Foo Bar Baz `), - ), - ).toThrowError(/Expected at most 1 paragraphs but got 3/); - }); + ), + ).toThrowError(/Expected at most 1 paragraphs but got 3/); + }); }); diff --git a/src/execution/utils/checkExpectations.ts b/src/execution/utils/checkExpectations.ts index 2fbe49451c..1a08c20dbd 100644 --- a/src/execution/utils/checkExpectations.ts +++ b/src/execution/utils/checkExpectations.ts @@ -1,7 +1,7 @@ -import { ExpectError } from '../../errors/ExpectError'; -import type { Expectations } from '../../pipeline/PipelineJson/Expectations'; -import type { ExpectationUnit } from '../../pipeline/PipelineJson/Expectations'; -import { CountUtils } from '../../utils/expectation-counters/index'; +import { ExpectError } from "../../errors/ExpectError"; +import type { Expectations } from "../../pipeline/PipelineJson/Expectations"; +import type { ExpectationUnit } from "../../pipeline/PipelineJson/Expectations"; +import { CountUtils } from "../../utils/expectation-counters/index"; /** * Function checkExpectations will check if the expectations on given value are met @@ -14,18 +14,25 @@ import { CountUtils } from '../../utils/expectation-counters/index'; * @returns {void} Nothing * @private internal function of `createPipelineExecutor` */ -export function checkExpectations(expectations: Expectations, value: string): void { - for (const [unit, { max, min }] of Object.entries(expectations)) { - const amount = CountUtils[unit.toUpperCase() as ExpectationUnit](value); +export function checkExpectations( + expectations: Expectations, + value: string, +): void { + for (const [unit, { max, min }] of Object.entries(expectations)) { + const amount = CountUtils[unit.toUpperCase() as ExpectationUnit](value); - if (min && amount < min) { - throw new ExpectError(`Expected at least ${min} ${unit} but got ${amount}`); - } /* not else */ + if (min && amount < min) { + throw new ExpectError( + `Expected at least ${min} ${unit} but got ${amount}`, + ); + } /* not else */ - if (max && amount > max) { - throw new ExpectError(`Expected at most ${max} ${unit} but got ${amount}`); - } - } + if (max && amount > max) { + throw new ExpectError( + `Expected at most ${max} ${unit} but got ${amount}`, + ); + } + } } /** @@ -38,16 +45,19 @@ export function checkExpectations(expectations: Expectations, value: string): vo * @returns {boolean} True if the expectations are met * @public exported from `@promptbook/core` */ -export function isPassingExpectations(expectations: Expectations, value: string): boolean { - try { - checkExpectations(expectations, value); - return true; - } catch (error) { - if (!(error instanceof ExpectError)) { - throw error; - } - return false; - } +export function isPassingExpectations( + expectations: Expectations, + value: string, +): boolean { + try { + checkExpectations(expectations, value); + return true; + } catch (error) { + if (!(error instanceof ExpectError)) { + throw error; + } + return false; + } } /** diff --git a/src/execution/utils/computeUsageCounts.ts b/src/execution/utils/computeUsageCounts.ts index ef441badff..4f21c35a9e 100644 --- a/src/execution/utils/computeUsageCounts.ts +++ b/src/execution/utils/computeUsageCounts.ts @@ -1,10 +1,10 @@ -import { countCharacters } from '../../utils/expectation-counters/countCharacters'; -import { countLines } from '../../utils/expectation-counters/countLines'; -import { countPages } from '../../utils/expectation-counters/countPages'; -import { countParagraphs } from '../../utils/expectation-counters/countParagraphs'; -import { countSentences } from '../../utils/expectation-counters/countSentences'; -import { countWords } from '../../utils/expectation-counters/countWords'; -import type { PromptResultUsageCounts } from '../PromptResultUsage'; +import { countCharacters } from "../../utils/expectation-counters/countCharacters"; +import { countLines } from "../../utils/expectation-counters/countLines"; +import { countPages } from "../../utils/expectation-counters/countPages"; +import { countParagraphs } from "../../utils/expectation-counters/countParagraphs"; +import { countSentences } from "../../utils/expectation-counters/countSentences"; +import { countWords } from "../../utils/expectation-counters/countWords"; +import type { PromptResultUsageCounts } from "../PromptResultUsage"; /** * Helper of usage compute @@ -14,13 +14,15 @@ import type { PromptResultUsageCounts } from '../PromptResultUsage'; * * @private internal utility of LlmExecutionTools */ -export function computeUsageCounts(content: string): Omit { - return { - charactersCount: { value: countCharacters(content) }, - wordsCount: { value: countWords(content) }, - sentencesCount: { value: countSentences(content) }, - linesCount: { value: countLines(content) }, - paragraphsCount: { value: countParagraphs(content) }, - pagesCount: { value: countPages(content) }, - }; +export function computeUsageCounts( + content: string, +): Omit { + return { + charactersCount: { value: countCharacters(content) }, + wordsCount: { value: countWords(content) }, + sentencesCount: { value: countSentences(content) }, + linesCount: { value: countLines(content) }, + paragraphsCount: { value: countParagraphs(content) }, + pagesCount: { value: countPages(content) }, + }; } diff --git a/src/execution/utils/forEachAsync.ts b/src/execution/utils/forEachAsync.ts index bb5d616045..9fc1a088a4 100644 --- a/src/execution/utils/forEachAsync.ts +++ b/src/execution/utils/forEachAsync.ts @@ -1,12 +1,12 @@ -import type { Promisable } from 'type-fest'; +import type { Promisable } from "type-fest"; type ForEachAsyncOptions = { - /** - * Maximum number of tasks running in parallel - * - * @default Infinity - */ - readonly maxParallelCount?: number; + /** + * Maximum number of tasks running in parallel + * + * @default Infinity + */ + readonly maxParallelCount?: number; }; /** @@ -19,29 +19,33 @@ type ForEachAsyncOptions = { * @deprecated [🪂] Use queues instead */ export async function forEachAsync( - array: ReadonlyArray, - options: ForEachAsyncOptions, - callbackfunction: (value: TItem, index: number, array: ReadonlyArray) => Promisable, + array: ReadonlyArray, + options: ForEachAsyncOptions, + callbackfunction: ( + value: TItem, + index: number, + array: ReadonlyArray, + ) => Promisable, ) { - const { maxParallelCount = Infinity } = options; - let index = 0; + const { maxParallelCount = Number.POSITIVE_INFINITY } = options; + let index = 0; - let runningTasks: Promisable[] = []; - const tasks: Promisable[] = []; - for (const item of array as ReadonlyArray) { - const currentIndex = index++; + let runningTasks: Promisable[] = []; + const tasks: Promisable[] = []; + for (const item of array as ReadonlyArray) { + const currentIndex = index++; - const task = callbackfunction(item, currentIndex, array); - tasks.push(task); - runningTasks.push(task); + const task = callbackfunction(item, currentIndex, array); + tasks.push(task); + runningTasks.push(task); - /* not await */ Promise.resolve(task).then(() => { - runningTasks = runningTasks.filter((t) => t !== task); - }); + /* not await */ Promise.resolve(task).then(() => { + runningTasks = runningTasks.filter((t) => t !== task); + }); - if (maxParallelCount < runningTasks.length) { - await Promise.race(runningTasks); - } - } - await Promise.all(tasks); + if (maxParallelCount < runningTasks.length) { + await Promise.race(runningTasks); + } + } + await Promise.all(tasks); } diff --git a/src/execution/utils/uncertainNumber.ts b/src/execution/utils/uncertainNumber.ts index 5f149600fa..3b62351451 100644 --- a/src/execution/utils/uncertainNumber.ts +++ b/src/execution/utils/uncertainNumber.ts @@ -1,4 +1,4 @@ -import type { UncertainNumber } from '../UncertainNumber'; +import type { UncertainNumber } from "../UncertainNumber"; /** * Make UncertainNumber @@ -7,10 +7,12 @@ import type { UncertainNumber } from '../UncertainNumber'; * * @private utility for initializating UncertainNumber */ -export function uncertainNumber(value?: number | typeof NaN | undefined | null): UncertainNumber { - if (value === null || value === undefined || Number.isNaN(value)) { - return { value: 0, isUncertain: true }; - } +export function uncertainNumber( + value?: number | typeof NaN | undefined | null, +): UncertainNumber { + if (value === null || value === undefined || Number.isNaN(value)) { + return { value: 0, isUncertain: true }; + } - return { value }; + return { value }; } diff --git a/src/execution/utils/usage-constants.ts b/src/execution/utils/usage-constants.ts index 51f604d7aa..a482c5d59f 100644 --- a/src/execution/utils/usage-constants.ts +++ b/src/execution/utils/usage-constants.ts @@ -1,5 +1,5 @@ -import { $deepFreeze } from '../../utils/serialization/$deepFreeze'; -import type { PromptResultUsage } from '../PromptResultUsage'; +import { $deepFreeze } from "../../utils/serialization/$deepFreeze"; +import type { PromptResultUsage } from "../PromptResultUsage"; /** * Represents the usage with no resources consumed @@ -7,25 +7,25 @@ import type { PromptResultUsage } from '../PromptResultUsage'; * @public exported from `@promptbook/core` */ export const ZERO_USAGE = $deepFreeze({ - price: { value: 0 }, - input: { - tokensCount: { value: 0 }, - charactersCount: { value: 0 }, - wordsCount: { value: 0 }, - sentencesCount: { value: 0 }, - linesCount: { value: 0 }, - paragraphsCount: { value: 0 }, - pagesCount: { value: 0 }, - }, - output: { - tokensCount: { value: 0 }, - charactersCount: { value: 0 }, - wordsCount: { value: 0 }, - sentencesCount: { value: 0 }, - linesCount: { value: 0 }, - paragraphsCount: { value: 0 }, - pagesCount: { value: 0 }, - }, + price: { value: 0 }, + input: { + tokensCount: { value: 0 }, + charactersCount: { value: 0 }, + wordsCount: { value: 0 }, + sentencesCount: { value: 0 }, + linesCount: { value: 0 }, + paragraphsCount: { value: 0 }, + pagesCount: { value: 0 }, + }, + output: { + tokensCount: { value: 0 }, + charactersCount: { value: 0 }, + wordsCount: { value: 0 }, + sentencesCount: { value: 0 }, + linesCount: { value: 0 }, + paragraphsCount: { value: 0 }, + pagesCount: { value: 0 }, + }, } as const satisfies PromptResultUsage); /** @@ -34,25 +34,25 @@ export const ZERO_USAGE = $deepFreeze({ * @public exported from `@promptbook/core` */ export const UNCERTAIN_USAGE = $deepFreeze({ - price: { value: 0, isUncertain: true }, - input: { - tokensCount: { value: 0, isUncertain: true }, - charactersCount: { value: 0, isUncertain: true }, - wordsCount: { value: 0, isUncertain: true }, - sentencesCount: { value: 0, isUncertain: true }, - linesCount: { value: 0, isUncertain: true }, - paragraphsCount: { value: 0, isUncertain: true }, - pagesCount: { value: 0, isUncertain: true }, - }, - output: { - tokensCount: { value: 0, isUncertain: true }, - charactersCount: { value: 0, isUncertain: true }, - wordsCount: { value: 0, isUncertain: true }, - sentencesCount: { value: 0, isUncertain: true }, - linesCount: { value: 0, isUncertain: true }, - paragraphsCount: { value: 0, isUncertain: true }, - pagesCount: { value: 0, isUncertain: true }, - }, + price: { value: 0, isUncertain: true }, + input: { + tokensCount: { value: 0, isUncertain: true }, + charactersCount: { value: 0, isUncertain: true }, + wordsCount: { value: 0, isUncertain: true }, + sentencesCount: { value: 0, isUncertain: true }, + linesCount: { value: 0, isUncertain: true }, + paragraphsCount: { value: 0, isUncertain: true }, + pagesCount: { value: 0, isUncertain: true }, + }, + output: { + tokensCount: { value: 0, isUncertain: true }, + charactersCount: { value: 0, isUncertain: true }, + wordsCount: { value: 0, isUncertain: true }, + sentencesCount: { value: 0, isUncertain: true }, + linesCount: { value: 0, isUncertain: true }, + paragraphsCount: { value: 0, isUncertain: true }, + pagesCount: { value: 0, isUncertain: true }, + }, } as const satisfies PromptResultUsage); /** diff --git a/src/execution/utils/usageToHuman.ts b/src/execution/utils/usageToHuman.ts index b0ad8995dc..f32ad04d7f 100644 --- a/src/execution/utils/usageToHuman.ts +++ b/src/execution/utils/usageToHuman.ts @@ -1,8 +1,8 @@ -import spaceTrim from 'spacetrim'; -import type { string_markdown } from '../../types/typeAliases'; -import type { PromptResultUsage } from '../PromptResultUsage'; -import type { UncertainNumber } from '../UncertainNumber'; -import { usageToWorktime } from './usageToWorktime'; +import spaceTrim from "spacetrim"; +import type { string_markdown } from "../../types/typeAliases"; +import type { PromptResultUsage } from "../PromptResultUsage"; +import type { UncertainNumber } from "../UncertainNumber"; +import { usageToWorktime } from "./usageToWorktime"; /** * Function `usageToHuman` will take usage and convert it to human readable report @@ -10,45 +10,49 @@ import { usageToWorktime } from './usageToWorktime'; * @public exported from `@promptbook/core` */ export function usageToHuman(usage: PromptResultUsage): string_markdown { - const reportItems: Array = []; - - const uncertainNumberToHuman = ({ value, isUncertain }: UncertainNumber) => - `${isUncertain ? 'approximately ' : ''}${Math.round(value * 100) / 100}`; - - if ( - usage.price.value > 0.01 - // <- TODO: [🍓][🧞‍♂️][👩🏽‍🤝‍🧑🏻] Configure negligible value - default value to config + value to `UsageToHumanSettings` - ) { - reportItems.push(`Cost ${uncertainNumberToHuman(usage.price)} USD`); - } else { - reportItems.push(`Negligible cost`); - } - - const worktime = usageToWorktime(usage); - if ( - worktime.value > - 1 / 60 - // <- TODO: [🍓][🧞‍♂️][👩🏽‍🤝‍🧑🏻] - ) { - reportItems.push(`Saved ${uncertainNumberToHuman(usageToWorktime(usage))} hours of human time`); - // TODO: [🍓][🧞‍♂️] Show minutes, seconds, days NOT 0.1 hours - } - - if (usage.output.charactersCount.value > 0) { - reportItems.push(`Written ${uncertainNumberToHuman(usage.output.charactersCount)} characters`); - } - - if (reportItems.length === 0) { - // Note: For negligible usage, we report at least something - reportItems.push('Negligible'); - } - - return spaceTrim( - (block) => ` + const reportItems: Array = []; + + const uncertainNumberToHuman = ({ value, isUncertain }: UncertainNumber) => + `${isUncertain ? "approximately " : ""}${Math.round(value * 100) / 100}`; + + if ( + usage.price.value > 0.01 + // <- TODO: [🍓][🧞‍♂️][👩🏽‍🤝‍🧑🏻] Configure negligible value - default value to config + value to `UsageToHumanSettings` + ) { + reportItems.push(`Cost ${uncertainNumberToHuman(usage.price)} USD`); + } else { + reportItems.push(`Negligible cost`); + } + + const worktime = usageToWorktime(usage); + if ( + worktime.value > + 1 / 60 + // <- TODO: [🍓][🧞‍♂️][👩🏽‍🤝‍🧑🏻] + ) { + reportItems.push( + `Saved ${uncertainNumberToHuman(usageToWorktime(usage))} hours of human time`, + ); + // TODO: [🍓][🧞‍♂️] Show minutes, seconds, days NOT 0.1 hours + } + + if (usage.output.charactersCount.value > 0) { + reportItems.push( + `Written ${uncertainNumberToHuman(usage.output.charactersCount)} characters`, + ); + } + + if (reportItems.length === 0) { + // Note: For negligible usage, we report at least something + reportItems.push("Negligible"); + } + + return spaceTrim( + (block) => ` Usage: - ${block(reportItems.map((item) => `- ${item}`).join('\n'))} + ${block(reportItems.map((item) => `- ${item}`).join("\n"))} `, - ); + ); } /** diff --git a/src/execution/utils/usageToWorktime.test.ts b/src/execution/utils/usageToWorktime.test.ts index 89f61b1729..4c66a757cb 100644 --- a/src/execution/utils/usageToWorktime.test.ts +++ b/src/execution/utils/usageToWorktime.test.ts @@ -1,57 +1,58 @@ -import { describe, expect, it } from '@jest/globals'; -import { ZERO_USAGE } from './usage-constants'; -import { usageToWorktime } from './usageToWorktime'; +import { describe, expect, it } from "@jest/globals"; +import { ZERO_USAGE } from "./usage-constants"; +import { usageToWorktime } from "./usageToWorktime"; -describe('how usageToWorktime works', () => { - it('no usage should return no time', () => expect(usageToWorktime(ZERO_USAGE)).toEqual({ value: 0 })); +describe("how usageToWorktime works", () => { + it("no usage should return no time", () => + expect(usageToWorktime(ZERO_USAGE)).toEqual({ value: 0 })); - it('should count worktime', () => - expect( - usageToWorktime({ - price: { value: 1 }, - input: { - tokensCount: { value: 0 }, - charactersCount: { value: 1 }, - wordsCount: { value: 20 }, - sentencesCount: { value: 3 }, - linesCount: { value: 4 }, - paragraphsCount: { value: 5 }, - pagesCount: { value: 6 }, - }, - output: { - tokensCount: { value: 3000 }, - charactersCount: { value: 1 }, - wordsCount: { value: 20 }, - sentencesCount: { value: 3 }, - linesCount: { value: 4 }, - paragraphsCount: { value: 5 }, - pagesCount: { value: 6 }, - }, - }), - ).toEqual({ value: 0.01 })); + it("should count worktime", () => + expect( + usageToWorktime({ + price: { value: 1 }, + input: { + tokensCount: { value: 0 }, + charactersCount: { value: 1 }, + wordsCount: { value: 20 }, + sentencesCount: { value: 3 }, + linesCount: { value: 4 }, + paragraphsCount: { value: 5 }, + pagesCount: { value: 6 }, + }, + output: { + tokensCount: { value: 3000 }, + charactersCount: { value: 1 }, + wordsCount: { value: 20 }, + sentencesCount: { value: 3 }, + linesCount: { value: 4 }, + paragraphsCount: { value: 5 }, + pagesCount: { value: 6 }, + }, + }), + ).toEqual({ value: 0.01 })); - it('should count uncertain worktime', () => - expect( - usageToWorktime({ - price: { value: 1, isUncertain: true }, - input: { - tokensCount: { value: 0, isUncertain: true }, - charactersCount: { value: 1, isUncertain: true }, - wordsCount: { value: 3000, isUncertain: true }, - sentencesCount: { value: 3, isUncertain: true }, - linesCount: { value: 4, isUncertain: true }, - paragraphsCount: { value: 5, isUncertain: true }, - pagesCount: { value: 6, isUncertain: true }, - }, - output: { - tokensCount: { value: 300, isUncertain: true }, - charactersCount: { value: 1, isUncertain: true }, - wordsCount: { value: 80000, isUncertain: true }, - sentencesCount: { value: 3, isUncertain: true }, - linesCount: { value: 4, isUncertain: true }, - paragraphsCount: { value: 5, isUncertain: true }, - pagesCount: { value: 6, isUncertain: true }, - }, - }), - ).toEqual({ value: 33.583333333333336, isUncertain: true })); + it("should count uncertain worktime", () => + expect( + usageToWorktime({ + price: { value: 1, isUncertain: true }, + input: { + tokensCount: { value: 0, isUncertain: true }, + charactersCount: { value: 1, isUncertain: true }, + wordsCount: { value: 3000, isUncertain: true }, + sentencesCount: { value: 3, isUncertain: true }, + linesCount: { value: 4, isUncertain: true }, + paragraphsCount: { value: 5, isUncertain: true }, + pagesCount: { value: 6, isUncertain: true }, + }, + output: { + tokensCount: { value: 300, isUncertain: true }, + charactersCount: { value: 1, isUncertain: true }, + wordsCount: { value: 80000, isUncertain: true }, + sentencesCount: { value: 3, isUncertain: true }, + linesCount: { value: 4, isUncertain: true }, + paragraphsCount: { value: 5, isUncertain: true }, + pagesCount: { value: 6, isUncertain: true }, + }, + }), + ).toEqual({ value: 33.583333333333336, isUncertain: true })); }); diff --git a/src/execution/utils/usageToWorktime.ts b/src/execution/utils/usageToWorktime.ts index 0c2734c7ea..8a3da95e7c 100644 --- a/src/execution/utils/usageToWorktime.ts +++ b/src/execution/utils/usageToWorktime.ts @@ -1,6 +1,6 @@ -import type { Writable } from 'type-fest'; -import type { PromptResultUsage } from '../PromptResultUsage'; -import type { UncertainNumber } from '../UncertainNumber'; +import type { Writable } from "type-fest"; +import type { PromptResultUsage } from "../PromptResultUsage"; +import type { UncertainNumber } from "../UncertainNumber"; /** * Function usageToWorktime will take usage and estimate saved worktime in hours of reading / writing @@ -12,17 +12,18 @@ import type { UncertainNumber } from '../UncertainNumber'; * @public exported from `@promptbook/core` */ export function usageToWorktime(usage: PromptResultUsage): UncertainNumber { - const value = - usage.input.wordsCount.value / (200 /* words per minute */ * 60) + - usage.output.wordsCount.value / (40 /* words per minute */ * 60); + const value = + usage.input.wordsCount.value / (200 /* words per minute */ * 60) + + usage.output.wordsCount.value / (40 /* words per minute */ * 60); - const isUncertain = usage.input.wordsCount.isUncertain || usage.output.wordsCount.isUncertain; + const isUncertain = + usage.input.wordsCount.isUncertain || usage.output.wordsCount.isUncertain; - const uncertainNumber: Writable = { value }; + const uncertainNumber: Writable = { value }; - if (isUncertain === true) { - uncertainNumber.isUncertain = true; - } + if (isUncertain === true) { + uncertainNumber.isUncertain = true; + } - return uncertainNumber; + return uncertainNumber; } diff --git a/src/expectations/drafts/isDomainNameFree.ts b/src/expectations/drafts/isDomainNameFree.ts index 5fc19406ae..3acaad2730 100644 --- a/src/expectations/drafts/isDomainNameFree.ts +++ b/src/expectations/drafts/isDomainNameFree.ts @@ -1,14 +1,17 @@ -import type { PromptbookFetch } from '../../execution/PromptbookFetch'; -import type { string_name } from '../../types/typeAliases'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; +import type { PromptbookFetch } from "../../execution/PromptbookFetch"; +import type { string_name } from "../../types/typeAliases"; +import { TODO_USE } from "../../utils/organization/TODO_USE"; /** * @private still in development */ -export async function isDomainNameFree(name: string_name, fetch: PromptbookFetch): Promise { - TODO_USE(name); - TODO_USE(fetch); - return false; +export async function isDomainNameFree( + name: string_name, + fetch: PromptbookFetch, +): Promise { + TODO_USE(name); + TODO_USE(fetch); + return false; } /** diff --git a/src/expectations/drafts/isGithubNameFree.ts b/src/expectations/drafts/isGithubNameFree.ts index 3a49bc3888..665d6187f4 100644 --- a/src/expectations/drafts/isGithubNameFree.ts +++ b/src/expectations/drafts/isGithubNameFree.ts @@ -1,43 +1,48 @@ -import type { PromptbookFetch } from '../../execution/PromptbookFetch'; -import type { string_name } from '../../types/typeAliases'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; +import type { PromptbookFetch } from "../../execution/PromptbookFetch"; +import type { string_name } from "../../types/typeAliases"; +import { TODO_USE } from "../../utils/organization/TODO_USE"; /** * @private still in development */ -export async function isGithubNameFree(name: string_name, fetch: PromptbookFetch): Promise { - TODO_USE(name); - const response = await fetch('https://github.com/organizations/check_name', { - headers: { - accept: '*/*', - 'accept-language': 'en,cs;q=0.9,en-US;q=0.8,cs-CZ;q=0.7', - 'cache-control': 'no-cache', - 'content-type': 'multipart/form-data; boundary=----WebKitFormBoundaryLj1jj6mDB8aoupcS', - pragma: 'no-cache', - priority: 'u=1, i', - 'sec-ch-ua': '"Google Chrome";v="125", "Chromium";v="125", "Not.A/Brand";v="24"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Windows"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - }, - referrer: - 'https://github.com/account/organizations/new?plan=free&ref_cta=Join%2520for%2520free&ref_loc=organization_plan%2520comparison&ref_page=%2Forganizations%2Fplan', - referrerPolicy: 'strict-origin-when-cross-origin', - body: '------WebKitFormBoundaryLj1jj6mDB8aoupcS\r\nContent-Disposition: form-data; name="authenticity_token"\r\n\r\nP1s9_auiZnFmQJ4U2Q7iZcHQA5ilpWWuR8MXZDSHheXPNsx9kTGWHpPjLsHzGTGAvdrPtfaV8C4VyXNQzV8YAg\r\n------WebKitFormBoundaryLj1jj6mDB8aoupcS\r\nContent-Disposition: form-data; name="value"\r\n\r\npromptbook\r\n------WebKitFormBoundaryLj1jj6mDB8aoupcS--\r\n', - method: 'POST', - mode: 'cors', - credentials: 'include', - }); +export async function isGithubNameFree( + name: string_name, + fetch: PromptbookFetch, +): Promise { + TODO_USE(name); + const response = await fetch("https://github.com/organizations/check_name", { + headers: { + accept: "*/*", + "accept-language": "en,cs;q=0.9,en-US;q=0.8,cs-CZ;q=0.7", + "cache-control": "no-cache", + "content-type": + "multipart/form-data; boundary=----WebKitFormBoundaryLj1jj6mDB8aoupcS", + pragma: "no-cache", + priority: "u=1, i", + "sec-ch-ua": + '"Google Chrome";v="125", "Chromium";v="125", "Not.A/Brand";v="24"', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"Windows"', + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-origin", + }, + referrer: + "https://github.com/account/organizations/new?plan=free&ref_cta=Join%2520for%2520free&ref_loc=organization_plan%2520comparison&ref_page=%2Forganizations%2Fplan", + referrerPolicy: "strict-origin-when-cross-origin", + body: '------WebKitFormBoundaryLj1jj6mDB8aoupcS\r\nContent-Disposition: form-data; name="authenticity_token"\r\n\r\nP1s9_auiZnFmQJ4U2Q7iZcHQA5ilpWWuR8MXZDSHheXPNsx9kTGWHpPjLsHzGTGAvdrPtfaV8C4VyXNQzV8YAg\r\n------WebKitFormBoundaryLj1jj6mDB8aoupcS\r\nContent-Disposition: form-data; name="value"\r\n\r\npromptbook\r\n------WebKitFormBoundaryLj1jj6mDB8aoupcS--\r\n', + method: "POST", + mode: "cors", + credentials: "include", + }); - console.log(response); + console.log(response); - const data = await response.text(); + const data = await response.text(); - console.log(data); + console.log(data); - return false; + return false; } /** diff --git a/src/formats/_common/FormatDefinition.ts b/src/formats/_common/FormatDefinition.ts index 65f7f9cef3..325b863358 100644 --- a/src/formats/_common/FormatDefinition.ts +++ b/src/formats/_common/FormatDefinition.ts @@ -1,7 +1,7 @@ -import type { string_mime_type, string_name } from '../../types/typeAliases'; -import type { string_SCREAMING_CASE } from '../../utils/normalization/normalizeTo_SCREAMING_CASE'; -import type { empty_object } from '../../utils/organization/empty_object'; -import type { FormatSubvalueDefinition } from './FormatSubvalueDefinition'; +import type { string_mime_type, string_name } from "../../types/typeAliases"; +import type { string_SCREAMING_CASE } from "../../utils/normalization/normalizeTo_SCREAMING_CASE"; +import type { empty_object } from "../../utils/organization/empty_object"; +import type { FormatSubvalueDefinition } from "./FormatSubvalueDefinition"; /** * A format definition is a set of functions that define how to validate, heal and convert response from LLM @@ -12,64 +12,74 @@ import type { FormatSubvalueDefinition } from './FormatSubvalueDefinition'; * @private still in development [🏢] */ export type FormatDefinition< - TValue extends TPartialValue, - TPartialValue extends string, - TSettings extends empty_object, - TSchema extends empty_object, + TValue extends TPartialValue, + TPartialValue extends string, + TSettings extends empty_object, + TSchema extends empty_object, > = { - /** - * The name of the format used in .book.md files - * - * @example "JSON" - */ - readonly formatName: string_name & string_SCREAMING_CASE; + /** + * The name of the format used in .book.md files + * + * @example "JSON" + */ + readonly formatName: string_name & string_SCREAMING_CASE; - /** - * Aliases for the `formatName` - */ - readonly aliases?: ReadonlyArray; + /** + * Aliases for the `formatName` + */ + readonly aliases?: ReadonlyArray; - /** - * The mime type of the format (if any) - * - * @example "application/json" - */ - readonly mimeType?: string_mime_type; + /** + * The mime type of the format (if any) + * + * @example "application/json" + */ + readonly mimeType?: string_mime_type; - /** - * Check if a value is fully valid - * - * @param value The value to check, for example "{\"foo\": true}" - * @param schema Optional schema to do extra validation - */ - isValid(value: string, settings?: TSettings, schema?: TSchema): value is TValue; + /** + * Check if a value is fully valid + * + * @param value The value to check, for example "{\"foo\": true}" + * @param schema Optional schema to do extra validation + */ + isValid( + value: string, + settings?: TSettings, + schema?: TSchema, + ): value is TValue; - /** - * Check if a first part of a value is valid - * - * @see https://github.com/webgptorg/promptbook/discussions/37 - * - * @param partialValue Partial value to check, for example "{\"foo\": t" - * @param schema Optional schema to do extra validation - */ - canBeValid(partialValue: string, settings?: TSettings, schema?: TSchema): partialValue is TPartialValue; + /** + * Check if a first part of a value is valid + * + * @see https://github.com/webgptorg/promptbook/discussions/37 + * + * @param partialValue Partial value to check, for example "{\"foo\": t" + * @param schema Optional schema to do extra validation + */ + canBeValid( + partialValue: string, + settings?: TSettings, + schema?: TSchema, + ): partialValue is TPartialValue; - /** - * Heal a value to make it valid if possible - * - * Note: This make sense in context of LLMs that often returns slightly invalid values - * @see https://github.com/webgptorg/promptbook/discussions/31 - * - * @param value The value to heal, for example "{foo: true}" - * @param scheme - * @throws {Error} If the value cannot be healed - */ - heal(value: string, settings?: TSettings, scheme?: TSchema): TValue; + /** + * Heal a value to make it valid if possible + * + * Note: This make sense in context of LLMs that often returns slightly invalid values + * @see https://github.com/webgptorg/promptbook/discussions/31 + * + * @param value The value to heal, for example "{foo: true}" + * @param scheme + * @throws {Error} If the value cannot be healed + */ + heal(value: string, settings?: TSettings, scheme?: TSchema): TValue; - /** - * @@@ - */ - readonly subvalueDefinitions: ReadonlyArray>; + /** + * @@@ + */ + readonly subvalueDefinitions: ReadonlyArray< + FormatSubvalueDefinition + >; }; /** diff --git a/src/formats/_common/FormatSubvalueDefinition.ts b/src/formats/_common/FormatSubvalueDefinition.ts index 984645764c..c9b0aac1b3 100644 --- a/src/formats/_common/FormatSubvalueDefinition.ts +++ b/src/formats/_common/FormatSubvalueDefinition.ts @@ -1,38 +1,41 @@ -import type { Promisable } from 'type-fest'; -import type { Parameters } from '../../types/typeAliases'; -import type { string_name } from '../../types/typeAliases'; -import type { string_parameter_name } from '../../types/typeAliases'; -import type { string_SCREAMING_CASE } from '../../utils/normalization/normalizeTo_SCREAMING_CASE'; -import type { empty_object } from '../../utils/organization/empty_object'; +import type { Promisable } from "type-fest"; +import type { Parameters } from "../../types/typeAliases"; +import type { string_name } from "../../types/typeAliases"; +import type { string_parameter_name } from "../../types/typeAliases"; +import type { string_SCREAMING_CASE } from "../../utils/normalization/normalizeTo_SCREAMING_CASE"; +import type { empty_object } from "../../utils/organization/empty_object"; /** * @@@ */ -export type FormatSubvalueDefinition = { - /** - * The name of the format used in .book.md files - * - * @example "CELL" - */ - readonly subvalueName: string_name & string_SCREAMING_CASE; +export type FormatSubvalueDefinition< + TValue extends string, + TSettings extends empty_object, +> = { + /** + * The name of the format used in .book.md files + * + * @example "CELL" + */ + readonly subvalueName: string_name & string_SCREAMING_CASE; - /** - * Aliases for the `subvalueName` - */ - readonly aliases?: ReadonlyArray; + /** + * Aliases for the `subvalueName` + */ + readonly aliases?: ReadonlyArray; - /** - * Maps values - * - * For example, if you have a JSON object and you want to map all values to uppercase - * Or iterate over all CSV cells @@@ - */ - mapValues( - value: TValue, - outputParameterName: string_parameter_name, - settings: TSettings, - mapCallback: (subvalues: Parameters, index: number) => Promisable, - ): Promise; + /** + * Maps values + * + * For example, if you have a JSON object and you want to map all values to uppercase + * Or iterate over all CSV cells @@@ + */ + mapValues( + value: TValue, + outputParameterName: string_parameter_name, + settings: TSettings, + mapCallback: (subvalues: Parameters, index: number) => Promisable, + ): Promise; }; /* diff --git a/src/formats/csv/CsvFormatDefinition.ts b/src/formats/csv/CsvFormatDefinition.ts index c532e8d590..512ae2ee35 100644 --- a/src/formats/csv/CsvFormatDefinition.ts +++ b/src/formats/csv/CsvFormatDefinition.ts @@ -1,12 +1,12 @@ -import { parse, unparse } from 'papaparse'; -import spaceTrim from 'spacetrim'; -import type { Parameters } from '../../types/typeAliases'; -import type { TODO_any } from '../../utils/organization/TODO_any'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; -import type { FormatDefinition } from '../_common/FormatDefinition'; -import { CsvFormatError } from './CsvFormatError'; -import type { CsvSettings } from './CsvSettings'; -import { MANDATORY_CSV_SETTINGS } from './CsvSettings'; +import { parse, unparse } from "papaparse"; +import spaceTrim from "spacetrim"; +import type { Parameters } from "../../types/typeAliases"; +import { TODO_USE } from "../../utils/organization/TODO_USE"; +import type { TODO_any } from "../../utils/organization/TODO_any"; +import type { FormatDefinition } from "../_common/FormatDefinition"; +import { CsvFormatError } from "./CsvFormatError"; +import type { CsvSettings } from "./CsvSettings"; +import { MANDATORY_CSV_SETTINGS } from "./CsvSettings"; /** * Definition for CSV spreadsheet @@ -15,52 +15,59 @@ import { MANDATORY_CSV_SETTINGS } from './CsvSettings'; * <- TODO: [🏢] Export from package `@promptbook/csv` */ export const CsvFormatDefinition: FormatDefinition< - string /* <- [0] */, - string /* <- [👨‍⚖️] */, - CsvSettings, - TODO_any /* <- TODO: Make CSV Schemas */ + string /* <- [0] */, + string /* <- [👨‍⚖️] */, + CsvSettings, + TODO_any /* <- TODO: Make CSV Schemas */ > = { - formatName: 'CSV', - - aliases: ['SPREADSHEET', 'TABLE'], - - isValid(value, settings, schema): value is string /* <- [0] */ { - // TODO: Implement CSV validation - TODO_USE(value /* <- TODO: Use value here */); - TODO_USE(settings /* <- TODO: Use settings here */); - TODO_USE(schema /* <- TODO: Use schema here */); - return true; - }, - - canBeValid(partialValue, settings, schema): partialValue is string /* <- [0] */ { - TODO_USE(partialValue /* <- TODO: Use partialValue here */); - TODO_USE(settings /* <- TODO: Use settings here */); - TODO_USE(schema /* <- TODO: Use schema here */); - return true; - }, - - heal(value, settings, schema) { - TODO_USE(value /* <- TODO: Use partialValue here */); - TODO_USE(settings /* <- TODO: Use settings here */); - TODO_USE(schema /* <- TODO: Use schema here */); - throw new Error('Not implemented'); - }, - - subvalueDefinitions: [ - { - subvalueName: 'ROW', - async mapValues(value, outputParameterName, settings, mapCallback) { - // TODO: [👨🏾‍🤝‍👨🏼] DRY csv parsing - const csv = parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS }); - - if (csv.errors.length !== 0) { - throw new CsvFormatError( - spaceTrim( - (block) => ` + formatName: "CSV", + + aliases: ["SPREADSHEET", "TABLE"], + + isValid(value, settings, schema): value is string /* <- [0] */ { + // TODO: Implement CSV validation + TODO_USE(value /* <- TODO: Use value here */); + TODO_USE(settings /* <- TODO: Use settings here */); + TODO_USE(schema /* <- TODO: Use schema here */); + return true; + }, + + canBeValid( + partialValue, + settings, + schema, + ): partialValue is string /* <- [0] */ { + TODO_USE(partialValue /* <- TODO: Use partialValue here */); + TODO_USE(settings /* <- TODO: Use settings here */); + TODO_USE(schema /* <- TODO: Use schema here */); + return true; + }, + + heal(value, settings, schema) { + TODO_USE(value /* <- TODO: Use partialValue here */); + TODO_USE(settings /* <- TODO: Use settings here */); + TODO_USE(schema /* <- TODO: Use schema here */); + throw new Error("Not implemented"); + }, + + subvalueDefinitions: [ + { + subvalueName: "ROW", + async mapValues(value, outputParameterName, settings, mapCallback) { + // TODO: [👨🏾‍🤝‍👨🏼] DRY csv parsing + const csv = parse(value, { + ...settings, + ...MANDATORY_CSV_SETTINGS, + }); + + if (csv.errors.length !== 0) { + throw new CsvFormatError( + spaceTrim( + (block) => ` CSV parsing error Error(s) from CSV parsing: - ${block(csv.errors.map((error) => error.message).join('\n\n'))} + ${block(csv.errors.map((error) => error.message).join("\n\n"))} The CSV setings: ${block(JSON.stringify({ ...settings, ...MANDATORY_CSV_SETTINGS }, null, 2))} @@ -68,42 +75,45 @@ export const CsvFormatDefinition: FormatDefinition< The CSV data: ${block(value)} `, - ), - ); - } - - const mappedData = await Promise.all( - csv.data.map(async (row, index) => { - if (row[outputParameterName]) { - throw new CsvFormatError( - `Can not overwrite existing column "${outputParameterName}" in CSV row`, - ); - } - - return { - ...row, - [outputParameterName]: await mapCallback(row, index), - }; - }), - ); - - return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }); - }, - }, - { - subvalueName: 'CELL', - async mapValues(value, outputParameterName, settings, mapCallback) { - // TODO: [👨🏾‍🤝‍👨🏼] DRY csv parsing - const csv = parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS }); - - if (csv.errors.length !== 0) { - throw new CsvFormatError( - spaceTrim( - (block) => ` + ), + ); + } + + const mappedData = await Promise.all( + csv.data.map(async (row, index) => { + if (row[outputParameterName]) { + throw new CsvFormatError( + `Can not overwrite existing column "${outputParameterName}" in CSV row`, + ); + } + + return { + ...row, + [outputParameterName]: await mapCallback(row, index), + }; + }), + ); + + return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }); + }, + }, + { + subvalueName: "CELL", + async mapValues(value, outputParameterName, settings, mapCallback) { + // TODO: [👨🏾‍🤝‍👨🏼] DRY csv parsing + const csv = parse(value, { + ...settings, + ...MANDATORY_CSV_SETTINGS, + }); + + if (csv.errors.length !== 0) { + throw new CsvFormatError( + spaceTrim( + (block) => ` CSV parsing error Error(s) from CSV parsing: - ${block(csv.errors.map((error) => error.message).join('\n\n'))} + ${block(csv.errors.map((error) => error.message).join("\n\n"))} The CSV setings: ${block(JSON.stringify({ ...settings, ...MANDATORY_CSV_SETTINGS }, null, 2))} @@ -111,25 +121,25 @@ export const CsvFormatDefinition: FormatDefinition< The CSV data: ${block(value)} `, - ), - ); - } - - const mappedData = await Promise.all( - csv.data.map(async (row, rowIndex) => { - return /* not await */ Promise.all( - Object.entries(row).map(async ([key, value], columnIndex) => { - const index = rowIndex * Object.keys(row).length + columnIndex; - return /* not await */ mapCallback({ [key]: value }, index); - }), - ); - }), - ); - - return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }); - }, - }, - ], + ), + ); + } + + const mappedData = await Promise.all( + csv.data.map(async (row, rowIndex) => { + return /* not await */ Promise.all( + Object.entries(row).map(async ([key, value], columnIndex) => { + const index = rowIndex * Object.keys(row).length + columnIndex; + return /* not await */ mapCallback({ [key]: value }, index); + }), + ); + }), + ); + + return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }); + }, + }, + ], }; /** diff --git a/src/formats/csv/CsvFormatError.ts b/src/formats/csv/CsvFormatError.ts index aa6842d68a..a337e8c74d 100644 --- a/src/formats/csv/CsvFormatError.ts +++ b/src/formats/csv/CsvFormatError.ts @@ -6,9 +6,9 @@ import { AbstractFormatError } from "../../errors/AbstractFormatError"; * @public exported from `@promptbook/core` */ export class CsvFormatError extends AbstractFormatError { - public readonly name = 'CsvFormatError'; - public constructor(message: string) { - super(message); - Object.setPrototypeOf(this, CsvFormatError.prototype); - } + public readonly name = "CsvFormatError"; + public constructor(message: string) { + super(message); + Object.setPrototypeOf(this, CsvFormatError.prototype); + } } diff --git a/src/formats/csv/CsvSettings.ts b/src/formats/csv/CsvSettings.ts index 012a681930..e629c9a4e8 100644 --- a/src/formats/csv/CsvSettings.ts +++ b/src/formats/csv/CsvSettings.ts @@ -1,12 +1,12 @@ -import type { ParseConfig, UnparseConfig } from 'papaparse'; +import type { ParseConfig, UnparseConfig } from "papaparse"; /** * @@@ */ export type CsvSettings = Pick< - ParseConfig & UnparseConfig, - 'delimiter' | 'quoteChar' | 'newline' | 'skipEmptyLines' - // <- TODO: List here more settings to configure CSV parsing + ParseConfig & UnparseConfig, + "delimiter" | "quoteChar" | "newline" | "skipEmptyLines" + // <- TODO: List here more settings to configure CSV parsing >; /** @@ -15,6 +15,6 @@ export type CsvSettings = Pick< * @public exported from `@promptbook/core` */ export const MANDATORY_CSV_SETTINGS = Object.freeze({ - header: true, - // encoding: 'utf-8', + header: true, + // encoding: 'utf-8', } as const) satisfies Omit; diff --git a/src/formats/index.ts b/src/formats/index.ts index 40f036aca3..6e34e5b259 100644 --- a/src/formats/index.ts +++ b/src/formats/index.ts @@ -1,7 +1,7 @@ -import { CsvFormatDefinition } from './csv/CsvFormatDefinition'; -import { JsonFormatDefinition } from './json/JsonFormatDefinition'; -import { TextFormatDefinition } from './text/TextFormatDefinition'; -import { XmlFormatDefinition } from './xml/XmlFormatDefinition'; +import { CsvFormatDefinition } from "./csv/CsvFormatDefinition"; +import { JsonFormatDefinition } from "./json/JsonFormatDefinition"; +import { TextFormatDefinition } from "./text/TextFormatDefinition"; +import { XmlFormatDefinition } from "./xml/XmlFormatDefinition"; /** * Definitions for all formats supported by Promptbook @@ -9,13 +9,12 @@ import { XmlFormatDefinition } from './xml/XmlFormatDefinition'; * @private internal index of `...` <- TODO [🏢] */ export const FORMAT_DEFINITIONS = [ - JsonFormatDefinition, - XmlFormatDefinition, - TextFormatDefinition, - CsvFormatDefinition, + JsonFormatDefinition, + XmlFormatDefinition, + TextFormatDefinition, + CsvFormatDefinition, ] as const; - /** * Note: [💞] Ignore a discrepancy between file name and entity name - */ \ No newline at end of file + */ diff --git a/src/formats/json/JsonFormatDefinition.ts b/src/formats/json/JsonFormatDefinition.ts index bd9c7d0718..59d005cac9 100644 --- a/src/formats/json/JsonFormatDefinition.ts +++ b/src/formats/json/JsonFormatDefinition.ts @@ -1,7 +1,7 @@ -import type { TODO_any } from '../../utils/organization/TODO_any'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; -import type { FormatDefinition } from '../_common/FormatDefinition'; -import { isValidJsonString } from './utils/isValidJsonString'; +import { TODO_USE } from "../../utils/organization/TODO_USE"; +import type { TODO_any } from "../../utils/organization/TODO_any"; +import type { FormatDefinition } from "../_common/FormatDefinition"; +import { isValidJsonString } from "./utils/isValidJsonString"; /** * Definition for JSON format @@ -9,36 +9,40 @@ import { isValidJsonString } from './utils/isValidJsonString'; * @private still in development [🏢] */ export const JsonFormatDefinition: FormatDefinition< - string /* <- [0] */, - string /* <- [👨‍⚖️] */, - TODO_any /* <- [1] */, - TODO_any /* <- [1] */ + string /* <- [0] */, + string /* <- [👨‍⚖️] */, + TODO_any /* <- [1] */, + TODO_any /* <- [1] */ > = { - formatName: 'JSON', + formatName: "JSON", - mimeType: 'application/json', + mimeType: "application/json", - isValid(value, settings, schema): value is string /* <- [0] */ { - TODO_USE(schema /* <- TODO: Use schema here */); - TODO_USE(settings /* <- TODO: Use settings here */); - return isValidJsonString(value); - }, + isValid(value, settings, schema): value is string /* <- [0] */ { + TODO_USE(schema /* <- TODO: Use schema here */); + TODO_USE(settings /* <- TODO: Use settings here */); + return isValidJsonString(value); + }, - canBeValid(partialValue, settings, schema): partialValue is string /* <- [0] */ { - TODO_USE(partialValue /* <- TODO: Use partialValue here */); - TODO_USE(settings /* <- TODO: Use settings here */); - TODO_USE(schema /* <- TODO: Use schema here */); - return true; - }, + canBeValid( + partialValue, + settings, + schema, + ): partialValue is string /* <- [0] */ { + TODO_USE(partialValue /* <- TODO: Use partialValue here */); + TODO_USE(settings /* <- TODO: Use settings here */); + TODO_USE(schema /* <- TODO: Use schema here */); + return true; + }, - heal(value, settings, schema) { - TODO_USE(value /* <- TODO: Use partialValue here */); - TODO_USE(settings /* <- TODO: Use settings here */); - TODO_USE(schema /* <- TODO: Use schema here */); - throw new Error('Not implemented'); - }, + heal(value, settings, schema) { + TODO_USE(value /* <- TODO: Use partialValue here */); + TODO_USE(settings /* <- TODO: Use settings here */); + TODO_USE(schema /* <- TODO: Use schema here */); + throw new Error("Not implemented"); + }, - subvalueDefinitions: [], + subvalueDefinitions: [], }; /** diff --git a/src/formats/json/utils/isValidJsonString.test.ts b/src/formats/json/utils/isValidJsonString.test.ts index f5b748ac64..d0b7950924 100644 --- a/src/formats/json/utils/isValidJsonString.test.ts +++ b/src/formats/json/utils/isValidJsonString.test.ts @@ -1,13 +1,13 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { isValidJsonString } from './isValidJsonString'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { isValidJsonString } from "./isValidJsonString"; -describe('how isValidJsonString works', () => { - it('should tell that valid json is valid', () => { - expect(isValidJsonString(`{"foo": "bar"}`)).toBe(true); - expect( - isValidJsonString( - spaceTrim(` +describe("how isValidJsonString works", () => { + it("should tell that valid json is valid", () => { + expect(isValidJsonString(`{"foo": "bar"}`)).toBe(true); + expect( + isValidJsonString( + spaceTrim(` { "foo": "bar", "bar": "baz", @@ -16,19 +16,19 @@ describe('how isValidJsonString works', () => { } } `), - ), - ).toBe(true); - }); + ), + ).toBe(true); + }); - it('should tell that non-json is not valid', () => { - expect( - isValidJsonString( - spaceTrim(` + it("should tell that non-json is not valid", () => { + expect( + isValidJsonString( + spaceTrim(` { "foo": "bar } `), - ), - ).toBe(false); - }); + ), + ).toBe(false); + }); }); diff --git a/src/formats/json/utils/isValidJsonString.ts b/src/formats/json/utils/isValidJsonString.ts index 5d704ed88b..5f5d0604eb 100644 --- a/src/formats/json/utils/isValidJsonString.ts +++ b/src/formats/json/utils/isValidJsonString.ts @@ -1,21 +1,21 @@ /** * Function isValidJsonString will tell you if the string is valid JSON or not - * + * * @public exported from `@promptbook/utils` */ export function isValidJsonString(value: string /* <- [👨‍⚖️] */): boolean { - try { - JSON.parse(value); - return true; - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } + try { + JSON.parse(value); + return true; + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } - if (error.message.includes('Unexpected token')) { - return false; - } + if (error.message.includes("Unexpected token")) { + return false; + } - return false; - } + return false; + } } diff --git a/src/formats/text/TextFormatDefinition.ts b/src/formats/text/TextFormatDefinition.ts index 8ad241cbb0..966212fb0d 100644 --- a/src/formats/text/TextFormatDefinition.ts +++ b/src/formats/text/TextFormatDefinition.ts @@ -1,6 +1,6 @@ -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { TODO_any } from '../../utils/organization/TODO_any'; -import type { FormatDefinition } from '../_common/FormatDefinition'; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { TODO_any } from "../../utils/organization/TODO_any"; +import type { FormatDefinition } from "../_common/FormatDefinition"; /** * Definition for any text - this will be always valid @@ -9,44 +9,51 @@ import type { FormatDefinition } from '../_common/FormatDefinition'; * * @public exported from `@promptbook/core` */ -export const TextFormatDefinition: FormatDefinition = { - formatName: 'TEXT', +export const TextFormatDefinition: FormatDefinition< + string, + string, + TODO_any /* <- [1] */, + TODO_any /* <- [1] */ +> = { + formatName: "TEXT", - isValid(value: string): value is string { - return typeof value === 'string'; - }, + isValid(value: string): value is string { + return typeof value === "string"; + }, - canBeValid(partialValue: string): partialValue is string { - return typeof partialValue === 'string'; - }, + canBeValid(partialValue: string): partialValue is string { + return typeof partialValue === "string"; + }, - heal() { - throw new UnexpectedError('It does not make sense to call `TextFormatDefinition.heal`'); - }, + heal() { + throw new UnexpectedError( + "It does not make sense to call `TextFormatDefinition.heal`", + ); + }, - subvalueDefinitions: [ - { - subvalueName: 'LINE', - async mapValues(value, outputParameterName, settings, mapCallback) { - const lines = value.split('\n'); - const mappedLines = await Promise.all( - lines.map((lineContent, lineNumber) => - // TODO: [🧠] Maybe option to skip empty line - /* not await */ mapCallback( - { - lineContent, + subvalueDefinitions: [ + { + subvalueName: "LINE", + async mapValues(value, outputParameterName, settings, mapCallback) { + const lines = value.split("\n"); + const mappedLines = await Promise.all( + lines.map((lineContent, lineNumber) => + // TODO: [🧠] Maybe option to skip empty line + /* not await */ mapCallback( + { + lineContent, - // TODO: [🧠] Maybe also put here `lineNumber` - }, - lineNumber, - ), - ), - ); - return mappedLines.join('\n'); - }, - }, - // <- TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues - ], + // TODO: [🧠] Maybe also put here `lineNumber` + }, + lineNumber, + ), + ), + ); + return mappedLines.join("\n"); + }, + }, + // <- TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues + ], }; /** diff --git a/src/formats/xml/XmlFormatDefinition.ts b/src/formats/xml/XmlFormatDefinition.ts index 9e2cf7ab77..ad1cbf8924 100644 --- a/src/formats/xml/XmlFormatDefinition.ts +++ b/src/formats/xml/XmlFormatDefinition.ts @@ -1,6 +1,6 @@ -import type { TODO_any } from '../../utils/organization/TODO_any'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; -import type { FormatDefinition } from '../_common/FormatDefinition'; +import { TODO_USE } from "../../utils/organization/TODO_USE"; +import type { TODO_any } from "../../utils/organization/TODO_any"; +import type { FormatDefinition } from "../_common/FormatDefinition"; /** * Definition for XML format @@ -8,37 +8,41 @@ import type { FormatDefinition } from '../_common/FormatDefinition'; * @private still in development [🏢] */ export const XmlFormatDefinition: FormatDefinition< - string /* <- [0] */, - string /* <- [👨‍⚖️] */, - TODO_any /* <- [1] */, - TODO_any /* <- [1] */ + string /* <- [0] */, + string /* <- [👨‍⚖️] */, + TODO_any /* <- [1] */, + TODO_any /* <- [1] */ > = { - formatName: 'XML', + formatName: "XML", - mimeType: 'application/xml', + mimeType: "application/xml", - isValid(value, settings, schema): value is string /* <- [0] */ { - TODO_USE(value /* <- TODO: Use value here */); - TODO_USE(settings /* <- TODO: Use settings here */); - TODO_USE(schema /* <- TODO: Use schema here */); - return true; - }, + isValid(value, settings, schema): value is string /* <- [0] */ { + TODO_USE(value /* <- TODO: Use value here */); + TODO_USE(settings /* <- TODO: Use settings here */); + TODO_USE(schema /* <- TODO: Use schema here */); + return true; + }, - canBeValid(partialValue, settings, schema): partialValue is string /* <- [0] */ { - TODO_USE(partialValue /* <- TODO: Use partialValue here */); - TODO_USE(settings /* <- TODO: Use settings here */); - TODO_USE(schema /* <- TODO: Use schema here */); - return true; - }, + canBeValid( + partialValue, + settings, + schema, + ): partialValue is string /* <- [0] */ { + TODO_USE(partialValue /* <- TODO: Use partialValue here */); + TODO_USE(settings /* <- TODO: Use settings here */); + TODO_USE(schema /* <- TODO: Use schema here */); + return true; + }, - heal(value, settings, schema) { - TODO_USE(value /* <- TODO: Use partialValue here */); - TODO_USE(settings /* <- TODO: Use settings here */); - TODO_USE(schema /* <- TODO: Use schema here */); - throw new Error('Not implemented'); - }, + heal(value, settings, schema) { + TODO_USE(value /* <- TODO: Use partialValue here */); + TODO_USE(settings /* <- TODO: Use settings here */); + TODO_USE(schema /* <- TODO: Use schema here */); + throw new Error("Not implemented"); + }, - subvalueDefinitions: [], + subvalueDefinitions: [], }; /** diff --git a/src/formfactors/_boilerplate/BoilerplateFormfactorDefinition.ts b/src/formfactors/_boilerplate/BoilerplateFormfactorDefinition.ts index e36c8a9aba..247669cb1c 100644 --- a/src/formfactors/_boilerplate/BoilerplateFormfactorDefinition.ts +++ b/src/formfactors/_boilerplate/BoilerplateFormfactorDefinition.ts @@ -1,4 +1,4 @@ -import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactorDefinition'; +import type { AbstractFormfactorDefinition } from "../_common/AbstractFormfactorDefinition"; /** * Boilerplate is form of app that @@@ @@ -6,18 +6,18 @@ import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactor * @public exported from `@promptbook/core` */ export const BoilerplateFormfactorDefinition = { - name: 'BOILERPLATE', - description: `@@@`, - documentationUrl: `https://github.com/webgptorg/promptbook/discussions/@@`, - // <- TODO: https://github.com/webgptorg/promptbook/discussions/new?category=concepts - // "🔠 Boilerplate Formfactor" + name: "BOILERPLATE", + description: `@@@`, + documentationUrl: `https://github.com/webgptorg/promptbook/discussions/@@`, + // <- TODO: https://github.com/webgptorg/promptbook/discussions/new?category=concepts + // "🔠 Boilerplate Formfactor" - pipelineInterface: { - inputParameters: [ - /* @@@ */ - ], - outputParameters: [ - /* @@@ */ - ], - }, + pipelineInterface: { + inputParameters: [ + /* @@@ */ + ], + outputParameters: [ + /* @@@ */ + ], + }, } as const satisfies AbstractFormfactorDefinition; diff --git a/src/formfactors/_common/AbstractFormfactorDefinition.ts b/src/formfactors/_common/AbstractFormfactorDefinition.ts index 1aa30e488f..9d1a9fe37e 100644 --- a/src/formfactors/_common/AbstractFormfactorDefinition.ts +++ b/src/formfactors/_common/AbstractFormfactorDefinition.ts @@ -1,8 +1,8 @@ -import type { PipelineInterface } from '../../pipeline/PipelineInterface/PipelineInterface'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_name } from '../../types/typeAliases'; -import type { string_promptbook_documentation_url } from '../../types/typeAliases'; -import type { string_SCREAMING_CASE } from '../../utils/normalization/normalizeTo_SCREAMING_CASE'; +import type { PipelineInterface } from "../../pipeline/PipelineInterface/PipelineInterface"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_name } from "../../types/typeAliases"; +import type { string_promptbook_documentation_url } from "../../types/typeAliases"; +import type { string_SCREAMING_CASE } from "../../utils/normalization/normalizeTo_SCREAMING_CASE"; /** * @@@ @@ -11,37 +11,37 @@ import type { string_SCREAMING_CASE } from '../../utils/normalization/normalizeT * @see https://github.com/webgptorg/promptbook/discussions/172 */ export type AbstractFormfactorDefinition = { - /** - * @@@ - */ - readonly name: string_name & string_SCREAMING_CASE; - - /** - * @@@ - */ - readonly aliasNames?: ReadonlyArray; - - /** - * @@@ - */ - readonly deprecatedNames?: ReadonlyArray; - - /** - * @@@ - */ - readonly description: string_markdown_text; - - /** - * @@@ - */ - readonly documentationUrl: string_promptbook_documentation_url; - - /** - * @@@ - */ - readonly pipelineInterface: PipelineInterface; - - // <- TODO: [🍼] Add here CLI interactive behavior definition + /** + * @@@ + */ + readonly name: string_name & string_SCREAMING_CASE; + + /** + * @@@ + */ + readonly aliasNames?: ReadonlyArray; + + /** + * @@@ + */ + readonly deprecatedNames?: ReadonlyArray; + + /** + * @@@ + */ + readonly description: string_markdown_text; + + /** + * @@@ + */ + readonly documentationUrl: string_promptbook_documentation_url; + + /** + * @@@ + */ + readonly pipelineInterface: PipelineInterface; + + // <- TODO: [🍼] Add here CLI interactive behavior definition }; /** diff --git a/src/formfactors/_common/FormfactorDefinition.ts b/src/formfactors/_common/FormfactorDefinition.ts index d5709f1647..07cd7851ca 100644 --- a/src/formfactors/_common/FormfactorDefinition.ts +++ b/src/formfactors/_common/FormfactorDefinition.ts @@ -1,4 +1,4 @@ -import { FORMFACTOR_DEFINITIONS } from '../index'; +import type { FORMFACTOR_DEFINITIONS } from "../index"; // <- TODO: Keep this a type import even if the entity is runtime but used as a type /** @@ -7,4 +7,4 @@ import { FORMFACTOR_DEFINITIONS } from '../index'; * Note: [🚉] This is fully serializable as JSON * @see https://github.com/webgptorg/promptbook/discussions/172 */ -export type FormfactorDefinition = typeof FORMFACTOR_DEFINITIONS[number]; +export type FormfactorDefinition = (typeof FORMFACTOR_DEFINITIONS)[number]; diff --git a/src/formfactors/_common/string_formfactor_name.ts b/src/formfactors/_common/string_formfactor_name.ts index 7255ad1197..4b098dfc66 100644 --- a/src/formfactors/_common/string_formfactor_name.ts +++ b/src/formfactors/_common/string_formfactor_name.ts @@ -1,6 +1,6 @@ -import type { FormfactorDefinition } from './FormfactorDefinition'; +import type { FormfactorDefinition } from "./FormfactorDefinition"; /** * @@@ */ -export type string_formfactor_name = FormfactorDefinition['name']; +export type string_formfactor_name = FormfactorDefinition["name"]; diff --git a/src/formfactors/chatbot/ChatbotFormfactorDefinition.ts b/src/formfactors/chatbot/ChatbotFormfactorDefinition.ts index fa5fbdaa8f..f19eb8dd36 100644 --- a/src/formfactors/chatbot/ChatbotFormfactorDefinition.ts +++ b/src/formfactors/chatbot/ChatbotFormfactorDefinition.ts @@ -1,4 +1,4 @@ -import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactorDefinition'; +import type { AbstractFormfactorDefinition } from "../_common/AbstractFormfactorDefinition"; /** * @@@ @@ -6,30 +6,50 @@ import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactor * @public exported from `@promptbook/core` */ export const ChatbotFormfactorDefinition = { - name: 'CHATBOT', - aliasNames: ['CHAT'], - description: `@@@`, - documentationUrl: `https://github.com/webgptorg/promptbook/discussions/174`, - pipelineInterface: { - inputParameters: [ - { - name: 'previousTitle', - description: `Previous title of the conversation`, - isInput: true, - isOutput: false, - }, - { - name: 'previousConversationSummary', - description: `Previous conversation summary`, - isInput: true, - isOutput: false, - }, - { name: 'userMessage', description: `User message`, isInput: true, isOutput: false }, - ], - outputParameters: [ - { name: 'title', description: `Title of the conversation`, isInput: false, isOutput: true }, - { name: 'conversationSummary', description: `Summary of the conversation`, isInput: false, isOutput: true }, - { name: 'chatbotResponse', description: `Chatbot response`, isInput: false, isOutput: true }, - ], - }, + name: "CHATBOT", + aliasNames: ["CHAT"], + description: `@@@`, + documentationUrl: `https://github.com/webgptorg/promptbook/discussions/174`, + pipelineInterface: { + inputParameters: [ + { + name: "previousTitle", + description: `Previous title of the conversation`, + isInput: true, + isOutput: false, + }, + { + name: "previousConversationSummary", + description: `Previous conversation summary`, + isInput: true, + isOutput: false, + }, + { + name: "userMessage", + description: `User message`, + isInput: true, + isOutput: false, + }, + ], + outputParameters: [ + { + name: "title", + description: `Title of the conversation`, + isInput: false, + isOutput: true, + }, + { + name: "conversationSummary", + description: `Summary of the conversation`, + isInput: false, + isOutput: true, + }, + { + name: "chatbotResponse", + description: `Chatbot response`, + isInput: false, + isOutput: true, + }, + ], + }, } as const satisfies AbstractFormfactorDefinition; diff --git a/src/formfactors/generator/GeneratorFormfactorDefinition.ts b/src/formfactors/generator/GeneratorFormfactorDefinition.ts index 30a52e7d12..e5581e7faf 100644 --- a/src/formfactors/generator/GeneratorFormfactorDefinition.ts +++ b/src/formfactors/generator/GeneratorFormfactorDefinition.ts @@ -1,4 +1,4 @@ -import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactorDefinition'; +import type { AbstractFormfactorDefinition } from "../_common/AbstractFormfactorDefinition"; /** * Generator is form of app that @@@ @@ -6,25 +6,25 @@ import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactor * @public exported from `@promptbook/core` */ export const GeneratorFormfactorDefinition = { - name: 'GENERATOR', - description: `Generates any kind (in HTML with possible scripts and css format) of content from input message`, - documentationUrl: `https://github.com/webgptorg/promptbook/discussions/184`, - pipelineInterface: { - inputParameters: [ - { - name: 'inputMessage', - description: `Input message to be image made from`, - isInput: true, - isOutput: false, - }, - ], - outputParameters: [ - { - name: 'result', - description: `Result in HTML to be shown to user`, - isInput: false, - isOutput: true, - }, - ], - }, + name: "GENERATOR", + description: `Generates any kind (in HTML with possible scripts and css format) of content from input message`, + documentationUrl: `https://github.com/webgptorg/promptbook/discussions/184`, + pipelineInterface: { + inputParameters: [ + { + name: "inputMessage", + description: `Input message to be image made from`, + isInput: true, + isOutput: false, + }, + ], + outputParameters: [ + { + name: "result", + description: `Result in HTML to be shown to user`, + isInput: false, + isOutput: true, + }, + ], + }, } as const satisfies AbstractFormfactorDefinition; diff --git a/src/formfactors/generic/GenericFormfactorDefinition.ts b/src/formfactors/generic/GenericFormfactorDefinition.ts index 58ec2d973e..1e4f87afff 100644 --- a/src/formfactors/generic/GenericFormfactorDefinition.ts +++ b/src/formfactors/generic/GenericFormfactorDefinition.ts @@ -1,5 +1,5 @@ -import { GENERIC_PIPELINE_INTERFACE } from '../../pipeline/PipelineInterface/constants'; -import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactorDefinition'; +import { GENERIC_PIPELINE_INTERFACE } from "../../pipeline/PipelineInterface/constants"; +import type { AbstractFormfactorDefinition } from "../_common/AbstractFormfactorDefinition"; /** * @@@ @@ -7,8 +7,8 @@ import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactor * @public exported from `@promptbook/core` */ export const GenericFormfactorDefinition = { - name: 'GENERIC', - description: `@@@`, - documentationUrl: `https://github.com/webgptorg/promptbook/discussions/173`, - pipelineInterface: GENERIC_PIPELINE_INTERFACE, + name: "GENERIC", + description: `@@@`, + documentationUrl: `https://github.com/webgptorg/promptbook/discussions/173`, + pipelineInterface: GENERIC_PIPELINE_INTERFACE, } as const satisfies AbstractFormfactorDefinition; diff --git a/src/formfactors/image-generator/ImageGeneratorFormfactorDefinition.ts b/src/formfactors/image-generator/ImageGeneratorFormfactorDefinition.ts index 7568822781..86de01cc83 100644 --- a/src/formfactors/image-generator/ImageGeneratorFormfactorDefinition.ts +++ b/src/formfactors/image-generator/ImageGeneratorFormfactorDefinition.ts @@ -1,4 +1,4 @@ -import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactorDefinition'; +import type { AbstractFormfactorDefinition } from "../_common/AbstractFormfactorDefinition"; /** * Image generator is form of app that generates image from input message @@ -6,25 +6,25 @@ import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactor * @public exported from `@promptbook/core` */ export const ImageGeneratorFormfactorDefinition = { - name: 'IMAGE_GENERATOR', - description: `Generates prompt for image generation from input message`, - documentationUrl: `https://github.com/webgptorg/promptbook/discussions/184`, - pipelineInterface: { - inputParameters: [ - { - name: 'inputMessage', - description: `Input message to be image made from`, - isInput: true, - isOutput: false, - }, - ], - outputParameters: [ - { - name: 'prompt', - description: `Prompt to be used for image generation`, - isInput: false, - isOutput: true, - }, - ], - }, + name: "IMAGE_GENERATOR", + description: `Generates prompt for image generation from input message`, + documentationUrl: `https://github.com/webgptorg/promptbook/discussions/184`, + pipelineInterface: { + inputParameters: [ + { + name: "inputMessage", + description: `Input message to be image made from`, + isInput: true, + isOutput: false, + }, + ], + outputParameters: [ + { + name: "prompt", + description: `Prompt to be used for image generation`, + isInput: false, + isOutput: true, + }, + ], + }, } as const satisfies AbstractFormfactorDefinition; diff --git a/src/formfactors/index.ts b/src/formfactors/index.ts index 39adb0a392..d641f23f1f 100644 --- a/src/formfactors/index.ts +++ b/src/formfactors/index.ts @@ -1,10 +1,10 @@ -import { ChatbotFormfactorDefinition } from './chatbot/ChatbotFormfactorDefinition'; -import { GeneratorFormfactorDefinition } from './generator/GeneratorFormfactorDefinition'; -import { GenericFormfactorDefinition } from './generic/GenericFormfactorDefinition'; -import { ImageGeneratorFormfactorDefinition } from './image-generator/ImageGeneratorFormfactorDefinition'; -import { MatcherFormfactorDefinition } from './matcher/MatcherFormfactorDefinition'; -import { SheetsFormfactorDefinition } from './sheets/SheetsFormfactorDefinition'; -import { TranslatorFormfactorDefinition } from './translator/TranslatorFormfactorDefinition'; +import { ChatbotFormfactorDefinition } from "./chatbot/ChatbotFormfactorDefinition"; +import { GeneratorFormfactorDefinition } from "./generator/GeneratorFormfactorDefinition"; +import { GenericFormfactorDefinition } from "./generic/GenericFormfactorDefinition"; +import { ImageGeneratorFormfactorDefinition } from "./image-generator/ImageGeneratorFormfactorDefinition"; +import { MatcherFormfactorDefinition } from "./matcher/MatcherFormfactorDefinition"; +import { SheetsFormfactorDefinition } from "./sheets/SheetsFormfactorDefinition"; +import { TranslatorFormfactorDefinition } from "./translator/TranslatorFormfactorDefinition"; /** * All available formfactor definitions @@ -12,13 +12,13 @@ import { TranslatorFormfactorDefinition } from './translator/TranslatorFormfacto * @public exported from `@promptbook/core` */ export const FORMFACTOR_DEFINITIONS = [ - GenericFormfactorDefinition, - ChatbotFormfactorDefinition, - TranslatorFormfactorDefinition, - SheetsFormfactorDefinition, - MatcherFormfactorDefinition, - GeneratorFormfactorDefinition, - ImageGeneratorFormfactorDefinition, + GenericFormfactorDefinition, + ChatbotFormfactorDefinition, + TranslatorFormfactorDefinition, + SheetsFormfactorDefinition, + MatcherFormfactorDefinition, + GeneratorFormfactorDefinition, + ImageGeneratorFormfactorDefinition, ] as const; /** diff --git a/src/formfactors/matcher/MatcherFormfactorDefinition.ts b/src/formfactors/matcher/MatcherFormfactorDefinition.ts index 71ca60e1f8..c4adf5a5ee 100644 --- a/src/formfactors/matcher/MatcherFormfactorDefinition.ts +++ b/src/formfactors/matcher/MatcherFormfactorDefinition.ts @@ -1,4 +1,4 @@ -import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactorDefinition'; +import type { AbstractFormfactorDefinition } from "../_common/AbstractFormfactorDefinition"; /** * Matcher is form of app that @@@ @@ -6,21 +6,22 @@ import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactor * @public exported from `@promptbook/core` */ export const MatcherFormfactorDefinition = { - name: 'EXPERIMENTAL_MATCHER', - description: `@@@`, - documentationUrl: `https://github.com/webgptorg/promptbook/discussions/177`, - pipelineInterface: { - inputParameters: [ - /* @@@ */ - { - name: 'nonce', - description: 'Just to prevent EXPERIMENTAL_MATCHER to be set as implicit formfactor', - isInput: true, - isOutput: false, - }, - ], - outputParameters: [ - /* @@@ */ - ], - }, + name: "EXPERIMENTAL_MATCHER", + description: `@@@`, + documentationUrl: `https://github.com/webgptorg/promptbook/discussions/177`, + pipelineInterface: { + inputParameters: [ + /* @@@ */ + { + name: "nonce", + description: + "Just to prevent EXPERIMENTAL_MATCHER to be set as implicit formfactor", + isInput: true, + isOutput: false, + }, + ], + outputParameters: [ + /* @@@ */ + ], + }, } as const satisfies AbstractFormfactorDefinition; diff --git a/src/formfactors/sheets/SheetsFormfactorDefinition.ts b/src/formfactors/sheets/SheetsFormfactorDefinition.ts index 1f6c765405..ee83ada98c 100644 --- a/src/formfactors/sheets/SheetsFormfactorDefinition.ts +++ b/src/formfactors/sheets/SheetsFormfactorDefinition.ts @@ -1,4 +1,4 @@ -import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactorDefinition'; +import type { AbstractFormfactorDefinition } from "../_common/AbstractFormfactorDefinition"; /** * Sheets is form of app that @@@ @@ -6,26 +6,26 @@ import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactor * @public exported from `@promptbook/core` */ export const SheetsFormfactorDefinition = { - name: 'SHEETS', - aliasNames: ['SHEETS', 'SHEET'], - description: `@@@`, - documentationUrl: `https://github.com/webgptorg/promptbook/discussions/176`, - pipelineInterface: { - inputParameters: [ - { - name: 'inputSheet', - description: `Input sheet to be processed as csv`, - isInput: true, - isOutput: false, - }, - ], - outputParameters: [ - { - name: 'outputSheet', - description: `Output sheet as csv`, - isInput: false, - isOutput: true, - }, - ], - }, + name: "SHEETS", + aliasNames: ["SHEETS", "SHEET"], + description: `@@@`, + documentationUrl: `https://github.com/webgptorg/promptbook/discussions/176`, + pipelineInterface: { + inputParameters: [ + { + name: "inputSheet", + description: `Input sheet to be processed as csv`, + isInput: true, + isOutput: false, + }, + ], + outputParameters: [ + { + name: "outputSheet", + description: `Output sheet as csv`, + isInput: false, + isOutput: true, + }, + ], + }, } as const satisfies AbstractFormfactorDefinition; diff --git a/src/formfactors/translator/TranslatorFormfactorDefinition.ts b/src/formfactors/translator/TranslatorFormfactorDefinition.ts index eee8c9f24d..c12c8c328a 100644 --- a/src/formfactors/translator/TranslatorFormfactorDefinition.ts +++ b/src/formfactors/translator/TranslatorFormfactorDefinition.ts @@ -1,4 +1,4 @@ -import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactorDefinition'; +import type { AbstractFormfactorDefinition } from "../_common/AbstractFormfactorDefinition"; /** * Translator is form of app that @@@ @@ -6,27 +6,27 @@ import type { AbstractFormfactorDefinition } from '../_common/AbstractFormfactor * @public exported from `@promptbook/core` */ export const TranslatorFormfactorDefinition = { - name: 'TRANSLATOR', - description: `@@@`, - documentationUrl: `https://github.com/webgptorg/promptbook/discussions/175`, - pipelineInterface: { - inputParameters: [ - { - name: 'inputMessage', - description: `Input message to be translated`, - isInput: true, - isOutput: false, - }, - ], - outputParameters: [ - { - name: 'outputMessage', - description: `Translated output message`, - isInput: false, - isOutput: true, - }, - ], - // <- TODO: [🤓] Maybe add {summary} - // <- TODO: [🧠] maybe change to {inputText}, {inputText} / or make system for any name of first input and first outpur parameter - }, + name: "TRANSLATOR", + description: `@@@`, + documentationUrl: `https://github.com/webgptorg/promptbook/discussions/175`, + pipelineInterface: { + inputParameters: [ + { + name: "inputMessage", + description: `Input message to be translated`, + isInput: true, + isOutput: false, + }, + ], + outputParameters: [ + { + name: "outputMessage", + description: `Translated output message`, + isInput: false, + isOutput: true, + }, + ], + // <- TODO: [🤓] Maybe add {summary} + // <- TODO: [🧠] maybe change to {inputText}, {inputText} / or make system for any name of first input and first outpur parameter + }, } as const satisfies AbstractFormfactorDefinition; diff --git a/src/globals.d.ts b/src/globals.d.ts index 692f996e91..eeda262a17 100644 --- a/src/globals.d.ts +++ b/src/globals.d.ts @@ -1,6 +1,6 @@ // Note: [📍] Allow here to import `.book` files -declare module '*.book'; -declare module '*.book.md'; -declare module '*.bookc'; +declare module "*.book"; +declare module "*.book.md"; +declare module "*.bookc"; // Note: When some module does not have types available, it is imported using `require` diff --git a/src/high-level-abstractions/_common/HighLevelAbstraction.ts b/src/high-level-abstractions/_common/HighLevelAbstraction.ts index 808d3d98fb..99d17c4121 100644 --- a/src/high-level-abstractions/_common/HighLevelAbstraction.ts +++ b/src/high-level-abstractions/_common/HighLevelAbstraction.ts @@ -1,4 +1,4 @@ -import type { $PipelineJson } from '../../commands/_common/types/CommandParser'; +import type { $PipelineJson } from "../../commands/_common/types/CommandParser"; /** * Used in `parsePipeline` @@ -6,14 +6,14 @@ import type { $PipelineJson } from '../../commands/_common/types/CommandParser'; * @private */ export type SyncHighLevelAbstraction = /* CommonHighLevelAbstraction & */ { - type: 'SYNC'; + type: "SYNC"; - /** - * Apply the abstraction to the `pipelineJson` - * - * Note: `$` is used to indicate that this function mutates given `pipelineJson` - */ - $applyToPipelineJson($pipelineJson: $PipelineJson): void; + /** + * Apply the abstraction to the `pipelineJson` + * + * Note: `$` is used to indicate that this function mutates given `pipelineJson` + */ + $applyToPipelineJson($pipelineJson: $PipelineJson): void; }; /* diff --git a/src/high-level-abstractions/implicit-formfactor/ImplicitFormfactorHla.ts b/src/high-level-abstractions/implicit-formfactor/ImplicitFormfactorHla.ts index e07bcfd908..97d14ee6bc 100644 --- a/src/high-level-abstractions/implicit-formfactor/ImplicitFormfactorHla.ts +++ b/src/high-level-abstractions/implicit-formfactor/ImplicitFormfactorHla.ts @@ -1,7 +1,7 @@ -import { isPipelineImplementingInterface } from '../../pipeline/PipelineInterface/isPipelineImplementingInterface'; -import type { $PipelineJson } from '../../commands/_common/types/CommandParser'; -import { FORMFACTOR_DEFINITIONS } from '../../formfactors/index'; -import type { SyncHighLevelAbstraction } from '../_common/HighLevelAbstraction'; +import type { $PipelineJson } from "../../commands/_common/types/CommandParser"; +import { FORMFACTOR_DEFINITIONS } from "../../formfactors/index"; +import { isPipelineImplementingInterface } from "../../pipeline/PipelineInterface/isPipelineImplementingInterface"; +import type { SyncHighLevelAbstraction } from "../_common/HighLevelAbstraction"; /** * Set formfactor based on the pipeline interface e @@ -9,32 +9,32 @@ import type { SyncHighLevelAbstraction } from '../_common/HighLevelAbstraction'; * @private */ export const ImplicitFormfactorHla = { - type: 'SYNC', - $applyToPipelineJson($pipelineJson: $PipelineJson): void { - if ($pipelineJson.formfactorName !== undefined) { - // Note: When formfactor is already set, do nothing - return; - } - for (const formfactorDefinition of FORMFACTOR_DEFINITIONS.filter( - ({ name }) => name !== 'GENERIC', - // <- Note: Skip GENERIC formfactor, it will be used as a fallback if no other formfactor is compatible - )) { - // <- Note: [♓️][💩] This is the order of the formfactors, make some explicit priority + type: "SYNC", + $applyToPipelineJson($pipelineJson: $PipelineJson): void { + if ($pipelineJson.formfactorName !== undefined) { + // Note: When formfactor is already set, do nothing + return; + } + for (const formfactorDefinition of FORMFACTOR_DEFINITIONS.filter( + ({ name }) => name !== "GENERIC", + // <- Note: Skip GENERIC formfactor, it will be used as a fallback if no other formfactor is compatible + )) { + // <- Note: [♓️][💩] This is the order of the formfactors, make some explicit priority - const { name, pipelineInterface } = formfactorDefinition; + const { name, pipelineInterface } = formfactorDefinition; - const isCompatible = isPipelineImplementingInterface({ - pipeline: { - formfactorName: name, - // <- Note: `formfactorName` has no role in `isPipelineImplementingInterface` - // but it is needed to satisfy the typescript + const isCompatible = isPipelineImplementingInterface({ + pipeline: { + formfactorName: name, + // <- Note: `formfactorName` has no role in `isPipelineImplementingInterface` + // but it is needed to satisfy the typescript - ...$pipelineJson, - }, - pipelineInterface, - }); + ...$pipelineJson, + }, + pipelineInterface, + }); - /*/ + /*/ console.log({ subject: `${$pipelineJson.title} implements ${name}`, pipelineTitle: $pipelineJson.title, @@ -45,10 +45,10 @@ export const ImplicitFormfactorHla = { }); /**/ - if (isCompatible) { - $pipelineJson.formfactorName = name; - return; - } - } - }, + if (isCompatible) { + $pipelineJson.formfactorName = name; + return; + } + } + }, } satisfies SyncHighLevelAbstraction; diff --git a/src/high-level-abstractions/index.ts b/src/high-level-abstractions/index.ts index 655bed6868..ac18990670 100644 --- a/src/high-level-abstractions/index.ts +++ b/src/high-level-abstractions/index.ts @@ -1,5 +1,5 @@ -import { ImplicitFormfactorHla } from './implicit-formfactor/ImplicitFormfactorHla'; -import { QuickChatbotHla } from './quick-chatbot/QuickChatbotHla'; +import { ImplicitFormfactorHla } from "./implicit-formfactor/ImplicitFormfactorHla"; +import { QuickChatbotHla } from "./quick-chatbot/QuickChatbotHla"; /** * All high-level abstractions @@ -7,9 +7,9 @@ import { QuickChatbotHla } from './quick-chatbot/QuickChatbotHla'; * @private internal index of `parsePipeline` (= used for sync) and `preparePipeline` (= used for async) */ export const HIGH_LEVEL_ABSTRACTIONS = [ - ImplicitFormfactorHla, - QuickChatbotHla, - // <- Note: [♓️][💩] This is the order of the application of high-level abstractions application on pipeline JSON + ImplicitFormfactorHla, + QuickChatbotHla, + // <- Note: [♓️][💩] This is the order of the application of high-level abstractions application on pipeline JSON ] as const; /** diff --git a/src/high-level-abstractions/quick-chatbot/QuickChatbotHla.ts b/src/high-level-abstractions/quick-chatbot/QuickChatbotHla.ts index 262e18e6bc..946dcd16f2 100644 --- a/src/high-level-abstractions/quick-chatbot/QuickChatbotHla.ts +++ b/src/high-level-abstractions/quick-chatbot/QuickChatbotHla.ts @@ -1,5 +1,5 @@ -import type { $PipelineJson } from '../../commands/_common/types/CommandParser'; -import type { SyncHighLevelAbstraction } from '../_common/HighLevelAbstraction'; +import type { $PipelineJson } from "../../commands/_common/types/CommandParser"; +import type { SyncHighLevelAbstraction } from "../_common/HighLevelAbstraction"; /** * Allow to define chatbot with no need to write full interface @@ -7,114 +7,119 @@ import type { SyncHighLevelAbstraction } from '../_common/HighLevelAbstraction'; * @private */ export const QuickChatbotHla = { - type: 'SYNC', - $applyToPipelineJson($pipelineJson: $PipelineJson): void { - if ($pipelineJson.tasks.length !== 0) { - // Note: When there are already tasks, do nothing - return; - } + type: "SYNC", + $applyToPipelineJson($pipelineJson: $PipelineJson): void { + if ($pipelineJson.tasks.length !== 0) { + // Note: When there are already tasks, do nothing + return; + } - if ($pipelineJson.parameters.length !== 0) { - // Note: When there are already parameters, do nothing - return; - } + if ($pipelineJson.parameters.length !== 0) { + // Note: When there are already parameters, do nothing + return; + } - if ($pipelineJson.personas.length === 0) { - // Note: When no personas defined, do nothing - return; - } + if ($pipelineJson.personas.length === 0) { + // Note: When no personas defined, do nothing + return; + } - const personaName = $pipelineJson.personas[0]!.name; + const personaName = $pipelineJson.personas[0]!.name; - $pipelineJson.formfactorName = 'CHATBOT'; + $pipelineJson.formfactorName = "CHATBOT"; - $pipelineJson.parameters.push( - { - name: 'previousTitle', - description: 'Previous title of the conversation', - isInput: true, - isOutput: false, - }, - { - name: 'previousConversationSummary', - description: 'Previous conversation summary', - isInput: true, - isOutput: false, - }, - { - name: 'userMessage', - description: 'User message', - isInput: true, - isOutput: false, - }, - { - name: 'title', - description: 'Title of the conversation', - isInput: false, - isOutput: true, - }, - { - name: 'conversationSummary', - description: 'Summary of the conversation', - isInput: false, - isOutput: true, - }, - { - name: 'chatbotResponse', - description: 'Chatbot response', - isInput: false, - isOutput: true, - exampleValues: [ - 'Hello, how can I help you?', - // <- TODO: [🧠] Make this dynamic, async, prepare-phase HLAs - ], - }, - ); + $pipelineJson.parameters.push( + { + name: "previousTitle", + description: "Previous title of the conversation", + isInput: true, + isOutput: false, + }, + { + name: "previousConversationSummary", + description: "Previous conversation summary", + isInput: true, + isOutput: false, + }, + { + name: "userMessage", + description: "User message", + isInput: true, + isOutput: false, + }, + { + name: "title", + description: "Title of the conversation", + isInput: false, + isOutput: true, + }, + { + name: "conversationSummary", + description: "Summary of the conversation", + isInput: false, + isOutput: true, + }, + { + name: "chatbotResponse", + description: "Chatbot response", + isInput: false, + isOutput: true, + exampleValues: [ + "Hello, how can I help you?", + // <- TODO: [🧠] Make this dynamic, async, prepare-phase HLAs + ], + }, + ); - // TODO: Use spaceTrim in multiline strings - $pipelineJson.tasks.push( - { - taskType: 'PROMPT_TASK', - name: 'create-an-answer', - title: 'Create an answer', - content: - 'Write a response to the user message:\n\n**Question from user**\n\n> {userMessage}\n\n**Previous conversation**\n\n> {previousConversationSummary}', - resultingParameterName: 'chatbotResponse', - personaName, - dependentParameterNames: [ - 'userMessage', - 'previousConversationSummary' /* TODO: [🧠][📛], 'knowledge'*/, - ], - // TODO: [🧠][📛] preparedContent: '{content}\n\n## Knowledge\n\n{knowledge}', - }, - { - taskType: 'PROMPT_TASK', - name: 'summarize-the-conversation', - title: 'Summarize the conversation', - content: - 'Summarize the conversation in a few words:\n\n## Rules\n\n- Summarise the text of the conversation in a few words\n- Convert the text to its basic idea\n- Imagine you are writing the headline or subject line of an email\n- Respond with a few words of summary only\n\n## Conversation\n\n**User:**\n\n> {userMessage}\n\n**You:**\n\n> {chatbotResponse}', - resultingParameterName: 'conversationSummary', - personaName, - expectations: { - words: { - min: 1, - max: 10, - }, - }, - dependentParameterNames: ['userMessage', 'chatbotResponse' /* TODO: [🧠][📛], 'knowledge'*/], - // TODO: [🧠][📛] preparedContent: '{content}\n\n## Knowledge\n\n{knowledge}', - }, - { - taskType: 'SIMPLE_TASK', - name: 'title', - title: 'Title', - content: '{conversationSummary}', - resultingParameterName: 'title', - dependentParameterNames: ['conversationSummary' /* TODO: [🧠][📛], 'knowledge'*/], - // TODO: [🧠][📛] preparedContent: '{content}\n\n## Knowledge\n\n{knowledge}', - }, - ); - }, + // TODO: Use spaceTrim in multiline strings + $pipelineJson.tasks.push( + { + taskType: "PROMPT_TASK", + name: "create-an-answer", + title: "Create an answer", + content: + "Write a response to the user message:\n\n**Question from user**\n\n> {userMessage}\n\n**Previous conversation**\n\n> {previousConversationSummary}", + resultingParameterName: "chatbotResponse", + personaName, + dependentParameterNames: [ + "userMessage", + "previousConversationSummary" /* TODO: [🧠][📛], 'knowledge'*/, + ], + // TODO: [🧠][📛] preparedContent: '{content}\n\n## Knowledge\n\n{knowledge}', + }, + { + taskType: "PROMPT_TASK", + name: "summarize-the-conversation", + title: "Summarize the conversation", + content: + "Summarize the conversation in a few words:\n\n## Rules\n\n- Summarise the text of the conversation in a few words\n- Convert the text to its basic idea\n- Imagine you are writing the headline or subject line of an email\n- Respond with a few words of summary only\n\n## Conversation\n\n**User:**\n\n> {userMessage}\n\n**You:**\n\n> {chatbotResponse}", + resultingParameterName: "conversationSummary", + personaName, + expectations: { + words: { + min: 1, + max: 10, + }, + }, + dependentParameterNames: [ + "userMessage", + "chatbotResponse" /* TODO: [🧠][📛], 'knowledge'*/, + ], + // TODO: [🧠][📛] preparedContent: '{content}\n\n## Knowledge\n\n{knowledge}', + }, + { + taskType: "SIMPLE_TASK", + name: "title", + title: "Title", + content: "{conversationSummary}", + resultingParameterName: "title", + dependentParameterNames: [ + "conversationSummary" /* TODO: [🧠][📛], 'knowledge'*/, + ], + // TODO: [🧠][📛] preparedContent: '{content}\n\n## Knowledge\n\n{knowledge}', + }, + ); + }, } satisfies SyncHighLevelAbstraction; /** diff --git a/src/llm-providers/_common/register/$llmToolsMetadataRegister.ts b/src/llm-providers/_common/register/$llmToolsMetadataRegister.ts index 77bca89bca..b0a83ad5f5 100644 --- a/src/llm-providers/_common/register/$llmToolsMetadataRegister.ts +++ b/src/llm-providers/_common/register/$llmToolsMetadataRegister.ts @@ -1,5 +1,5 @@ -import { $Register } from '../../../utils/$Register'; -import type { LlmToolsMetadata } from './LlmToolsMetadata'; +import { $Register } from "../../../utils/$Register"; +import type { LlmToolsMetadata } from "./LlmToolsMetadata"; /** * @@@ @@ -8,10 +8,10 @@ import type { LlmToolsMetadata } from './LlmToolsMetadata'; * @singleton Only one instance of each register is created per build, but thare can be more @@@ * @public exported from `@promptbook/core` */ -export const $llmToolsMetadataRegister = new $Register('llm_tools_metadata'); - - +export const $llmToolsMetadataRegister = new $Register( + "llm_tools_metadata", +); /** * TODO: [®] DRY Register logic - */ \ No newline at end of file + */ diff --git a/src/llm-providers/_common/register/$llmToolsRegister.ts b/src/llm-providers/_common/register/$llmToolsRegister.ts index 3d718d4d7f..59677b5aef 100644 --- a/src/llm-providers/_common/register/$llmToolsRegister.ts +++ b/src/llm-providers/_common/register/$llmToolsRegister.ts @@ -1,5 +1,5 @@ -import type { LlmExecutionToolsConstructor } from '../../../execution/LlmExecutionToolsConstructor'; -import { $Register } from '../../../utils/$Register'; +import type { LlmExecutionToolsConstructor } from "../../../execution/LlmExecutionToolsConstructor"; +import { $Register } from "../../../utils/$Register"; /** * @@@ @@ -8,9 +8,10 @@ import { $Register } from '../../../utils/$Register'; * @singleton Only one instance of each register is created per build, but thare can be more @@@ * @public exported from `@promptbook/core` */ -export const $llmToolsRegister = new $Register('llm_execution_tools_constructors'); - +export const $llmToolsRegister = new $Register( + "llm_execution_tools_constructors", +); /** * TODO: [®] DRY Register logic - */ \ No newline at end of file + */ diff --git a/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.ts b/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.ts index f4e9a96bc0..adf25ee840 100644 --- a/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.ts +++ b/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.ts @@ -1,15 +1,15 @@ -import * as dotenv from 'dotenv'; -import { join } from 'path'; -import { LOOP_LIMIT } from '../../../config'; -import { EnvironmentMismatchError } from '../../../errors/EnvironmentMismatchError'; -import { $provideFilesystemForNode } from '../../../scrapers/_common/register/$provideFilesystemForNode'; -import type { string_name } from '../../../types/typeAliases'; -import { $isRunningInNode } from '../../../utils/environment/$isRunningInNode'; -import { isFileExisting } from '../../../utils/files/isFileExisting'; -import { isRootPath } from '../../../utils/validators/filePath/isRootPath'; -import { $llmToolsMetadataRegister } from './$llmToolsMetadataRegister'; -import { $setUsedEnvFilename } from './$registeredLlmToolsMessage'; -import type { LlmToolsConfiguration } from './LlmToolsConfiguration'; +import { join } from "path"; +import * as dotenv from "dotenv"; +import { LOOP_LIMIT } from "../../../config"; +import { EnvironmentMismatchError } from "../../../errors/EnvironmentMismatchError"; +import { $provideFilesystemForNode } from "../../../scrapers/_common/register/$provideFilesystemForNode"; +import type { string_name } from "../../../types/typeAliases"; +import { $isRunningInNode } from "../../../utils/environment/$isRunningInNode"; +import { isFileExisting } from "../../../utils/files/isFileExisting"; +import { isRootPath } from "../../../utils/validators/filePath/isRootPath"; +import { $llmToolsMetadataRegister } from "./$llmToolsMetadataRegister"; +import { $setUsedEnvFilename } from "./$registeredLlmToolsMessage"; +import type { LlmToolsConfiguration } from "./LlmToolsConfiguration"; /** * @@@ @@ -25,52 +25,61 @@ import type { LlmToolsConfiguration } from './LlmToolsConfiguration'; * @public exported from `@promptbook/node` */ export async function $provideLlmToolsConfigurationFromEnv(): Promise { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError('Function `$provideLlmToolsFromEnv` works only in Node.js environment'); - } + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Function `$provideLlmToolsFromEnv` works only in Node.js environment", + ); + } - const envFilePatterns = [ - '.env', - '.env.test', - '.env.local', - '.env.development.local', - '.env.development', + const envFilePatterns = [ + ".env", + ".env.test", + ".env.local", + ".env.development.local", + ".env.development", - '.env.production.local', - '.env.production', - '.env.prod.local', - '.env.prod', + ".env.production.local", + ".env.production", + ".env.prod.local", + ".env.prod", - // <- TODO: Maybe add more patterns - ]; + // <- TODO: Maybe add more patterns + ]; - let rootDirname = process.cwd(); + let rootDirname = process.cwd(); - up_to_root: for (let i = 0; i < LOOP_LIMIT; i++) { - for (const pattern of envFilePatterns) { - const envFilename = join(rootDirname, pattern); + up_to_root: for (let i = 0; i < LOOP_LIMIT; i++) { + for (const pattern of envFilePatterns) { + const envFilename = join(rootDirname, pattern); - if (await isFileExisting(envFilename, $provideFilesystemForNode())) { - $setUsedEnvFilename(envFilename); - dotenv.config({ path: envFilename }); - break up_to_root; - } - } + if (await isFileExisting(envFilename, $provideFilesystemForNode())) { + $setUsedEnvFilename(envFilename); + dotenv.config({ path: envFilename }); + break up_to_root; + } + } - if (isRootPath(rootDirname)) { - break up_to_root; - } + if (isRootPath(rootDirname)) { + break; + } - // Note: If the directory does not exist, try the parent directory - rootDirname = join(rootDirname, '..'); - } + // Note: If the directory does not exist, try the parent directory + rootDirname = join(rootDirname, ".."); + } - const llmToolsConfiguration: LlmToolsConfiguration = $llmToolsMetadataRegister - .list() - .map((metadata) => metadata.createConfigurationFromEnv(process.env as Record)) - .filter((configuration): configuration is LlmToolsConfiguration[number] => configuration !== null); + const llmToolsConfiguration: LlmToolsConfiguration = $llmToolsMetadataRegister + .list() + .map((metadata) => + metadata.createConfigurationFromEnv( + process.env as Record, + ), + ) + .filter( + (configuration): configuration is LlmToolsConfiguration[number] => + configuration !== null, + ); - return llmToolsConfiguration; + return llmToolsConfiguration; } /** diff --git a/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.ts b/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.ts index 65bd234e13..d1447eba3f 100644 --- a/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.ts +++ b/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.ts @@ -1,27 +1,28 @@ -import { join } from 'path'; -import '../../../_packages/cli.index'; // <- Note: Really importing core index to register all the LLM providers -import { DEFAULT_EXECUTION_CACHE_DIRNAME } from '../../../config'; -import { IS_COST_PREVENTED } from '../../../config'; -import { EnvironmentMismatchError } from '../../../errors/EnvironmentMismatchError'; -import type { LlmExecutionTools } from '../../../execution/LlmExecutionTools'; -import { $provideFilesystemForNode } from '../../../scrapers/_common/register/$provideFilesystemForNode'; -import { FileCacheStorage } from '../../../storage/file-cache-storage/FileCacheStorage'; -import { $isRunningInNode } from '../../../utils/environment/$isRunningInNode'; -import { cacheLlmTools } from '../utils/cache/cacheLlmTools'; -import { countTotalUsage } from '../utils/count-total-usage/countTotalUsage'; -import { limitTotalUsage } from '../utils/count-total-usage/limitTotalUsage'; -import type { LlmExecutionToolsWithTotalUsage } from '../utils/count-total-usage/LlmExecutionToolsWithTotalUsage'; -import { $provideLlmToolsFromEnv } from './$provideLlmToolsFromEnv'; -import type { CreateLlmToolsFromConfigurationOptions } from './createLlmToolsFromConfiguration'; +import { join } from "path"; +import "../../../_packages/cli.index"; // <- Note: Really importing core index to register all the LLM providers +import { DEFAULT_EXECUTION_CACHE_DIRNAME } from "../../../config"; +import { IS_COST_PREVENTED } from "../../../config"; +import { EnvironmentMismatchError } from "../../../errors/EnvironmentMismatchError"; +import type { LlmExecutionTools } from "../../../execution/LlmExecutionTools"; +import { $provideFilesystemForNode } from "../../../scrapers/_common/register/$provideFilesystemForNode"; +import { FileCacheStorage } from "../../../storage/file-cache-storage/FileCacheStorage"; +import { $isRunningInNode } from "../../../utils/environment/$isRunningInNode"; +import { cacheLlmTools } from "../utils/cache/cacheLlmTools"; +import type { LlmExecutionToolsWithTotalUsage } from "../utils/count-total-usage/LlmExecutionToolsWithTotalUsage"; +import { countTotalUsage } from "../utils/count-total-usage/countTotalUsage"; +import { limitTotalUsage } from "../utils/count-total-usage/limitTotalUsage"; +import { $provideLlmToolsFromEnv } from "./$provideLlmToolsFromEnv"; +import type { CreateLlmToolsFromConfigurationOptions } from "./createLlmToolsFromConfiguration"; -type GetLlmToolsForTestingAndScriptsAndPlaygroundOptions = CreateLlmToolsFromConfigurationOptions & { - /** - * @@@ - * - * @default false - */ - isCacheReloaded?: boolean; -}; +type GetLlmToolsForTestingAndScriptsAndPlaygroundOptions = + CreateLlmToolsFromConfigurationOptions & { + /** + * @@@ + * + * @default false + */ + isCacheReloaded?: boolean; + }; /** * Returns LLM tools for testing purposes @@ -29,37 +30,38 @@ type GetLlmToolsForTestingAndScriptsAndPlaygroundOptions = CreateLlmToolsFromCon * @private within the repository - JUST FOR TESTS, SCRIPTS AND PLAYGROUND */ export async function $provideLlmToolsForTestingAndScriptsAndPlayground( - options?: GetLlmToolsForTestingAndScriptsAndPlaygroundOptions, + options?: GetLlmToolsForTestingAndScriptsAndPlaygroundOptions, ): Promise { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError( - 'Function `$provideLlmToolsForTestingAndScriptsAndPlayground` works only in Node.js environment', - ); - } + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Function `$provideLlmToolsForTestingAndScriptsAndPlayground` works only in Node.js environment", + ); + } - const { isCacheReloaded = false, ...restOptions } = options ?? {}; + const { isCacheReloaded = false, ...restOptions } = options ?? {}; - const llmTools: LlmExecutionTools = await $provideLlmToolsFromEnv(restOptions); - const llmToolsWithUsage = !IS_COST_PREVENTED - ? countTotalUsage(llmTools) - : // <- Note: for example here we don`t want the [🌯] - limitTotalUsage(llmTools); - // <- Note: `limitTotalUsage` will do everything as `countTotalUsage` and adds usage limit - // <- Note: for example here we don`t want the [🌯] + const llmTools: LlmExecutionTools = + await $provideLlmToolsFromEnv(restOptions); + const llmToolsWithUsage = !IS_COST_PREVENTED + ? countTotalUsage(llmTools) + : // <- Note: for example here we don`t want the [🌯] + limitTotalUsage(llmTools); + // <- Note: `limitTotalUsage` will do everything as `countTotalUsage` and adds usage limit + // <- Note: for example here we don`t want the [🌯] - return cacheLlmTools(llmToolsWithUsage, { - storage: new FileCacheStorage( - { fs: $provideFilesystemForNode() }, - { - rootFolderPath: join( - process.cwd(), - DEFAULT_EXECUTION_CACHE_DIRNAME, - // <- TODO: [🦒] Allow to override (pass different value into the function) - ), - }, - ), - isCacheReloaded: isCacheReloaded, - }); + return cacheLlmTools(llmToolsWithUsage, { + storage: new FileCacheStorage( + { fs: $provideFilesystemForNode() }, + { + rootFolderPath: join( + process.cwd(), + DEFAULT_EXECUTION_CACHE_DIRNAME, + // <- TODO: [🦒] Allow to override (pass different value into the function) + ), + }, + ), + isCacheReloaded: isCacheReloaded, + }); } /** diff --git a/src/llm-providers/_common/register/$provideLlmToolsForWizzardOrCli.ts b/src/llm-providers/_common/register/$provideLlmToolsForWizzardOrCli.ts index 83fee95067..5e89e9c2d5 100644 --- a/src/llm-providers/_common/register/$provideLlmToolsForWizzardOrCli.ts +++ b/src/llm-providers/_common/register/$provideLlmToolsForWizzardOrCli.ts @@ -1,14 +1,14 @@ -import { join } from 'path'; -import { DEFAULT_EXECUTION_CACHE_DIRNAME } from '../../../config'; -import { EnvironmentMismatchError } from '../../../errors/EnvironmentMismatchError'; -import { $provideFilesystemForNode } from '../../../scrapers/_common/register/$provideFilesystemForNode'; -import { FileCacheStorage } from '../../../storage/file-cache-storage/FileCacheStorage'; -import { $isRunningInNode } from '../../../utils/environment/$isRunningInNode'; -import { cacheLlmTools } from '../utils/cache/cacheLlmTools'; -import type { CacheLlmToolsOptions } from '../utils/cache/CacheLlmToolsOptions'; -import { countTotalUsage } from '../utils/count-total-usage/countTotalUsage'; -import type { LlmExecutionToolsWithTotalUsage } from '../utils/count-total-usage/LlmExecutionToolsWithTotalUsage'; -import { $provideLlmToolsFromEnv } from './$provideLlmToolsFromEnv'; +import { join } from "path"; +import { DEFAULT_EXECUTION_CACHE_DIRNAME } from "../../../config"; +import { EnvironmentMismatchError } from "../../../errors/EnvironmentMismatchError"; +import { $provideFilesystemForNode } from "../../../scrapers/_common/register/$provideFilesystemForNode"; +import { FileCacheStorage } from "../../../storage/file-cache-storage/FileCacheStorage"; +import { $isRunningInNode } from "../../../utils/environment/$isRunningInNode"; +import type { CacheLlmToolsOptions } from "../utils/cache/CacheLlmToolsOptions"; +import { cacheLlmTools } from "../utils/cache/cacheLlmTools"; +import type { LlmExecutionToolsWithTotalUsage } from "../utils/count-total-usage/LlmExecutionToolsWithTotalUsage"; +import { countTotalUsage } from "../utils/count-total-usage/countTotalUsage"; +import { $provideLlmToolsFromEnv } from "./$provideLlmToolsFromEnv"; /** * Returns LLM tools for CLI @@ -16,34 +16,34 @@ import { $provideLlmToolsFromEnv } from './$provideLlmToolsFromEnv'; * @private within the repository - for CLI utils */ export async function $provideLlmToolsForWizzardOrCli( - options?: Pick, + options?: Pick, ): Promise { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError( - 'Function `$provideLlmToolsForWizzardOrCli` works only in Node.js environment', - ); - } + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Function `$provideLlmToolsForWizzardOrCli` works only in Node.js environment", + ); + } - const { isCacheReloaded } = options ?? {}; + const { isCacheReloaded } = options ?? {}; - return cacheLlmTools( - countTotalUsage( - // <- Note: for example here we don`t want the [🌯] - await $provideLlmToolsFromEnv(), - ), - { - storage: new FileCacheStorage( - { fs: $provideFilesystemForNode() }, - { - rootFolderPath: join( - process.cwd(), - DEFAULT_EXECUTION_CACHE_DIRNAME, // <- TODO: [🦒] Allow to override (pass different value into the function) - ), - }, - ), - isCacheReloaded, - }, - ); + return cacheLlmTools( + countTotalUsage( + // <- Note: for example here we don`t want the [🌯] + await $provideLlmToolsFromEnv(), + ), + { + storage: new FileCacheStorage( + { fs: $provideFilesystemForNode() }, + { + rootFolderPath: join( + process.cwd(), + DEFAULT_EXECUTION_CACHE_DIRNAME, // <- TODO: [🦒] Allow to override (pass different value into the function) + ), + }, + ), + isCacheReloaded, + }, + ); } /** diff --git a/src/llm-providers/_common/register/$provideLlmToolsFromEnv.ts b/src/llm-providers/_common/register/$provideLlmToolsFromEnv.ts index 22b3e6d546..6dcc09f0f3 100644 --- a/src/llm-providers/_common/register/$provideLlmToolsFromEnv.ts +++ b/src/llm-providers/_common/register/$provideLlmToolsFromEnv.ts @@ -1,13 +1,13 @@ -import spaceTrim from 'spacetrim'; -import { EnvironmentMismatchError } from '../../../errors/EnvironmentMismatchError'; -import { UnexpectedError } from '../../../errors/UnexpectedError'; -import { $isRunningInNode } from '../../../utils/environment/$isRunningInNode'; -import { MultipleLlmExecutionTools } from '../../multiple/MultipleLlmExecutionTools'; -import { $llmToolsMetadataRegister } from './$llmToolsMetadataRegister'; -import { $provideLlmToolsConfigurationFromEnv } from './$provideLlmToolsConfigurationFromEnv'; -import { $registeredLlmToolsMessage } from './$registeredLlmToolsMessage'; -import type { CreateLlmToolsFromConfigurationOptions } from './createLlmToolsFromConfiguration'; -import { createLlmToolsFromConfiguration } from './createLlmToolsFromConfiguration'; +import spaceTrim from "spacetrim"; +import { EnvironmentMismatchError } from "../../../errors/EnvironmentMismatchError"; +import { UnexpectedError } from "../../../errors/UnexpectedError"; +import { $isRunningInNode } from "../../../utils/environment/$isRunningInNode"; +import type { MultipleLlmExecutionTools } from "../../multiple/MultipleLlmExecutionTools"; +import { $llmToolsMetadataRegister } from "./$llmToolsMetadataRegister"; +import { $provideLlmToolsConfigurationFromEnv } from "./$provideLlmToolsConfigurationFromEnv"; +import { $registeredLlmToolsMessage } from "./$registeredLlmToolsMessage"; +import type { CreateLlmToolsFromConfigurationOptions } from "./createLlmToolsFromConfiguration"; +import { createLlmToolsFromConfiguration } from "./createLlmToolsFromConfiguration"; /** * @@@ @@ -25,40 +25,42 @@ import { createLlmToolsFromConfiguration } from './createLlmToolsFromConfigurati * @public exported from `@promptbook/node` */ export async function $provideLlmToolsFromEnv( - options: CreateLlmToolsFromConfigurationOptions = {}, + options: CreateLlmToolsFromConfigurationOptions = {}, ): Promise { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError('Function `$provideLlmToolsFromEnv` works only in Node.js environment'); - } + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Function `$provideLlmToolsFromEnv` works only in Node.js environment", + ); + } - const configuration = await $provideLlmToolsConfigurationFromEnv(); + const configuration = await $provideLlmToolsConfigurationFromEnv(); - if (configuration.length === 0) { - if ($llmToolsMetadataRegister.list().length === 0) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + if (configuration.length === 0) { + if ($llmToolsMetadataRegister.list().length === 0) { + throw new UnexpectedError( + spaceTrim( + (block) => ` No LLM tools registered, this is probably a bug in the Promptbook library ${block($registeredLlmToolsMessage())}} `, - ), - ); - } + ), + ); + } - // TODO: [🥃] - throw new Error( - spaceTrim( - (block) => ` + // TODO: [🥃] + throw new Error( + spaceTrim( + (block) => ` No LLM tools found in the environment ${block($registeredLlmToolsMessage())}} `, - ), - ); - } + ), + ); + } - return createLlmToolsFromConfiguration(configuration, options); + return createLlmToolsFromConfiguration(configuration, options); } /** diff --git a/src/llm-providers/_common/register/$registeredLlmToolsMessage.ts b/src/llm-providers/_common/register/$registeredLlmToolsMessage.ts index 75b1facd5f..41659c6ba8 100644 --- a/src/llm-providers/_common/register/$registeredLlmToolsMessage.ts +++ b/src/llm-providers/_common/register/$registeredLlmToolsMessage.ts @@ -1,14 +1,14 @@ -import colors from 'colors'; -import spaceTrim from 'spacetrim'; -import type { string_filename } from '../../../types/typeAliases'; -import type { string_markdown } from '../../../types/typeAliases'; -import type { string_name } from '../../../types/typeAliases'; -import type { Registered } from '../../../utils/$Register'; -import { $isRunningInNode } from '../../../utils/environment/$isRunningInNode'; -import { just } from '../../../utils/organization/just'; -import { $llmToolsMetadataRegister } from './$llmToolsMetadataRegister'; -import { $llmToolsRegister } from './$llmToolsRegister'; -import type { LlmToolsMetadata } from './LlmToolsMetadata'; +import colors from "colors"; +import spaceTrim from "spacetrim"; +import type { string_filename } from "../../../types/typeAliases"; +import type { string_markdown } from "../../../types/typeAliases"; +import type { string_name } from "../../../types/typeAliases"; +import type { Registered } from "../../../utils/$Register"; +import { $isRunningInNode } from "../../../utils/environment/$isRunningInNode"; +import { just } from "../../../utils/organization/just"; +import { $llmToolsMetadataRegister } from "./$llmToolsMetadataRegister"; +import { $llmToolsRegister } from "./$llmToolsRegister"; +import type { LlmToolsMetadata } from "./LlmToolsMetadata"; /** * Path to the `.env` file which was used to configure LLM tools @@ -25,7 +25,7 @@ let $usedEnvFilename: string | null = null; * @private internal log of `$provideLlmToolsConfigurationFromEnv` and `$registeredLlmToolsMessage` */ export function $setUsedEnvFilename(filepath: string_filename) { - $usedEnvFilename = filepath; + $usedEnvFilename = filepath; } /** @@ -36,160 +36,201 @@ export function $setUsedEnvFilename(filepath: string_filename) { * @private internal function of `createLlmToolsFromConfiguration` and `$provideLlmToolsFromEnv` */ export function $registeredLlmToolsMessage(): string_markdown { - let env: Record; - if ($isRunningInNode()) { - env = process.env as Record; - // <- TODO: [⚛] Some DRY way how to get to `process.env` and pass it into functions - ACRY search for `env` - } else { - env = {}; - } - - /** - * Mixes registered LLM tools from $llmToolsMetadataRegister and $llmToolsRegister - */ - const all: Array>> = []; - - for (const { title, packageName, className, envVariables } of $llmToolsMetadataRegister.list()) { - if (all.some((item) => item.packageName === packageName && item.className === className)) { - continue; - } - all.push({ title, packageName, className, envVariables }); - } - - for (const { packageName, className } of $llmToolsRegister.list()) { - if (all.some((item) => item.packageName === packageName && item.className === className)) { - continue; - } - all.push({ packageName, className }); - } - - const metadata = all.map((metadata) => { - const isMetadataAviailable = $llmToolsMetadataRegister - .list() - .find( - ({ packageName, className }) => - metadata.packageName === packageName && metadata.className === className, - ); - - const isInstalled = $llmToolsRegister - .list() - .find( - ({ packageName, className }) => - metadata.packageName === packageName && metadata.className === className, - ); - - const isFullyConfigured = - metadata.envVariables?.every((envVariableName) => env[envVariableName] !== undefined) || false; - const isPartiallyConfigured = - metadata.envVariables?.some((envVariableName) => env[envVariableName] !== undefined) || false; - // <- Note: [🗨] - - return { ...metadata, isMetadataAviailable, isInstalled, isFullyConfigured, isPartiallyConfigured }; - }); - - const usedEnvMessage = - $usedEnvFilename === null ? `Unknown \`.env\` file` : `Used \`.env\` file:\n${$usedEnvFilename}`; - - if (metadata.length === 0) { - return spaceTrim( - (block) => ` + let env: Record; + if ($isRunningInNode()) { + env = process.env as Record; + // <- TODO: [⚛] Some DRY way how to get to `process.env` and pass it into functions - ACRY search for `env` + } else { + env = {}; + } + + /** + * Mixes registered LLM tools from $llmToolsMetadataRegister and $llmToolsRegister + */ + const all: Array< + Registered & Partial> + > = []; + + for (const { + title, + packageName, + className, + envVariables, + } of $llmToolsMetadataRegister.list()) { + if ( + all.some( + (item) => + item.packageName === packageName && item.className === className, + ) + ) { + continue; + } + all.push({ title, packageName, className, envVariables }); + } + + for (const { packageName, className } of $llmToolsRegister.list()) { + if ( + all.some( + (item) => + item.packageName === packageName && item.className === className, + ) + ) { + continue; + } + all.push({ packageName, className }); + } + + const metadata = all.map((metadata) => { + const isMetadataAviailable = $llmToolsMetadataRegister + .list() + .find( + ({ packageName, className }) => + metadata.packageName === packageName && + metadata.className === className, + ); + + const isInstalled = $llmToolsRegister + .list() + .find( + ({ packageName, className }) => + metadata.packageName === packageName && + metadata.className === className, + ); + + const isFullyConfigured = + metadata.envVariables?.every( + (envVariableName) => env[envVariableName] !== undefined, + ) || false; + const isPartiallyConfigured = + metadata.envVariables?.some( + (envVariableName) => env[envVariableName] !== undefined, + ) || false; + // <- Note: [🗨] + + return { + ...metadata, + isMetadataAviailable, + isInstalled, + isFullyConfigured, + isPartiallyConfigured, + }; + }); + + const usedEnvMessage = + $usedEnvFilename === null + ? `Unknown \`.env\` file` + : `Used \`.env\` file:\n${$usedEnvFilename}`; + + if (metadata.length === 0) { + return spaceTrim( + (block) => ` No LLM providers are available. ${block(usedEnvMessage)} `, - ); - } + ); + } - return spaceTrim( - (block) => ` + return spaceTrim( + (block) => ` ${block(usedEnvMessage)} Relevant environment variables: ${block( - Object.keys(env) - .filter((envVariableName) => - metadata.some(({ envVariables }) => envVariables?.includes(envVariableName)), - ) - .map((envVariableName) => `- \`${envVariableName}\``) - .join('\n'), - )} + Object.keys(env) + .filter((envVariableName) => + metadata.some(({ envVariables }) => + envVariables?.includes(envVariableName), + ), + ) + .map((envVariableName) => `- \`${envVariableName}\``) + .join("\n"), + )} Available LLM providers are: ${block( - metadata - .map( - ( - { - title, - packageName, - className, - envVariables, - isMetadataAviailable, - isInstalled, - isFullyConfigured, - isPartiallyConfigured, - }, - i, - ) => { - const morePieces: Array = []; - - if (just(false)) { - // Keep for prettier formatting - } else if (!isMetadataAviailable && !isInstalled) { - // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found - morePieces.push(`Not installed and no metadata, looks like a unexpected behavior`); - } else if (isMetadataAviailable && !isInstalled) { - // TODO: [�][�] - morePieces.push(`Not installed`); - } else if (!isMetadataAviailable && isInstalled) { - morePieces.push(`No metadata but installed, looks like a unexpected behavior`); - } else if (isMetadataAviailable && isInstalled) { - morePieces.push(`Installed`); - } else { - morePieces.push(`unknown state, looks like a unexpected behavior`); - } /* not else */ - - if (isFullyConfigured) { - morePieces.push(`Configured`); - } else if (isPartiallyConfigured) { - morePieces.push( - `Partially confugured, missing ${envVariables - ?.filter((envVariable) => env[envVariable] === undefined) - .join(' + ')}`, - ); - } else { - if (envVariables !== null) { - morePieces.push( - `Not configured, to configure set env ${envVariables?.join(' + ')}`, - ); - } else { - morePieces.push(`Not configured`); // <- Note: Can not be configured via environment variables - } - } - - let providerMessage = spaceTrim(` + metadata + .map( + ( + { + title, + packageName, + className, + envVariables, + isMetadataAviailable, + isInstalled, + isFullyConfigured, + isPartiallyConfigured, + }, + i, + ) => { + const morePieces: Array = []; + + if (just(false)) { + // Keep for prettier formatting + } else if (!isMetadataAviailable && !isInstalled) { + // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found + morePieces.push( + `Not installed and no metadata, looks like a unexpected behavior`, + ); + } else if (isMetadataAviailable && !isInstalled) { + // TODO: [�][�] + morePieces.push(`Not installed`); + } else if (!isMetadataAviailable && isInstalled) { + morePieces.push( + `No metadata but installed, looks like a unexpected behavior`, + ); + } else if (isMetadataAviailable && isInstalled) { + morePieces.push(`Installed`); + } else { + morePieces.push( + `unknown state, looks like a unexpected behavior`, + ); + } /* not else */ + + if (isFullyConfigured) { + morePieces.push(`Configured`); + } else if (isPartiallyConfigured) { + morePieces.push( + `Partially confugured, missing ${envVariables + ?.filter( + (envVariable) => env[envVariable] === undefined, + ) + .join(" + ")}`, + ); + } else { + if (envVariables !== null) { + morePieces.push( + `Not configured, to configure set env ${envVariables?.join(" + ")}`, + ); + } else { + morePieces.push(`Not configured`); // <- Note: Can not be configured via environment variables + } + } + + let providerMessage = spaceTrim(` ${i + 1}) **${title}** \`${className}\` from \`${packageName}\` - ${morePieces.join('; ')} + ${morePieces.join("; ")} `); - if ($isRunningInNode) { - if (isInstalled && isFullyConfigured) { - providerMessage = colors.green(providerMessage); - } else if (isInstalled && isPartiallyConfigured) { - providerMessage = colors.yellow(providerMessage); - } else { - providerMessage = colors.gray(providerMessage); - } - } - - return providerMessage; - }, - ) - .join('\n'), - )} + if ($isRunningInNode) { + if (isInstalled && isFullyConfigured) { + providerMessage = colors.green(providerMessage); + } else if (isInstalled && isPartiallyConfigured) { + providerMessage = colors.yellow(providerMessage); + } else { + providerMessage = colors.gray(providerMessage); + } + } + + return providerMessage; + }, + ) + .join("\n"), + )} `, - ); + ); } /** diff --git a/src/llm-providers/_common/register/LlmToolsConfiguration.ts b/src/llm-providers/_common/register/LlmToolsConfiguration.ts index 43b0d8fdea..ae142fc36e 100644 --- a/src/llm-providers/_common/register/LlmToolsConfiguration.ts +++ b/src/llm-providers/_common/register/LlmToolsConfiguration.ts @@ -1,6 +1,6 @@ -import type { string_title } from '../../../types/typeAliases'; -import type { Registered } from '../../../utils/$Register'; -import type { LlmToolsOptions } from './LlmToolsOptions'; +import type { string_title } from "../../../types/typeAliases"; +import type { Registered } from "../../../utils/$Register"; +import type { LlmToolsOptions } from "./LlmToolsOptions"; /** * @@@ @@ -8,17 +8,17 @@ import type { LlmToolsOptions } from './LlmToolsOptions'; * @@@ `LlmToolsMetadata` vs `LlmToolsConfiguration` vs `LlmToolsOptions` (vs `Registered`) */ export type LlmToolsConfiguration = ReadonlyArray< - Registered & { - /** - * @@@ - */ - readonly title: string_title; + Registered & { + /** + * @@@ + */ + readonly title: string_title; - /** - * @@@ - */ - readonly options: LlmToolsOptions; - } + /** + * @@@ + */ + readonly options: LlmToolsOptions; + } >; /** diff --git a/src/llm-providers/_common/register/LlmToolsMetadata.ts b/src/llm-providers/_common/register/LlmToolsMetadata.ts index 908b6d1f44..1ced30876b 100644 --- a/src/llm-providers/_common/register/LlmToolsMetadata.ts +++ b/src/llm-providers/_common/register/LlmToolsMetadata.ts @@ -1,8 +1,8 @@ -import type { string_name } from '../../../types/typeAliases'; -import type { string_title } from '../../../types/typeAliases'; -import type { Registered } from '../../../utils/$Register'; -import type { string_SCREAMING_CASE } from '../../../utils/normalization/normalizeTo_SCREAMING_CASE'; -import type { LlmToolsConfiguration } from './LlmToolsConfiguration'; +import type { string_name } from "../../../types/typeAliases"; +import type { string_title } from "../../../types/typeAliases"; +import type { Registered } from "../../../utils/$Register"; +import type { string_SCREAMING_CASE } from "../../../utils/normalization/normalizeTo_SCREAMING_CASE"; +import type { LlmToolsConfiguration } from "./LlmToolsConfiguration"; /** * @@@ @@ -10,28 +10,32 @@ import type { LlmToolsConfiguration } from './LlmToolsConfiguration'; * @@@ `LlmToolsMetadata` vs `LlmToolsConfiguration` vs `LlmToolsOptions` (vs `Registered`) */ export type LlmToolsMetadata = Registered & { - /** - * @@@ - */ - readonly title: string_title; + /** + * @@@ + */ + readonly title: string_title; - /** - * List of environment variables that can be used to configure the provider - * - * If `[]`, empty array, it means that the provider is available automatically without any configuration - * If `null`, it means that the provider can not be configured via environment variables - */ - readonly envVariables: ReadonlyArray | null; + /** + * List of environment variables that can be used to configure the provider + * + * If `[]`, empty array, it means that the provider is available automatically without any configuration + * If `null`, it means that the provider can not be configured via environment variables + */ + readonly envVariables: ReadonlyArray< + string_name & string_SCREAMING_CASE + > | null; - /** - * @@@ - */ - getBoilerplateConfiguration(): LlmToolsConfiguration[number]; + /** + * @@@ + */ + getBoilerplateConfiguration(): LlmToolsConfiguration[number]; - /** - * @@@ - */ - createConfigurationFromEnv(env: Record): LlmToolsConfiguration[number] | null; + /** + * @@@ + */ + createConfigurationFromEnv( + env: Record, + ): LlmToolsConfiguration[number] | null; }; /** diff --git a/src/llm-providers/_common/register/LlmToolsOptions.ts b/src/llm-providers/_common/register/LlmToolsOptions.ts index ee5c45b4e6..c08011084a 100644 --- a/src/llm-providers/_common/register/LlmToolsOptions.ts +++ b/src/llm-providers/_common/register/LlmToolsOptions.ts @@ -1,4 +1,4 @@ -import type { TODO_object } from '../../../utils/organization/TODO_object'; +import type { TODO_object } from "../../../utils/organization/TODO_object"; /** * @@@ diff --git a/src/llm-providers/_common/register/createLlmToolsFromConfiguration.test.ts b/src/llm-providers/_common/register/createLlmToolsFromConfiguration.test.ts index 37248b5bd0..3ad58746c7 100644 --- a/src/llm-providers/_common/register/createLlmToolsFromConfiguration.test.ts +++ b/src/llm-providers/_common/register/createLlmToolsFromConfiguration.test.ts @@ -1,26 +1,26 @@ -import { describe, expect, it } from '@jest/globals'; -import { keepUnused } from '../../../utils/organization/keepUnused'; +import { describe, expect, it } from "@jest/globals"; +import { keepUnused } from "../../../utils/organization/keepUnused"; //import { _GoogleRegistration } from '../../google/register-constructor'; // <- Note: [㊗] Registering the provider -import { _OpenAiRegistration } from '../../openai/register-constructor'; // <- Note: [㊗] Registering the provider -import { createLlmToolsFromConfiguration } from './createLlmToolsFromConfiguration'; +import { _OpenAiRegistration } from "../../openai/register-constructor"; // <- Note: [㊗] Registering the provider +import { createLlmToolsFromConfiguration } from "./createLlmToolsFromConfiguration"; keepUnused(_OpenAiRegistration); // keepUnused(_GoogleRegistration); // <- Note: `GoogleExecutionTools` are not supported in Jest environment -describe('how createLlmToolsFromConfiguration works', () => { - it('should create LLM configuration', () => { - expect(() => - createLlmToolsFromConfiguration([ - { - title: 'Open AI (boilerplate)', - packageName: '@promptbook/openai', - className: 'OpenAiExecutionTools', - options: { - apiKey: 'sk-xxxxxxfakekey', - }, - }, - /* +describe("how createLlmToolsFromConfiguration works", () => { + it("should create LLM configuration", () => { + expect(() => + createLlmToolsFromConfiguration([ + { + title: "Open AI (boilerplate)", + packageName: "@promptbook/openai", + className: "OpenAiExecutionTools", + options: { + apiKey: "sk-xxxxxxfakekey", + }, + }, + /* { title: 'Google Gemini (boilerplate)', packageName: '@promptbook/google', @@ -28,7 +28,7 @@ describe('how createLlmToolsFromConfiguration works', () => { options: { apiKey: 'AIxxxxxxfakekeys' }, }, */ - ]), - ).not.toThrow(); - }); + ]), + ).not.toThrow(); + }); }); diff --git a/src/llm-providers/_common/register/createLlmToolsFromConfiguration.ts b/src/llm-providers/_common/register/createLlmToolsFromConfiguration.ts index 8bc7c868f7..d6a97ccf00 100644 --- a/src/llm-providers/_common/register/createLlmToolsFromConfiguration.ts +++ b/src/llm-providers/_common/register/createLlmToolsFromConfiguration.ts @@ -1,13 +1,13 @@ -import spaceTrim from 'spacetrim'; -import { DEFAULT_IS_VERBOSE } from '../../../config'; -import type { LlmExecutionTools } from '../../../execution/LlmExecutionTools'; -import type { string_user_id } from '../../../types/typeAliases'; -import type { TODO_any } from '../../../utils/organization/TODO_any'; -import { joinLlmExecutionTools } from '../../multiple/joinLlmExecutionTools'; -import { MultipleLlmExecutionTools } from '../../multiple/MultipleLlmExecutionTools'; -import { $llmToolsRegister } from './$llmToolsRegister'; -import { $registeredLlmToolsMessage } from './$registeredLlmToolsMessage'; -import type { LlmToolsConfiguration } from './LlmToolsConfiguration'; +import spaceTrim from "spacetrim"; +import { DEFAULT_IS_VERBOSE } from "../../../config"; +import type { LlmExecutionTools } from "../../../execution/LlmExecutionTools"; +import type { string_user_id } from "../../../types/typeAliases"; +import type { TODO_any } from "../../../utils/organization/TODO_any"; +import type { MultipleLlmExecutionTools } from "../../multiple/MultipleLlmExecutionTools"; +import { joinLlmExecutionTools } from "../../multiple/joinLlmExecutionTools"; +import { $llmToolsRegister } from "./$llmToolsRegister"; +import { $registeredLlmToolsMessage } from "./$registeredLlmToolsMessage"; +import type { LlmToolsConfiguration } from "./LlmToolsConfiguration"; /** * Options for `$provideLlmToolsFromEnv` @@ -15,19 +15,19 @@ import type { LlmToolsConfiguration } from './LlmToolsConfiguration'; * @private internal type for `$provideLlmToolsFromEnv` and `$provideLlmToolsForTestingAndScriptsAndPlayground` */ export type CreateLlmToolsFromConfigurationOptions = { - /** - * This will will be passed to the created `LlmExecutionTools` - * - * @default false - */ - isVerbose?: boolean; + /** + * This will will be passed to the created `LlmExecutionTools` + * + * @default false + */ + isVerbose?: boolean; - /** - * Identifier of the end user - * - * Note: This is passed to the LLM tools providers to identify misuse - */ - readonly userId?: string_user_id ; + /** + * Identifier of the end user + * + * Note: This is passed to the LLM tools providers to identify misuse + */ + readonly userId?: string_user_id; }; /** @@ -39,26 +39,28 @@ export type CreateLlmToolsFromConfigurationOptions = { * @public exported from `@promptbook/core` */ export function createLlmToolsFromConfiguration( - configuration: LlmToolsConfiguration, - options: CreateLlmToolsFromConfigurationOptions = {}, + configuration: LlmToolsConfiguration, + options: CreateLlmToolsFromConfigurationOptions = {}, ): MultipleLlmExecutionTools { - const { isVerbose = DEFAULT_IS_VERBOSE, userId } = options; + const { isVerbose = DEFAULT_IS_VERBOSE, userId } = options; - const llmTools: ReadonlyArray = configuration.map((llmConfiguration: TODO_any) => { - const registeredItem = $llmToolsRegister - .list() - .find( - ({ packageName, className }) => - llmConfiguration.packageName === packageName && llmConfiguration.className === className, - ); + const llmTools: ReadonlyArray = configuration.map( + (llmConfiguration: TODO_any) => { + const registeredItem = $llmToolsRegister + .list() + .find( + ({ packageName, className }) => + llmConfiguration.packageName === packageName && + llmConfiguration.className === className, + ); - if (registeredItem === undefined) { - throw new Error( - spaceTrim( - (block) => ` + if (registeredItem === undefined) { + throw new Error( + spaceTrim( + (block) => ` There is no constructor for LLM provider \`${llmConfiguration.className}\` from \`${ - llmConfiguration.packageName - }\` + llmConfiguration.packageName + }\` You have probably forgotten install and import the provider package. To fix this issue, you can: @@ -74,18 +76,19 @@ export function createLlmToolsFromConfiguration( ${block($registeredLlmToolsMessage())} `, - ), - ); - } + ), + ); + } - return registeredItem({ - isVerbose, - userId, - ...llmConfiguration.options, - }); - }); + return registeredItem({ + isVerbose, + userId, + ...llmConfiguration.options, + }); + }, + ); - return joinLlmExecutionTools(...llmTools); + return joinLlmExecutionTools(...llmTools); } /** diff --git a/src/llm-providers/_common/utils/cache/CacheItem.ts b/src/llm-providers/_common/utils/cache/CacheItem.ts index 5bbdf14abe..50870373c0 100644 --- a/src/llm-providers/_common/utils/cache/CacheItem.ts +++ b/src/llm-providers/_common/utils/cache/CacheItem.ts @@ -1,31 +1,31 @@ -import type { PromptResult } from '../../../../execution/PromptResult'; -import type { Prompt } from '../../../../types/Prompt'; -import type { string_date_iso8601 } from '../../../../types/typeAliases'; -import type { string_promptbook_version } from '../../../../version'; +import type { PromptResult } from "../../../../execution/PromptResult"; +import type { Prompt } from "../../../../types/Prompt"; +import type { string_date_iso8601 } from "../../../../types/typeAliases"; +import type { string_promptbook_version } from "../../../../version"; /** * @@@ */ export type CacheItem = { - /** - * @@@ - */ - date: string_date_iso8601; + /** + * @@@ + */ + date: string_date_iso8601; - /** - * @@@ - */ - promptbookVersion?: string_promptbook_version; + /** + * @@@ + */ + promptbookVersion?: string_promptbook_version; - /** - * @@@ - */ - prompt: Prompt; + /** + * @@@ + */ + prompt: Prompt; - /** - * @@@ - */ - promptResult: PromptResult; + /** + * @@@ + */ + promptResult: PromptResult; }; /** diff --git a/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.ts b/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.ts index a47dc51e3a..2d2d2ae6e0 100644 --- a/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.ts +++ b/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.ts @@ -1,22 +1,22 @@ -import type { PromptbookStorage } from '../../../../storage/_common/PromptbookStorage'; -import type { CacheItem } from './CacheItem'; +import type { PromptbookStorage } from "../../../../storage/_common/PromptbookStorage"; +import type { CacheItem } from "./CacheItem"; /** * @@@ */ export type CacheLlmToolsOptions = { - /** - * @@@ - * - * @default MemoryStorage - */ - storage: PromptbookStorage; + /** + * @@@ + * + * @default MemoryStorage + */ + storage: PromptbookStorage; - /** - * @@@ - * - * @default false - */ - isCacheReloaded?: boolean; - // <- TODO: [🎅] Maybe unite with `intermediateFilesStrategy` and change to `cacheStrategy` + /** + * @@@ + * + * @default false + */ + isCacheReloaded?: boolean; + // <- TODO: [🎅] Maybe unite with `intermediateFilesStrategy` and change to `cacheStrategy` }; diff --git a/src/llm-providers/_common/utils/cache/cacheLlmTools.ts b/src/llm-providers/_common/utils/cache/cacheLlmTools.ts index 9879ac76aa..ef08d0f8cb 100644 --- a/src/llm-providers/_common/utils/cache/cacheLlmTools.ts +++ b/src/llm-providers/_common/utils/cache/cacheLlmTools.ts @@ -1,21 +1,21 @@ -import hexEncoder from 'crypto-js/enc-hex'; -import sha256 from 'crypto-js/sha256'; -import type { Promisable } from 'type-fest'; -import { MAX_FILENAME_LENGTH } from '../../../../config'; -import { PipelineExecutionError } from '../../../../errors/PipelineExecutionError'; -import type { AvailableModel } from '../../../../execution/AvailableModel'; -import type { LlmExecutionTools } from '../../../../execution/LlmExecutionTools'; -import type { ChatPromptResult } from '../../../../execution/PromptResult'; -import type { CompletionPromptResult } from '../../../../execution/PromptResult'; -import type { EmbeddingPromptResult } from '../../../../execution/PromptResult'; -import { MemoryStorage } from '../../../../storage/memory/MemoryStorage'; -import type { Prompt } from '../../../../types/Prompt'; -import { $getCurrentDate } from '../../../../utils/$getCurrentDate'; -import { titleToName } from '../../../../utils/normalization/titleToName'; -import type { really_any } from '../../../../utils/organization/really_any'; -import type { TODO_any } from '../../../../utils/organization/TODO_any'; -import { PROMPTBOOK_ENGINE_VERSION } from '../../../../version'; -import type { CacheLlmToolsOptions } from './CacheLlmToolsOptions'; +import hexEncoder from "crypto-js/enc-hex"; +import sha256 from "crypto-js/sha256"; +import type { Promisable } from "type-fest"; +import { MAX_FILENAME_LENGTH } from "../../../../config"; +import { PipelineExecutionError } from "../../../../errors/PipelineExecutionError"; +import type { AvailableModel } from "../../../../execution/AvailableModel"; +import type { LlmExecutionTools } from "../../../../execution/LlmExecutionTools"; +import type { ChatPromptResult } from "../../../../execution/PromptResult"; +import type { CompletionPromptResult } from "../../../../execution/PromptResult"; +import type { EmbeddingPromptResult } from "../../../../execution/PromptResult"; +import { MemoryStorage } from "../../../../storage/memory/MemoryStorage"; +import type { Prompt } from "../../../../types/Prompt"; +import { $getCurrentDate } from "../../../../utils/$getCurrentDate"; +import { titleToName } from "../../../../utils/normalization/titleToName"; +import type { TODO_any } from "../../../../utils/organization/TODO_any"; +import type { really_any } from "../../../../utils/organization/really_any"; +import { PROMPTBOOK_ENGINE_VERSION } from "../../../../version"; +import type { CacheLlmToolsOptions } from "./CacheLlmToolsOptions"; /** * Intercepts LLM tools and counts total usage of the tools @@ -27,105 +27,113 @@ import type { CacheLlmToolsOptions } from './CacheLlmToolsOptions'; * @public exported from `@promptbook/core` */ export function cacheLlmTools( - llmTools: TLlmTools, - options: Partial = {}, + llmTools: TLlmTools, + options: Partial = {}, ): TLlmTools { - const { storage = new MemoryStorage(), isCacheReloaded = false } = options; - - const proxyTools: TLlmTools = { - ...llmTools, - // <- Note: [🥫] - - get title() { - // TODO: [🧠] Maybe put here some suffix - return llmTools.title; - }, - - get description() { - // TODO: [🧠] Maybe put here some suffix - return llmTools.description; - }, - - listModels(): Promisable> { - // TODO: [🧠] Should be model listing also cached? - return /* not await */ llmTools.listModels(); - }, - }; - - const callCommonModel = async (prompt: Prompt): Promise => { - const { parameters, content, modelRequirements } = prompt; - // <- Note: These are relevant things from the prompt that the cache key should depend on. - - const key = titleToName( - prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) + - '-' + - sha256( - hexEncoder.parse(JSON.stringify({ parameters, content, modelRequirements })), - ).toString(/* hex */), - // <- TODO: [🥬] Encapsulate sha256 to some private utility function - ); - - const cacheItem = !isCacheReloaded ? await storage.getItem(key) : null; - - if (cacheItem) { - return cacheItem.promptResult as ChatPromptResult; - } - - let promptResult: TODO_any; - variant: switch (prompt.modelRequirements.modelVariant) { - case 'CHAT': - promptResult = await llmTools.callChatModel!(prompt); - break variant; - case 'COMPLETION': - promptResult = await llmTools.callCompletionModel!(prompt); - break variant; - - case 'EMBEDDING': - promptResult = await llmTools.callEmbeddingModel!(prompt); - break variant; - - // <- case [🤖]: - - default: - throw new PipelineExecutionError( - `Unknown model variant "${(prompt as really_any).modelRequirements.modelVariant}"`, - ); - } - - // TODO: [🧠] !!5 How to do timing in mixed cache / non-cache situation - // promptResult.timing: FromtoItems - - await storage.setItem(key, { - date: $getCurrentDate(), - promptbookVersion: PROMPTBOOK_ENGINE_VERSION, - prompt, - promptResult, - }); - - return promptResult; - }; - - if (llmTools.callChatModel !== undefined) { - proxyTools.callChatModel = async (prompt: Prompt): Promise => { - return /* not await */ callCommonModel(prompt); - }; - } - - if (llmTools.callCompletionModel !== undefined) { - proxyTools.callCompletionModel = async (prompt: Prompt): Promise => { - return /* not await */ callCommonModel(prompt); - }; - } - - if (llmTools.callEmbeddingModel !== undefined) { - proxyTools.callEmbeddingModel = async (prompt: Prompt): Promise => { - return /* not await */ callCommonModel(prompt); - }; - } - - // <- Note: [🤖] - - return proxyTools; + const { storage = new MemoryStorage(), isCacheReloaded = false } = options; + + const proxyTools: TLlmTools = { + ...llmTools, + // <- Note: [🥫] + + get title() { + // TODO: [🧠] Maybe put here some suffix + return llmTools.title; + }, + + get description() { + // TODO: [🧠] Maybe put here some suffix + return llmTools.description; + }, + + listModels(): Promisable> { + // TODO: [🧠] Should be model listing also cached? + return /* not await */ llmTools.listModels(); + }, + }; + + const callCommonModel = async (prompt: Prompt): Promise => { + const { parameters, content, modelRequirements } = prompt; + // <- Note: These are relevant things from the prompt that the cache key should depend on. + + const key = titleToName( + prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) + + "-" + + sha256( + hexEncoder.parse( + JSON.stringify({ parameters, content, modelRequirements }), + ), + ).toString(/* hex */), + // <- TODO: [🥬] Encapsulate sha256 to some private utility function + ); + + const cacheItem = !isCacheReloaded ? await storage.getItem(key) : null; + + if (cacheItem) { + return cacheItem.promptResult as ChatPromptResult; + } + + let promptResult: TODO_any; + switch (prompt.modelRequirements.modelVariant) { + case "CHAT": + promptResult = await llmTools.callChatModel!(prompt); + break; + case "COMPLETION": + promptResult = await llmTools.callCompletionModel!(prompt); + break; + + case "EMBEDDING": + promptResult = await llmTools.callEmbeddingModel!(prompt); + break; + + // <- case [🤖]: + + default: + throw new PipelineExecutionError( + `Unknown model variant "${(prompt as really_any).modelRequirements.modelVariant}"`, + ); + } + + // TODO: [🧠] !!5 How to do timing in mixed cache / non-cache situation + // promptResult.timing: FromtoItems + + await storage.setItem(key, { + date: $getCurrentDate(), + promptbookVersion: PROMPTBOOK_ENGINE_VERSION, + prompt, + promptResult, + }); + + return promptResult; + }; + + if (llmTools.callChatModel !== undefined) { + proxyTools.callChatModel = async ( + prompt: Prompt, + ): Promise => { + return /* not await */ callCommonModel(prompt); + }; + } + + if (llmTools.callCompletionModel !== undefined) { + proxyTools.callCompletionModel = async ( + prompt: Prompt, + ): Promise => { + return /* not await */ callCommonModel(prompt); + }; + } + + if (llmTools.callEmbeddingModel !== undefined) { + proxyTools.callEmbeddingModel = async ( + prompt: Prompt, + ): Promise => { + return /* not await */ callCommonModel(prompt); + }; + } + + // <- Note: [🤖] + + return proxyTools; } /** diff --git a/src/llm-providers/_common/utils/count-total-usage/LlmExecutionToolsWithTotalUsage.ts b/src/llm-providers/_common/utils/count-total-usage/LlmExecutionToolsWithTotalUsage.ts index 0e9877982b..08790f5637 100644 --- a/src/llm-providers/_common/utils/count-total-usage/LlmExecutionToolsWithTotalUsage.ts +++ b/src/llm-providers/_common/utils/count-total-usage/LlmExecutionToolsWithTotalUsage.ts @@ -1,14 +1,14 @@ -import type { LlmExecutionTools } from '../../../../execution/LlmExecutionTools'; -import type { PromptResultUsage } from '../../../../execution/PromptResultUsage'; +import type { LlmExecutionTools } from "../../../../execution/LlmExecutionTools"; +import type { PromptResultUsage } from "../../../../execution/PromptResultUsage"; /** * LLM tools with option to get total usage of the execution */ export type LlmExecutionToolsWithTotalUsage = LlmExecutionTools & { - /** - * Get total cost of the execution up to this point - */ - getTotalUsage(): PromptResultUsage; + /** + * Get total cost of the execution up to this point + */ + getTotalUsage(): PromptResultUsage; }; /** diff --git a/src/llm-providers/_common/utils/count-total-usage/countTotalUsage.ts b/src/llm-providers/_common/utils/count-total-usage/countTotalUsage.ts index 7b9ed9a291..9f3cba20ec 100644 --- a/src/llm-providers/_common/utils/count-total-usage/countTotalUsage.ts +++ b/src/llm-providers/_common/utils/count-total-usage/countTotalUsage.ts @@ -1,16 +1,16 @@ -import type { Promisable } from 'type-fest'; -import type { AvailableModel } from '../../../../execution/AvailableModel'; -import type { LlmExecutionTools } from '../../../../execution/LlmExecutionTools'; -import type { ChatPromptResult } from '../../../../execution/PromptResult'; -import type { CompletionPromptResult } from '../../../../execution/PromptResult'; -import type { EmbeddingPromptResult } from '../../../../execution/PromptResult'; -import type { PromptResultUsage } from '../../../../execution/PromptResultUsage'; -import { addUsage } from '../../../../execution/utils/addUsage'; -import { ZERO_USAGE } from '../../../../execution/utils/usage-constants'; -import type { ChatPrompt } from '../../../../types/Prompt'; -import type { CompletionPrompt } from '../../../../types/Prompt'; -import type { EmbeddingPrompt } from '../../../../types/Prompt'; -import type { LlmExecutionToolsWithTotalUsage } from './LlmExecutionToolsWithTotalUsage'; +import type { Promisable } from "type-fest"; +import type { AvailableModel } from "../../../../execution/AvailableModel"; +import type { LlmExecutionTools } from "../../../../execution/LlmExecutionTools"; +import type { ChatPromptResult } from "../../../../execution/PromptResult"; +import type { CompletionPromptResult } from "../../../../execution/PromptResult"; +import type { EmbeddingPromptResult } from "../../../../execution/PromptResult"; +import type { PromptResultUsage } from "../../../../execution/PromptResultUsage"; +import { addUsage } from "../../../../execution/utils/addUsage"; +import { ZERO_USAGE } from "../../../../execution/utils/usage-constants"; +import type { ChatPrompt } from "../../../../types/Prompt"; +import type { CompletionPrompt } from "../../../../types/Prompt"; +import type { EmbeddingPrompt } from "../../../../types/Prompt"; +import type { LlmExecutionToolsWithTotalUsage } from "./LlmExecutionToolsWithTotalUsage"; /** * Intercepts LLM tools and counts total usage of the tools @@ -19,64 +19,72 @@ import type { LlmExecutionToolsWithTotalUsage } from './LlmExecutionToolsWithTot * @returns LLM tools with same functionality with added total cost counting * @public exported from `@promptbook/core` */ -export function countTotalUsage(llmTools: LlmExecutionTools): LlmExecutionToolsWithTotalUsage { - let totalUsage: PromptResultUsage = ZERO_USAGE; +export function countTotalUsage( + llmTools: LlmExecutionTools, +): LlmExecutionToolsWithTotalUsage { + let totalUsage: PromptResultUsage = ZERO_USAGE; - const proxyTools: LlmExecutionToolsWithTotalUsage = { - get title() { - // TODO: [🧠] Maybe put here some suffix - return llmTools.title; - }, + const proxyTools: LlmExecutionToolsWithTotalUsage = { + get title() { + // TODO: [🧠] Maybe put here some suffix + return llmTools.title; + }, - get description() { - // TODO: [🧠] Maybe put here some suffix - return llmTools.description; - }, + get description() { + // TODO: [🧠] Maybe put here some suffix + return llmTools.description; + }, - async checkConfiguration(): Promise { - return /* not await */ llmTools.checkConfiguration(); - }, + async checkConfiguration(): Promise { + return /* not await */ llmTools.checkConfiguration(); + }, - listModels(): Promisable> { - return /* not await */ llmTools.listModels(); - }, + listModels(): Promisable> { + return /* not await */ llmTools.listModels(); + }, - getTotalUsage() { - // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied - return totalUsage; - }, - }; + getTotalUsage() { + // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied + return totalUsage; + }, + }; - if (llmTools.callChatModel !== undefined) { - proxyTools.callChatModel = async (prompt: ChatPrompt): Promise => { - // console.info('[🚕] callChatModel through countTotalUsage'); - const promptResult = await llmTools.callChatModel!(prompt); - totalUsage = addUsage(totalUsage, promptResult.usage); - return promptResult; - }; - } + if (llmTools.callChatModel !== undefined) { + proxyTools.callChatModel = async ( + prompt: ChatPrompt, + ): Promise => { + // console.info('[🚕] callChatModel through countTotalUsage'); + const promptResult = await llmTools.callChatModel!(prompt); + totalUsage = addUsage(totalUsage, promptResult.usage); + return promptResult; + }; + } - if (llmTools.callCompletionModel !== undefined) { - proxyTools.callCompletionModel = async (prompt: CompletionPrompt): Promise => { - // console.info('[🚕] callCompletionModel through countTotalUsage'); - const promptResult = await llmTools.callCompletionModel!(prompt); - totalUsage = addUsage(totalUsage, promptResult.usage); - return promptResult; - }; - } + if (llmTools.callCompletionModel !== undefined) { + proxyTools.callCompletionModel = async ( + prompt: CompletionPrompt, + ): Promise => { + // console.info('[🚕] callCompletionModel through countTotalUsage'); + const promptResult = await llmTools.callCompletionModel!(prompt); + totalUsage = addUsage(totalUsage, promptResult.usage); + return promptResult; + }; + } - if (llmTools.callEmbeddingModel !== undefined) { - proxyTools.callEmbeddingModel = async (prompt: EmbeddingPrompt): Promise => { - // console.info('[🚕] callEmbeddingModel through countTotalUsage'); - const promptResult = await llmTools.callEmbeddingModel!(prompt); - totalUsage = addUsage(totalUsage, promptResult.usage); - return promptResult; - }; - } + if (llmTools.callEmbeddingModel !== undefined) { + proxyTools.callEmbeddingModel = async ( + prompt: EmbeddingPrompt, + ): Promise => { + // console.info('[🚕] callEmbeddingModel through countTotalUsage'); + const promptResult = await llmTools.callEmbeddingModel!(prompt); + totalUsage = addUsage(totalUsage, promptResult.usage); + return promptResult; + }; + } - // <- Note: [🤖] + // <- Note: [🤖] - return proxyTools; + return proxyTools; } /** diff --git a/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.ts b/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.ts index e037b0d7c0..c9fd51ea24 100644 --- a/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.ts +++ b/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.ts @@ -1,38 +1,38 @@ -import { LimitReachedError } from '../../../../errors/LimitReachedError'; -import { NotYetImplementedError } from '../../../../errors/NotYetImplementedError'; -import type { LlmExecutionTools } from '../../../../execution/LlmExecutionTools'; -import type { ChatPromptResult } from '../../../../execution/PromptResult'; -import type { CompletionPromptResult } from '../../../../execution/PromptResult'; -import type { EmbeddingPromptResult } from '../../../../execution/PromptResult'; -import type { PromptResultUsage } from '../../../../execution/PromptResultUsage'; -import { ZERO_USAGE } from '../../../../execution/utils/usage-constants'; -import { MemoryStorage } from '../../../../storage/memory/MemoryStorage'; -import type { PromptbookStorage } from '../../../../storage/_common/PromptbookStorage'; -import type { ChatPrompt } from '../../../../types/Prompt'; -import type { CompletionPrompt } from '../../../../types/Prompt'; -import type { EmbeddingPrompt } from '../../../../types/Prompt'; -import type { TODO_any } from '../../../../utils/organization/TODO_any'; -import { TODO_USE } from '../../../../utils/organization/TODO_USE'; -import { countTotalUsage } from './countTotalUsage'; -import type { LlmExecutionToolsWithTotalUsage } from './LlmExecutionToolsWithTotalUsage'; +import { LimitReachedError } from "../../../../errors/LimitReachedError"; +import { NotYetImplementedError } from "../../../../errors/NotYetImplementedError"; +import type { LlmExecutionTools } from "../../../../execution/LlmExecutionTools"; +import type { ChatPromptResult } from "../../../../execution/PromptResult"; +import type { CompletionPromptResult } from "../../../../execution/PromptResult"; +import type { EmbeddingPromptResult } from "../../../../execution/PromptResult"; +import type { PromptResultUsage } from "../../../../execution/PromptResultUsage"; +import { ZERO_USAGE } from "../../../../execution/utils/usage-constants"; +import type { PromptbookStorage } from "../../../../storage/_common/PromptbookStorage"; +import { MemoryStorage } from "../../../../storage/memory/MemoryStorage"; +import type { ChatPrompt } from "../../../../types/Prompt"; +import type { CompletionPrompt } from "../../../../types/Prompt"; +import type { EmbeddingPrompt } from "../../../../types/Prompt"; +import { TODO_USE } from "../../../../utils/organization/TODO_USE"; +import type { TODO_any } from "../../../../utils/organization/TODO_any"; +import type { LlmExecutionToolsWithTotalUsage } from "./LlmExecutionToolsWithTotalUsage"; +import { countTotalUsage } from "./countTotalUsage"; /** * Options for `limitTotalUsage` */ type LimitTotalUsageOptions = { - /** - * @@@ - * - * @default ZERO_USAGE - */ - maxTotalUsage: PromptResultUsage; + /** + * @@@ + * + * @default ZERO_USAGE + */ + maxTotalUsage: PromptResultUsage; - /** - * @@@ - * - * @default MemoryStorage - */ - storage: PromptbookStorage; + /** + * @@@ + * + * @default MemoryStorage + */ + storage: PromptbookStorage; }; /** @@ -41,49 +41,59 @@ type LimitTotalUsageOptions = { * @public exported from `@promptbook/core` */ export function limitTotalUsage( - llmTools: LlmExecutionTools, - options: Partial = {}, + llmTools: LlmExecutionTools, + options: Partial = {}, ): LlmExecutionToolsWithTotalUsage { - const { maxTotalUsage = ZERO_USAGE, storage = new MemoryStorage() } = options; + const { maxTotalUsage = ZERO_USAGE, storage = new MemoryStorage() } = options; - TODO_USE(storage); + TODO_USE(storage); - const proxyTools = countTotalUsage(llmTools); + const proxyTools = countTotalUsage(llmTools); - if (maxTotalUsage.price.value !== 0) { - throw new NotYetImplementedError('`limitTotalUsage` is not yet implemented for non-zero price'); + if (maxTotalUsage.price.value !== 0) { + throw new NotYetImplementedError( + "`limitTotalUsage` is not yet implemented for non-zero price", + ); - // TODO: "Cannot call `callChatModel` because the total cost limit is reached" - } + // TODO: "Cannot call `callChatModel` because the total cost limit is reached" + } - if (proxyTools.callChatModel !== undefined) { - proxyTools.callChatModel = async (prompt: ChatPrompt): Promise => { - TODO_USE(prompt); - throw new LimitReachedError('Cannot call `callChatModel` because you are not allowed to spend any cost'); - }; - } + if (proxyTools.callChatModel !== undefined) { + proxyTools.callChatModel = async ( + prompt: ChatPrompt, + ): Promise => { + TODO_USE(prompt); + throw new LimitReachedError( + "Cannot call `callChatModel` because you are not allowed to spend any cost", + ); + }; + } - if (proxyTools.callCompletionModel !== undefined) { - proxyTools.callCompletionModel = async (prompt: CompletionPrompt): Promise => { - TODO_USE(prompt); - throw new LimitReachedError( - 'Cannot call `callCompletionModel` because you are not allowed to spend any cost', - ); - }; - } + if (proxyTools.callCompletionModel !== undefined) { + proxyTools.callCompletionModel = async ( + prompt: CompletionPrompt, + ): Promise => { + TODO_USE(prompt); + throw new LimitReachedError( + "Cannot call `callCompletionModel` because you are not allowed to spend any cost", + ); + }; + } - if (proxyTools.callEmbeddingModel !== undefined) { - proxyTools.callEmbeddingModel = async (prompt: EmbeddingPrompt): Promise => { - TODO_USE(prompt); - throw new LimitReachedError( - 'Cannot call `callEmbeddingModel` because you are not allowed to spend any cost', - ); - }; - } + if (proxyTools.callEmbeddingModel !== undefined) { + proxyTools.callEmbeddingModel = async ( + prompt: EmbeddingPrompt, + ): Promise => { + TODO_USE(prompt); + throw new LimitReachedError( + "Cannot call `callEmbeddingModel` because you are not allowed to spend any cost", + ); + }; + } - // <- Note: [🤖] + // <- Note: [🤖] - return proxyTools; + return proxyTools; } /** diff --git a/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.ts b/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.ts index 1742a45292..50c3369105 100644 --- a/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.ts +++ b/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.ts @@ -1,26 +1,26 @@ -import type { ClientOptions } from '@anthropic-ai/sdk'; -import Anthropic from '@anthropic-ai/sdk'; -import type { MessageCreateParamsNonStreaming } from '@anthropic-ai/sdk/resources'; -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import spaceTrim from 'spacetrim'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { AvailableModel } from '../../execution/AvailableModel'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import type { ChatPromptResult } from '../../execution/PromptResult'; -import type { Prompt } from '../../types/Prompt'; -import type { string_date_iso8601 } from '../../types/typeAliases'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_model_name } from '../../types/typeAliases'; -import type { string_title } from '../../types/typeAliases'; -import { $getCurrentDate } from '../../utils/$getCurrentDate'; -import type { really_any } from '../../utils/organization/really_any'; -import { templateParameters } from '../../utils/parameters/templateParameters'; -import { exportJson } from '../../utils/serialization/exportJson'; -import { ANTHROPIC_CLAUDE_MODELS } from './anthropic-claude-models'; -import type { AnthropicClaudeExecutionToolsDirectOptions } from './AnthropicClaudeExecutionToolsOptions'; -import { computeAnthropicClaudeUsage } from './computeAnthropicClaudeUsage'; +import type { ClientOptions } from "@anthropic-ai/sdk"; +import Anthropic from "@anthropic-ai/sdk"; +import type { MessageCreateParamsNonStreaming } from "@anthropic-ai/sdk/resources"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import spaceTrim from "spacetrim"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { AvailableModel } from "../../execution/AvailableModel"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import type { ChatPromptResult } from "../../execution/PromptResult"; +import type { Prompt } from "../../types/Prompt"; +import type { string_date_iso8601 } from "../../types/typeAliases"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_model_name } from "../../types/typeAliases"; +import type { string_title } from "../../types/typeAliases"; +import { $getCurrentDate } from "../../utils/$getCurrentDate"; +import type { really_any } from "../../utils/organization/really_any"; +import { templateParameters } from "../../utils/parameters/templateParameters"; +import { exportJson } from "../../utils/serialization/exportJson"; +import type { AnthropicClaudeExecutionToolsDirectOptions } from "./AnthropicClaudeExecutionToolsOptions"; +import { ANTHROPIC_CLAUDE_MODELS } from "./anthropic-claude-models"; +import { computeAnthropicClaudeUsage } from "./computeAnthropicClaudeUsage"; /** * Execution Tools for calling Anthropic Claude API. @@ -28,153 +28,182 @@ import { computeAnthropicClaudeUsage } from './computeAnthropicClaudeUsage'; * @public exported from `@promptbook/anthropic-claude` * @deprecated use `createAnthropicClaudeExecutionTools` instead */ -export class AnthropicClaudeExecutionTools implements LlmExecutionTools /* <- TODO: [🍚] `, Destroyable` */ { - /** - * Anthropic Claude API client. - */ - private client: Anthropic | null = null; - - /** - * Creates Anthropic Claude Execution Tools. - * - * @param options which are relevant are directly passed to the Anthropic Claude client - */ - public constructor(protected readonly options: AnthropicClaudeExecutionToolsDirectOptions = { isProxied: false }) {} - - public get title(): string_title & string_markdown_text { - return 'Anthropic Claude'; - } - - public get description(): string_markdown { - return 'Use all models provided by Anthropic Claude'; - } - - public async getClient(): Promise { - if (this.client === null) { - // Note: Passing only Anthropic Claude relevant options to Anthropic constructor - const anthropicOptions: ClientOptions = { ...this.options }; - delete (anthropicOptions as really_any).isVerbose; - delete (anthropicOptions as really_any).isProxied; - this.client = new Anthropic(anthropicOptions); - } - - return this.client; - } - - /** - * Check the `options` passed to `constructor` - */ - public async checkConfiguration(): Promise { - await this.getClient(); - // TODO: [🎍] Do here a real check that API is online, working and API key is correct - } - - /** - * List all available Anthropic Claude models that can be used - */ - public listModels(): ReadonlyArray { - return ANTHROPIC_CLAUDE_MODELS; - } - - /** - * Calls Anthropic Claude API to use a chat model. - */ - public async callChatModel( - prompt: Pick, - ): Promise { - if (this.options.isVerbose) { - console.info('💬 Anthropic Claude callChatModel call'); - } - - const { content, parameters, modelRequirements } = prompt; - - const client = await this.getClient(); - - // TODO: [☂] Use here more modelRequirements - if (modelRequirements.modelVariant !== 'CHAT') { - throw new PipelineExecutionError('Use callChatModel only for CHAT variant'); - } - - const modelName = modelRequirements.modelName || this.getDefaultChatModel().modelName; - - const rawPromptContent = templateParameters(content, { ...parameters, modelName }); - const rawRequest: MessageCreateParamsNonStreaming = { - model: modelRequirements.modelName || this.getDefaultChatModel().modelName, - max_tokens: modelRequirements.maxTokens || 4096, - // <- TODO: [🌾] Make some global max cap for maxTokens - temperature: modelRequirements.temperature, - system: modelRequirements.systemMessage, - - // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools - // <- Note: [🧆] - messages: [ - { - role: 'user', - content: rawPromptContent, - // <- TODO: [🧠][♏] Maybe if expecting JSON and its not specified in prompt content, append the instructions - // @see https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/increase-consistency#specify-the-desired-output-format - }, - ], - // TODO: Is here some equivalent of user identification?> user: this.options.user, - }; - const start: string_date_iso8601 = $getCurrentDate(); - let complete: string_date_iso8601; - - if (this.options.isVerbose) { - console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4)); - } - const rawResponse = await client.messages.create(rawRequest).catch((error) => { - if (this.options.isVerbose) { - console.info(colors.bgRed('error'), error); - } - throw error; - }); - - if (this.options.isVerbose) { - console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4)); - } - - if (!rawResponse.content[0]) { - throw new PipelineExecutionError('No content from Anthropic Claude'); - } - - if (rawResponse.content.length > 1) { - throw new PipelineExecutionError('More than one content blocks from Anthropic Claude'); - } - - const contentBlock = rawResponse.content[0]; - - if (contentBlock.type !== 'text') { - throw new PipelineExecutionError(`Returned content is not "text" type but "${contentBlock.type}"`); - } - - const resultContent = contentBlock.text; - - // eslint-disable-next-line prefer-const - complete = $getCurrentDate(); - const usage = computeAnthropicClaudeUsage(rawPromptContent || '', resultContent || '', rawResponse); - - return exportJson({ - name: 'promptResult', - message: `Result of \`AzureOpenAiExecutionTools.callChatModel\``, - order: [], - value: { - content: resultContent, - modelName: rawResponse.model, - timing: { - start, - complete, - }, - usage, - rawPromptContent, - rawRequest, - rawResponse, - // <- [🗯] - }, - }); - } - - /* +export class AnthropicClaudeExecutionTools + implements LlmExecutionTools /* <- TODO: [🍚] `, Destroyable` */ +{ + /** + * Anthropic Claude API client. + */ + private client: Anthropic | null = null; + + /** + * Creates Anthropic Claude Execution Tools. + * + * @param options which are relevant are directly passed to the Anthropic Claude client + */ + public constructor( + protected readonly options: AnthropicClaudeExecutionToolsDirectOptions = { + isProxied: false, + }, + ) {} + + public get title(): string_title & string_markdown_text { + return "Anthropic Claude"; + } + + public get description(): string_markdown { + return "Use all models provided by Anthropic Claude"; + } + + public async getClient(): Promise { + if (this.client === null) { + // Note: Passing only Anthropic Claude relevant options to Anthropic constructor + const anthropicOptions: ClientOptions = { ...this.options }; + delete (anthropicOptions as really_any).isVerbose; + delete (anthropicOptions as really_any).isProxied; + this.client = new Anthropic(anthropicOptions); + } + + return this.client; + } + + /** + * Check the `options` passed to `constructor` + */ + public async checkConfiguration(): Promise { + await this.getClient(); + // TODO: [🎍] Do here a real check that API is online, working and API key is correct + } + + /** + * List all available Anthropic Claude models that can be used + */ + public listModels(): ReadonlyArray { + return ANTHROPIC_CLAUDE_MODELS; + } + + /** + * Calls Anthropic Claude API to use a chat model. + */ + public async callChatModel( + prompt: Pick, + ): Promise { + if (this.options.isVerbose) { + console.info("💬 Anthropic Claude callChatModel call"); + } + + const { content, parameters, modelRequirements } = prompt; + + const client = await this.getClient(); + + // TODO: [☂] Use here more modelRequirements + if (modelRequirements.modelVariant !== "CHAT") { + throw new PipelineExecutionError( + "Use callChatModel only for CHAT variant", + ); + } + + const modelName = + modelRequirements.modelName || this.getDefaultChatModel().modelName; + + const rawPromptContent = templateParameters(content, { + ...parameters, + modelName, + }); + const rawRequest: MessageCreateParamsNonStreaming = { + model: + modelRequirements.modelName || this.getDefaultChatModel().modelName, + max_tokens: modelRequirements.maxTokens || 4096, + // <- TODO: [🌾] Make some global max cap for maxTokens + temperature: modelRequirements.temperature, + system: modelRequirements.systemMessage, + + // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools + // <- Note: [🧆] + messages: [ + { + role: "user", + content: rawPromptContent, + // <- TODO: [🧠][♏] Maybe if expecting JSON and its not specified in prompt content, append the instructions + // @see https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/increase-consistency#specify-the-desired-output-format + }, + ], + // TODO: Is here some equivalent of user identification?> user: this.options.user, + }; + const start: string_date_iso8601 = $getCurrentDate(); + let complete: string_date_iso8601; + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("rawRequest"), + JSON.stringify(rawRequest, null, 4), + ); + } + const rawResponse = await client.messages + .create(rawRequest) + .catch((error) => { + if (this.options.isVerbose) { + console.info(colors.bgRed("error"), error); + } + throw error; + }); + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("rawResponse"), + JSON.stringify(rawResponse, null, 4), + ); + } + + if (!rawResponse.content[0]) { + throw new PipelineExecutionError("No content from Anthropic Claude"); + } + + if (rawResponse.content.length > 1) { + throw new PipelineExecutionError( + "More than one content blocks from Anthropic Claude", + ); + } + + const contentBlock = rawResponse.content[0]; + + if (contentBlock.type !== "text") { + throw new PipelineExecutionError( + `Returned content is not "text" type but "${contentBlock.type}"`, + ); + } + + const resultContent = contentBlock.text; + + // eslint-disable-next-line prefer-const + complete = $getCurrentDate(); + const usage = computeAnthropicClaudeUsage( + rawPromptContent || "", + resultContent || "", + rawResponse, + ); + + return exportJson({ + name: "promptResult", + message: `Result of \`AzureOpenAiExecutionTools.callChatModel\``, + order: [], + value: { + content: resultContent, + modelName: rawResponse.model, + timing: { + start, + complete, + }, + usage, + rawPromptContent, + rawRequest, + rawResponse, + // <- [🗯] + }, + }); + } + + /* TODO: [👏] public async callCompletionModel( prompt: Pick, @@ -252,38 +281,40 @@ export class AnthropicClaudeExecutionTools implements LlmExecutionTools /* <- TO } */ - // <- Note: [🤖] callXxxModel - - /** - * Get the model that should be used as default - */ - private getDefaultModel(defaultModelName: string_model_name): AvailableModel { - const model = ANTHROPIC_CLAUDE_MODELS.find(({ modelName }) => modelName.startsWith(defaultModelName)); - if (model === undefined) { - throw new UnexpectedError( - spaceTrim( - (block) => - ` + // <- Note: [🤖] callXxxModel + + /** + * Get the model that should be used as default + */ + private getDefaultModel(defaultModelName: string_model_name): AvailableModel { + const model = ANTHROPIC_CLAUDE_MODELS.find(({ modelName }) => + modelName.startsWith(defaultModelName), + ); + if (model === undefined) { + throw new UnexpectedError( + spaceTrim( + (block) => + ` Cannot find model in OpenAI models with name "${defaultModelName}" which should be used as default. Available models: - ${block(ANTHROPIC_CLAUDE_MODELS.map(({ modelName }) => `- "${modelName}"`).join('\n'))} + ${block(ANTHROPIC_CLAUDE_MODELS.map(({ modelName }) => `- "${modelName}"`).join("\n"))} `, - ), - ); - } - return model; - } - - /** - * Default model for chat variant. - */ - private getDefaultChatModel(): AvailableModel { - return this.getDefaultModel('claude-3-5-sonnet'); - } - - // <- Note: [🤖] getDefaultXxxModel + ), + ); + } + return model; + } + + /** + * Default model for chat variant. + */ + private getDefaultChatModel(): AvailableModel { + return this.getDefaultModel("claude-3-5-sonnet"); + } + + // <- Note: [🤖] getDefaultXxxModel } /** diff --git a/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.ts b/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.ts index bdbc868f47..df791fcc23 100644 --- a/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.ts +++ b/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.ts @@ -1,6 +1,6 @@ -import type { ClientOptions } from '@anthropic-ai/sdk'; -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; -import type { RemoteClientOptions } from '../../remote-server/types/RemoteClientOptions'; +import type { ClientOptions } from "@anthropic-ai/sdk"; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; +import type { RemoteClientOptions } from "../../remote-server/types/RemoteClientOptions"; /** * Options for `AnthropicClaudeExecutionTools` @@ -9,8 +9,8 @@ import type { RemoteClientOptions } from '../../remote-server/types/RemoteClient * @public exported from `@promptbook/anthropic-claude` */ export type AnthropicClaudeExecutionToolsOptions = - | AnthropicClaudeExecutionToolsDirectOptions - | AnthropicClaudeExecutionToolsProxiedOptions; + | AnthropicClaudeExecutionToolsDirectOptions + | AnthropicClaudeExecutionToolsProxiedOptions; /** * Options for directly used `AnthropicClaudeExecutionTools` @@ -19,9 +19,9 @@ export type AnthropicClaudeExecutionToolsOptions = * @public exported from `@promptbook/anthropic-claude` */ export type AnthropicClaudeExecutionToolsDirectOptions = CommonToolsOptions & - ClientOptions & { - isProxied?: false; - }; + ClientOptions & { + isProxied?: false; + }; /** * Options for proxied `AnthropicClaudeExecutionTools` @@ -30,9 +30,9 @@ export type AnthropicClaudeExecutionToolsDirectOptions = CommonToolsOptions & * @public exported from `@promptbook/anthropic-claude` */ export type AnthropicClaudeExecutionToolsProxiedOptions = CommonToolsOptions & - ClientOptions & { - isProxied: true; - } & Pick, 'remoteUrl' | 'path'>; + ClientOptions & { + isProxied: true; + } & Pick, "remoteUrl" | "path">; /** * TODO: [🧠][🤺] Pass `userId` diff --git a/src/llm-providers/anthropic-claude/anthropic-claude-models.ts b/src/llm-providers/anthropic-claude/anthropic-claude-models.ts index e7bb2bdd1a..f845c3d4d9 100644 --- a/src/llm-providers/anthropic-claude/anthropic-claude-models.ts +++ b/src/llm-providers/anthropic-claude/anthropic-claude-models.ts @@ -1,7 +1,7 @@ -import type { AvailableModel } from '../../execution/AvailableModel'; -import type { number_usd } from '../../types/typeAliases'; -import { exportJson } from '../../utils/serialization/exportJson'; -import { computeUsage } from '../openai/computeUsage'; +import type { AvailableModel } from "../../execution/AvailableModel"; +import type { number_usd } from "../../types/typeAliases"; +import { exportJson } from "../../utils/serialization/exportJson"; +import { computeUsage } from "../openai/computeUsage"; /** * List of available Anthropic Claude models with pricing @@ -12,81 +12,81 @@ import { computeUsage } from '../openai/computeUsage'; * @public exported from `@promptbook/anthropic-claude` */ export const ANTHROPIC_CLAUDE_MODELS: ReadonlyArray< - AvailableModel & { - pricing?: { - readonly prompt: number_usd; - readonly output: number_usd; - }; - } + AvailableModel & { + pricing?: { + readonly prompt: number_usd; + readonly output: number_usd; + }; + } > = exportJson({ - name: 'ANTHROPIC_CLAUDE_MODELS', - value: [ - { - modelVariant: 'CHAT', - modelTitle: 'Claude 3.5 Sonnet', - modelName: 'claude-3-5-sonnet-20240620', - pricing: { - prompt: computeUsage(`$3.00 / 1M tokens`), - output: computeUsage(`$15.00 / 1M tokens`), - }, - }, - { - modelVariant: 'CHAT', - modelTitle: 'Claude 3 Opus', - modelName: 'claude-3-opus-20240229', - pricing: { - prompt: computeUsage(`$15.00 / 1M tokens`), - output: computeUsage(`$75.00 / 1M tokens`), - }, - }, - { - modelVariant: 'CHAT', - modelTitle: 'Claude 3 Sonnet', - modelName: 'claude-3-sonnet-20240229', - pricing: { - prompt: computeUsage(`$3.00 / 1M tokens`), - output: computeUsage(`$15.00 / 1M tokens`), - }, - }, - { - modelVariant: 'CHAT', - modelTitle: 'Claude 3 Haiku', - modelName: ' claude-3-haiku-20240307', - pricing: { - prompt: computeUsage(`$0.25 / 1M tokens`), - output: computeUsage(`$1.25 / 1M tokens`), - }, - }, - { - modelVariant: 'CHAT', - modelTitle: 'Claude 2.1', - modelName: 'claude-2.1', - pricing: { - prompt: computeUsage(`$8.00 / 1M tokens`), - output: computeUsage(`$24.00 / 1M tokens`), - }, - }, - { - modelVariant: 'CHAT', - modelTitle: 'Claude 2', - modelName: 'claude-2.0', - pricing: { - prompt: computeUsage(`$8.00 / 1M tokens`), - output: computeUsage(`$24.00 / 1M tokens`), - }, - }, - { - modelVariant: 'CHAT', - modelTitle: ' Claude Instant 1.2', - modelName: 'claude-instant-1.2', - pricing: { - prompt: computeUsage(`$0.80 / 1M tokens`), - output: computeUsage(`$2.40 / 1M tokens`), - }, - }, + name: "ANTHROPIC_CLAUDE_MODELS", + value: [ + { + modelVariant: "CHAT", + modelTitle: "Claude 3.5 Sonnet", + modelName: "claude-3-5-sonnet-20240620", + pricing: { + prompt: computeUsage(`$3.00 / 1M tokens`), + output: computeUsage(`$15.00 / 1M tokens`), + }, + }, + { + modelVariant: "CHAT", + modelTitle: "Claude 3 Opus", + modelName: "claude-3-opus-20240229", + pricing: { + prompt: computeUsage(`$15.00 / 1M tokens`), + output: computeUsage(`$75.00 / 1M tokens`), + }, + }, + { + modelVariant: "CHAT", + modelTitle: "Claude 3 Sonnet", + modelName: "claude-3-sonnet-20240229", + pricing: { + prompt: computeUsage(`$3.00 / 1M tokens`), + output: computeUsage(`$15.00 / 1M tokens`), + }, + }, + { + modelVariant: "CHAT", + modelTitle: "Claude 3 Haiku", + modelName: " claude-3-haiku-20240307", + pricing: { + prompt: computeUsage(`$0.25 / 1M tokens`), + output: computeUsage(`$1.25 / 1M tokens`), + }, + }, + { + modelVariant: "CHAT", + modelTitle: "Claude 2.1", + modelName: "claude-2.1", + pricing: { + prompt: computeUsage(`$8.00 / 1M tokens`), + output: computeUsage(`$24.00 / 1M tokens`), + }, + }, + { + modelVariant: "CHAT", + modelTitle: "Claude 2", + modelName: "claude-2.0", + pricing: { + prompt: computeUsage(`$8.00 / 1M tokens`), + output: computeUsage(`$24.00 / 1M tokens`), + }, + }, + { + modelVariant: "CHAT", + modelTitle: " Claude Instant 1.2", + modelName: "claude-instant-1.2", + pricing: { + prompt: computeUsage(`$0.80 / 1M tokens`), + output: computeUsage(`$2.40 / 1M tokens`), + }, + }, - // TODO: [main] !!3 Claude 1 and 2 has also completion versions - ask Hoagy - ], + // TODO: [main] !!3 Claude 1 and 2 has also completion versions - ask Hoagy + ], }); /** diff --git a/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.test.ts b/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.test.ts index 5991342c27..bd6cf6a037 100644 --- a/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.test.ts +++ b/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.test.ts @@ -1,70 +1,70 @@ -import { describe, expect, it } from '@jest/globals'; -import { computeAnthropicClaudeUsage } from './computeAnthropicClaudeUsage'; +import { describe, expect, it } from "@jest/globals"; +import { computeAnthropicClaudeUsage } from "./computeAnthropicClaudeUsage"; -describe('how computeAnthropicClaudeUsage works', () => { - // TODO: [🐞] Test Chat and Completion models +describe("how computeAnthropicClaudeUsage works", () => { + // TODO: [🐞] Test Chat and Completion models - it('should compute a Anthropic Claude usage of embedding model', () => { - expect( - computeAnthropicClaudeUsage('', '', { - model: 'claude-3-opus-20240229', - usage: { - input_tokens: 7, - output_tokens: 3, - }, - }), - ).toEqual({ - input: { - charactersCount: { - value: 0, - }, - linesCount: { - value: 0, - }, - pagesCount: { - value: 0, - }, - paragraphsCount: { - value: 0, - }, - sentencesCount: { - value: 0, - }, - tokensCount: { - value: 7, - }, - wordsCount: { - value: 0, - }, - }, - output: { - charactersCount: { - value: 0, - }, - linesCount: { - value: 0, - }, - pagesCount: { - value: 0, - }, - paragraphsCount: { - value: 0, - }, - sentencesCount: { - value: 0, - }, - tokensCount: { - value: 3, - }, - wordsCount: { - value: 0, - }, - }, - price: { - value: 0.00033, - }, - }); - }); + it("should compute a Anthropic Claude usage of embedding model", () => { + expect( + computeAnthropicClaudeUsage("", "", { + model: "claude-3-opus-20240229", + usage: { + input_tokens: 7, + output_tokens: 3, + }, + }), + ).toEqual({ + input: { + charactersCount: { + value: 0, + }, + linesCount: { + value: 0, + }, + pagesCount: { + value: 0, + }, + paragraphsCount: { + value: 0, + }, + sentencesCount: { + value: 0, + }, + tokensCount: { + value: 7, + }, + wordsCount: { + value: 0, + }, + }, + output: { + charactersCount: { + value: 0, + }, + linesCount: { + value: 0, + }, + pagesCount: { + value: 0, + }, + paragraphsCount: { + value: 0, + }, + sentencesCount: { + value: 0, + }, + tokensCount: { + value: 3, + }, + wordsCount: { + value: 0, + }, + }, + price: { + value: 0.00033, + }, + }); + }); }); /** diff --git a/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.ts b/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.ts index 458dec9a95..4eab74cdd7 100644 --- a/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.ts +++ b/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.ts @@ -1,12 +1,12 @@ -import type Anthropic from '@anthropic-ai/sdk'; -import type { PartialDeep } from 'type-fest'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import type { PromptResultUsage } from '../../execution/PromptResultUsage'; -import type { UncertainNumber } from '../../execution/UncertainNumber'; -import { computeUsageCounts } from '../../execution/utils/computeUsageCounts'; -import { uncertainNumber } from '../../execution/utils/uncertainNumber'; -import type { Prompt } from '../../types/Prompt'; -import { ANTHROPIC_CLAUDE_MODELS } from './anthropic-claude-models'; +import type Anthropic from "@anthropic-ai/sdk"; +import type { PartialDeep } from "type-fest"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import type { PromptResultUsage } from "../../execution/PromptResultUsage"; +import type { UncertainNumber } from "../../execution/UncertainNumber"; +import { computeUsageCounts } from "../../execution/utils/computeUsageCounts"; +import { uncertainNumber } from "../../execution/utils/uncertainNumber"; +import type { Prompt } from "../../types/Prompt"; +import { ANTHROPIC_CLAUDE_MODELS } from "./anthropic-claude-models"; /** * Computes the usage of the Anthropic Claude API based on the response from Anthropic Claude @@ -18,42 +18,51 @@ import { ANTHROPIC_CLAUDE_MODELS } from './anthropic-claude-models'; * @private internal utility of `AnthropicClaudeExecutionTools` */ export function computeAnthropicClaudeUsage( - promptContent: Prompt['content'], // <- Note: Intentionally using [] to access type properties to bring jsdoc from Prompt/PromptResult to consumer - resultContent: string, - rawResponse: PartialDeep>, + promptContent: Prompt["content"], // <- Note: Intentionally using [] to access type properties to bring jsdoc from Prompt/PromptResult to consumer + resultContent: string, + rawResponse: PartialDeep>, ): PromptResultUsage { - if (rawResponse.usage === undefined) { - throw new PipelineExecutionError('The usage is not defined in the response from Anthropic Claude'); - } - - if (rawResponse.usage?.input_tokens === undefined) { - throw new PipelineExecutionError('In Anthropic Claude response `usage.prompt_tokens` not defined'); - } - - const inputTokens = rawResponse.usage.input_tokens; - const outputTokens = rawResponse.usage?.output_tokens || 0; - - const modelInfo = ANTHROPIC_CLAUDE_MODELS.find((model) => model.modelName === rawResponse.model); - - let price: UncertainNumber; - - if (modelInfo === undefined || modelInfo.pricing === undefined) { - price = uncertainNumber(); - } else { - price = uncertainNumber(inputTokens * modelInfo.pricing.prompt + outputTokens * modelInfo.pricing.output); - } - - return { - price, - input: { - tokensCount: uncertainNumber(rawResponse.usage.input_tokens), - ...computeUsageCounts(promptContent), - }, - output: { - tokensCount: uncertainNumber(outputTokens), - ...computeUsageCounts(resultContent), - }, - }; + if (rawResponse.usage === undefined) { + throw new PipelineExecutionError( + "The usage is not defined in the response from Anthropic Claude", + ); + } + + if (rawResponse.usage?.input_tokens === undefined) { + throw new PipelineExecutionError( + "In Anthropic Claude response `usage.prompt_tokens` not defined", + ); + } + + const inputTokens = rawResponse.usage.input_tokens; + const outputTokens = rawResponse.usage?.output_tokens || 0; + + const modelInfo = ANTHROPIC_CLAUDE_MODELS.find( + (model) => model.modelName === rawResponse.model, + ); + + let price: UncertainNumber; + + if (modelInfo === undefined || modelInfo.pricing === undefined) { + price = uncertainNumber(); + } else { + price = uncertainNumber( + inputTokens * modelInfo.pricing.prompt + + outputTokens * modelInfo.pricing.output, + ); + } + + return { + price, + input: { + tokensCount: uncertainNumber(rawResponse.usage.input_tokens), + ...computeUsageCounts(promptContent), + }, + output: { + tokensCount: uncertainNumber(outputTokens), + ...computeUsageCounts(resultContent), + }, + }; } /** diff --git a/src/llm-providers/anthropic-claude/createAnthropicClaudeExecutionTools.ts b/src/llm-providers/anthropic-claude/createAnthropicClaudeExecutionTools.ts index 2626434a73..1d798ca160 100644 --- a/src/llm-providers/anthropic-claude/createAnthropicClaudeExecutionTools.ts +++ b/src/llm-providers/anthropic-claude/createAnthropicClaudeExecutionTools.ts @@ -1,7 +1,7 @@ -import type { LlmExecutionToolsConstructor } from '../../execution/LlmExecutionToolsConstructor'; -import { RemoteLlmExecutionTools } from '../remote/RemoteLlmExecutionTools'; -import { AnthropicClaudeExecutionTools } from './AnthropicClaudeExecutionTools'; -import type { AnthropicClaudeExecutionToolsOptions } from './AnthropicClaudeExecutionToolsOptions'; +import type { LlmExecutionToolsConstructor } from "../../execution/LlmExecutionToolsConstructor"; +import { RemoteLlmExecutionTools } from "../remote/RemoteLlmExecutionTools"; +import { AnthropicClaudeExecutionTools } from "./AnthropicClaudeExecutionTools"; +import type { AnthropicClaudeExecutionToolsOptions } from "./AnthropicClaudeExecutionToolsOptions"; /** * Execution Tools for calling Anthropic Claude API. @@ -9,33 +9,35 @@ import type { AnthropicClaudeExecutionToolsOptions } from './AnthropicClaudeExec * @public exported from `@promptbook/anthropic-claude` */ export const createAnthropicClaudeExecutionTools = Object.assign( - (options: AnthropicClaudeExecutionToolsOptions): AnthropicClaudeExecutionTools | RemoteLlmExecutionTools => { - if (options.isProxied) { - return new RemoteLlmExecutionTools({ - ...options, - identification: { - isAnonymous: true, - llmToolsConfiguration: [ - { - title: 'Anthropic Claude (proxied)', - packageName: '@promptbook/anthropic-claude', - className: 'AnthropicClaudeExecutionTools', - options: { - ...options, - isProxied: false, - }, - }, - ], - }, - }); - } + ( + options: AnthropicClaudeExecutionToolsOptions, + ): AnthropicClaudeExecutionTools | RemoteLlmExecutionTools => { + if (options.isProxied) { + return new RemoteLlmExecutionTools({ + ...options, + identification: { + isAnonymous: true, + llmToolsConfiguration: [ + { + title: "Anthropic Claude (proxied)", + packageName: "@promptbook/anthropic-claude", + className: "AnthropicClaudeExecutionTools", + options: { + ...options, + isProxied: false, + }, + }, + ], + }, + }); + } - return new AnthropicClaudeExecutionTools(options); - }, - { - packageName: '@promptbook/anthropic-claude', - className: 'AnthropicClaudeExecutionTools', - }, + return new AnthropicClaudeExecutionTools(options); + }, + { + packageName: "@promptbook/anthropic-claude", + className: "AnthropicClaudeExecutionTools", + }, ) satisfies LlmExecutionToolsConstructor; /** diff --git a/src/llm-providers/anthropic-claude/playground/playground.ts b/src/llm-providers/anthropic-claude/playground/playground.ts index ec9a5d136d..ebc4ee319b 100644 --- a/src/llm-providers/anthropic-claude/playground/playground.ts +++ b/src/llm-providers/anthropic-claude/playground/playground.ts @@ -1,68 +1,71 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { usageToHuman } from '../../../execution/utils/usageToHuman'; -import type { Prompt } from '../../../types/Prompt'; -import { keepUnused } from '../../../utils/organization/keepUnused'; -import { createAnthropicClaudeExecutionTools } from '../createAnthropicClaudeExecutionTools'; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { usageToHuman } from "../../../execution/utils/usageToHuman"; +import type { Prompt } from "../../../types/Prompt"; +import { keepUnused } from "../../../utils/organization/keepUnused"; +import { createAnthropicClaudeExecutionTools } from "../createAnthropicClaudeExecutionTools"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 Anthropic Claude Playground`); + console.info(`🧸 Anthropic Claude Playground`); - // Do here stuff you want to test - //========================================> + // Do here stuff you want to test + //========================================> - const anthropicClaudeExecutionTools = createAnthropicClaudeExecutionTools({ - // isProxied: true, - // remoteUrl: DEFAULT_REMOTE_URL, - // path: DEFAULT_REMOTE_URL_PATH, // <- [🧜‍♂️] - isVerbose: true, - apiKey: process.env.ANTHROPIC_CLAUDE_API_KEY!, - }); + const anthropicClaudeExecutionTools = createAnthropicClaudeExecutionTools({ + // isProxied: true, + // remoteUrl: DEFAULT_REMOTE_URL, + // path: DEFAULT_REMOTE_URL_PATH, // <- [🧜‍♂️] + isVerbose: true, + apiKey: process.env.ANTHROPIC_CLAUDE_API_KEY!, + }); - keepUnused(anthropicClaudeExecutionTools); - keepUnused(usageToHuman); - keepUnused(); + keepUnused(anthropicClaudeExecutionTools); + keepUnused(usageToHuman); + keepUnused(); - /*/ + /*/ const models = await anthropicClaudeExecutionTools.listModels(); console.info({ models }); /**/ - /**/ - const chatPrompt = { - title: 'Poem about Prague', - parameters: {}, - content: `Write me something about Prague`, - modelRequirements: { - modelVariant: 'CHAT', - systemMessage: 'You are an assistant who only speaks in rhymes.', - temperature: 1, - }, - format: 'JSON', - } as const satisfies Prompt; - const chatPromptResult = await anthropicClaudeExecutionTools.callChatModel(chatPrompt); - console.info({ chatPromptResult }); - console.info(colors.cyan(usageToHuman(chatPromptResult.usage))); - console.info(colors.bgBlue(' User: ') + colors.blue(chatPrompt.content)); - console.info(colors.bgGreen(' Chat: ') + colors.green(chatPromptResult.content)); - /**/ - - /*/ + /**/ + const chatPrompt = { + title: "Poem about Prague", + parameters: {}, + content: `Write me something about Prague`, + modelRequirements: { + modelVariant: "CHAT", + systemMessage: "You are an assistant who only speaks in rhymes.", + temperature: 1, + }, + format: "JSON", + } as const satisfies Prompt; + const chatPromptResult = + await anthropicClaudeExecutionTools.callChatModel(chatPrompt); + console.info({ chatPromptResult }); + console.info(colors.cyan(usageToHuman(chatPromptResult.usage))); + console.info(colors.bgBlue(" User: ") + colors.blue(chatPrompt.content)); + console.info( + colors.bgGreen(" Chat: ") + colors.green(chatPromptResult.content), + ); + /**/ + + /*/ // TODO: [👏] Make Claude completion models work const completionPrompt = { title: 'Hello', @@ -78,19 +81,19 @@ async function playground() { console.info(chalk.green(completionPrompt.content + completionPromptResult.content)); /**/ - /*/ + /*/ // TODO: Test Translations in playground /**/ - /*/ + /*/ // TODO: Test Embeddings in playground /**/ - /*/ + /*/ // <- Note: [🤖] Test here new model variant if needed /**/ - //========================================/ + //========================================/ } /** diff --git a/src/llm-providers/anthropic-claude/playground/tsconfig.json b/src/llm-providers/anthropic-claude/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/llm-providers/anthropic-claude/playground/tsconfig.json +++ b/src/llm-providers/anthropic-claude/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/llm-providers/anthropic-claude/register-configuration.ts b/src/llm-providers/anthropic-claude/register-configuration.ts index 533331e6f1..bd31a8e244 100644 --- a/src/llm-providers/anthropic-claude/register-configuration.ts +++ b/src/llm-providers/anthropic-claude/register-configuration.ts @@ -1,9 +1,9 @@ -import { DEFAULT_REMOTE_URL } from '../../config'; -import { DEFAULT_REMOTE_URL_PATH } from '../../config'; -import type { string_name } from '../../types/typeAliases'; -import type { Registration } from '../../utils/$Register'; -import { $llmToolsMetadataRegister } from '../_common/register/$llmToolsMetadataRegister'; -import type { LlmToolsConfiguration } from '../_common/register/LlmToolsConfiguration'; +import { DEFAULT_REMOTE_URL } from "../../config"; +import { DEFAULT_REMOTE_URL_PATH } from "../../config"; +import type { string_name } from "../../types/typeAliases"; +import type { Registration } from "../../utils/$Register"; +import { $llmToolsMetadataRegister } from "../_common/register/$llmToolsMetadataRegister"; +import type { LlmToolsConfiguration } from "../_common/register/LlmToolsConfiguration"; /** * Registration of LLM provider metadata @@ -14,44 +14,46 @@ import type { LlmToolsConfiguration } from '../_common/register/LlmToolsConfigur * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _AnthropicClaudeMetadataRegistration: Registration = $llmToolsMetadataRegister.register({ - title: 'Anthropic Claude', - packageName: '@promptbook/anthropic-claude', - className: 'AnthropicClaudeExecutionTools', - envVariables: ['ANTHROPIC_CLAUDE_API_KEY'], +export const _AnthropicClaudeMetadataRegistration: Registration = + $llmToolsMetadataRegister.register({ + title: "Anthropic Claude", + packageName: "@promptbook/anthropic-claude", + className: "AnthropicClaudeExecutionTools", + envVariables: ["ANTHROPIC_CLAUDE_API_KEY"], - getBoilerplateConfiguration(): LlmToolsConfiguration[number] { - return { - title: 'Anthropic Claude (boilerplate)', - packageName: '@promptbook/anthropic-claude', - className: 'AnthropicClaudeExecutionTools', - options: { - apiKey: 'sk-ant-api03-', - isProxied: true, - remoteUrl: DEFAULT_REMOTE_URL, - path: DEFAULT_REMOTE_URL_PATH, - }, - }; - }, + getBoilerplateConfiguration(): LlmToolsConfiguration[number] { + return { + title: "Anthropic Claude (boilerplate)", + packageName: "@promptbook/anthropic-claude", + className: "AnthropicClaudeExecutionTools", + options: { + apiKey: "sk-ant-api03-", + isProxied: true, + remoteUrl: DEFAULT_REMOTE_URL, + path: DEFAULT_REMOTE_URL_PATH, + }, + }; + }, - createConfigurationFromEnv(env: Record): LlmToolsConfiguration[number] | null { - // Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically - if (typeof env.ANTHROPIC_CLAUDE_API_KEY === 'string') { - return { - title: 'Claude (from env)', - packageName: '@promptbook/anthropic-claude', - className: 'AnthropicClaudeExecutionTools', - options: { - apiKey: env.ANTHROPIC_CLAUDE_API_KEY!, - }, - }; - } - - return null; - }, -}); + createConfigurationFromEnv( + env: Record, + ): LlmToolsConfiguration[number] | null { + // Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically + if (typeof env.ANTHROPIC_CLAUDE_API_KEY === "string") { + return { + title: "Claude (from env)", + packageName: "@promptbook/anthropic-claude", + className: "AnthropicClaudeExecutionTools", + options: { + apiKey: env.ANTHROPIC_CLAUDE_API_KEY!, + }, + }; + } + return null; + }, + }); /** * Note: [💞] Ignore a discrepancy between file name and entity name - */ \ No newline at end of file + */ diff --git a/src/llm-providers/anthropic-claude/register-constructor.ts b/src/llm-providers/anthropic-claude/register-constructor.ts index aa9830bb9a..cc30553009 100644 --- a/src/llm-providers/anthropic-claude/register-constructor.ts +++ b/src/llm-providers/anthropic-claude/register-constructor.ts @@ -1,6 +1,6 @@ -import type { Registration } from '../../utils/$Register'; -import { $llmToolsRegister } from '../_common/register/$llmToolsRegister'; -import { createAnthropicClaudeExecutionTools } from './createAnthropicClaudeExecutionTools'; +import type { Registration } from "../../utils/$Register"; +import { $llmToolsRegister } from "../_common/register/$llmToolsRegister"; +import { createAnthropicClaudeExecutionTools } from "./createAnthropicClaudeExecutionTools"; /** * Registration of LLM provider @@ -12,9 +12,8 @@ import { createAnthropicClaudeExecutionTools } from './createAnthropicClaudeExec * @public exported from `@promptbook/cli` * */ -export const _AnthropicClaudeRegistration: Registration = $llmToolsRegister.register( - createAnthropicClaudeExecutionTools, -); +export const _AnthropicClaudeRegistration: Registration = + $llmToolsRegister.register(createAnthropicClaudeExecutionTools); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.ts b/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.ts index 9be9825514..1ee3601b64 100644 --- a/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.ts +++ b/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.ts @@ -1,27 +1,27 @@ -import { AzureKeyCredential, OpenAIClient } from '@azure/openai'; -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { CONNECTION_TIMEOUT_MS } from '../../config'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { AvailableModel } from '../../execution/AvailableModel'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import type { ChatPromptResult } from '../../execution/PromptResult'; -import type { CompletionPromptResult } from '../../execution/PromptResult'; -import type { PromptResultUsage } from '../../execution/PromptResultUsage'; -import { computeUsageCounts } from '../../execution/utils/computeUsageCounts'; -import { uncertainNumber } from '../../execution/utils/uncertainNumber'; -import type { Prompt } from '../../types/Prompt'; -import type { string_completion_prompt } from '../../types/typeAliases'; -import type { string_date_iso8601 } from '../../types/typeAliases'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_title } from '../../types/typeAliases'; -import { $getCurrentDate } from '../../utils/$getCurrentDate'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import { templateParameters } from '../../utils/parameters/templateParameters'; -import { exportJson } from '../../utils/serialization/exportJson'; -import { OPENAI_MODELS } from '../openai/openai-models'; -import type { AzureOpenAiExecutionToolsOptions } from './AzureOpenAiExecutionToolsOptions'; +import { AzureKeyCredential, OpenAIClient } from "@azure/openai"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { CONNECTION_TIMEOUT_MS } from "../../config"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { AvailableModel } from "../../execution/AvailableModel"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import type { ChatPromptResult } from "../../execution/PromptResult"; +import type { CompletionPromptResult } from "../../execution/PromptResult"; +import type { PromptResultUsage } from "../../execution/PromptResultUsage"; +import { computeUsageCounts } from "../../execution/utils/computeUsageCounts"; +import { uncertainNumber } from "../../execution/utils/uncertainNumber"; +import type { Prompt } from "../../types/Prompt"; +import type { string_completion_prompt } from "../../types/typeAliases"; +import type { string_date_iso8601 } from "../../types/typeAliases"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_title } from "../../types/typeAliases"; +import { $getCurrentDate } from "../../utils/$getCurrentDate"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import { templateParameters } from "../../utils/parameters/templateParameters"; +import { exportJson } from "../../utils/serialization/exportJson"; +import { OPENAI_MODELS } from "../openai/openai-models"; +import type { AzureOpenAiExecutionToolsOptions } from "./AzureOpenAiExecutionToolsOptions"; keepTypeImported(); @@ -30,329 +30,378 @@ keepTypeImported(); * * @public exported from `@promptbook/azure-openai` */ -export class AzureOpenAiExecutionTools implements LlmExecutionTools /* <- TODO: [🍚] `, Destroyable` */ { - /** - * OpenAI Azure API client. - */ - private client: OpenAIClient | null = null; - - /** - * Creates OpenAI Execution Tools. - * - * @param options which are relevant are directly passed to the OpenAI client - */ - public constructor(protected readonly options: AzureOpenAiExecutionToolsOptions) {} - - public get title(): string_title & string_markdown_text { - return 'Azure OpenAI'; - } - - public get description(): string_markdown { - return 'Use all models trained by OpenAI provided by Azure'; - } - - public async getClient(): Promise { - if (this.client === null) { - this.client = new OpenAIClient( - `https://${this.options.resourceName}.openai.azure.com/`, - new AzureKeyCredential(this.options.apiKey), - ); - } - - return this.client; - } - - /** - * Check the `options` passed to `constructor` - */ - public async checkConfiguration(): Promise { - await this.getClient(); - // TODO: [🎍] Do here a real check that API is online, working and API key is correct - } - - /** - * List all available Azure OpenAI models that can be used - */ - public async listModels(): Promise> { - // TODO: [main] !!3 Do here some filtering which models are really available as deployment - // @see https://management.azure.com/subscriptions/subscriptionId/resourceGroups/resourceGroupName/providers/Microsoft.CognitiveServices/accounts/accountName/deployments?api-version=2023-05-01 - return OPENAI_MODELS.map( - ({ - modelTitle, - modelName, - - modelVariant, - }) => ({ - modelTitle: `Azure ${modelTitle}`, - modelName, - modelVariant, - }), - ); - } - - /** - * Calls OpenAI API to use a chat model. - */ - public async callChatModel( - prompt: Pick, - ): Promise { - if (this.options.isVerbose) { - console.info('💬 OpenAI callChatModel call'); - } - - const { content, parameters, modelRequirements } = prompt; - - const client = await this.getClient(); - - // TODO: [☂] Use here more modelRequirements - if (modelRequirements.modelVariant !== 'CHAT') { - throw new PipelineExecutionError('Use callChatModel only for CHAT variant'); - } - - try { - const modelName = prompt.modelRequirements.modelName || this.options.deploymentName; - const modelSettings = { - maxTokens: modelRequirements.maxTokens, - // <- TODO: [🌾] Make some global max cap for maxTokens - temperature: modelRequirements.temperature, - user: this.options.userId?.toString(), - // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools - // <- Note: [🧆] - }; // <- TODO: [💩] TODO: Guard here types better - - const rawPromptContent = templateParameters(content, { ...parameters, modelName }); - const messages = [ - ...(modelRequirements.systemMessage === undefined - ? [] - : ([ - { - role: 'system', - content: modelRequirements.systemMessage, - }, - ] as const)), - { - role: 'user', - content: rawPromptContent, - }, - ]; - - const start: string_date_iso8601 = $getCurrentDate(); - let complete: string_date_iso8601; - - if (this.options.isVerbose) { - console.info(colors.bgWhite('messages'), JSON.stringify(messages, null, 4)); - } - - const rawRequest = [modelName, messages, modelSettings] as const; - const rawResponse = await this.withTimeout(client.getChatCompletions(...rawRequest)).catch((error) => { - if (this.options.isVerbose) { - console.info(colors.bgRed('error'), error); - } - throw error; - }); - - if (this.options.isVerbose) { - console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4)); - } - - if (!rawResponse.choices[0]) { - throw new PipelineExecutionError('No choises from Azure OpenAI'); - } - - if (rawResponse.choices.length > 1) { - // TODO: This should be maybe only warning - throw new PipelineExecutionError('More than one choise from Azure OpenAI'); - } - - if (!rawResponse.choices[0].message || !rawResponse.choices[0].message.content) { - throw new PipelineExecutionError('Empty response from Azure OpenAI'); - } - - const resultContent = rawResponse.choices[0].message.content; - // eslint-disable-next-line prefer-const - complete = $getCurrentDate(); - const usage = { - price: uncertainNumber() /* <- TODO: [🐞] Compute usage */, - input: { - tokensCount: uncertainNumber(rawResponse.usage?.promptTokens), - ...computeUsageCounts(prompt.content), - }, - output: { - tokensCount: uncertainNumber(rawResponse.usage?.completionTokens), - ...computeUsageCounts(prompt.content), - }, - } satisfies PromptResultUsage; /* <- Note: [🤛] */ - - return exportJson({ - name: 'promptResult', - message: `Result of \`AzureOpenAiExecutionTools.callChatModel\``, - order: [], - value: { - content: resultContent, - modelName, - timing: { - start, - complete, - }, - usage, - rawPromptContent, - rawRequest, - rawResponse: { - ...rawResponse, - created: rawResponse.created.toISOString(), - // <- TODO: Put `created` at begining - }, - // <- [🗯] - }, - }); - } catch (error) { - throw this.transformAzureError(error as { code: string; message: string }); - } - } - - /** - * Calls Azure OpenAI API to use a complete model. - */ - public async callCompletionModel( - prompt: Pick, - ): Promise { - if (this.options.isVerbose) { - console.info('🖋 OpenAI callCompletionModel call'); - } - - const { content, parameters, modelRequirements } = prompt; - - const client = await this.getClient(); - - // TODO: [☂] Use here more modelRequirements - if (modelRequirements.modelVariant !== 'COMPLETION') { - throw new PipelineExecutionError('Use callCompletionModel only for COMPLETION variant'); - } - - try { - const modelName = prompt.modelRequirements.modelName || this.options.deploymentName; - const modelSettings = { - maxTokens: modelRequirements.maxTokens || 2000, // <- Note: [🌾] 2000 is for lagacy reasons - // <- TODO: [🌾] Make some global max cap for maxTokens - temperature: modelRequirements.temperature, - user: this.options.userId?.toString(), - // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools - // <- Note: [🧆] - }; // <- TODO: [💩] Guard here types better - - const start: string_date_iso8601 = $getCurrentDate(); - let complete: string_date_iso8601; - - if (this.options.isVerbose) { - console.info(colors.bgWhite('content'), JSON.stringify(content, null, 4)); - console.info(colors.bgWhite('parameters'), JSON.stringify(parameters, null, 4)); - } - - const rawPromptContent = templateParameters(content, { ...parameters, modelName }); - const rawRequest = [ - modelName, - [rawPromptContent] as Array, - modelSettings, - ] as const; - - const rawResponse = await this.withTimeout(client.getCompletions(...rawRequest)).catch((error) => { - if (this.options.isVerbose) { - console.info(colors.bgRed('error'), error); - } - throw error; - }); - - if (this.options.isVerbose) { - console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4)); - } - - if (!rawResponse.choices[0]) { - throw new PipelineExecutionError('No choises from OpenAI'); - } - - if (rawResponse.choices.length > 1) { - // TODO: This should be maybe only warning - throw new PipelineExecutionError('More than one choise from OpenAI'); - } - - const resultContent = rawResponse.choices[0].text; - // eslint-disable-next-line prefer-const - complete = $getCurrentDate(); - - const usage = { - price: uncertainNumber() /* <- TODO: [🐞] Compute usage */, - input: { - tokensCount: uncertainNumber(rawResponse.usage?.promptTokens), - ...computeUsageCounts(prompt.content), - }, - output: { - tokensCount: uncertainNumber(rawResponse.usage?.completionTokens), - ...computeUsageCounts(prompt.content), - }, - } satisfies PromptResultUsage; /* <- Note: [🤛] */ - - return exportJson({ - name: 'promptResult', - message: `Result of \`AzureOpenAiExecutionTools.callCompletionModel\``, - order: [], - value: { - content: resultContent, - modelName, - timing: { - start, - complete, - }, - usage, - rawPromptContent, - rawRequest, - rawResponse: { - ...rawResponse, - created: rawResponse.created.toISOString(), - // <- TODO: Put `created` at begining - }, - // <- [🗯] - }, - }); - } catch (error) { - throw this.transformAzureError(error as { code: string; message: string }); - } - } - - // <- Note: [🤖] callXxxModel - - /** - * Library `@azure/openai` has bug/weird behavior that it does not throw error but hangs forever - * - * This method wraps the promise with timeout - */ - private withTimeout(promise: Promise): Promise { - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - reject(new PipelineExecutionError('Timeout')); - }, CONNECTION_TIMEOUT_MS); - - promise.then((result) => { - clearTimeout(timeout); - resolve(result); - }, reject); - }); - } - - /** - * Changes Azure error (which is not propper Error but object) to propper Error - */ - private transformAzureError(azureError: { code: string; message: string }): Error { - if (azureError instanceof UnexpectedError) { - return azureError; - } - - if (typeof azureError !== 'object' || azureError === null) { - return new PipelineExecutionError(`Unknown Azure OpenAI error`); - } - - const { code, message } = azureError; - return new PipelineExecutionError(`${code || '(No Azure error code)'}: ${message}`); - } +export class AzureOpenAiExecutionTools + implements LlmExecutionTools /* <- TODO: [🍚] `, Destroyable` */ +{ + /** + * OpenAI Azure API client. + */ + private client: OpenAIClient | null = null; + + /** + * Creates OpenAI Execution Tools. + * + * @param options which are relevant are directly passed to the OpenAI client + */ + public constructor( + protected readonly options: AzureOpenAiExecutionToolsOptions, + ) {} + + public get title(): string_title & string_markdown_text { + return "Azure OpenAI"; + } + + public get description(): string_markdown { + return "Use all models trained by OpenAI provided by Azure"; + } + + public async getClient(): Promise { + if (this.client === null) { + this.client = new OpenAIClient( + `https://${this.options.resourceName}.openai.azure.com/`, + new AzureKeyCredential(this.options.apiKey), + ); + } + + return this.client; + } + + /** + * Check the `options` passed to `constructor` + */ + public async checkConfiguration(): Promise { + await this.getClient(); + // TODO: [🎍] Do here a real check that API is online, working and API key is correct + } + + /** + * List all available Azure OpenAI models that can be used + */ + public async listModels(): Promise> { + // TODO: [main] !!3 Do here some filtering which models are really available as deployment + // @see https://management.azure.com/subscriptions/subscriptionId/resourceGroups/resourceGroupName/providers/Microsoft.CognitiveServices/accounts/accountName/deployments?api-version=2023-05-01 + return OPENAI_MODELS.map( + ({ + modelTitle, + modelName, + + modelVariant, + }) => ({ + modelTitle: `Azure ${modelTitle}`, + modelName, + modelVariant, + }), + ); + } + + /** + * Calls OpenAI API to use a chat model. + */ + public async callChatModel( + prompt: Pick, + ): Promise { + if (this.options.isVerbose) { + console.info("💬 OpenAI callChatModel call"); + } + + const { content, parameters, modelRequirements } = prompt; + + const client = await this.getClient(); + + // TODO: [☂] Use here more modelRequirements + if (modelRequirements.modelVariant !== "CHAT") { + throw new PipelineExecutionError( + "Use callChatModel only for CHAT variant", + ); + } + + try { + const modelName = + prompt.modelRequirements.modelName || this.options.deploymentName; + const modelSettings = { + maxTokens: modelRequirements.maxTokens, + // <- TODO: [🌾] Make some global max cap for maxTokens + temperature: modelRequirements.temperature, + user: this.options.userId?.toString(), + // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools + // <- Note: [🧆] + }; // <- TODO: [💩] TODO: Guard here types better + + const rawPromptContent = templateParameters(content, { + ...parameters, + modelName, + }); + const messages = [ + ...(modelRequirements.systemMessage === undefined + ? [] + : ([ + { + role: "system", + content: modelRequirements.systemMessage, + }, + ] as const)), + { + role: "user", + content: rawPromptContent, + }, + ]; + + const start: string_date_iso8601 = $getCurrentDate(); + let complete: string_date_iso8601; + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("messages"), + JSON.stringify(messages, null, 4), + ); + } + + const rawRequest = [modelName, messages, modelSettings] as const; + const rawResponse = await this.withTimeout( + client.getChatCompletions(...rawRequest), + ).catch((error) => { + if (this.options.isVerbose) { + console.info(colors.bgRed("error"), error); + } + throw error; + }); + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("rawResponse"), + JSON.stringify(rawResponse, null, 4), + ); + } + + if (!rawResponse.choices[0]) { + throw new PipelineExecutionError("No choises from Azure OpenAI"); + } + + if (rawResponse.choices.length > 1) { + // TODO: This should be maybe only warning + throw new PipelineExecutionError( + "More than one choise from Azure OpenAI", + ); + } + + if ( + !rawResponse.choices[0].message || + !rawResponse.choices[0].message.content + ) { + throw new PipelineExecutionError("Empty response from Azure OpenAI"); + } + + const resultContent = rawResponse.choices[0].message.content; + // eslint-disable-next-line prefer-const + complete = $getCurrentDate(); + const usage = { + price: uncertainNumber() /* <- TODO: [🐞] Compute usage */, + input: { + tokensCount: uncertainNumber(rawResponse.usage?.promptTokens), + ...computeUsageCounts(prompt.content), + }, + output: { + tokensCount: uncertainNumber(rawResponse.usage?.completionTokens), + ...computeUsageCounts(prompt.content), + }, + } satisfies PromptResultUsage; /* <- Note: [🤛] */ + + return exportJson({ + name: "promptResult", + message: `Result of \`AzureOpenAiExecutionTools.callChatModel\``, + order: [], + value: { + content: resultContent, + modelName, + timing: { + start, + complete, + }, + usage, + rawPromptContent, + rawRequest, + rawResponse: { + ...rawResponse, + created: rawResponse.created.toISOString(), + // <- TODO: Put `created` at begining + }, + // <- [🗯] + }, + }); + } catch (error) { + throw this.transformAzureError( + error as { code: string; message: string }, + ); + } + } + + /** + * Calls Azure OpenAI API to use a complete model. + */ + public async callCompletionModel( + prompt: Pick, + ): Promise { + if (this.options.isVerbose) { + console.info("🖋 OpenAI callCompletionModel call"); + } + + const { content, parameters, modelRequirements } = prompt; + + const client = await this.getClient(); + + // TODO: [☂] Use here more modelRequirements + if (modelRequirements.modelVariant !== "COMPLETION") { + throw new PipelineExecutionError( + "Use callCompletionModel only for COMPLETION variant", + ); + } + + try { + const modelName = + prompt.modelRequirements.modelName || this.options.deploymentName; + const modelSettings = { + maxTokens: modelRequirements.maxTokens || 2000, // <- Note: [🌾] 2000 is for lagacy reasons + // <- TODO: [🌾] Make some global max cap for maxTokens + temperature: modelRequirements.temperature, + user: this.options.userId?.toString(), + // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools + // <- Note: [🧆] + }; // <- TODO: [💩] Guard here types better + + const start: string_date_iso8601 = $getCurrentDate(); + let complete: string_date_iso8601; + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("content"), + JSON.stringify(content, null, 4), + ); + console.info( + colors.bgWhite("parameters"), + JSON.stringify(parameters, null, 4), + ); + } + + const rawPromptContent = templateParameters(content, { + ...parameters, + modelName, + }); + const rawRequest = [ + modelName, + [rawPromptContent] as Array, + modelSettings, + ] as const; + + const rawResponse = await this.withTimeout( + client.getCompletions(...rawRequest), + ).catch((error) => { + if (this.options.isVerbose) { + console.info(colors.bgRed("error"), error); + } + throw error; + }); + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("rawResponse"), + JSON.stringify(rawResponse, null, 4), + ); + } + + if (!rawResponse.choices[0]) { + throw new PipelineExecutionError("No choises from OpenAI"); + } + + if (rawResponse.choices.length > 1) { + // TODO: This should be maybe only warning + throw new PipelineExecutionError("More than one choise from OpenAI"); + } + + const resultContent = rawResponse.choices[0].text; + // eslint-disable-next-line prefer-const + complete = $getCurrentDate(); + + const usage = { + price: uncertainNumber() /* <- TODO: [🐞] Compute usage */, + input: { + tokensCount: uncertainNumber(rawResponse.usage?.promptTokens), + ...computeUsageCounts(prompt.content), + }, + output: { + tokensCount: uncertainNumber(rawResponse.usage?.completionTokens), + ...computeUsageCounts(prompt.content), + }, + } satisfies PromptResultUsage; /* <- Note: [🤛] */ + + return exportJson({ + name: "promptResult", + message: `Result of \`AzureOpenAiExecutionTools.callCompletionModel\``, + order: [], + value: { + content: resultContent, + modelName, + timing: { + start, + complete, + }, + usage, + rawPromptContent, + rawRequest, + rawResponse: { + ...rawResponse, + created: rawResponse.created.toISOString(), + // <- TODO: Put `created` at begining + }, + // <- [🗯] + }, + }); + } catch (error) { + throw this.transformAzureError( + error as { code: string; message: string }, + ); + } + } + + // <- Note: [🤖] callXxxModel + + /** + * Library `@azure/openai` has bug/weird behavior that it does not throw error but hangs forever + * + * This method wraps the promise with timeout + */ + private withTimeout(promise: Promise): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new PipelineExecutionError("Timeout")); + }, CONNECTION_TIMEOUT_MS); + + promise.then((result) => { + clearTimeout(timeout); + resolve(result); + }, reject); + }); + } + + /** + * Changes Azure error (which is not propper Error but object) to propper Error + */ + private transformAzureError(azureError: { + code: string; + message: string; + }): Error { + if (azureError instanceof UnexpectedError) { + return azureError; + } + + if (typeof azureError !== "object" || azureError === null) { + return new PipelineExecutionError(`Unknown Azure OpenAI error`); + } + + const { code, message } = azureError; + return new PipelineExecutionError( + `${code || "(No Azure error code)"}: ${message}`, + ); + } } /** diff --git a/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.ts b/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.ts index b650833047..ee090d6318 100644 --- a/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.ts +++ b/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.ts @@ -1,7 +1,7 @@ -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; -import type { string_name } from '../../types/typeAliases'; -import type { string_token } from '../../types/typeAliases'; -import type { string_user_id } from '../../types/typeAliases'; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; +import type { string_name } from "../../types/typeAliases"; +import type { string_token } from "../../types/typeAliases"; +import type { string_user_id } from "../../types/typeAliases"; /** * Options for `AzureOpenAiExecutionTools` @@ -10,33 +10,33 @@ import type { string_user_id } from '../../types/typeAliases'; * @public exported from `@promptbook/azure-openai` */ export type AzureOpenAiExecutionToolsOptions = CommonToolsOptions & { - /** - * The resource name of the Azure OpenAI resource - * - * Note: Typically you have one resource and multiple deployments. - */ - readonly resourceName: string_name; + /** + * The resource name of the Azure OpenAI resource + * + * Note: Typically you have one resource and multiple deployments. + */ + readonly resourceName: string_name; - /** - * The deployment name - * - * Note: If you specify modelName in prompt, it will be used instead of deploymentName - * Note: This is kind of a modelName in OpenAI terms - * Note: Typically you have one resource and multiple deployments. - */ - readonly deploymentName: string_name; + /** + * The deployment name + * + * Note: If you specify modelName in prompt, it will be used instead of deploymentName + * Note: This is kind of a modelName in OpenAI terms + * Note: Typically you have one resource and multiple deployments. + */ + readonly deploymentName: string_name; - /** - * The API key of the Azure OpenAI resource - */ - readonly apiKey: string_token; + /** + * The API key of the Azure OpenAI resource + */ + readonly apiKey: string_token; - /** - * A unique identifier representing your end-user, which can help Azure OpenAI to monitor - * and detect abuse. - * - * @see https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids (document from OpenAI not Azure, but same concept) - */ - readonly userId?: string_user_id; - // <- TODO: [🧠][🤺] Maybe allow overriding of `userId` for each prompt + /** + * A unique identifier representing your end-user, which can help Azure OpenAI to monitor + * and detect abuse. + * + * @see https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids (document from OpenAI not Azure, but same concept) + */ + readonly userId?: string_user_id; + // <- TODO: [🧠][🤺] Maybe allow overriding of `userId` for each prompt }; diff --git a/src/llm-providers/azure-openai/createAzureOpenAiExecutionTools.ts b/src/llm-providers/azure-openai/createAzureOpenAiExecutionTools.ts index 5af7ee2229..706b3aecd2 100644 --- a/src/llm-providers/azure-openai/createAzureOpenAiExecutionTools.ts +++ b/src/llm-providers/azure-openai/createAzureOpenAiExecutionTools.ts @@ -1,6 +1,6 @@ -import type { LlmExecutionToolsConstructor } from '../../execution/LlmExecutionToolsConstructor'; -import { AzureOpenAiExecutionTools } from './AzureOpenAiExecutionTools'; -import type { AzureOpenAiExecutionToolsOptions } from './AzureOpenAiExecutionToolsOptions'; +import type { LlmExecutionToolsConstructor } from "../../execution/LlmExecutionToolsConstructor"; +import { AzureOpenAiExecutionTools } from "./AzureOpenAiExecutionTools"; +import type { AzureOpenAiExecutionToolsOptions } from "./AzureOpenAiExecutionToolsOptions"; /** * Execution Tools for calling Azure OpenAI API @@ -8,13 +8,13 @@ import type { AzureOpenAiExecutionToolsOptions } from './AzureOpenAiExecutionToo * @public exported from `@promptbook/azure-openai` */ export const createAzureOpenAiExecutionTools = Object.assign( - (options: AzureOpenAiExecutionToolsOptions): AzureOpenAiExecutionTools => { - return new AzureOpenAiExecutionTools(options); - }, - { - packageName: '@promptbook/azure-openai', - className: 'AzureOpenAiExecutionTools', - }, + (options: AzureOpenAiExecutionToolsOptions): AzureOpenAiExecutionTools => { + return new AzureOpenAiExecutionTools(options); + }, + { + packageName: "@promptbook/azure-openai", + className: "AzureOpenAiExecutionTools", + }, ) satisfies LlmExecutionToolsConstructor; /** diff --git a/src/llm-providers/azure-openai/playground/playground.ts b/src/llm-providers/azure-openai/playground/playground.ts index 84a874ce9a..9382e77988 100644 --- a/src/llm-providers/azure-openai/playground/playground.ts +++ b/src/llm-providers/azure-openai/playground/playground.ts @@ -1,70 +1,73 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { usageToHuman } from '../../../execution/utils/usageToHuman'; -import type { Prompt } from '../../../types/Prompt'; -import { keepUnused } from '../../../utils/organization/keepUnused'; -import { AzureOpenAiExecutionTools } from '../AzureOpenAiExecutionTools'; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { usageToHuman } from "../../../execution/utils/usageToHuman"; +import type { Prompt } from "../../../types/Prompt"; +import { keepUnused } from "../../../utils/organization/keepUnused"; +import { AzureOpenAiExecutionTools } from "../AzureOpenAiExecutionTools"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 Azure OpenAI Playground`); - - // Do here stuff you want to test - //========================================> - - const azureOpenAiExecutionTools = new AzureOpenAiExecutionTools( - // <- TODO: [🧱] Implement in a functional (not new Class) way - { - isVerbose: true, - userId: 'playground', - resourceName: process.env.AZUREOPENAI_RESOURCE_NAME!, - deploymentName: process.env.AZUREOPENAI_DEPLOYMENT_NAME!, - apiKey: process.env.AZUREOPENAI_API_KEY!, - }, - ); - - keepUnused(azureOpenAiExecutionTools); - keepUnused(usageToHuman); - keepUnused(); - - /*/ + console.info(`🧸 Azure OpenAI Playground`); + + // Do here stuff you want to test + //========================================> + + const azureOpenAiExecutionTools = new AzureOpenAiExecutionTools( + // <- TODO: [🧱] Implement in a functional (not new Class) way + { + isVerbose: true, + userId: "playground", + resourceName: process.env.AZUREOPENAI_RESOURCE_NAME!, + deploymentName: process.env.AZUREOPENAI_DEPLOYMENT_NAME!, + apiKey: process.env.AZUREOPENAI_API_KEY!, + }, + ); + + keepUnused(azureOpenAiExecutionTools); + keepUnused(usageToHuman); + keepUnused(); + + /*/ const models = await azureOpenAiExecutionTools.listModels(); console.info({ models }); /**/ - /**/ - const chatPrompt = { - title: 'Poem about Prague', - parameters: {}, - content: `Write me something about Prague`, - modelRequirements: { - modelVariant: 'CHAT', - systemMessage: 'You are an assistant who only speaks in rhymes.', - temperature: 1.5, - }, - } as const satisfies Prompt; - const chatPromptResult = await azureOpenAiExecutionTools.callChatModel(chatPrompt); - console.info({ chatPromptResult }); - console.info(colors.cyan(usageToHuman(chatPromptResult.usage))); - console.info(colors.bgBlue(' User: ') + colors.blue(chatPrompt.content)); - console.info(colors.bgGreen(' Chat: ') + colors.green(chatPromptResult.content)); - /**/ - - /*/ + /**/ + const chatPrompt = { + title: "Poem about Prague", + parameters: {}, + content: `Write me something about Prague`, + modelRequirements: { + modelVariant: "CHAT", + systemMessage: "You are an assistant who only speaks in rhymes.", + temperature: 1.5, + }, + } as const satisfies Prompt; + const chatPromptResult = + await azureOpenAiExecutionTools.callChatModel(chatPrompt); + console.info({ chatPromptResult }); + console.info(colors.cyan(usageToHuman(chatPromptResult.usage))); + console.info(colors.bgBlue(" User: ") + colors.blue(chatPrompt.content)); + console.info( + colors.bgGreen(" Chat: ") + colors.green(chatPromptResult.content), + ); + /**/ + + /*/ const completionPrompt = { title: 'Hello', parameters: {}, @@ -79,19 +82,19 @@ async function playground() { console.info(chalk.green(completionPrompt.content + completionPromptResult.content)); /**/ - /*/ + /*/ // TODO: Test Translations in playground /**/ - /*/ + /*/ // TODO: Test Embeddings in playground /**/ - /*/ + /*/ // <- Note: [🤖] Test here new model variant if needed /**/ - //========================================/ + //========================================/ } /** diff --git a/src/llm-providers/azure-openai/playground/tsconfig.json b/src/llm-providers/azure-openai/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/llm-providers/azure-openai/playground/tsconfig.json +++ b/src/llm-providers/azure-openai/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/llm-providers/azure-openai/register-configuration.ts b/src/llm-providers/azure-openai/register-configuration.ts index 5cb5173d13..46309088db 100644 --- a/src/llm-providers/azure-openai/register-configuration.ts +++ b/src/llm-providers/azure-openai/register-configuration.ts @@ -1,7 +1,7 @@ -import type { string_name } from '../../types/typeAliases'; -import type { Registration } from '../../utils/$Register'; -import { $llmToolsMetadataRegister } from '../_common/register/$llmToolsMetadataRegister'; -import type { LlmToolsConfiguration } from '../_common/register/LlmToolsConfiguration'; +import type { string_name } from "../../types/typeAliases"; +import type { Registration } from "../../utils/$Register"; +import { $llmToolsMetadataRegister } from "../_common/register/$llmToolsMetadataRegister"; +import type { LlmToolsConfiguration } from "../_common/register/LlmToolsConfiguration"; /** * Registration of LLM provider metadata @@ -12,47 +12,54 @@ import type { LlmToolsConfiguration } from '../_common/register/LlmToolsConfigur * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _AzureOpenAiMetadataRegistration: Registration = $llmToolsMetadataRegister.register({ - title: 'Azure Open AI', - packageName: '@promptbook/azure-openai', - className: 'AzureOpenAiExecutionTools', - envVariables: ['AZUREOPENAI_RESOURCE_NAME', 'AZUREOPENAI_DEPLOYMENT_NAME', 'AZUREOPENAI_API_KEY'], +export const _AzureOpenAiMetadataRegistration: Registration = + $llmToolsMetadataRegister.register({ + title: "Azure Open AI", + packageName: "@promptbook/azure-openai", + className: "AzureOpenAiExecutionTools", + envVariables: [ + "AZUREOPENAI_RESOURCE_NAME", + "AZUREOPENAI_DEPLOYMENT_NAME", + "AZUREOPENAI_API_KEY", + ], - getBoilerplateConfiguration(): LlmToolsConfiguration[number] { - return { - title: 'Azure Open AI (boilerplate)', - packageName: '@promptbook/azure-openai', - className: 'AzureOpenAiExecutionTools', - options: { - apiKey: 'sk-', - }, - }; - }, + getBoilerplateConfiguration(): LlmToolsConfiguration[number] { + return { + title: "Azure Open AI (boilerplate)", + packageName: "@promptbook/azure-openai", + className: "AzureOpenAiExecutionTools", + options: { + apiKey: "sk-", + }, + }; + }, - createConfigurationFromEnv(env: Record): LlmToolsConfiguration[number] | null { - // Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically - if ( - typeof env.AZUREOPENAI_RESOURCE_NAME === 'string' && - typeof env.AZUREOPENAI_DEPLOYMENT_NAME === 'string' && - typeof env.AZUREOPENAI_API_KEY === 'string' - ) { - return { - title: 'Azure Open AI (from env)', - packageName: '@promptbook/azure-openai', - className: 'AzureOpenAiExecutionTools', - options: { - resourceName: env.AZUREOPENAI_RESOURCE_NAME, - deploymentName: env.AZUREOPENAI_DEPLOYMENT_NAME, - apiKey: env.AZUREOPENAI_API_KEY, - }, - }; - } else if ( - typeof env.AZUREOPENAI_RESOURCE_NAME === 'string' || - typeof env.AZUREOPENAI_DEPLOYMENT_NAME === 'string' || - typeof env.AZUREOPENAI_API_KEY === 'string' - ) { - return null; - /* + createConfigurationFromEnv( + env: Record, + ): LlmToolsConfiguration[number] | null { + // Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically + if ( + typeof env.AZUREOPENAI_RESOURCE_NAME === "string" && + typeof env.AZUREOPENAI_DEPLOYMENT_NAME === "string" && + typeof env.AZUREOPENAI_API_KEY === "string" + ) { + return { + title: "Azure Open AI (from env)", + packageName: "@promptbook/azure-openai", + className: "AzureOpenAiExecutionTools", + options: { + resourceName: env.AZUREOPENAI_RESOURCE_NAME, + deploymentName: env.AZUREOPENAI_DEPLOYMENT_NAME, + apiKey: env.AZUREOPENAI_API_KEY, + }, + }; + } else if ( + typeof env.AZUREOPENAI_RESOURCE_NAME === "string" || + typeof env.AZUREOPENAI_DEPLOYMENT_NAME === "string" || + typeof env.AZUREOPENAI_API_KEY === "string" + ) { + return null; + /* Note: [🗨] Partial configuration is handled more gracefully elsewhere > throw new Error( > spaceTrim(` @@ -68,11 +75,11 @@ export const _AzureOpenAiMetadataRegistration: Registration = $llmToolsMetadataR > `), > ); */ - } + } - return null; - }, -}); + return null; + }, + }); /** * Note: [💞] Ignore a discrepancy between file name and entity name diff --git a/src/llm-providers/azure-openai/register-constructor.ts b/src/llm-providers/azure-openai/register-constructor.ts index db9fbc6856..b3d7387c18 100644 --- a/src/llm-providers/azure-openai/register-constructor.ts +++ b/src/llm-providers/azure-openai/register-constructor.ts @@ -1,6 +1,6 @@ -import type { Registration } from '../../utils/$Register'; -import { $llmToolsRegister } from '../_common/register/$llmToolsRegister'; -import { createAzureOpenAiExecutionTools } from './createAzureOpenAiExecutionTools'; +import type { Registration } from "../../utils/$Register"; +import { $llmToolsRegister } from "../_common/register/$llmToolsRegister"; +import { createAzureOpenAiExecutionTools } from "./createAzureOpenAiExecutionTools"; /** * Registration of LLM provider @@ -11,7 +11,8 @@ import { createAzureOpenAiExecutionTools } from './createAzureOpenAiExecutionToo * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _AzureOpenAiRegistration: Registration = $llmToolsRegister.register(createAzureOpenAiExecutionTools); +export const _AzureOpenAiRegistration: Registration = + $llmToolsRegister.register(createAzureOpenAiExecutionTools); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/llm-providers/deepseek/DeepseekExecutionToolsOptions.ts b/src/llm-providers/deepseek/DeepseekExecutionToolsOptions.ts index 062391662f..418c3f393b 100644 --- a/src/llm-providers/deepseek/DeepseekExecutionToolsOptions.ts +++ b/src/llm-providers/deepseek/DeepseekExecutionToolsOptions.ts @@ -1,5 +1,5 @@ -import type { createDeepSeek } from '@ai-sdk/deepseek'; -import type { VercelExecutionToolsOptions } from '../vercel/VercelExecutionToolsOptions'; +import type { createDeepSeek } from "@ai-sdk/deepseek"; +import type { VercelExecutionToolsOptions } from "../vercel/VercelExecutionToolsOptions"; /** * Options for `DeepseekExecutionTools` @@ -8,7 +8,7 @@ import type { VercelExecutionToolsOptions } from '../vercel/VercelExecutionTools * @public exported from `@promptbook/deepseek` */ export type DeepseekExecutionToolsOptions = Omit< - VercelExecutionToolsOptions, - 'title' | 'description' | 'vercelProvider' | 'availableModels' + VercelExecutionToolsOptions, + "title" | "description" | "vercelProvider" | "availableModels" > & - Parameters[0]; + Parameters[0]; diff --git a/src/llm-providers/deepseek/createDeepseekExecutionTools.ts b/src/llm-providers/deepseek/createDeepseekExecutionTools.ts index 9768a57a6b..981ebc7fba 100644 --- a/src/llm-providers/deepseek/createDeepseekExecutionTools.ts +++ b/src/llm-providers/deepseek/createDeepseekExecutionTools.ts @@ -1,8 +1,8 @@ -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import type { LlmExecutionToolsConstructor } from '../../execution/LlmExecutionToolsConstructor'; -import { $isRunningInJest } from '../../utils/environment/$isRunningInJest'; -import { createExecutionToolsFromVercelProvider } from '../vercel/createExecutionToolsFromVercelProvider'; -import type { DeepseekExecutionToolsOptions } from './DeepseekExecutionToolsOptions'; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import type { LlmExecutionToolsConstructor } from "../../execution/LlmExecutionToolsConstructor"; +import { $isRunningInJest } from "../../utils/environment/$isRunningInJest"; +import { createExecutionToolsFromVercelProvider } from "../vercel/createExecutionToolsFromVercelProvider"; +import type { DeepseekExecutionToolsOptions } from "./DeepseekExecutionToolsOptions"; /** * Execution Tools for calling Deepseek API. @@ -10,38 +10,40 @@ import type { DeepseekExecutionToolsOptions } from './DeepseekExecutionToolsOpti * @public exported from `@promptbook/deepseek` */ export const createDeepseekExecutionTools = Object.assign( - (options: DeepseekExecutionToolsOptions): LlmExecutionTools => { - if ($isRunningInJest()) { - // Note: [🔘] - throw new Error('DeepseekExecutionTools are not supported in Jest environment'); - } + (options: DeepseekExecutionToolsOptions): LlmExecutionTools => { + if ($isRunningInJest()) { + // Note: [🔘] + throw new Error( + "DeepseekExecutionTools are not supported in Jest environment", + ); + } - // Note: [🔘] Maybe there is same compatibility problem as in '@ai-sdk/deepseek' - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { createDeepSeek } = require('@ai-sdk/deepseek'); + // Note: [🔘] Maybe there is same compatibility problem as in '@ai-sdk/deepseek' + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { createDeepSeek } = require("@ai-sdk/deepseek"); - const deepseekVercelProvider = createDeepSeek({ - ...options, - // apiKey: process.env.DEEPSEEK_GENERATIVE_AI_API_KEY, - }); + const deepseekVercelProvider = createDeepSeek({ + ...options, + // apiKey: process.env.DEEPSEEK_GENERATIVE_AI_API_KEY, + }); - return createExecutionToolsFromVercelProvider({ - title: 'Deepseek', - description: 'Implementation of Deepseek models', - vercelProvider: deepseekVercelProvider, - availableModels: [ - // TODO: [🕘] Maybe list models in same way as in other providers - in separate file with metadata - 'deepseek-chat', - 'deepseek-reasoner', - // <- TODO: How picking of the default model looks like in `createExecutionToolsFromVercelProvider` - ].map((modelName) => ({ modelName, modelVariant: 'CHAT' })), - ...options, - }); - }, - { - packageName: '@promptbook/deepseek', - className: 'DeepseekExecutionTools', - }, + return createExecutionToolsFromVercelProvider({ + title: "Deepseek", + description: "Implementation of Deepseek models", + vercelProvider: deepseekVercelProvider, + availableModels: [ + // TODO: [🕘] Maybe list models in same way as in other providers - in separate file with metadata + "deepseek-chat", + "deepseek-reasoner", + // <- TODO: How picking of the default model looks like in `createExecutionToolsFromVercelProvider` + ].map((modelName) => ({ modelName, modelVariant: "CHAT" })), + ...options, + }); + }, + { + packageName: "@promptbook/deepseek", + className: "DeepseekExecutionTools", + }, ) satisfies LlmExecutionToolsConstructor; /** diff --git a/src/llm-providers/deepseek/register-configuration.ts b/src/llm-providers/deepseek/register-configuration.ts index 122959067b..bd4816d632 100644 --- a/src/llm-providers/deepseek/register-configuration.ts +++ b/src/llm-providers/deepseek/register-configuration.ts @@ -1,8 +1,8 @@ -import type { string_name } from '../../types/typeAliases'; -import type { Registration } from '../../utils/$Register'; -import { $isRunningInJest } from '../../utils/environment/$isRunningInJest'; -import { $llmToolsMetadataRegister } from '../_common/register/$llmToolsMetadataRegister'; -import type { LlmToolsConfiguration } from '../_common/register/LlmToolsConfiguration'; +import type { string_name } from "../../types/typeAliases"; +import type { Registration } from "../../utils/$Register"; +import { $isRunningInJest } from "../../utils/environment/$isRunningInJest"; +import { $llmToolsMetadataRegister } from "../_common/register/$llmToolsMetadataRegister"; +import type { LlmToolsConfiguration } from "../_common/register/LlmToolsConfiguration"; /** * Registration of LLM provider metadata @@ -13,47 +13,50 @@ import type { LlmToolsConfiguration } from '../_common/register/LlmToolsConfigur * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _DeepseekMetadataRegistration: Registration = $llmToolsMetadataRegister.register({ - title: 'Deepseek', - packageName: '@promptbook/deepseek', - className: 'DeepseekExecutionTools', - envVariables: ['DEEPSEEK_GENERATIVE_AI_API_KEY'], +export const _DeepseekMetadataRegistration: Registration = + $llmToolsMetadataRegister.register({ + title: "Deepseek", + packageName: "@promptbook/deepseek", + className: "DeepseekExecutionTools", + envVariables: ["DEEPSEEK_GENERATIVE_AI_API_KEY"], - getBoilerplateConfiguration(): LlmToolsConfiguration[number] { - return { - title: 'Deepseek (boilerplate)', - packageName: '@promptbook/deepseek', - className: 'DeepseekExecutionTools', - options: { - apiKey: 'AI', - }, - }; - }, + getBoilerplateConfiguration(): LlmToolsConfiguration[number] { + return { + title: "Deepseek (boilerplate)", + packageName: "@promptbook/deepseek", + className: "DeepseekExecutionTools", + options: { + apiKey: "AI", + }, + }; + }, - createConfigurationFromEnv(env: Record): LlmToolsConfiguration[number] | null { - if ( - $isRunningInJest() - // <- TODO: Maybe check `env.JEST_WORKER_ID` directly here or pass `env` into `$isRunningInJest` - ) { - // Note: [🔘] Maybe same problem as Gemini - return null; - } + createConfigurationFromEnv( + env: Record, + ): LlmToolsConfiguration[number] | null { + if ( + $isRunningInJest() + // <- TODO: Maybe check `env.JEST_WORKER_ID` directly here or pass `env` into `$isRunningInJest` + ) { + // Note: [🔘] Maybe same problem as Gemini + return null; + } - // Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically - if (typeof env.DEEPSEEK_GENERATIVE_AI_API_KEY === 'string') { - return { - title: 'Deepseek (from env)', - packageName: '@promptbook/deepseek', - className: 'DeepseekExecutionTools', - options: { - apiKey: env.DEEPSEEK_GENERATIVE_AI_API_KEY!, - }, - }; - } + // Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically + if (typeof env.DEEPSEEK_GENERATIVE_AI_API_KEY === "string") { + return { + title: "Deepseek (from env)", + packageName: "@promptbook/deepseek", + className: "DeepseekExecutionTools", + options: { + apiKey: env.DEEPSEEK_GENERATIVE_AI_API_KEY!, + }, + }; + } - return null; - }, -}); + return null; + }, + }); /** * Note: [💞] Ignore a discrepancy between file name and entity name diff --git a/src/llm-providers/deepseek/register-constructor.ts b/src/llm-providers/deepseek/register-constructor.ts index 035ec664a4..a3a8ac2222 100644 --- a/src/llm-providers/deepseek/register-constructor.ts +++ b/src/llm-providers/deepseek/register-constructor.ts @@ -1,6 +1,6 @@ -import type { Registration } from '../../utils/$Register'; -import { $llmToolsRegister } from '../_common/register/$llmToolsRegister'; -import { createDeepseekExecutionTools } from './createDeepseekExecutionTools'; +import type { Registration } from "../../utils/$Register"; +import { $llmToolsRegister } from "../_common/register/$llmToolsRegister"; +import { createDeepseekExecutionTools } from "./createDeepseekExecutionTools"; /** * Registration of LLM provider @@ -11,7 +11,9 @@ import { createDeepseekExecutionTools } from './createDeepseekExecutionTools'; * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _DeepseekRegistration: Registration = $llmToolsRegister.register(createDeepseekExecutionTools); +export const _DeepseekRegistration: Registration = $llmToolsRegister.register( + createDeepseekExecutionTools, +); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/llm-providers/google/GoogleExecutionToolsOptions.ts b/src/llm-providers/google/GoogleExecutionToolsOptions.ts index 3f609f4de8..363513f4d8 100644 --- a/src/llm-providers/google/GoogleExecutionToolsOptions.ts +++ b/src/llm-providers/google/GoogleExecutionToolsOptions.ts @@ -1,5 +1,5 @@ -import type { createGoogleGenerativeAI } from '@ai-sdk/google'; -import type { VercelExecutionToolsOptions } from '../vercel/VercelExecutionToolsOptions'; +import type { createGoogleGenerativeAI } from "@ai-sdk/google"; +import type { VercelExecutionToolsOptions } from "../vercel/VercelExecutionToolsOptions"; /** * Options for `GoogleExecutionTools` @@ -8,7 +8,7 @@ import type { VercelExecutionToolsOptions } from '../vercel/VercelExecutionTools * @public exported from `@promptbook/google` */ export type GoogleExecutionToolsOptions = Omit< - VercelExecutionToolsOptions, - 'title' | 'description' | 'vercelProvider' | 'availableModels' + VercelExecutionToolsOptions, + "title" | "description" | "vercelProvider" | "availableModels" > & - Parameters[0]; + Parameters[0]; diff --git a/src/llm-providers/google/createGoogleExecutionTools.ts b/src/llm-providers/google/createGoogleExecutionTools.ts index 43c27a9331..c376f84863 100644 --- a/src/llm-providers/google/createGoogleExecutionTools.ts +++ b/src/llm-providers/google/createGoogleExecutionTools.ts @@ -1,8 +1,8 @@ -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import type { LlmExecutionToolsConstructor } from '../../execution/LlmExecutionToolsConstructor'; -import { $isRunningInJest } from '../../utils/environment/$isRunningInJest'; -import { createExecutionToolsFromVercelProvider } from '../vercel/createExecutionToolsFromVercelProvider'; -import type { GoogleExecutionToolsOptions } from './GoogleExecutionToolsOptions'; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import type { LlmExecutionToolsConstructor } from "../../execution/LlmExecutionToolsConstructor"; +import { $isRunningInJest } from "../../utils/environment/$isRunningInJest"; +import { createExecutionToolsFromVercelProvider } from "../vercel/createExecutionToolsFromVercelProvider"; +import type { GoogleExecutionToolsOptions } from "./GoogleExecutionToolsOptions"; /** * Execution Tools for calling Google Gemini API. @@ -10,50 +10,52 @@ import type { GoogleExecutionToolsOptions } from './GoogleExecutionToolsOptions' * @public exported from `@promptbook/google` */ export const createGoogleExecutionTools = Object.assign( - (options: GoogleExecutionToolsOptions): LlmExecutionTools => { - if ($isRunningInJest()) { - // Note: [🔘] - throw new Error('GoogleExecutionTools are not supported in Jest environment'); - } + (options: GoogleExecutionToolsOptions): LlmExecutionTools => { + if ($isRunningInJest()) { + // Note: [🔘] + throw new Error( + "GoogleExecutionTools are not supported in Jest environment", + ); + } - // Note: [🔘] There is a compatibility when using import from '@ai-sdk/google' - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { createGoogleGenerativeAI } = require('@ai-sdk/google'); + // Note: [🔘] There is a compatibility when using import from '@ai-sdk/google' + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { createGoogleGenerativeAI } = require("@ai-sdk/google"); - const googleGeminiVercelProvider = createGoogleGenerativeAI({ - ...options, - /// apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY, - }); + const googleGeminiVercelProvider = createGoogleGenerativeAI({ + ...options, + /// apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY, + }); - return createExecutionToolsFromVercelProvider({ - title: 'Google', - description: 'Implementation of Google models', - vercelProvider: googleGeminiVercelProvider, - availableModels: [ - // TODO: [🕘] Maybe list models in same way as in other providers - in separate file with metadata - 'gemini-1.5-flash', - 'gemini-1.5-flash-latest', - 'gemini-1.5-flash-001', - 'gemini-1.5-flash-002', - 'gemini-1.5-flash-exp-0827', - 'gemini-1.5-flash-8b', - 'gemini-1.5-flash-8b-latest', - 'gemini-1.5-flash-8b-exp-0924', - 'gemini-1.5-flash-8b-exp-0827', - 'gemini-1.5-pro-latest', - 'gemini-1.5-pro', - 'gemini-1.5-pro-001', - 'gemini-1.5-pro-002', - 'gemini-1.5-pro-exp-0827', - 'gemini-1.0-pro', - ].map((modelName) => ({ modelName, modelVariant: 'CHAT' })), - ...options, - }); - }, - { - packageName: '@promptbook/google', - className: 'GoogleExecutionTools', - }, + return createExecutionToolsFromVercelProvider({ + title: "Google", + description: "Implementation of Google models", + vercelProvider: googleGeminiVercelProvider, + availableModels: [ + // TODO: [🕘] Maybe list models in same way as in other providers - in separate file with metadata + "gemini-1.5-flash", + "gemini-1.5-flash-latest", + "gemini-1.5-flash-001", + "gemini-1.5-flash-002", + "gemini-1.5-flash-exp-0827", + "gemini-1.5-flash-8b", + "gemini-1.5-flash-8b-latest", + "gemini-1.5-flash-8b-exp-0924", + "gemini-1.5-flash-8b-exp-0827", + "gemini-1.5-pro-latest", + "gemini-1.5-pro", + "gemini-1.5-pro-001", + "gemini-1.5-pro-002", + "gemini-1.5-pro-exp-0827", + "gemini-1.0-pro", + ].map((modelName) => ({ modelName, modelVariant: "CHAT" })), + ...options, + }); + }, + { + packageName: "@promptbook/google", + className: "GoogleExecutionTools", + }, ) satisfies LlmExecutionToolsConstructor; /** diff --git a/src/llm-providers/google/register-configuration.ts b/src/llm-providers/google/register-configuration.ts index 4bcf39bfc2..137b44166f 100644 --- a/src/llm-providers/google/register-configuration.ts +++ b/src/llm-providers/google/register-configuration.ts @@ -1,8 +1,8 @@ -import type { string_name } from '../../types/typeAliases'; -import type { Registration } from '../../utils/$Register'; -import { $isRunningInJest } from '../../utils/environment/$isRunningInJest'; -import { $llmToolsMetadataRegister } from '../_common/register/$llmToolsMetadataRegister'; -import type { LlmToolsConfiguration } from '../_common/register/LlmToolsConfiguration'; +import type { string_name } from "../../types/typeAliases"; +import type { Registration } from "../../utils/$Register"; +import { $isRunningInJest } from "../../utils/environment/$isRunningInJest"; +import { $llmToolsMetadataRegister } from "../_common/register/$llmToolsMetadataRegister"; +import type { LlmToolsConfiguration } from "../_common/register/LlmToolsConfiguration"; /** * Registration of LLM provider metadata @@ -13,47 +13,50 @@ import type { LlmToolsConfiguration } from '../_common/register/LlmToolsConfigur * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _GoogleMetadataRegistration: Registration = $llmToolsMetadataRegister.register({ - title: 'Google Gemini', - packageName: '@promptbook/google', - className: 'GoogleExecutionTools', - envVariables: ['GOOGLE_GENERATIVE_AI_API_KEY'], +export const _GoogleMetadataRegistration: Registration = + $llmToolsMetadataRegister.register({ + title: "Google Gemini", + packageName: "@promptbook/google", + className: "GoogleExecutionTools", + envVariables: ["GOOGLE_GENERATIVE_AI_API_KEY"], - getBoilerplateConfiguration(): LlmToolsConfiguration[number] { - return { - title: 'Google Gemini (boilerplate)', - packageName: '@promptbook/google', - className: 'GoogleExecutionTools', - options: { - apiKey: 'AI', - }, - }; - }, + getBoilerplateConfiguration(): LlmToolsConfiguration[number] { + return { + title: "Google Gemini (boilerplate)", + packageName: "@promptbook/google", + className: "GoogleExecutionTools", + options: { + apiKey: "AI", + }, + }; + }, - createConfigurationFromEnv(env: Record): LlmToolsConfiguration[number] | null { - if ( - $isRunningInJest() - // <- TODO: Maybe check `env.JEST_WORKER_ID` directly here or pass `env` into `$isRunningInJest` - ) { - // Note: [🔘] Gemini makes problems in Jest environment - return null; - } + createConfigurationFromEnv( + env: Record, + ): LlmToolsConfiguration[number] | null { + if ( + $isRunningInJest() + // <- TODO: Maybe check `env.JEST_WORKER_ID` directly here or pass `env` into `$isRunningInJest` + ) { + // Note: [🔘] Gemini makes problems in Jest environment + return null; + } - // Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically - if (typeof env.GOOGLE_GENERATIVE_AI_API_KEY === 'string') { - return { - title: 'Google Gemini (from env)', - packageName: '@promptbook/google', - className: 'GoogleExecutionTools', - options: { - apiKey: env.GOOGLE_GENERATIVE_AI_API_KEY!, - }, - }; - } + // Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically + if (typeof env.GOOGLE_GENERATIVE_AI_API_KEY === "string") { + return { + title: "Google Gemini (from env)", + packageName: "@promptbook/google", + className: "GoogleExecutionTools", + options: { + apiKey: env.GOOGLE_GENERATIVE_AI_API_KEY!, + }, + }; + } - return null; - }, -}); + return null; + }, + }); /** * Note: [💞] Ignore a discrepancy between file name and entity name diff --git a/src/llm-providers/google/register-constructor.ts b/src/llm-providers/google/register-constructor.ts index 914337af7a..6a93d0e941 100644 --- a/src/llm-providers/google/register-constructor.ts +++ b/src/llm-providers/google/register-constructor.ts @@ -1,6 +1,6 @@ -import type { Registration } from '../../utils/$Register'; -import { $llmToolsRegister } from '../_common/register/$llmToolsRegister'; -import { createGoogleExecutionTools } from './createGoogleExecutionTools'; +import type { Registration } from "../../utils/$Register"; +import { $llmToolsRegister } from "../_common/register/$llmToolsRegister"; +import { createGoogleExecutionTools } from "./createGoogleExecutionTools"; /** * Registration of LLM provider @@ -11,7 +11,9 @@ import { createGoogleExecutionTools } from './createGoogleExecutionTools'; * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _GoogleRegistration: Registration = $llmToolsRegister.register(createGoogleExecutionTools); +export const _GoogleRegistration: Registration = $llmToolsRegister.register( + createGoogleExecutionTools, +); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/llm-providers/mocked/$fakeTextToExpectations.ts b/src/llm-providers/mocked/$fakeTextToExpectations.ts index 248b4443ae..5b6ca5101a 100644 --- a/src/llm-providers/mocked/$fakeTextToExpectations.ts +++ b/src/llm-providers/mocked/$fakeTextToExpectations.ts @@ -1,12 +1,12 @@ -import { LoremIpsum } from 'lorem-ipsum'; -import { spaceTrim } from 'spacetrim'; -import { CHARACTER_LOOP_LIMIT } from '../../config'; -import { LimitReachedError } from '../../errors/LimitReachedError'; -import type { ScriptExecutionTools } from '../../execution/ScriptExecutionTools'; -import { isPassingExpectations } from '../../execution/utils/checkExpectations'; -import type { Expectations } from '../../pipeline/PipelineJson/Expectations'; -import { JavascriptExecutionTools } from '../../scripting/javascript/JavascriptExecutionTools'; -import type { string_postprocessing_function_name } from '../../types/typeAliases'; +import { LoremIpsum } from "lorem-ipsum"; +import { spaceTrim } from "spacetrim"; +import { CHARACTER_LOOP_LIMIT } from "../../config"; +import { LimitReachedError } from "../../errors/LimitReachedError"; +import type { ScriptExecutionTools } from "../../execution/ScriptExecutionTools"; +import { isPassingExpectations } from "../../execution/utils/checkExpectations"; +import type { Expectations } from "../../pipeline/PipelineJson/Expectations"; +import { JavascriptExecutionTools } from "../../scripting/javascript/JavascriptExecutionTools"; +import type { string_postprocessing_function_name } from "../../types/typeAliases"; /** * Gets the expectations and creates a fake text that meets the expectations @@ -18,50 +18,51 @@ import type { string_postprocessing_function_name } from '../../types/typeAliase * @private internal utility for MockedFackedLlmExecutionTools */ export async function $fakeTextToExpectations( - expectations: Expectations, - postprocessingFunctionNames?: ReadonlyArray, + expectations: Expectations, + postprocessingFunctionNames?: ReadonlyArray, ): Promise { - const lorem = new LoremIpsum({ - wordsPerSentence: { min: 5, max: 15 }, - sentencesPerParagraph: { min: 5, max: 15 }, - }); - let loremText = ''; - let text = ''; + const lorem = new LoremIpsum({ + wordsPerSentence: { min: 5, max: 15 }, + sentencesPerParagraph: { min: 5, max: 15 }, + }); + let loremText = ""; + let text = ""; - for (let loopLimit = CHARACTER_LOOP_LIMIT; loopLimit-- > 0; ) { - let textToCheck = text; + for (let loopLimit = CHARACTER_LOOP_LIMIT; loopLimit-- > 0; ) { + let textToCheck = text; - // TODO: DRY [☯] - let scriptTools: ScriptExecutionTools | null = null; - for (const postprocessingFunctionName of postprocessingFunctionNames || []) { - if (scriptTools === null) { - scriptTools = new JavascriptExecutionTools(); - } - textToCheck = await scriptTools.execute({ - scriptLanguage: `javascript` /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */, - script: `${postprocessingFunctionName}(result)`, - parameters: { - result: textToCheck || '', - // Note: No ...parametersForTask, because working with result only - }, - }); - } + // TODO: DRY [☯] + let scriptTools: ScriptExecutionTools | null = null; + for (const postprocessingFunctionName of postprocessingFunctionNames || + []) { + if (scriptTools === null) { + scriptTools = new JavascriptExecutionTools(); + } + textToCheck = await scriptTools.execute({ + scriptLanguage: `javascript` /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */, + script: `${postprocessingFunctionName}(result)`, + parameters: { + result: textToCheck || "", + // Note: No ...parametersForTask, because working with result only + }, + }); + } - if (isPassingExpectations(expectations, textToCheck)) { - return text; // <- Note: Returning the text because the postprocessing - } + if (isPassingExpectations(expectations, textToCheck)) { + return text; // <- Note: Returning the text because the postprocessing + } - if (loremText === '') { - loremText = lorem.generateParagraphs(1) + '\n\n'; - } + if (loremText === "") { + loremText = lorem.generateParagraphs(1) + "\n\n"; + } - text += loremText.substring(0, 1); - loremText = loremText.substring(1); - } + text += loremText.substring(0, 1); + loremText = loremText.substring(1); + } - throw new LimitReachedError( - spaceTrim( - (block) => ` + throw new LimitReachedError( + spaceTrim( + (block) => ` Can not generate fake text to met the expectations Loop limit reached @@ -72,8 +73,8 @@ export async function $fakeTextToExpectations( ${block(text)} `, - ), - ); + ), + ); } /** diff --git a/src/llm-providers/mocked/MockedEchoLlmExecutionTools.ts b/src/llm-providers/mocked/MockedEchoLlmExecutionTools.ts index 0d5a36ed43..7ae6bf835c 100644 --- a/src/llm-providers/mocked/MockedEchoLlmExecutionTools.ts +++ b/src/llm-providers/mocked/MockedEchoLlmExecutionTools.ts @@ -1,145 +1,153 @@ -import { spaceTrim } from 'spacetrim'; -import type { AvailableModel } from '../../execution/AvailableModel'; -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import type { ChatPromptResult } from '../../execution/PromptResult'; -import type { CompletionPromptResult } from '../../execution/PromptResult'; -import { ZERO_USAGE } from '../../execution/utils/usage-constants'; -import type { Prompt } from '../../types/Prompt'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_title } from '../../types/typeAliases'; -import { $getCurrentDate } from '../../utils/$getCurrentDate'; -import { templateParameters } from '../../utils/parameters/templateParameters'; -import { exportJson } from '../../utils/serialization/exportJson'; +import { spaceTrim } from "spacetrim"; +import type { AvailableModel } from "../../execution/AvailableModel"; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import type { ChatPromptResult } from "../../execution/PromptResult"; +import type { CompletionPromptResult } from "../../execution/PromptResult"; +import { ZERO_USAGE } from "../../execution/utils/usage-constants"; +import type { Prompt } from "../../types/Prompt"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_title } from "../../types/typeAliases"; +import { $getCurrentDate } from "../../utils/$getCurrentDate"; +import { templateParameters } from "../../utils/parameters/templateParameters"; +import { exportJson } from "../../utils/serialization/exportJson"; /** * Mocked execution Tools for just echoing the requests for testing purposes. * * @public exported from `@promptbook/fake-llm` */ -export class MockedEchoLlmExecutionTools implements LlmExecutionTools /* <- TODO: [🍚] `, Destroyable` */ { - public constructor(protected readonly options: CommonToolsOptions = {}) {} +export class MockedEchoLlmExecutionTools + implements LlmExecutionTools /* <- TODO: [🍚] `, Destroyable` */ +{ + public constructor(protected readonly options: CommonToolsOptions = {}) {} - public get title(): string_title & string_markdown_text { - return 'Mocked echo'; - } + public get title(): string_title & string_markdown_text { + return "Mocked echo"; + } - public get description(): string_markdown { - return 'What you say is whay you get - just for testing'; - } + public get description(): string_markdown { + return "What you say is whay you get - just for testing"; + } - /** - * Does nothing, just to implement the interface - */ - public checkConfiguration(): void {} + /** + * Does nothing, just to implement the interface + */ + public checkConfiguration(): void {} - /** - * List all available mocked-models that can be used - */ - public listModels(): ReadonlyArray { - return [ - { - modelTitle: 'Echo chat', - modelName: 'mocked-echo', - modelVariant: 'CHAT', - }, - { - modelTitle: 'Echo completion', - modelName: 'mocked-echo', - modelVariant: 'COMPLETION', - }, - // <- Note: [🤖] - ]; - } + /** + * List all available mocked-models that can be used + */ + public listModels(): ReadonlyArray { + return [ + { + modelTitle: "Echo chat", + modelName: "mocked-echo", + modelVariant: "CHAT", + }, + { + modelTitle: "Echo completion", + modelName: "mocked-echo", + modelVariant: "COMPLETION", + }, + // <- Note: [🤖] + ]; + } - /** - * Mocks chat model - */ - public async callChatModel( - prompt: Pick, - ): Promise { - if (this.options.isVerbose) { - console.info('💬 Mocked callChatModel call'); - } + /** + * Mocks chat model + */ + public async callChatModel( + prompt: Pick, + ): Promise { + if (this.options.isVerbose) { + console.info("💬 Mocked callChatModel call"); + } - const modelName = 'mocked-echo'; - const rawPromptContent = templateParameters(prompt.content, { ...prompt.parameters, modelName }); + const modelName = "mocked-echo"; + const rawPromptContent = templateParameters(prompt.content, { + ...prompt.parameters, + modelName, + }); - const usage = ZERO_USAGE; - // <- TODO: [🧠] Compute here at least words, characters,... etc + const usage = ZERO_USAGE; + // <- TODO: [🧠] Compute here at least words, characters,... etc - return exportJson({ - name: 'promptResult', - message: `Result of \`MockedEchoLlmExecutionTools.callChatModel\``, - order: [], - value: { - content: spaceTrim( - (block) => ` + return exportJson({ + name: "promptResult", + message: `Result of \`MockedEchoLlmExecutionTools.callChatModel\``, + order: [], + value: { + content: spaceTrim( + (block) => ` You said: ${block(rawPromptContent)} `, - ), - modelName, - timing: { - start: $getCurrentDate(), - complete: $getCurrentDate(), - }, - usage, - rawPromptContent, - rawRequest: null, - rawResponse: { - note: 'This is mocked echo', - }, - // <- [🗯] - }, - }); - } + ), + modelName, + timing: { + start: $getCurrentDate(), + complete: $getCurrentDate(), + }, + usage, + rawPromptContent, + rawRequest: null, + rawResponse: { + note: "This is mocked echo", + }, + // <- [🗯] + }, + }); + } - /** - * Mocks completion model - */ - public async callCompletionModel( - prompt: Pick, - ): Promise { - if (this.options.isVerbose) { - console.info('🖋 Mocked callCompletionModel call'); - } + /** + * Mocks completion model + */ + public async callCompletionModel( + prompt: Pick, + ): Promise { + if (this.options.isVerbose) { + console.info("🖋 Mocked callCompletionModel call"); + } - const modelName = 'mocked-echo'; - const rawPromptContent = templateParameters(prompt.content, { ...prompt.parameters, modelName }); + const modelName = "mocked-echo"; + const rawPromptContent = templateParameters(prompt.content, { + ...prompt.parameters, + modelName, + }); - const usage = ZERO_USAGE; - // <- TODO: [🧠] Compute here at least words, characters,... etc + const usage = ZERO_USAGE; + // <- TODO: [🧠] Compute here at least words, characters,... etc - return exportJson({ - name: 'promptResult', - message: `Result of \`MockedEchoLlmExecutionTools.callCompletionModel\``, - order: [], - value: { - content: spaceTrim( - (block) => ` + return exportJson({ + name: "promptResult", + message: `Result of \`MockedEchoLlmExecutionTools.callCompletionModel\``, + order: [], + value: { + content: spaceTrim( + (block) => ` ${block(rawPromptContent)} And so on... `, - ), - modelName, - timing: { - start: $getCurrentDate(), - complete: $getCurrentDate(), - }, - usage, - rawPromptContent, - rawRequest: null, - rawResponse: { - note: 'This is mocked echo', - }, - // <- [🗯] - }, - }); - } + ), + modelName, + timing: { + start: $getCurrentDate(), + complete: $getCurrentDate(), + }, + usage, + rawPromptContent, + rawRequest: null, + rawResponse: { + note: "This is mocked echo", + }, + // <- [🗯] + }, + }); + } - // <- Note: [🤖] callXxxModel + // <- Note: [🤖] callXxxModel } /** diff --git a/src/llm-providers/mocked/MockedFackedLlmExecutionTools.ts b/src/llm-providers/mocked/MockedFackedLlmExecutionTools.ts index a7066a1d4a..f8a2a9adf8 100644 --- a/src/llm-providers/mocked/MockedFackedLlmExecutionTools.ts +++ b/src/llm-providers/mocked/MockedFackedLlmExecutionTools.ts @@ -1,20 +1,20 @@ -import type { AvailableModel } from '../../execution/AvailableModel'; -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; -import type { EmbeddingVector } from '../../execution/EmbeddingVector'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import type { ChatPromptResult } from '../../execution/PromptResult'; -import type { CompletionPromptResult } from '../../execution/PromptResult'; -import type { EmbeddingPromptResult } from '../../execution/PromptResult'; -import { ZERO_USAGE } from '../../execution/utils/usage-constants'; -import type { Prompt } from '../../types/Prompt'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_title } from '../../types/typeAliases'; -import { $getCurrentDate } from '../../utils/$getCurrentDate'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import { templateParameters } from '../../utils/parameters/templateParameters'; -import { exportJson } from '../../utils/serialization/exportJson'; -import { $fakeTextToExpectations } from './$fakeTextToExpectations'; +import type { AvailableModel } from "../../execution/AvailableModel"; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; +import type { EmbeddingVector } from "../../execution/EmbeddingVector"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import type { ChatPromptResult } from "../../execution/PromptResult"; +import type { CompletionPromptResult } from "../../execution/PromptResult"; +import type { EmbeddingPromptResult } from "../../execution/PromptResult"; +import { ZERO_USAGE } from "../../execution/utils/usage-constants"; +import type { Prompt } from "../../types/Prompt"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_title } from "../../types/typeAliases"; +import { $getCurrentDate } from "../../utils/$getCurrentDate"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import { templateParameters } from "../../utils/parameters/templateParameters"; +import { exportJson } from "../../utils/serialization/exportJson"; +import { $fakeTextToExpectations } from "./$fakeTextToExpectations"; keepTypeImported(); @@ -23,156 +23,178 @@ keepTypeImported(); * * @public exported from `@promptbook/fake-llm` */ -export class MockedFackedLlmExecutionTools implements LlmExecutionTools /* <- TODO: [🍚] `, Destroyable` */ { - public constructor(protected readonly options: CommonToolsOptions = {}) {} - - public get title(): string_title & string_markdown_text { - return 'Mocked facked'; - } - - public get description(): string_markdown { - return 'Use faked lorem ipsum data - just for testing'; - } - - /** - * Does nothing, just to implement the interface - */ - public checkConfiguration(): void {} - - /** - * List all available fake-models that can be used - */ - public listModels(): ReadonlyArray { - return [ - { - modelTitle: 'Fake chat', - modelName: 'mocked-echo', - modelVariant: 'CHAT', - }, - { - modelTitle: 'Fake completion', - modelName: 'mocked-echo', - modelVariant: 'COMPLETION', - }, - // <- Note: [🤖] - ]; - } - - /** - * Fakes chat model - */ - public async callChatModel( - prompt: Pick< - Prompt, - 'content' | 'parameters' | 'modelRequirements' | 'expectations' | 'postprocessingFunctionNames' - >, - ): Promise { - if (this.options.isVerbose) { - console.info('💬 Mocked faked prompt', prompt); - } - - const modelName = 'mocked-facked'; - const rawPromptContent = templateParameters(prompt.content, { ...prompt.parameters, modelName }); - - const usage = ZERO_USAGE; - // <- TODO: [🧠] Compute here at least words, characters,... etc - - const content = await $fakeTextToExpectations( - prompt.expectations || { - sentences: { min: 1, max: 1 }, - }, - prompt.postprocessingFunctionNames, - ); - - const result = { - content, - modelName, - timing: { - start: $getCurrentDate(), - complete: $getCurrentDate(), - }, - usage, - rawPromptContent, - rawRequest: null, - rawResponse: { - note: 'This is mocked echo', - }, - // <- [🗯] - } satisfies ChatPromptResult & CompletionPromptResult; - - if (this.options.isVerbose) { - console.info('💬 Mocked faked result', result); - } - - return exportJson({ - name: 'promptResult', - message: `Result of \`MockedFackedLlmExecutionTools.callChatModel\``, - order: [], - value: result, - }); - } - - /** - * Fakes completion model - */ - public async callCompletionModel( - prompt: Pick< - Prompt, - 'content' | 'parameters' | 'modelRequirements' | 'expectations' | 'postprocessingFunctionNames' - >, - ): Promise { - return this.callChatModel(prompt); - } - - /** - * Fakes embedding model - */ - public async callEmbeddingModel( - prompt: Pick< - Prompt, - 'content' | 'parameters' | 'modelRequirements' | 'expectations' | 'postprocessingFunctionNames' - >, - ): Promise { - const modelName = 'mocked-facked'; - const rawPromptContent = templateParameters(prompt.content, { ...prompt.parameters, modelName }); - const content = new Array(1024) - .fill(0) - .map(() => Math.random() * 2 - 1) satisfies EmbeddingVector; /* <- Note: [🤛] */ - - const usage = ZERO_USAGE; - // <- TODO: [🧠] Compute here at least words, characters,... etc - - // TODO: Make content vector exactly length of 1 - - const result = { - content, - modelName, - timing: { - start: $getCurrentDate(), - complete: $getCurrentDate(), - }, - usage, - rawPromptContent, - rawRequest: null, - rawResponse: { - note: 'This is mocked embedding', - }, - // <- [🗯] - } satisfies EmbeddingPromptResult; - - if (this.options.isVerbose) { - console.info('💬 Mocked faked result', result); - } - - return exportJson({ - name: 'promptResult', - message: `Result of \`MockedFackedLlmExecutionTools.callEmbeddingModel\``, - order: [], - value: result, - }); - } - - // <- Note: [🤖] callXxxModel +export class MockedFackedLlmExecutionTools + implements LlmExecutionTools /* <- TODO: [🍚] `, Destroyable` */ +{ + public constructor(protected readonly options: CommonToolsOptions = {}) {} + + public get title(): string_title & string_markdown_text { + return "Mocked facked"; + } + + public get description(): string_markdown { + return "Use faked lorem ipsum data - just for testing"; + } + + /** + * Does nothing, just to implement the interface + */ + public checkConfiguration(): void {} + + /** + * List all available fake-models that can be used + */ + public listModels(): ReadonlyArray { + return [ + { + modelTitle: "Fake chat", + modelName: "mocked-echo", + modelVariant: "CHAT", + }, + { + modelTitle: "Fake completion", + modelName: "mocked-echo", + modelVariant: "COMPLETION", + }, + // <- Note: [🤖] + ]; + } + + /** + * Fakes chat model + */ + public async callChatModel( + prompt: Pick< + Prompt, + | "content" + | "parameters" + | "modelRequirements" + | "expectations" + | "postprocessingFunctionNames" + >, + ): Promise { + if (this.options.isVerbose) { + console.info("💬 Mocked faked prompt", prompt); + } + + const modelName = "mocked-facked"; + const rawPromptContent = templateParameters(prompt.content, { + ...prompt.parameters, + modelName, + }); + + const usage = ZERO_USAGE; + // <- TODO: [🧠] Compute here at least words, characters,... etc + + const content = await $fakeTextToExpectations( + prompt.expectations || { + sentences: { min: 1, max: 1 }, + }, + prompt.postprocessingFunctionNames, + ); + + const result = { + content, + modelName, + timing: { + start: $getCurrentDate(), + complete: $getCurrentDate(), + }, + usage, + rawPromptContent, + rawRequest: null, + rawResponse: { + note: "This is mocked echo", + }, + // <- [🗯] + } satisfies ChatPromptResult & CompletionPromptResult; + + if (this.options.isVerbose) { + console.info("💬 Mocked faked result", result); + } + + return exportJson({ + name: "promptResult", + message: `Result of \`MockedFackedLlmExecutionTools.callChatModel\``, + order: [], + value: result, + }); + } + + /** + * Fakes completion model + */ + public async callCompletionModel( + prompt: Pick< + Prompt, + | "content" + | "parameters" + | "modelRequirements" + | "expectations" + | "postprocessingFunctionNames" + >, + ): Promise { + return this.callChatModel(prompt); + } + + /** + * Fakes embedding model + */ + public async callEmbeddingModel( + prompt: Pick< + Prompt, + | "content" + | "parameters" + | "modelRequirements" + | "expectations" + | "postprocessingFunctionNames" + >, + ): Promise { + const modelName = "mocked-facked"; + const rawPromptContent = templateParameters(prompt.content, { + ...prompt.parameters, + modelName, + }); + const content = new Array(1024) + .fill(0) + .map( + () => Math.random() * 2 - 1, + ) satisfies EmbeddingVector; /* <- Note: [🤛] */ + + const usage = ZERO_USAGE; + // <- TODO: [🧠] Compute here at least words, characters,... etc + + // TODO: Make content vector exactly length of 1 + + const result = { + content, + modelName, + timing: { + start: $getCurrentDate(), + complete: $getCurrentDate(), + }, + usage, + rawPromptContent, + rawRequest: null, + rawResponse: { + note: "This is mocked embedding", + }, + // <- [🗯] + } satisfies EmbeddingPromptResult; + + if (this.options.isVerbose) { + console.info("💬 Mocked faked result", result); + } + + return exportJson({ + name: "promptResult", + message: `Result of \`MockedFackedLlmExecutionTools.callEmbeddingModel\``, + order: [], + value: result, + }); + } + + // <- Note: [🤖] callXxxModel } /** diff --git a/src/llm-providers/mocked/test/fakeTextToExpectations.test.ts b/src/llm-providers/mocked/test/fakeTextToExpectations.test.ts index 8974f33ba0..88886b2b96 100644 --- a/src/llm-providers/mocked/test/fakeTextToExpectations.test.ts +++ b/src/llm-providers/mocked/test/fakeTextToExpectations.test.ts @@ -1,77 +1,83 @@ -import { describe, expect, it } from '@jest/globals'; -import { $fakeTextToExpectations } from '../$fakeTextToExpectations'; -import { checkExpectations } from '../../../execution/utils/checkExpectations'; -import type { Expectations } from '../../../pipeline/PipelineJson/Expectations'; +import { describe, expect, it } from "@jest/globals"; +import { $fakeTextToExpectations } from "../$fakeTextToExpectations"; +import { checkExpectations } from "../../../execution/utils/checkExpectations"; +import type { Expectations } from "../../../pipeline/PipelineJson/Expectations"; -describe('how $fakeTextToExpectations works', () => { - it('should fake the text with characters expectation', () => - expect(async () => { - const expectations = { characters: { min: 10, max: 50 } } satisfies Expectations; - const text = await $fakeTextToExpectations(expectations); - checkExpectations(expectations, text); - }).not.toThrow()); +describe("how $fakeTextToExpectations works", () => { + it("should fake the text with characters expectation", () => + expect(async () => { + const expectations = { + characters: { min: 10, max: 50 }, + } satisfies Expectations; + const text = await $fakeTextToExpectations(expectations); + checkExpectations(expectations, text); + }).not.toThrow()); - it('should fake the text with words expectation', () => - expect(async () => { - const expectations = { words: { min: 1, max: 5 } } satisfies Expectations; - const text = await $fakeTextToExpectations(expectations); - checkExpectations(expectations, text); - }).not.toThrow()); + it("should fake the text with words expectation", () => + expect(async () => { + const expectations = { words: { min: 1, max: 5 } } satisfies Expectations; + const text = await $fakeTextToExpectations(expectations); + checkExpectations(expectations, text); + }).not.toThrow()); - it('should fake the text with sentences expectation', () => - expect(async () => { - const expectations = { sentences: { min: 1, max: 5 } } satisfies Expectations; - const text = await $fakeTextToExpectations(expectations); - checkExpectations(expectations, text); - }).not.toThrow()); + it("should fake the text with sentences expectation", () => + expect(async () => { + const expectations = { + sentences: { min: 1, max: 5 }, + } satisfies Expectations; + const text = await $fakeTextToExpectations(expectations); + checkExpectations(expectations, text); + }).not.toThrow()); - it('should fake the text with paragraphs expectation', () => - expect(async () => { - const expectations = { paragraphs: { min: 1, max: 5 } } satisfies Expectations; - const text = await $fakeTextToExpectations(expectations); - checkExpectations(expectations, text); - }).not.toThrow()); + it("should fake the text with paragraphs expectation", () => + expect(async () => { + const expectations = { + paragraphs: { min: 1, max: 5 }, + } satisfies Expectations; + const text = await $fakeTextToExpectations(expectations); + checkExpectations(expectations, text); + }).not.toThrow()); - it('should fake the text with lines expectation', () => - expect(async () => { - const expectations = { lines: { min: 2 } } satisfies Expectations; - const text = await $fakeTextToExpectations(expectations); - checkExpectations(expectations, text); - }).not.toThrow()); + it("should fake the text with lines expectation", () => + expect(async () => { + const expectations = { lines: { min: 2 } } satisfies Expectations; + const text = await $fakeTextToExpectations(expectations); + checkExpectations(expectations, text); + }).not.toThrow()); - it('should fake the text with pages expectation', () => - expect(async () => { - const expectations = { pages: { min: 2 } } satisfies Expectations; - const text = await $fakeTextToExpectations(expectations); - checkExpectations(expectations, text); - }).not.toThrow()); + it("should fake the text with pages expectation", () => + expect(async () => { + const expectations = { pages: { min: 2 } } satisfies Expectations; + const text = await $fakeTextToExpectations(expectations); + checkExpectations(expectations, text); + }).not.toThrow()); - it('should fake the text with both characters, words and sentences expectation', () => - expect(async () => { - const expectations = { - characters: { min: 10, max: 500 }, - words: { min: 1, max: 5 }, - sentences: { min: 1, max: 5 }, - } satisfies Expectations; - const text = await $fakeTextToExpectations(expectations); - checkExpectations(expectations, text); - }).not.toThrow()); + it("should fake the text with both characters, words and sentences expectation", () => + expect(async () => { + const expectations = { + characters: { min: 10, max: 500 }, + words: { min: 1, max: 5 }, + sentences: { min: 1, max: 5 }, + } satisfies Expectations; + const text = await $fakeTextToExpectations(expectations); + checkExpectations(expectations, text); + }).not.toThrow()); - it('should fake the text with both words and lines expectation', () => - expect(async () => { - const expectations = { - words: { - min: 10, - }, - lines: { - min: 3, - }, - } satisfies Expectations; - const text = await $fakeTextToExpectations(expectations); - checkExpectations(expectations, text); - }).not.toThrow()); + it("should fake the text with both words and lines expectation", () => + expect(async () => { + const expectations = { + words: { + min: 10, + }, + lines: { + min: 3, + }, + } satisfies Expectations; + const text = await $fakeTextToExpectations(expectations); + checkExpectations(expectations, text); + }).not.toThrow()); - /* + /* TODO: it('should fail on contradictory expectations', () => expect(async () => { diff --git a/src/llm-providers/mocked/test/faked-completion.test.ts b/src/llm-providers/mocked/test/faked-completion.test.ts index 1d2e443486..fcbabfde9b 100644 --- a/src/llm-providers/mocked/test/faked-completion.test.ts +++ b/src/llm-providers/mocked/test/faked-completion.test.ts @@ -1,94 +1,96 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { compilePipeline } from '../../../conversion/compilePipeline'; -import { CallbackInterfaceTools } from '../../../dialogs/callback/CallbackInterfaceTools'; -import { createPipelineExecutor } from '../../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import type { PipelineString } from '../../../pipeline/PipelineString'; -import { PROMPTBOOK_ENGINE_VERSION } from '../../../version'; -import { MockedFackedLlmExecutionTools } from '../MockedFackedLlmExecutionTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { compilePipeline } from "../../../conversion/compilePipeline"; +import { CallbackInterfaceTools } from "../../../dialogs/callback/CallbackInterfaceTools"; +import { createPipelineExecutor } from "../../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import type { PipelineString } from "../../../pipeline/PipelineString"; +import { PROMPTBOOK_ENGINE_VERSION } from "../../../version"; +import { MockedFackedLlmExecutionTools } from "../MockedFackedLlmExecutionTools"; -describe('createPipelineExecutor + MockedFackedLlmExecutionTools with example completion prompt', () => { - it('should work when every INPUT PARAMETER defined', async () => { - const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({ thing: 'a cup of coffee' }).asPromise()).resolves.toMatchObject({ - outputParameters: { - response: /.*/, - }, - }); - }); +describe("createPipelineExecutor + MockedFackedLlmExecutionTools with example completion prompt", () => { + it("should work when every INPUT PARAMETER defined", async () => { + const pipelineExecutor = await getPipelineExecutor(); + expect( + pipelineExecutor({ thing: "a cup of coffee" }).asPromise(), + ).resolves.toMatchObject({ + outputParameters: { + response: /.*/, + }, + }); + }); - it('should fail when some INPUT PARAMETER is missing', async () => { - const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({}).asPromise()).resolves.toMatchObject({ - isSuccessful: false, - errors: [/Parameter `{thing}` is required as an input parameter/i], - executionReport: { - title: 'Example prompt', - description: 'Show how to use a simple completion prompt', - promptExecutions: [], - pipelineUrl: 'https://promptbook.studio/examples/pipeline.book', - promptbookRequestedVersion: '1.0.0', - promptbookUsedVersion: PROMPTBOOK_ENGINE_VERSION, - }, - outputParameters: {}, - usage: { - input: { - charactersCount: { - value: 0, - }, - linesCount: { - value: 0, - }, - pagesCount: { - value: 0, - }, - paragraphsCount: { - value: 0, - }, - sentencesCount: { - value: 0, - }, - tokensCount: { - value: 0, - }, - wordsCount: { - value: 0, - }, - }, - output: { - charactersCount: { - value: 0, - }, - linesCount: { - value: 0, - }, - pagesCount: { - value: 0, - }, - paragraphsCount: { - value: 0, - }, - sentencesCount: { - value: 0, - }, - tokensCount: { - value: 0, - }, - wordsCount: { - value: 0, - }, - }, - price: { - value: 0, - }, - }, - }); - }); + it("should fail when some INPUT PARAMETER is missing", async () => { + const pipelineExecutor = await getPipelineExecutor(); + expect(pipelineExecutor({}).asPromise()).resolves.toMatchObject({ + isSuccessful: false, + errors: [/Parameter `{thing}` is required as an input parameter/i], + executionReport: { + title: "Example prompt", + description: "Show how to use a simple completion prompt", + promptExecutions: [], + pipelineUrl: "https://promptbook.studio/examples/pipeline.book", + promptbookRequestedVersion: "1.0.0", + promptbookUsedVersion: PROMPTBOOK_ENGINE_VERSION, + }, + outputParameters: {}, + usage: { + input: { + charactersCount: { + value: 0, + }, + linesCount: { + value: 0, + }, + pagesCount: { + value: 0, + }, + paragraphsCount: { + value: 0, + }, + sentencesCount: { + value: 0, + }, + tokensCount: { + value: 0, + }, + wordsCount: { + value: 0, + }, + }, + output: { + charactersCount: { + value: 0, + }, + linesCount: { + value: 0, + }, + pagesCount: { + value: 0, + }, + paragraphsCount: { + value: 0, + }, + sentencesCount: { + value: 0, + }, + tokensCount: { + value: 0, + }, + wordsCount: { + value: 0, + }, + }, + price: { + value: 0, + }, + }, + }); + }); }); async function getPipelineExecutor() { - const pipeline = await compilePipeline( - spaceTrim(` + const pipeline = await compilePipeline( + spaceTrim(` # Example prompt Show how to use a simple completion prompt @@ -113,23 +115,23 @@ async function getPipelineExecutor() { -> {response} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - ); - const pipelineExecutor = createPipelineExecutor({ - pipeline, - tools: { - llm: new MockedFackedLlmExecutionTools( - // <- TODO: [🧱] Implement in a functional (not new Class) way - { isVerbose: true }, - ), - script: [], - userInterface: new CallbackInterfaceTools({ - isVerbose: true, - async callback() { - return 'Hello'; - }, - }), - }, - }); - return pipelineExecutor; + // <- TODO: [📼] Use`book\`` string literal notation + ); + const pipelineExecutor = createPipelineExecutor({ + pipeline, + tools: { + llm: new MockedFackedLlmExecutionTools( + // <- TODO: [🧱] Implement in a functional (not new Class) way + { isVerbose: true }, + ), + script: [], + userInterface: new CallbackInterfaceTools({ + isVerbose: true, + async callback() { + return "Hello"; + }, + }), + }, + }); + return pipelineExecutor; } diff --git a/src/llm-providers/mocked/test/joker.test.ts b/src/llm-providers/mocked/test/joker.test.ts index d8446ca5af..f916792844 100644 --- a/src/llm-providers/mocked/test/joker.test.ts +++ b/src/llm-providers/mocked/test/joker.test.ts @@ -1,41 +1,45 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { compilePipeline } from '../../../conversion/compilePipeline'; -import { CallbackInterfaceTools } from '../../../dialogs/callback/CallbackInterfaceTools'; -import { createPipelineExecutor } from '../../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import type { PipelineString } from '../../../pipeline/PipelineString'; -import { MockedEchoLlmExecutionTools } from '../MockedEchoLlmExecutionTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { compilePipeline } from "../../../conversion/compilePipeline"; +import { CallbackInterfaceTools } from "../../../dialogs/callback/CallbackInterfaceTools"; +import { createPipelineExecutor } from "../../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import type { PipelineString } from "../../../pipeline/PipelineString"; +import { MockedEchoLlmExecutionTools } from "../MockedEchoLlmExecutionTools"; -describe('createPipelineExecutor + MockedEchoExecutionTools with example chat prompt', () => { - it('should work when joker is used', async () => { - const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({ yourName: 'Good name' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - name: 'Good name', - }, - }); - }); +describe("createPipelineExecutor + MockedEchoExecutionTools with example chat prompt", () => { + it("should work when joker is used", async () => { + const pipelineExecutor = await getPipelineExecutor(); + expect( + pipelineExecutor({ yourName: "Good name" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + name: "Good name", + }, + }); + }); - it('should work when joker is NOT used', async () => { - const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({ yourName: 'Badname' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - name: spaceTrim(` + it("should work when joker is NOT used", async () => { + const pipelineExecutor = await getPipelineExecutor(); + expect( + pipelineExecutor({ yourName: "Badname" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + name: spaceTrim(` You said: Write some name for Badname `), - }, - }); - }); + }, + }); + }); }); async function getPipelineExecutor() { - const pipeline = await compilePipeline( - spaceTrim(` + const pipeline = await compilePipeline( + spaceTrim(` # ✨ Example: Jokers - MODEL VARIANT Chat @@ -54,21 +58,21 @@ async function getPipelineExecutor() { -> {name} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - ); - return createPipelineExecutor({ - pipeline, - tools: { - llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), - script: [], - userInterface: new CallbackInterfaceTools({ - isVerbose: true, - async callback() { - return 'Hello'; - }, - }), - }, - }); + // <- TODO: [📼] Use`book\`` string literal notation + ); + return createPipelineExecutor({ + pipeline, + tools: { + llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), + script: [], + userInterface: new CallbackInterfaceTools({ + isVerbose: true, + async callback() { + return "Hello"; + }, + }), + }, + }); } /** diff --git a/src/llm-providers/mocked/test/mocked-chat.test.ts b/src/llm-providers/mocked/test/mocked-chat.test.ts index 2c08bed226..4be42d7f54 100644 --- a/src/llm-providers/mocked/test/mocked-chat.test.ts +++ b/src/llm-providers/mocked/test/mocked-chat.test.ts @@ -1,103 +1,105 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { compilePipeline } from '../../../conversion/compilePipeline'; -import { CallbackInterfaceTools } from '../../../dialogs/callback/CallbackInterfaceTools'; -import { createPipelineExecutor } from '../../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import type { PipelineString } from '../../../pipeline/PipelineString'; -import { PROMPTBOOK_ENGINE_VERSION } from '../../../version'; -import { MockedEchoLlmExecutionTools } from '../MockedEchoLlmExecutionTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { compilePipeline } from "../../../conversion/compilePipeline"; +import { CallbackInterfaceTools } from "../../../dialogs/callback/CallbackInterfaceTools"; +import { createPipelineExecutor } from "../../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import type { PipelineString } from "../../../pipeline/PipelineString"; +import { PROMPTBOOK_ENGINE_VERSION } from "../../../version"; +import { MockedEchoLlmExecutionTools } from "../MockedEchoLlmExecutionTools"; -describe('createPipelineExecutor + MockedEchoLlmExecutionTools with example chat prompt', () => { - it('should work when every INPUT PARAMETER defined', async () => { - const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({ thing: 'a cup of coffee' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - executionReport: { - title: 'Example prompt', - promptbookRequestedVersion: '1.0.0', - pipelineUrl: 'https://promptbook.studio/examples/pipeline.book', - promptbookUsedVersion: PROMPTBOOK_ENGINE_VERSION, - }, - outputParameters: { - response: spaceTrim(` +describe("createPipelineExecutor + MockedEchoLlmExecutionTools with example chat prompt", () => { + it("should work when every INPUT PARAMETER defined", async () => { + const pipelineExecutor = await getPipelineExecutor(); + expect( + pipelineExecutor({ thing: "a cup of coffee" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + executionReport: { + title: "Example prompt", + promptbookRequestedVersion: "1.0.0", + pipelineUrl: "https://promptbook.studio/examples/pipeline.book", + promptbookUsedVersion: PROMPTBOOK_ENGINE_VERSION, + }, + outputParameters: { + response: spaceTrim(` You said: One day I went to the shop and bought a cup of coffee. Now I have a cup of coffee. `), - }, - }); - }); + }, + }); + }); - it('should fail when some INPUT PARAMETER is missing', async () => { - const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({}).asPromise()).resolves.toMatchObject({ - isSuccessful: false, - errors: [/Parameter `{thing}` is required as an input parameter/i], - executionReport: { - title: 'Example prompt', - description: 'Show how to use a simple chat prompt', - promptExecutions: [], - pipelineUrl: 'https://promptbook.studio/examples/pipeline.book', - promptbookRequestedVersion: '1.0.0', - promptbookUsedVersion: PROMPTBOOK_ENGINE_VERSION, - }, - outputParameters: {}, - usage: { - input: { - charactersCount: { - value: 0, - }, - linesCount: { - value: 0, - }, - pagesCount: { - value: 0, - }, - paragraphsCount: { - value: 0, - }, - sentencesCount: { - value: 0, - }, - tokensCount: { - value: 0, - }, - wordsCount: { - value: 0, - }, - }, - output: { - charactersCount: { - value: 0, - }, - linesCount: { - value: 0, - }, - pagesCount: { - value: 0, - }, - paragraphsCount: { - value: 0, - }, - sentencesCount: { - value: 0, - }, - tokensCount: { - value: 0, - }, - wordsCount: { - value: 0, - }, - }, - price: { - value: 0, - }, - }, - }); - }); + it("should fail when some INPUT PARAMETER is missing", async () => { + const pipelineExecutor = await getPipelineExecutor(); + expect(pipelineExecutor({}).asPromise()).resolves.toMatchObject({ + isSuccessful: false, + errors: [/Parameter `{thing}` is required as an input parameter/i], + executionReport: { + title: "Example prompt", + description: "Show how to use a simple chat prompt", + promptExecutions: [], + pipelineUrl: "https://promptbook.studio/examples/pipeline.book", + promptbookRequestedVersion: "1.0.0", + promptbookUsedVersion: PROMPTBOOK_ENGINE_VERSION, + }, + outputParameters: {}, + usage: { + input: { + charactersCount: { + value: 0, + }, + linesCount: { + value: 0, + }, + pagesCount: { + value: 0, + }, + paragraphsCount: { + value: 0, + }, + sentencesCount: { + value: 0, + }, + tokensCount: { + value: 0, + }, + wordsCount: { + value: 0, + }, + }, + output: { + charactersCount: { + value: 0, + }, + linesCount: { + value: 0, + }, + pagesCount: { + value: 0, + }, + paragraphsCount: { + value: 0, + }, + sentencesCount: { + value: 0, + }, + tokensCount: { + value: 0, + }, + wordsCount: { + value: 0, + }, + }, + price: { + value: 0, + }, + }, + }); + }); - /* + /* TODO: [🧠] Should be this failing or not? it('should fail when there is INPUT PARAMETER extra', () => { expect(pipelineExecutor({ thing: 'a cup of coffee', sound: 'Meow!' }).asPromise()).rejects.toThrowError(/Parameter \{sound\} should not be defined/i); @@ -106,8 +108,8 @@ describe('createPipelineExecutor + MockedEchoLlmExecutionTools with example chat }); async function getPipelineExecutor() { - const pipeline = await compilePipeline( - spaceTrim(` + const pipeline = await compilePipeline( + spaceTrim(` # Example prompt Show how to use a simple chat prompt @@ -128,22 +130,22 @@ async function getPipelineExecutor() { -> {response} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - ); - const pipelineExecutor = createPipelineExecutor({ - pipeline, - tools: { - llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), - script: [], - userInterface: new CallbackInterfaceTools({ - isVerbose: true, - async callback() { - return 'Hello'; - }, - }), - }, - }); - return pipelineExecutor; + // <- TODO: [📼] Use`book\`` string literal notation + ); + const pipelineExecutor = createPipelineExecutor({ + pipeline, + tools: { + llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), + script: [], + userInterface: new CallbackInterfaceTools({ + isVerbose: true, + async callback() { + return "Hello"; + }, + }), + }, + }); + return pipelineExecutor; } /** diff --git a/src/llm-providers/mocked/test/mocked-completion.test.ts b/src/llm-providers/mocked/test/mocked-completion.test.ts index 36556bb506..4217926214 100644 --- a/src/llm-providers/mocked/test/mocked-completion.test.ts +++ b/src/llm-providers/mocked/test/mocked-completion.test.ts @@ -1,95 +1,97 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { compilePipeline } from '../../../conversion/compilePipeline'; -import { CallbackInterfaceTools } from '../../../dialogs/callback/CallbackInterfaceTools'; -import { createPipelineExecutor } from '../../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import type { PipelineString } from '../../../pipeline/PipelineString'; -import { PROMPTBOOK_ENGINE_VERSION } from '../../../version'; -import { MockedEchoLlmExecutionTools } from '../MockedEchoLlmExecutionTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { compilePipeline } from "../../../conversion/compilePipeline"; +import { CallbackInterfaceTools } from "../../../dialogs/callback/CallbackInterfaceTools"; +import { createPipelineExecutor } from "../../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import type { PipelineString } from "../../../pipeline/PipelineString"; +import { PROMPTBOOK_ENGINE_VERSION } from "../../../version"; +import { MockedEchoLlmExecutionTools } from "../MockedEchoLlmExecutionTools"; -describe('createPipelineExecutor + MockedEchoLlmExecutionTools with example completion prompt', () => { - it('should work when every INPUT PARAMETER defined', async () => { - const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({ thing: 'a cup of coffee' }).asPromise()).resolves.toMatchObject({ - outputParameters: { - response: spaceTrim(` +describe("createPipelineExecutor + MockedEchoLlmExecutionTools with example completion prompt", () => { + it("should work when every INPUT PARAMETER defined", async () => { + const pipelineExecutor = await getPipelineExecutor(); + expect( + pipelineExecutor({ thing: "a cup of coffee" }).asPromise(), + ).resolves.toMatchObject({ + outputParameters: { + response: spaceTrim(` One day I went to the shop and bought a cup of coffee. Now I have a cup of coffee. And so on... `), - }, - }); - }); + }, + }); + }); - it('should fail when some INPUT PARAMETER is missing', async () => { - const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({}).asPromise()).resolves.toMatchObject({ - isSuccessful: false, - errors: [/Parameter `{thing}` is required as an input parameter/i], - executionReport: { - title: 'Example prompt', - description: 'Show how to use a simple completion prompt', - promptExecutions: [], - pipelineUrl: 'https://promptbook.studio/examples/pipeline.book', - promptbookRequestedVersion: '1.0.0', - promptbookUsedVersion: PROMPTBOOK_ENGINE_VERSION, - }, - outputParameters: {}, - usage: { - input: { - charactersCount: { - value: 0, - }, - linesCount: { - value: 0, - }, - pagesCount: { - value: 0, - }, - paragraphsCount: { - value: 0, - }, - sentencesCount: { - value: 0, - }, - tokensCount: { - value: 0, - }, - wordsCount: { - value: 0, - }, - }, - output: { - charactersCount: { - value: 0, - }, - linesCount: { - value: 0, - }, - pagesCount: { - value: 0, - }, - paragraphsCount: { - value: 0, - }, - sentencesCount: { - value: 0, - }, - tokensCount: { - value: 0, - }, - wordsCount: { - value: 0, - }, - }, - price: { - value: 0, - }, - }, - }); - }); + it("should fail when some INPUT PARAMETER is missing", async () => { + const pipelineExecutor = await getPipelineExecutor(); + expect(pipelineExecutor({}).asPromise()).resolves.toMatchObject({ + isSuccessful: false, + errors: [/Parameter `{thing}` is required as an input parameter/i], + executionReport: { + title: "Example prompt", + description: "Show how to use a simple completion prompt", + promptExecutions: [], + pipelineUrl: "https://promptbook.studio/examples/pipeline.book", + promptbookRequestedVersion: "1.0.0", + promptbookUsedVersion: PROMPTBOOK_ENGINE_VERSION, + }, + outputParameters: {}, + usage: { + input: { + charactersCount: { + value: 0, + }, + linesCount: { + value: 0, + }, + pagesCount: { + value: 0, + }, + paragraphsCount: { + value: 0, + }, + sentencesCount: { + value: 0, + }, + tokensCount: { + value: 0, + }, + wordsCount: { + value: 0, + }, + }, + output: { + charactersCount: { + value: 0, + }, + linesCount: { + value: 0, + }, + pagesCount: { + value: 0, + }, + paragraphsCount: { + value: 0, + }, + sentencesCount: { + value: 0, + }, + tokensCount: { + value: 0, + }, + wordsCount: { + value: 0, + }, + }, + price: { + value: 0, + }, + }, + }); + }); - /* + /* TODO: [🧠] Should be this failing or not? it('should fail when there is INPUT PARAMETER extra', () => { expect(pipelineExecutor({ thing: 'a cup of coffee', sound: 'Meow!' }).asPromise()).rejects.toThrowError(/Parameter \{sound\} should not be defined/i); @@ -98,8 +100,8 @@ describe('createPipelineExecutor + MockedEchoLlmExecutionTools with example comp }); async function getPipelineExecutor() { - const pipeline = await compilePipeline( - spaceTrim(` + const pipeline = await compilePipeline( + spaceTrim(` # Example prompt Show how to use a simple completion prompt @@ -121,22 +123,22 @@ async function getPipelineExecutor() { -> {response} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - ); - const pipelineExecutor = createPipelineExecutor({ - pipeline, - tools: { - llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), - script: [], - userInterface: new CallbackInterfaceTools({ - isVerbose: true, - async callback() { - return 'Hello'; - }, - }), - }, - }); - return pipelineExecutor; + // <- TODO: [📼] Use`book\`` string literal notation + ); + const pipelineExecutor = createPipelineExecutor({ + pipeline, + tools: { + llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), + script: [], + userInterface: new CallbackInterfaceTools({ + isVerbose: true, + async callback() { + return "Hello"; + }, + }), + }, + }); + return pipelineExecutor; } /** diff --git a/src/llm-providers/multiple/MultipleLlmExecutionTools.ts b/src/llm-providers/multiple/MultipleLlmExecutionTools.ts index 22635647d1..26a2fcdac5 100644 --- a/src/llm-providers/multiple/MultipleLlmExecutionTools.ts +++ b/src/llm-providers/multiple/MultipleLlmExecutionTools.ts @@ -1,17 +1,26 @@ -import spaceTrim from 'spacetrim'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { AvailableModel } from '../../execution/AvailableModel'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; +import spaceTrim from "spacetrim"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { AvailableModel } from "../../execution/AvailableModel"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; import type { - ChatPromptResult, - CompletionPromptResult, - EmbeddingPromptResult, - PromptResult, -} from '../../execution/PromptResult'; -import type { ChatPrompt, CompletionPrompt, EmbeddingPrompt, Prompt } from '../../types/Prompt'; -import type { string_markdown, string_markdown_text, string_title } from '../../types/typeAliases'; -import type { really_any } from '../../utils/organization/really_any'; + ChatPromptResult, + CompletionPromptResult, + EmbeddingPromptResult, + PromptResult, +} from "../../execution/PromptResult"; +import type { + ChatPrompt, + CompletionPrompt, + EmbeddingPrompt, + Prompt, +} from "../../types/Prompt"; +import type { + string_markdown, + string_markdown_text, + string_title, +} from "../../types/typeAliases"; +import type { really_any } from "../../utils/organization/really_any"; /** * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally. @@ -19,170 +28,183 @@ import type { really_any } from '../../utils/organization/really_any'; * Note: Internal utility of `joinLlmExecutionTools` but exposed type * @public exported from `@promptbook/core` */ -export class MultipleLlmExecutionTools implements LlmExecutionTools /* <- TODO: [🍚] `, Destroyable` */ { - /** - * Array of execution tools in order of priority - */ - public readonly llmExecutionTools: ReadonlyArray; - - /** - * Gets array of execution tools in order of priority - */ - public constructor(...llmExecutionTools: ReadonlyArray) { - this.llmExecutionTools = llmExecutionTools; - } - - public get title(): string_title & string_markdown_text { - return 'Multiple LLM Providers'; - } - - public get description(): string_markdown { - return this.llmExecutionTools.map(({ title }, index) => `${index + 1}) \`${title}\``).join('\n'); - } - - /** - * Check the configuration of all execution tools - */ - public async checkConfiguration(): Promise { - // TODO: Maybe do it in parallel - for (const llmExecutionTools of this.llmExecutionTools) { - await llmExecutionTools.checkConfiguration(); - } - } - - /** - * List all available models that can be used - * This lists is a combination of all available models from all execution tools - */ - public async listModels(): Promise> { - const availableModels: Array = []; - - for (const llmExecutionTools of this.llmExecutionTools) { - // TODO: [🪂] Obtain models in parallel - const models = await llmExecutionTools.listModels(); - availableModels.push(...models); - } - - return availableModels; - } - - /** - * Calls the best available chat model - */ - public callChatModel(prompt: ChatPrompt): Promise { - return this.callCommonModel(prompt) as Promise; - } - - /** - * Calls the best available completion model - */ - public callCompletionModel(prompt: CompletionPrompt): Promise { - return this.callCommonModel(prompt) as Promise; - } - - /** - * Calls the best available embedding model - */ - public callEmbeddingModel(prompt: EmbeddingPrompt): Promise { - return this.callCommonModel(prompt) as Promise; - } - - // <- Note: [🤖] - - /** - * Calls the best available model - * - * Note: This should be private or protected but is public to be usable with duck typing - */ - public async callCommonModel(prompt: Prompt): Promise { - const errors: Array<{ llmExecutionTools: LlmExecutionTools; error: Error }> = []; - - llm: for (const llmExecutionTools of this.llmExecutionTools) { - try { - variant: switch (prompt.modelRequirements.modelVariant) { - case 'CHAT': - if (llmExecutionTools.callChatModel === undefined) { - continue llm; - } - - return await llmExecutionTools.callChatModel(prompt); - break variant; - case 'COMPLETION': - if (llmExecutionTools.callCompletionModel === undefined) { - continue llm; - } - - return await llmExecutionTools.callCompletionModel(prompt); - break variant; - - case 'EMBEDDING': - if (llmExecutionTools.callEmbeddingModel === undefined) { - continue llm; - } - - return await llmExecutionTools.callEmbeddingModel(prompt); - break variant; - - // <- case [🤖]: - - default: - throw new UnexpectedError( - `Unknown model variant "${(prompt as really_any).modelRequirements.modelVariant}"`, - ); - } - } catch (error) { - if (!(error instanceof Error) || error instanceof UnexpectedError) { - throw error; - } - - errors.push({ llmExecutionTools, error }); - } - } - - if (errors.length === 1) { - throw errors[0]; - } else if (errors.length > 1) { - throw new PipelineExecutionError( - // TODO: Tell which execution tools failed like - // 1) OpenAI throw PipelineExecutionError: Parameter `{knowledge}` is not defined - // 2) AnthropicClaude throw PipelineExecutionError: Parameter `{knowledge}` is not defined - // 3) ... - spaceTrim( - (block) => ` +export class MultipleLlmExecutionTools + implements LlmExecutionTools /* <- TODO: [🍚] `, Destroyable` */ +{ + /** + * Array of execution tools in order of priority + */ + public readonly llmExecutionTools: ReadonlyArray; + + /** + * Gets array of execution tools in order of priority + */ + public constructor(...llmExecutionTools: ReadonlyArray) { + this.llmExecutionTools = llmExecutionTools; + } + + public get title(): string_title & string_markdown_text { + return "Multiple LLM Providers"; + } + + public get description(): string_markdown { + return this.llmExecutionTools + .map(({ title }, index) => `${index + 1}) \`${title}\``) + .join("\n"); + } + + /** + * Check the configuration of all execution tools + */ + public async checkConfiguration(): Promise { + // TODO: Maybe do it in parallel + for (const llmExecutionTools of this.llmExecutionTools) { + await llmExecutionTools.checkConfiguration(); + } + } + + /** + * List all available models that can be used + * This lists is a combination of all available models from all execution tools + */ + public async listModels(): Promise> { + const availableModels: Array = []; + + for (const llmExecutionTools of this.llmExecutionTools) { + // TODO: [🪂] Obtain models in parallel + const models = await llmExecutionTools.listModels(); + availableModels.push(...models); + } + + return availableModels; + } + + /** + * Calls the best available chat model + */ + public callChatModel(prompt: ChatPrompt): Promise { + return this.callCommonModel(prompt) as Promise; + } + + /** + * Calls the best available completion model + */ + public callCompletionModel( + prompt: CompletionPrompt, + ): Promise { + return this.callCommonModel(prompt) as Promise; + } + + /** + * Calls the best available embedding model + */ + public callEmbeddingModel( + prompt: EmbeddingPrompt, + ): Promise { + return this.callCommonModel(prompt) as Promise; + } + + // <- Note: [🤖] + + /** + * Calls the best available model + * + * Note: This should be private or protected but is public to be usable with duck typing + */ + public async callCommonModel(prompt: Prompt): Promise { + const errors: Array<{ + llmExecutionTools: LlmExecutionTools; + error: Error; + }> = []; + + llm: for (const llmExecutionTools of this.llmExecutionTools) { + try { + switch (prompt.modelRequirements.modelVariant) { + case "CHAT": + if (llmExecutionTools.callChatModel === undefined) { + continue llm; + } + + return await llmExecutionTools.callChatModel(prompt); + break; + case "COMPLETION": + if (llmExecutionTools.callCompletionModel === undefined) { + continue llm; + } + + return await llmExecutionTools.callCompletionModel(prompt); + break; + + case "EMBEDDING": + if (llmExecutionTools.callEmbeddingModel === undefined) { + continue llm; + } + + return await llmExecutionTools.callEmbeddingModel(prompt); + break; + + // <- case [🤖]: + + default: + throw new UnexpectedError( + `Unknown model variant "${(prompt as really_any).modelRequirements.modelVariant}"`, + ); + } + } catch (error) { + if (!(error instanceof Error) || error instanceof UnexpectedError) { + throw error; + } + + errors.push({ llmExecutionTools, error }); + } + } + + if (errors.length === 1) { + throw errors[0]; + } else if (errors.length > 1) { + throw new PipelineExecutionError( + // TODO: Tell which execution tools failed like + // 1) OpenAI throw PipelineExecutionError: Parameter `{knowledge}` is not defined + // 2) AnthropicClaude throw PipelineExecutionError: Parameter `{knowledge}` is not defined + // 3) ... + spaceTrim( + (block) => ` All execution tools failed: ${block( - errors - .map( - ({ error, llmExecutionTools }, i) => - `${i + 1}) **${llmExecutionTools.title}** thrown **${ - error.name || 'Error' - }:** ${error.message}`, - ) - .join('\n'), - )} + errors + .map( + ({ error, llmExecutionTools }, i) => + `${i + 1}) **${llmExecutionTools.title}** thrown **${ + error.name || "Error" + }:** ${error.message}`, + ) + .join("\n"), + )} `, - ), - ); - } else if (this.llmExecutionTools.length === 0) { - throw new PipelineExecutionError(`You have not provided any \`LlmExecutionTools\``); - } else { - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + ), + ); + } else if (this.llmExecutionTools.length === 0) { + throw new PipelineExecutionError( + `You have not provided any \`LlmExecutionTools\``, + ); + } else { + throw new PipelineExecutionError( + spaceTrim( + (block) => ` You have not provided any \`LlmExecutionTools\` that support model variant "${ - prompt.modelRequirements.modelVariant - }" + prompt.modelRequirements.modelVariant + }" Available \`LlmExecutionTools\`: ${block(this.description)} `, - ), - ); - } - } + ), + ); + } + } } /** diff --git a/src/llm-providers/multiple/joinLlmExecutionTools.ts b/src/llm-providers/multiple/joinLlmExecutionTools.ts index dda5f73d06..910fded2ab 100644 --- a/src/llm-providers/multiple/joinLlmExecutionTools.ts +++ b/src/llm-providers/multiple/joinLlmExecutionTools.ts @@ -1,6 +1,6 @@ -import spaceTrim from 'spacetrim'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import { MultipleLlmExecutionTools } from './MultipleLlmExecutionTools'; +import spaceTrim from "spacetrim"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import { MultipleLlmExecutionTools } from "./MultipleLlmExecutionTools"; /** * Joins multiple LLM Execution Tools into one @@ -18,20 +18,20 @@ import { MultipleLlmExecutionTools } from './MultipleLlmExecutionTools'; * @public exported from `@promptbook/core` */ export function joinLlmExecutionTools( - ...llmExecutionTools: ReadonlyArray + ...llmExecutionTools: ReadonlyArray ): MultipleLlmExecutionTools { - if (llmExecutionTools.length === 0) { - const warningMessage = spaceTrim(` + if (llmExecutionTools.length === 0) { + const warningMessage = spaceTrim(` You have not provided any \`LlmExecutionTools\` This means that you won't be able to execute any prompts that require large language models like GPT-4 or Anthropic's Claude. Technically, it's not an error, but it's probably not what you want because it does not make sense to use Promptbook without language models. `); - // TODO: [🟥] Detect browser / node and make it colorfull - console.warn(warningMessage); + // TODO: [🟥] Detect browser / node and make it colorfull + console.warn(warningMessage); - /* + /* return { async listModels() { // TODO: [🟥] Detect browser / node and make it colorfull @@ -51,9 +51,9 @@ export function joinLlmExecutionTools( }, }; */ - } + } - return new MultipleLlmExecutionTools(...llmExecutionTools); + return new MultipleLlmExecutionTools(...llmExecutionTools); } /** diff --git a/src/llm-providers/multiple/playground/playground.ts b/src/llm-providers/multiple/playground/playground.ts index bd37c88239..b2fe31ccd6 100644 --- a/src/llm-providers/multiple/playground/playground.ts +++ b/src/llm-providers/multiple/playground/playground.ts @@ -1,81 +1,81 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { embeddingVectorToString } from '../../../execution/embeddingVectorToString'; -import { usageToHuman } from '../../../execution/utils/usageToHuman'; -import { JavascriptExecutionTools } from '../../../scripting/javascript/JavascriptExecutionTools'; -import type { Prompt } from '../../../types/Prompt'; -import { keepUnused } from '../../../utils/organization/keepUnused'; -import { AnthropicClaudeExecutionTools } from '../../anthropic-claude/AnthropicClaudeExecutionTools'; -import { AzureOpenAiExecutionTools } from '../../azure-openai/AzureOpenAiExecutionTools'; -import { OpenAiExecutionTools } from '../../openai/OpenAiExecutionTools'; -import { joinLlmExecutionTools } from '../joinLlmExecutionTools'; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { embeddingVectorToString } from "../../../execution/embeddingVectorToString"; +import { usageToHuman } from "../../../execution/utils/usageToHuman"; +import { JavascriptExecutionTools } from "../../../scripting/javascript/JavascriptExecutionTools"; +import type { Prompt } from "../../../types/Prompt"; +import { keepUnused } from "../../../utils/organization/keepUnused"; +import { AnthropicClaudeExecutionTools } from "../../anthropic-claude/AnthropicClaudeExecutionTools"; +import { AzureOpenAiExecutionTools } from "../../azure-openai/AzureOpenAiExecutionTools"; +import { OpenAiExecutionTools } from "../../openai/OpenAiExecutionTools"; +import { joinLlmExecutionTools } from "../joinLlmExecutionTools"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 Multiple LLMs Playground`); - - // Do here stuff you want to test - //========================================> - - const tools = { - llm: [ - new OpenAiExecutionTools( - // <- TODO: [🧱] Implement in a functional (not new Class) way - { - isVerbose: true, - userId: 'playground', - apiKey: process.env.OPENAI_API_KEY!, - }, - ), - new AnthropicClaudeExecutionTools( - // <- TODO: [🧱] Implement in a functional (not new Class) way - { - isVerbose: true, - apiKey: process.env.ANTHROPIC_CLAUDE_API_KEY!, - }, - ), - new AzureOpenAiExecutionTools( - // <- TODO: [🧱] Implement in a functional (not new Class) way - { - isVerbose: true, - userId: 'playground', - resourceName: process.env.AZUREOPENAI_RESOURCE_NAME!, - deploymentName: process.env.AZUREOPENAI_DEPLOYMENT_NAME!, - apiKey: process.env.AZUREOPENAI_API_KEY!, - }, - ), - // TODO: [🦻] Add langtail - ], - script: [new JavascriptExecutionTools()], - }; - const llmTools = joinLlmExecutionTools(...tools.llm); - - keepUnused(llmTools); - keepUnused(embeddingVectorToString); - keepUnused(usageToHuman); - keepUnused(); - - /*/ + console.info(`🧸 Multiple LLMs Playground`); + + // Do here stuff you want to test + //========================================> + + const tools = { + llm: [ + new OpenAiExecutionTools( + // <- TODO: [🧱] Implement in a functional (not new Class) way + { + isVerbose: true, + userId: "playground", + apiKey: process.env.OPENAI_API_KEY!, + }, + ), + new AnthropicClaudeExecutionTools( + // <- TODO: [🧱] Implement in a functional (not new Class) way + { + isVerbose: true, + apiKey: process.env.ANTHROPIC_CLAUDE_API_KEY!, + }, + ), + new AzureOpenAiExecutionTools( + // <- TODO: [🧱] Implement in a functional (not new Class) way + { + isVerbose: true, + userId: "playground", + resourceName: process.env.AZUREOPENAI_RESOURCE_NAME!, + deploymentName: process.env.AZUREOPENAI_DEPLOYMENT_NAME!, + apiKey: process.env.AZUREOPENAI_API_KEY!, + }, + ), + // TODO: [🦻] Add langtail + ], + script: [new JavascriptExecutionTools()], + }; + const llmTools = joinLlmExecutionTools(...tools.llm); + + keepUnused(llmTools); + keepUnused(embeddingVectorToString); + keepUnused(usageToHuman); + keepUnused(); + + /*/ const models = await llmTools.listModels(); console.info(llmTools.title, llmTools.description); console.info({ models }); /**/ - /*/ + /*/ const chatPrompt = { title: 'Hello', parameters: {}, @@ -92,7 +92,7 @@ async function playground() { console.info(chalk.bgGreen(' Completion: ') + chalk.green(chatPromptResult.content)); /**/ - /*/ + /*/ const completionPrompt = { title: 'Hello', parameters: {}, @@ -107,11 +107,11 @@ async function playground() { console.info(chalk.green(completionPrompt.content + completionPromptResult.content)); /**/ - /*/ + /*/ // TODO: Test Translations in playground /**/ - /*/ + /*/ const prompt = { title: 'Embedding Prompt', parameters: {}, @@ -128,11 +128,11 @@ async function playground() { console.info(chalk.bgGreen(' Embedding: ') + chalk.green(embeddingVectorToString(promptResult.content))); /**/ - /*/ + /*/ // <- Note: [🤖] Test here new model variant if needed /**/ - //========================================/ + //========================================/ } /** diff --git a/src/llm-providers/multiple/playground/tsconfig.json b/src/llm-providers/multiple/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/llm-providers/multiple/playground/tsconfig.json +++ b/src/llm-providers/multiple/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/llm-providers/openai/OpenAiAssistantExecutionTools.ts b/src/llm-providers/openai/OpenAiAssistantExecutionTools.ts index 3fd290a02a..9fa4d64557 100644 --- a/src/llm-providers/openai/OpenAiAssistantExecutionTools.ts +++ b/src/llm-providers/openai/OpenAiAssistantExecutionTools.ts @@ -1,22 +1,22 @@ -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import OpenAI from 'openai'; -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import type { ChatPromptResult } from '../../execution/PromptResult'; -import { UNCERTAIN_USAGE } from '../../execution/utils/usage-constants'; -import type { ModelRequirements } from '../../types/ModelRequirements'; -import type { Prompt } from '../../types/Prompt'; -import type { string_date_iso8601 } from '../../types/typeAliases'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_title } from '../../types/typeAliases'; -import type { string_token } from '../../types/typeAliases'; -import { $getCurrentDate } from '../../utils/$getCurrentDate'; -import { templateParameters } from '../../utils/parameters/templateParameters'; -import { exportJson } from '../../utils/serialization/exportJson'; -import type { OpenAiAssistantExecutionToolsOptions } from './OpenAiAssistantExecutionToolsOptions'; -import { OpenAiExecutionTools } from './OpenAiExecutionTools'; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import type OpenAI from "openai"; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import type { ChatPromptResult } from "../../execution/PromptResult"; +import { UNCERTAIN_USAGE } from "../../execution/utils/usage-constants"; +import type { ModelRequirements } from "../../types/ModelRequirements"; +import type { Prompt } from "../../types/Prompt"; +import type { string_date_iso8601 } from "../../types/typeAliases"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_title } from "../../types/typeAliases"; +import type { string_token } from "../../types/typeAliases"; +import { $getCurrentDate } from "../../utils/$getCurrentDate"; +import { templateParameters } from "../../utils/parameters/templateParameters"; +import { exportJson } from "../../utils/serialization/exportJson"; +import type { OpenAiAssistantExecutionToolsOptions } from "./OpenAiAssistantExecutionToolsOptions"; +import { OpenAiExecutionTools } from "./OpenAiExecutionTools"; /** * Execution Tools for calling OpenAI API Assistants @@ -25,55 +25,70 @@ import { OpenAiExecutionTools } from './OpenAiExecutionTools'; * * @public exported from `@promptbook/openai` */ -export class OpenAiAssistantExecutionTools extends OpenAiExecutionTools implements LlmExecutionTools { - /* <- TODO: [🍚] `, Destroyable` */ - private readonly assistantId: string_token; - - /** - * Creates OpenAI Execution Tools. - * - * @param options which are relevant are directly passed to the OpenAI client - */ - public constructor(options: OpenAiAssistantExecutionToolsOptions) { - super(options); - this.assistantId = options.assistantId; - } - - public get title(): string_title & string_markdown_text { - return 'OpenAI Assistant'; - } - - public get description(): string_markdown { - return 'Use single assistant provided by OpenAI'; - } - - /** - * Calls OpenAI API to use a chat model. - */ - public async callChatModel( - prompt: Pick, - ): Promise { - if (this.options.isVerbose) { - console.info('💬 OpenAI callChatModel call', { prompt }); - } - - const { content, parameters, modelRequirements /*, format*/ } = prompt; - - const client = await this.getClient(); - - // TODO: [☂] Use here more modelRequirements - if (modelRequirements.modelVariant !== 'CHAT') { - throw new PipelineExecutionError('Use callChatModel only for CHAT variant'); - } - - // TODO: [👨‍👨‍👧‍👧] Remove: - for (const key of ['maxTokens', 'modelName', 'seed', 'temperature'] as Array) { - if (modelRequirements[key] !== undefined) { - throw new NotYetImplementedError(`In \`OpenAiAssistantExecutionTools\` you cannot specify \`${key}\``); - } - } - - /* +export class OpenAiAssistantExecutionTools + extends OpenAiExecutionTools + implements LlmExecutionTools +{ + /* <- TODO: [🍚] `, Destroyable` */ + private readonly assistantId: string_token; + + /** + * Creates OpenAI Execution Tools. + * + * @param options which are relevant are directly passed to the OpenAI client + */ + public constructor(options: OpenAiAssistantExecutionToolsOptions) { + super(options); + this.assistantId = options.assistantId; + } + + public get title(): string_title & string_markdown_text { + return "OpenAI Assistant"; + } + + public get description(): string_markdown { + return "Use single assistant provided by OpenAI"; + } + + /** + * Calls OpenAI API to use a chat model. + */ + public async callChatModel( + prompt: Pick< + Prompt, + "content" | "parameters" | "modelRequirements" | "format" + >, + ): Promise { + if (this.options.isVerbose) { + console.info("💬 OpenAI callChatModel call", { prompt }); + } + + const { content, parameters, modelRequirements /*, format*/ } = prompt; + + const client = await this.getClient(); + + // TODO: [☂] Use here more modelRequirements + if (modelRequirements.modelVariant !== "CHAT") { + throw new PipelineExecutionError( + "Use callChatModel only for CHAT variant", + ); + } + + // TODO: [👨‍👨‍👧‍👧] Remove: + for (const key of [ + "maxTokens", + "modelName", + "seed", + "temperature", + ] as Array) { + if (modelRequirements[key] !== undefined) { + throw new NotYetImplementedError( + `In \`OpenAiAssistantExecutionTools\` you cannot specify \`${key}\``, + ); + } + } + + /* TODO: [👨‍👨‍👧‍👧] Implement all of this for Assistants const modelName = modelRequirements.modelName || this.getDefaultChatModel().modelName; const modelSettings = { @@ -94,125 +109,133 @@ export class OpenAiAssistantExecutionTools extends OpenAiExecutionTools implemen } */ - // <- TODO: [🚸] Not all models are compatible with JSON mode - // > 'response_format' of type 'json_object' is not supported with this model. - - const rawPromptContent = templateParameters(content, { - ...parameters, - modelName: 'assistant', - // <- [🧠] What is the best value here - }); - const rawRequest: OpenAI.Beta.ThreadCreateAndRunStreamParams = { - // TODO: [👨‍👨‍👧‍👧] ...modelSettings, - // TODO: [👨‍👨‍👧‍👧][🧠] What about system message for assistants, does it make sence - combination of OpenAI assistants with Promptbook Personas - - assistant_id: this.assistantId, - thread: { - messages: [ - // TODO: [🗯] Allow threads to be passed - { role: 'user', content: rawPromptContent }, - ], - }, - - // <- TODO: Add user identification here> user: this.options.user, - }; - const start: string_date_iso8601 = $getCurrentDate(); - let complete: string_date_iso8601; - - if (this.options.isVerbose) { - console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4)); - } - - const stream = await client.beta.threads.createAndRunStream(rawRequest); - - stream.on('connect', () => { - if (this.options.isVerbose) { - console.info('connect', stream.currentEvent); - } - }); - - stream.on('messageDelta', (messageDelta) => { - if ( - this.options.isVerbose && - messageDelta && - messageDelta.content && - messageDelta.content[0] && - messageDelta.content[0].type === 'text' - ) { - console.info('messageDelta', messageDelta.content[0].text?.value); - } - - // <- TODO: [🐚] Make streaming and running tasks working - }); - - stream.on('messageCreated', (message) => { - if (this.options.isVerbose) { - console.info('messageCreated', message); - } - }); - - stream.on('messageDone', (message) => { - if (this.options.isVerbose) { - console.info('messageDone', message); - } - }); - - const rawResponse = await stream.finalMessages(); - - if (this.options.isVerbose) { - console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4)); - } - - if (rawResponse.length !== 1) { - throw new PipelineExecutionError(`There is NOT 1 BUT ${rawResponse.length} finalMessages from OpenAI`); - } - - if (rawResponse[0]!.content.length !== 1) { - throw new PipelineExecutionError( - `There is NOT 1 BUT ${rawResponse[0]!.content.length} finalMessages content from OpenAI`, - ); - } - - if (rawResponse[0]!.content[0]?.type !== 'text') { - throw new PipelineExecutionError( - `There is NOT 'text' BUT ${rawResponse[0]!.content[0]?.type} finalMessages content type from OpenAI`, - ); - } - - const resultContent = rawResponse[0]!.content[0]?.text.value; - // <- TODO: [🧠] There are also annotations, maybe use them - - // eslint-disable-next-line prefer-const - complete = $getCurrentDate(); - const usage = UNCERTAIN_USAGE; - // <- TODO: [🥘] Compute real usage for assistant - // ?> const usage = computeOpenAiUsage(content, resultContent || '', rawResponse); - - if (resultContent === null) { - throw new PipelineExecutionError('No response message from OpenAI'); - } - - return exportJson({ - name: 'promptResult', - message: `Result of \`OpenAiAssistantExecutionTools.callChatModel\``, - order: [], - value: { - content: resultContent, - modelName: 'assistant', - // <- TODO: [🥘] Detect used model in assistant - // ?> model: rawResponse.model || modelName, - timing: { - start, - complete, - }, - usage, - rawPromptContent, - rawRequest, - rawResponse, - // <- [🗯] - }, - }); - } + // <- TODO: [🚸] Not all models are compatible with JSON mode + // > 'response_format' of type 'json_object' is not supported with this model. + + const rawPromptContent = templateParameters(content, { + ...parameters, + modelName: "assistant", + // <- [🧠] What is the best value here + }); + const rawRequest: OpenAI.Beta.ThreadCreateAndRunStreamParams = { + // TODO: [👨‍👨‍👧‍👧] ...modelSettings, + // TODO: [👨‍👨‍👧‍👧][🧠] What about system message for assistants, does it make sence - combination of OpenAI assistants with Promptbook Personas + + assistant_id: this.assistantId, + thread: { + messages: [ + // TODO: [🗯] Allow threads to be passed + { role: "user", content: rawPromptContent }, + ], + }, + + // <- TODO: Add user identification here> user: this.options.user, + }; + const start: string_date_iso8601 = $getCurrentDate(); + let complete: string_date_iso8601; + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("rawRequest"), + JSON.stringify(rawRequest, null, 4), + ); + } + + const stream = await client.beta.threads.createAndRunStream(rawRequest); + + stream.on("connect", () => { + if (this.options.isVerbose) { + console.info("connect", stream.currentEvent); + } + }); + + stream.on("messageDelta", (messageDelta) => { + if ( + this.options.isVerbose && + messageDelta && + messageDelta.content && + messageDelta.content[0] && + messageDelta.content[0].type === "text" + ) { + console.info("messageDelta", messageDelta.content[0].text?.value); + } + + // <- TODO: [🐚] Make streaming and running tasks working + }); + + stream.on("messageCreated", (message) => { + if (this.options.isVerbose) { + console.info("messageCreated", message); + } + }); + + stream.on("messageDone", (message) => { + if (this.options.isVerbose) { + console.info("messageDone", message); + } + }); + + const rawResponse = await stream.finalMessages(); + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("rawResponse"), + JSON.stringify(rawResponse, null, 4), + ); + } + + if (rawResponse.length !== 1) { + throw new PipelineExecutionError( + `There is NOT 1 BUT ${rawResponse.length} finalMessages from OpenAI`, + ); + } + + if (rawResponse[0]!.content.length !== 1) { + throw new PipelineExecutionError( + `There is NOT 1 BUT ${rawResponse[0]!.content.length} finalMessages content from OpenAI`, + ); + } + + if (rawResponse[0]!.content[0]?.type !== "text") { + throw new PipelineExecutionError( + `There is NOT 'text' BUT ${rawResponse[0]!.content[0]?.type} finalMessages content type from OpenAI`, + ); + } + + const resultContent = rawResponse[0]!.content[0]?.text.value; + // <- TODO: [🧠] There are also annotations, maybe use them + + // eslint-disable-next-line prefer-const + complete = $getCurrentDate(); + const usage = UNCERTAIN_USAGE; + // <- TODO: [🥘] Compute real usage for assistant + // ?> const usage = computeOpenAiUsage(content, resultContent || '', rawResponse); + + if (resultContent === null) { + throw new PipelineExecutionError("No response message from OpenAI"); + } + + return exportJson({ + name: "promptResult", + message: `Result of \`OpenAiAssistantExecutionTools.callChatModel\``, + order: [], + value: { + content: resultContent, + modelName: "assistant", + // <- TODO: [🥘] Detect used model in assistant + // ?> model: rawResponse.model || modelName, + timing: { + start, + complete, + }, + usage, + rawPromptContent, + rawRequest, + rawResponse, + // <- [🗯] + }, + }); + } } /** diff --git a/src/llm-providers/openai/OpenAiAssistantExecutionToolsOptions.ts b/src/llm-providers/openai/OpenAiAssistantExecutionToolsOptions.ts index 7bfbdb2316..b254864f9c 100644 --- a/src/llm-providers/openai/OpenAiAssistantExecutionToolsOptions.ts +++ b/src/llm-providers/openai/OpenAiAssistantExecutionToolsOptions.ts @@ -1,6 +1,6 @@ -import type { ClientOptions } from 'openai'; -import type { string_token } from '../../types/typeAliases'; -import type { OpenAiExecutionToolsOptions } from './OpenAiExecutionToolsOptions'; +import type { ClientOptions } from "openai"; +import type { string_token } from "../../types/typeAliases"; +import type { OpenAiExecutionToolsOptions } from "./OpenAiExecutionToolsOptions"; /** * Options for `OpenAiAssistantExecutionTools` @@ -8,10 +8,10 @@ import type { OpenAiExecutionToolsOptions } from './OpenAiExecutionToolsOptions' * @public exported from `@promptbook/openai` */ export type OpenAiAssistantExecutionToolsOptions = OpenAiExecutionToolsOptions & - ClientOptions & { - /** - * Which assistant to use - */ - assistantId: string_token; - // <- TODO: [🧠] This should be maybe more like model for each prompt? - }; + ClientOptions & { + /** + * Which assistant to use + */ + assistantId: string_token; + // <- TODO: [🧠] This should be maybe more like model for each prompt? + }; diff --git a/src/llm-providers/openai/OpenAiExecutionTools.ts b/src/llm-providers/openai/OpenAiExecutionTools.ts index e04d116e98..046d5d60ba 100644 --- a/src/llm-providers/openai/OpenAiExecutionTools.ts +++ b/src/llm-providers/openai/OpenAiExecutionTools.ts @@ -1,91 +1,95 @@ -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import type { ClientOptions } from 'openai'; -import OpenAI from 'openai'; -import spaceTrim from 'spacetrim'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { AvailableModel } from '../../execution/AvailableModel'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import type { ChatPromptResult } from '../../execution/PromptResult'; -import type { CompletionPromptResult } from '../../execution/PromptResult'; -import type { EmbeddingPromptResult } from '../../execution/PromptResult'; -import type { Prompt } from '../../types/Prompt'; -import type { string_date_iso8601 } from '../../types/typeAliases'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_model_name } from '../../types/typeAliases'; -import type { string_title } from '../../types/typeAliases'; -import type { string_token } from '../../types/typeAliases'; -import { $getCurrentDate } from '../../utils/$getCurrentDate'; -import type { really_any } from '../../utils/organization/really_any'; -import { templateParameters } from '../../utils/parameters/templateParameters'; -import { exportJson } from '../../utils/serialization/exportJson'; -import { computeOpenAiUsage } from './computeOpenAiUsage'; -import { OPENAI_MODELS } from './openai-models'; -import { OpenAiAssistantExecutionTools } from './OpenAiAssistantExecutionTools'; -import type { OpenAiExecutionToolsOptions } from './OpenAiExecutionToolsOptions'; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import type { ClientOptions } from "openai"; +import OpenAI from "openai"; +import spaceTrim from "spacetrim"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { AvailableModel } from "../../execution/AvailableModel"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import type { ChatPromptResult } from "../../execution/PromptResult"; +import type { CompletionPromptResult } from "../../execution/PromptResult"; +import type { EmbeddingPromptResult } from "../../execution/PromptResult"; +import type { Prompt } from "../../types/Prompt"; +import type { string_date_iso8601 } from "../../types/typeAliases"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_model_name } from "../../types/typeAliases"; +import type { string_title } from "../../types/typeAliases"; +import type { string_token } from "../../types/typeAliases"; +import { $getCurrentDate } from "../../utils/$getCurrentDate"; +import type { really_any } from "../../utils/organization/really_any"; +import { templateParameters } from "../../utils/parameters/templateParameters"; +import { exportJson } from "../../utils/serialization/exportJson"; +import { OpenAiAssistantExecutionTools } from "./OpenAiAssistantExecutionTools"; +import type { OpenAiExecutionToolsOptions } from "./OpenAiExecutionToolsOptions"; +import { computeOpenAiUsage } from "./computeOpenAiUsage"; +import { OPENAI_MODELS } from "./openai-models"; /** * Execution Tools for calling OpenAI API * * @public exported from `@promptbook/openai` */ -export class OpenAiExecutionTools implements LlmExecutionTools /* <- TODO: [🍚] `, Destroyable` */ { - /** - * OpenAI API client. - */ - private client: OpenAI | null = null; - - /** - * Creates OpenAI Execution Tools. - * - * @param options which are relevant are directly passed to the OpenAI client - */ - public constructor(protected readonly options: OpenAiExecutionToolsOptions) {} - - public get title(): string_title & string_markdown_text { - return 'OpenAI'; - } - - public get description(): string_markdown { - return 'Use all models provided by OpenAI'; - } - - public async getClient(): Promise { - if (this.client === null) { - // Note: Passing only OpenAI relevant options to OpenAI constructor - const openAiOptions: really_any = { ...this.options }; - delete openAiOptions.isVerbose; - delete openAiOptions.userId; - this.client = new OpenAI(openAiOptions as ClientOptions); - } - - return this.client; - } - - /** - * Create (sub)tools for calling OpenAI API Assistants - * - * @param assistantId Which assistant to use - * @returns Tools for calling OpenAI API Assistants with same token - */ - public createAssistantSubtools(assistantId: string_token): OpenAiAssistantExecutionTools { - return new OpenAiAssistantExecutionTools({ ...this.options, assistantId }); - } - - /** - * Check the `options` passed to `constructor` - */ - public async checkConfiguration(): Promise { - await this.getClient(); - // TODO: [🎍] Do here a real check that API is online, working and API key is correct - } - - /** - * List all available OpenAI models that can be used - */ - public listModels(): ReadonlyArray { - /* +export class OpenAiExecutionTools + implements LlmExecutionTools /* <- TODO: [🍚] `, Destroyable` */ +{ + /** + * OpenAI API client. + */ + private client: OpenAI | null = null; + + /** + * Creates OpenAI Execution Tools. + * + * @param options which are relevant are directly passed to the OpenAI client + */ + public constructor(protected readonly options: OpenAiExecutionToolsOptions) {} + + public get title(): string_title & string_markdown_text { + return "OpenAI"; + } + + public get description(): string_markdown { + return "Use all models provided by OpenAI"; + } + + public async getClient(): Promise { + if (this.client === null) { + // Note: Passing only OpenAI relevant options to OpenAI constructor + const openAiOptions: really_any = { ...this.options }; + delete openAiOptions.isVerbose; + delete openAiOptions.userId; + this.client = new OpenAI(openAiOptions as ClientOptions); + } + + return this.client; + } + + /** + * Create (sub)tools for calling OpenAI API Assistants + * + * @param assistantId Which assistant to use + * @returns Tools for calling OpenAI API Assistants with same token + */ + public createAssistantSubtools( + assistantId: string_token, + ): OpenAiAssistantExecutionTools { + return new OpenAiAssistantExecutionTools({ ...this.options, assistantId }); + } + + /** + * Check the `options` passed to `constructor` + */ + public async checkConfiguration(): Promise { + await this.getClient(); + // TODO: [🎍] Do here a real check that API is online, working and API key is correct + } + + /** + * List all available OpenAI models that can be used + */ + public listModels(): ReadonlyArray { + /* Note: Dynamic lising of the models const models = await this.openai.models.list({}); @@ -93,338 +97,392 @@ export class OpenAiExecutionTools implements LlmExecutionTools /* <- TODO: [🍚 console.log(models.data); */ - return OPENAI_MODELS; - } - - /** - * Calls OpenAI API to use a chat model. - */ - public async callChatModel( - prompt: Pick, - ): Promise { - if (this.options.isVerbose) { - console.info('💬 OpenAI callChatModel call', { prompt }); - } - - const { content, parameters, modelRequirements, format } = prompt; - - const client = await this.getClient(); - - // TODO: [☂] Use here more modelRequirements - if (modelRequirements.modelVariant !== 'CHAT') { - throw new PipelineExecutionError('Use callChatModel only for CHAT variant'); - } - - const modelName = modelRequirements.modelName || this.getDefaultChatModel().modelName; - const modelSettings = { - model: modelName, - max_tokens: modelRequirements.maxTokens, - // <- TODO: [🌾] Make some global max cap for maxTokens - - temperature: modelRequirements.temperature, - - // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools - // <- Note: [🧆] - } as OpenAI.Chat.Completions.CompletionCreateParamsNonStreaming; // <- TODO: [💩] Guard here types better - - if (format === 'JSON') { - modelSettings.response_format = { - type: 'json_object', - }; - } - - // <- TODO: [🚸] Not all models are compatible with JSON mode - // > 'response_format' of type 'json_object' is not supported with this model. - - const rawPromptContent = templateParameters(content, { ...parameters, modelName }); - const rawRequest: OpenAI.Chat.Completions.CompletionCreateParamsNonStreaming = { - ...modelSettings, - messages: [ - ...(modelRequirements.systemMessage === undefined - ? [] - : ([ - { - role: 'system', - content: modelRequirements.systemMessage, - }, - ] as const)), - { - role: 'user', - content: rawPromptContent, - }, - ], - user: this.options.userId?.toString(), - }; - const start: string_date_iso8601 = $getCurrentDate(); - let complete: string_date_iso8601; - - if (this.options.isVerbose) { - console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4)); - } - const rawResponse = await client.chat.completions.create(rawRequest).catch((error) => { - if (this.options.isVerbose) { - console.info(colors.bgRed('error'), error); - } - throw error; - }); - - if (this.options.isVerbose) { - console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4)); - } - - if (!rawResponse.choices[0]) { - throw new PipelineExecutionError('No choises from OpenAI'); - } - - if (rawResponse.choices.length > 1) { - // TODO: This should be maybe only warning - throw new PipelineExecutionError('More than one choise from OpenAI'); - } - - const resultContent = rawResponse.choices[0].message.content; - // eslint-disable-next-line prefer-const - complete = $getCurrentDate(); - const usage = computeOpenAiUsage(content || '', resultContent || '', rawResponse); - - if (resultContent === null) { - throw new PipelineExecutionError('No response message from OpenAI'); - } - - return exportJson({ - name: 'promptResult', - message: `Result of \`OpenAiExecutionTools.callChatModel\``, - order: [], - value: { - content: resultContent, - modelName: rawResponse.model || modelName, - timing: { - start, - complete, - }, - usage, - rawPromptContent, - rawRequest, - rawResponse, - // <- [🗯] - }, - }); - } - - /** - * Calls OpenAI API to use a complete model. - */ - public async callCompletionModel( - prompt: Pick, - ): Promise { - if (this.options.isVerbose) { - console.info('🖋 OpenAI callCompletionModel call', { prompt }); - } - - const { content, parameters, modelRequirements } = prompt; - - const client = await this.getClient(); - - // TODO: [☂] Use here more modelRequirements - if (modelRequirements.modelVariant !== 'COMPLETION') { - throw new PipelineExecutionError('Use callCompletionModel only for COMPLETION variant'); - } - - const modelName = modelRequirements.modelName || this.getDefaultCompletionModel().modelName; - const modelSettings = { - model: modelName, - max_tokens: modelRequirements.maxTokens || 2000, // <- Note: [🌾] 2000 is for lagacy reasons - // <- TODO: [🌾] Make some global max cap for maxTokens - temperature: modelRequirements.temperature, - - // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools - // <- Note: [🧆] - }; - - const rawPromptContent = templateParameters(content, { ...parameters, modelName }); - const rawRequest: OpenAI.Completions.CompletionCreateParamsNonStreaming = { - ...modelSettings, - prompt: rawPromptContent, - user: this.options.userId?.toString(), - }; - const start: string_date_iso8601 = $getCurrentDate(); - let complete: string_date_iso8601; - - if (this.options.isVerbose) { - console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4)); - } - const rawResponse = await client.completions.create(rawRequest).catch((error) => { - if (this.options.isVerbose) { - console.info(colors.bgRed('error'), error); - } - throw error; - }); - - if (this.options.isVerbose) { - console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4)); - } - - if (!rawResponse.choices[0]) { - throw new PipelineExecutionError('No choises from OpenAI'); - } - - if (rawResponse.choices.length > 1) { - // TODO: This should be maybe only warning - throw new PipelineExecutionError('More than one choise from OpenAI'); - } - - const resultContent = rawResponse.choices[0].text; - // eslint-disable-next-line prefer-const - complete = $getCurrentDate(); - const usage = computeOpenAiUsage(content || '', resultContent || '', rawResponse); - - return exportJson({ - name: 'promptResult', - message: `Result of \`OpenAiExecutionTools.callCompletionModel\``, - order: [], - value: { - content: resultContent, - modelName: rawResponse.model || modelName, - timing: { - start, - complete, - }, - usage, - rawPromptContent, - rawRequest, - rawResponse, - // <- [🗯] - }, - }); - } - - /** - * Calls OpenAI API to use a embedding model - */ - public async callEmbeddingModel( - prompt: Pick, - ): Promise { - if (this.options.isVerbose) { - console.info('🖋 OpenAI embedding call', { prompt }); - } - - const { content, parameters, modelRequirements } = prompt; - - const client = await this.getClient(); - - // TODO: [☂] Use here more modelRequirements - if (modelRequirements.modelVariant !== 'EMBEDDING') { - throw new PipelineExecutionError('Use embed only for EMBEDDING variant'); - } - - const modelName = modelRequirements.modelName || this.getDefaultEmbeddingModel().modelName; - - const rawPromptContent = templateParameters(content, { ...parameters, modelName }); - const rawRequest: OpenAI.Embeddings.EmbeddingCreateParams = { - input: rawPromptContent, - model: modelName, - }; - - const start: string_date_iso8601 = $getCurrentDate(); - let complete: string_date_iso8601; - - if (this.options.isVerbose) { - console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4)); - } - - const rawResponse = await client.embeddings.create(rawRequest).catch((error) => { - if (this.options.isVerbose) { - console.info(colors.bgRed('error'), error); - } - throw error; - }); - - if (this.options.isVerbose) { - console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4)); - } - - if (rawResponse.data.length !== 1) { - throw new PipelineExecutionError( - `Expected exactly 1 data item in response, got ${rawResponse.data.length}`, - ); - } - - const resultContent = rawResponse.data[0]!.embedding; - - // eslint-disable-next-line prefer-const - complete = $getCurrentDate(); - const usage = computeOpenAiUsage( - content || '', - '', - // <- Note: Embedding does not have result content - rawResponse, - ); - - return exportJson({ - name: 'promptResult', - message: `Result of \`OpenAiExecutionTools.callEmbeddingModel\``, - order: [], - value: { - content: resultContent, - modelName: rawResponse.model || modelName, - timing: { - start, - complete, - }, - usage, - rawPromptContent, - rawRequest, - rawResponse, - // <- [🗯] - }, - }); - } - - // <- Note: [🤖] callXxxModel - - /** - * Get the model that should be used as default - */ - private getDefaultModel(defaultModelName: string_model_name): AvailableModel { - const model = OPENAI_MODELS.find(({ modelName }) => modelName === defaultModelName); - if (model === undefined) { - throw new UnexpectedError( - spaceTrim( - (block) => - ` + return OPENAI_MODELS; + } + + /** + * Calls OpenAI API to use a chat model. + */ + public async callChatModel( + prompt: Pick< + Prompt, + "content" | "parameters" | "modelRequirements" | "format" + >, + ): Promise { + if (this.options.isVerbose) { + console.info("💬 OpenAI callChatModel call", { prompt }); + } + + const { content, parameters, modelRequirements, format } = prompt; + + const client = await this.getClient(); + + // TODO: [☂] Use here more modelRequirements + if (modelRequirements.modelVariant !== "CHAT") { + throw new PipelineExecutionError( + "Use callChatModel only for CHAT variant", + ); + } + + const modelName = + modelRequirements.modelName || this.getDefaultChatModel().modelName; + const modelSettings = { + model: modelName, + max_tokens: modelRequirements.maxTokens, + // <- TODO: [🌾] Make some global max cap for maxTokens + + temperature: modelRequirements.temperature, + + // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools + // <- Note: [🧆] + } as OpenAI.Chat.Completions.CompletionCreateParamsNonStreaming; // <- TODO: [💩] Guard here types better + + if (format === "JSON") { + modelSettings.response_format = { + type: "json_object", + }; + } + + // <- TODO: [🚸] Not all models are compatible with JSON mode + // > 'response_format' of type 'json_object' is not supported with this model. + + const rawPromptContent = templateParameters(content, { + ...parameters, + modelName, + }); + const rawRequest: OpenAI.Chat.Completions.CompletionCreateParamsNonStreaming = + { + ...modelSettings, + messages: [ + ...(modelRequirements.systemMessage === undefined + ? [] + : ([ + { + role: "system", + content: modelRequirements.systemMessage, + }, + ] as const)), + { + role: "user", + content: rawPromptContent, + }, + ], + user: this.options.userId?.toString(), + }; + const start: string_date_iso8601 = $getCurrentDate(); + let complete: string_date_iso8601; + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("rawRequest"), + JSON.stringify(rawRequest, null, 4), + ); + } + const rawResponse = await client.chat.completions + .create(rawRequest) + .catch((error) => { + if (this.options.isVerbose) { + console.info(colors.bgRed("error"), error); + } + throw error; + }); + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("rawResponse"), + JSON.stringify(rawResponse, null, 4), + ); + } + + if (!rawResponse.choices[0]) { + throw new PipelineExecutionError("No choises from OpenAI"); + } + + if (rawResponse.choices.length > 1) { + // TODO: This should be maybe only warning + throw new PipelineExecutionError("More than one choise from OpenAI"); + } + + const resultContent = rawResponse.choices[0].message.content; + // eslint-disable-next-line prefer-const + complete = $getCurrentDate(); + const usage = computeOpenAiUsage( + content || "", + resultContent || "", + rawResponse, + ); + + if (resultContent === null) { + throw new PipelineExecutionError("No response message from OpenAI"); + } + + return exportJson({ + name: "promptResult", + message: `Result of \`OpenAiExecutionTools.callChatModel\``, + order: [], + value: { + content: resultContent, + modelName: rawResponse.model || modelName, + timing: { + start, + complete, + }, + usage, + rawPromptContent, + rawRequest, + rawResponse, + // <- [🗯] + }, + }); + } + + /** + * Calls OpenAI API to use a complete model. + */ + public async callCompletionModel( + prompt: Pick, + ): Promise { + if (this.options.isVerbose) { + console.info("🖋 OpenAI callCompletionModel call", { prompt }); + } + + const { content, parameters, modelRequirements } = prompt; + + const client = await this.getClient(); + + // TODO: [☂] Use here more modelRequirements + if (modelRequirements.modelVariant !== "COMPLETION") { + throw new PipelineExecutionError( + "Use callCompletionModel only for COMPLETION variant", + ); + } + + const modelName = + modelRequirements.modelName || this.getDefaultCompletionModel().modelName; + const modelSettings = { + model: modelName, + max_tokens: modelRequirements.maxTokens || 2000, // <- Note: [🌾] 2000 is for lagacy reasons + // <- TODO: [🌾] Make some global max cap for maxTokens + temperature: modelRequirements.temperature, + + // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools + // <- Note: [🧆] + }; + + const rawPromptContent = templateParameters(content, { + ...parameters, + modelName, + }); + const rawRequest: OpenAI.Completions.CompletionCreateParamsNonStreaming = { + ...modelSettings, + prompt: rawPromptContent, + user: this.options.userId?.toString(), + }; + const start: string_date_iso8601 = $getCurrentDate(); + let complete: string_date_iso8601; + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("rawRequest"), + JSON.stringify(rawRequest, null, 4), + ); + } + const rawResponse = await client.completions + .create(rawRequest) + .catch((error) => { + if (this.options.isVerbose) { + console.info(colors.bgRed("error"), error); + } + throw error; + }); + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("rawResponse"), + JSON.stringify(rawResponse, null, 4), + ); + } + + if (!rawResponse.choices[0]) { + throw new PipelineExecutionError("No choises from OpenAI"); + } + + if (rawResponse.choices.length > 1) { + // TODO: This should be maybe only warning + throw new PipelineExecutionError("More than one choise from OpenAI"); + } + + const resultContent = rawResponse.choices[0].text; + // eslint-disable-next-line prefer-const + complete = $getCurrentDate(); + const usage = computeOpenAiUsage( + content || "", + resultContent || "", + rawResponse, + ); + + return exportJson({ + name: "promptResult", + message: `Result of \`OpenAiExecutionTools.callCompletionModel\``, + order: [], + value: { + content: resultContent, + modelName: rawResponse.model || modelName, + timing: { + start, + complete, + }, + usage, + rawPromptContent, + rawRequest, + rawResponse, + // <- [🗯] + }, + }); + } + + /** + * Calls OpenAI API to use a embedding model + */ + public async callEmbeddingModel( + prompt: Pick, + ): Promise { + if (this.options.isVerbose) { + console.info("🖋 OpenAI embedding call", { prompt }); + } + + const { content, parameters, modelRequirements } = prompt; + + const client = await this.getClient(); + + // TODO: [☂] Use here more modelRequirements + if (modelRequirements.modelVariant !== "EMBEDDING") { + throw new PipelineExecutionError("Use embed only for EMBEDDING variant"); + } + + const modelName = + modelRequirements.modelName || this.getDefaultEmbeddingModel().modelName; + + const rawPromptContent = templateParameters(content, { + ...parameters, + modelName, + }); + const rawRequest: OpenAI.Embeddings.EmbeddingCreateParams = { + input: rawPromptContent, + model: modelName, + }; + + const start: string_date_iso8601 = $getCurrentDate(); + let complete: string_date_iso8601; + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("rawRequest"), + JSON.stringify(rawRequest, null, 4), + ); + } + + const rawResponse = await client.embeddings + .create(rawRequest) + .catch((error) => { + if (this.options.isVerbose) { + console.info(colors.bgRed("error"), error); + } + throw error; + }); + + if (this.options.isVerbose) { + console.info( + colors.bgWhite("rawResponse"), + JSON.stringify(rawResponse, null, 4), + ); + } + + if (rawResponse.data.length !== 1) { + throw new PipelineExecutionError( + `Expected exactly 1 data item in response, got ${rawResponse.data.length}`, + ); + } + + const resultContent = rawResponse.data[0]!.embedding; + + // eslint-disable-next-line prefer-const + complete = $getCurrentDate(); + const usage = computeOpenAiUsage( + content || "", + "", + // <- Note: Embedding does not have result content + rawResponse, + ); + + return exportJson({ + name: "promptResult", + message: `Result of \`OpenAiExecutionTools.callEmbeddingModel\``, + order: [], + value: { + content: resultContent, + modelName: rawResponse.model || modelName, + timing: { + start, + complete, + }, + usage, + rawPromptContent, + rawRequest, + rawResponse, + // <- [🗯] + }, + }); + } + + // <- Note: [🤖] callXxxModel + + /** + * Get the model that should be used as default + */ + private getDefaultModel(defaultModelName: string_model_name): AvailableModel { + const model = OPENAI_MODELS.find( + ({ modelName }) => modelName === defaultModelName, + ); + if (model === undefined) { + throw new UnexpectedError( + spaceTrim( + (block) => + ` Cannot find model in OpenAI models with name "${defaultModelName}" which should be used as default. Available models: - ${block(OPENAI_MODELS.map(({ modelName }) => `- "${modelName}"`).join('\n'))} + ${block(OPENAI_MODELS.map(({ modelName }) => `- "${modelName}"`).join("\n"))} `, - ), - ); - } - return model; - } - - /** - * Default model for chat variant. - */ - private getDefaultChatModel(): AvailableModel { - return this.getDefaultModel('gpt-4o'); - } - - /** - * Default model for completion variant. - */ - private getDefaultCompletionModel(): AvailableModel { - return this.getDefaultModel('gpt-3.5-turbo-instruct'); - } - - /** - * Default model for completion variant. - */ - private getDefaultEmbeddingModel(): AvailableModel { - return this.getDefaultModel('text-embedding-3-large'); - } - - // <- Note: [🤖] getDefaultXxxModel + ), + ); + } + return model; + } + + /** + * Default model for chat variant. + */ + private getDefaultChatModel(): AvailableModel { + return this.getDefaultModel("gpt-4o"); + } + + /** + * Default model for completion variant. + */ + private getDefaultCompletionModel(): AvailableModel { + return this.getDefaultModel("gpt-3.5-turbo-instruct"); + } + + /** + * Default model for completion variant. + */ + private getDefaultEmbeddingModel(): AvailableModel { + return this.getDefaultModel("text-embedding-3-large"); + } + + // <- Note: [🤖] getDefaultXxxModel } /** diff --git a/src/llm-providers/openai/OpenAiExecutionToolsOptions.ts b/src/llm-providers/openai/OpenAiExecutionToolsOptions.ts index 8530554480..78fc13017a 100644 --- a/src/llm-providers/openai/OpenAiExecutionToolsOptions.ts +++ b/src/llm-providers/openai/OpenAiExecutionToolsOptions.ts @@ -1,5 +1,5 @@ -import type { ClientOptions } from 'openai'; -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; +import type { ClientOptions } from "openai"; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; /** * Options for `OpenAiExecutionTools` diff --git a/src/llm-providers/openai/computeOpenAiUsage.test.ts b/src/llm-providers/openai/computeOpenAiUsage.test.ts index 034372ec2f..e010eec0ff 100644 --- a/src/llm-providers/openai/computeOpenAiUsage.test.ts +++ b/src/llm-providers/openai/computeOpenAiUsage.test.ts @@ -1,70 +1,70 @@ -import { describe, expect, it } from '@jest/globals'; -import { computeOpenAiUsage } from './computeOpenAiUsage'; +import { describe, expect, it } from "@jest/globals"; +import { computeOpenAiUsage } from "./computeOpenAiUsage"; -describe('how computeOpenAiUsage works', () => { - // TODO: [🐞] Test Chat and Completion models +describe("how computeOpenAiUsage works", () => { + // TODO: [🐞] Test Chat and Completion models - it('should compute a OpenAI usage of embedding model', () => { - expect( - computeOpenAiUsage('', '', { - model: 'text-embedding-3-large', - usage: { - prompt_tokens: 29, - // total_tokens: 29, - }, - }), - ).toEqual({ - input: { - charactersCount: { - value: 0, - }, - linesCount: { - value: 0, - }, - pagesCount: { - value: 0, - }, - paragraphsCount: { - value: 0, - }, - sentencesCount: { - value: 0, - }, - tokensCount: { - value: 29, - }, - wordsCount: { - value: 0, - }, - }, - output: { - charactersCount: { - value: 0, - }, - linesCount: { - value: 0, - }, - pagesCount: { - value: 0, - }, - paragraphsCount: { - value: 0, - }, - sentencesCount: { - value: 0, - }, - tokensCount: { - value: 0, - }, - wordsCount: { - value: 0, - }, - }, - price: { - value: 0.00000377, - }, - }); - }); + it("should compute a OpenAI usage of embedding model", () => { + expect( + computeOpenAiUsage("", "", { + model: "text-embedding-3-large", + usage: { + prompt_tokens: 29, + // total_tokens: 29, + }, + }), + ).toEqual({ + input: { + charactersCount: { + value: 0, + }, + linesCount: { + value: 0, + }, + pagesCount: { + value: 0, + }, + paragraphsCount: { + value: 0, + }, + sentencesCount: { + value: 0, + }, + tokensCount: { + value: 29, + }, + wordsCount: { + value: 0, + }, + }, + output: { + charactersCount: { + value: 0, + }, + linesCount: { + value: 0, + }, + pagesCount: { + value: 0, + }, + paragraphsCount: { + value: 0, + }, + sentencesCount: { + value: 0, + }, + tokensCount: { + value: 0, + }, + wordsCount: { + value: 0, + }, + }, + price: { + value: 0.00000377, + }, + }); + }); }); /** diff --git a/src/llm-providers/openai/computeOpenAiUsage.ts b/src/llm-providers/openai/computeOpenAiUsage.ts index e09e88e8b2..60e4f7435d 100644 --- a/src/llm-providers/openai/computeOpenAiUsage.ts +++ b/src/llm-providers/openai/computeOpenAiUsage.ts @@ -1,12 +1,12 @@ -import type OpenAI from 'openai'; -import type { PartialDeep } from 'type-fest'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import type { PromptResultUsage } from '../../execution/PromptResultUsage'; -import type { UncertainNumber } from '../../execution/UncertainNumber'; -import { computeUsageCounts } from '../../execution/utils/computeUsageCounts'; -import { uncertainNumber } from '../../execution/utils/uncertainNumber'; -import type { Prompt } from '../../types/Prompt'; -import { OPENAI_MODELS } from './openai-models'; +import type OpenAI from "openai"; +import type { PartialDeep } from "type-fest"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import type { PromptResultUsage } from "../../execution/PromptResultUsage"; +import type { UncertainNumber } from "../../execution/UncertainNumber"; +import { computeUsageCounts } from "../../execution/utils/computeUsageCounts"; +import { uncertainNumber } from "../../execution/utils/uncertainNumber"; +import type { Prompt } from "../../types/Prompt"; +import { OPENAI_MODELS } from "./openai-models"; /** * Computes the usage of the OpenAI API based on the response from OpenAI @@ -18,50 +18,61 @@ import { OPENAI_MODELS } from './openai-models'; * @private internal utility of `OpenAiExecutionTools` */ export function computeOpenAiUsage( - promptContent: Prompt['content'], // <- Note: Intentionally using [] to access type properties to bring jsdoc from Prompt/PromptResult to consumer - resultContent: string, - rawResponse: PartialDeep< - Pick< - | OpenAI.Chat.Completions.ChatCompletion - | OpenAI.Completions.Completion - | OpenAI.Embeddings.CreateEmbeddingResponse, - // [🗯] | OpenAI.Beta.Threads.Messages.Message, - 'model' | 'usage' - > - >, + promptContent: Prompt["content"], // <- Note: Intentionally using [] to access type properties to bring jsdoc from Prompt/PromptResult to consumer + resultContent: string, + rawResponse: PartialDeep< + Pick< + | OpenAI.Chat.Completions.ChatCompletion + | OpenAI.Completions.Completion + | OpenAI.Embeddings.CreateEmbeddingResponse, + // [🗯] | OpenAI.Beta.Threads.Messages.Message, + "model" | "usage" + > + >, ): PromptResultUsage { - if (rawResponse.usage === undefined) { - throw new PipelineExecutionError('The usage is not defined in the response from OpenAI'); - } + if (rawResponse.usage === undefined) { + throw new PipelineExecutionError( + "The usage is not defined in the response from OpenAI", + ); + } - if (rawResponse.usage?.prompt_tokens === undefined) { - throw new PipelineExecutionError('In OpenAI response `usage.prompt_tokens` not defined'); - } + if (rawResponse.usage?.prompt_tokens === undefined) { + throw new PipelineExecutionError( + "In OpenAI response `usage.prompt_tokens` not defined", + ); + } - const inputTokens = rawResponse.usage.prompt_tokens; - const outputTokens = (rawResponse as OpenAI.Chat.Completions.ChatCompletion).usage?.completion_tokens || 0; + const inputTokens = rawResponse.usage.prompt_tokens; + const outputTokens = + (rawResponse as OpenAI.Chat.Completions.ChatCompletion).usage + ?.completion_tokens || 0; - const modelInfo = OPENAI_MODELS.find((model) => model.modelName === rawResponse.model); + const modelInfo = OPENAI_MODELS.find( + (model) => model.modelName === rawResponse.model, + ); - let price: UncertainNumber; + let price: UncertainNumber; - if (modelInfo === undefined || modelInfo.pricing === undefined) { - price = uncertainNumber(); - } else { - price = uncertainNumber(inputTokens * modelInfo.pricing.prompt + outputTokens * modelInfo.pricing.output); - } + if (modelInfo === undefined || modelInfo.pricing === undefined) { + price = uncertainNumber(); + } else { + price = uncertainNumber( + inputTokens * modelInfo.pricing.prompt + + outputTokens * modelInfo.pricing.output, + ); + } - return { - price, - input: { - tokensCount: uncertainNumber(rawResponse.usage.prompt_tokens), - ...computeUsageCounts(promptContent), - }, - output: { - tokensCount: uncertainNumber(outputTokens), - ...computeUsageCounts(resultContent), - }, - }; + return { + price, + input: { + tokensCount: uncertainNumber(rawResponse.usage.prompt_tokens), + ...computeUsageCounts(promptContent), + }, + output: { + tokensCount: uncertainNumber(outputTokens), + ...computeUsageCounts(resultContent), + }, + }; } /** diff --git a/src/llm-providers/openai/computeUsage.test.ts b/src/llm-providers/openai/computeUsage.test.ts index 4a1b02141f..b76dbaafc2 100644 --- a/src/llm-providers/openai/computeUsage.test.ts +++ b/src/llm-providers/openai/computeUsage.test.ts @@ -1,11 +1,11 @@ -import { describe, expect, it } from '@jest/globals'; -import { computeUsage } from './computeUsage'; +import { describe, expect, it } from "@jest/globals"; +import { computeUsage } from "./computeUsage"; -describe('how computeUsage works', () => { - it('should compute a usage', () => { - expect(computeUsage(`$2.00 / 1M tokens`)).toBe(0.000002); - expect(computeUsage(`$1.50 / 1M tokens`)).toBe(0.0000015); - expect(computeUsage(`$5.00 / 1M tokens`)).toBe(0.000005); - expect(computeUsage(`$30.00 / 1M tokens`)).toBe(0.00003); - }); +describe("how computeUsage works", () => { + it("should compute a usage", () => { + expect(computeUsage(`$2.00 / 1M tokens`)).toBe(0.000002); + expect(computeUsage(`$1.50 / 1M tokens`)).toBe(0.0000015); + expect(computeUsage(`$5.00 / 1M tokens`)).toBe(0.000005); + expect(computeUsage(`$30.00 / 1M tokens`)).toBe(0.00003); + }); }); diff --git a/src/llm-providers/openai/computeUsage.ts b/src/llm-providers/openai/computeUsage.ts index ee232d1ad2..8bd148b089 100644 --- a/src/llm-providers/openai/computeUsage.ts +++ b/src/llm-providers/openai/computeUsage.ts @@ -14,7 +14,11 @@ type string_model_price = `$${number}.${number} / ${number}M tokens`; * @private within the repository, used only as internal helper for `OPENAI_MODELS` */ export function computeUsage(value: string_model_price): number { - const [price, tokens] = value.split(' / '); + const [price, tokens] = value.split(" / "); - return parseFloat(price!.replace('$', '')) / parseFloat(tokens!.replace('M tokens', '')) / 1000000; + return ( + Number.parseFloat(price!.replace("$", "")) / + Number.parseFloat(tokens!.replace("M tokens", "")) / + 1000000 + ); } diff --git a/src/llm-providers/openai/createOpenAiAssistantExecutionTools.ts b/src/llm-providers/openai/createOpenAiAssistantExecutionTools.ts index af839fb225..875df48137 100644 --- a/src/llm-providers/openai/createOpenAiAssistantExecutionTools.ts +++ b/src/llm-providers/openai/createOpenAiAssistantExecutionTools.ts @@ -1,8 +1,8 @@ -import type { LlmExecutionToolsConstructor } from '../../execution/LlmExecutionToolsConstructor'; -import { $isRunningInBrowser } from '../../utils/environment/$isRunningInBrowser'; -import { $isRunningInWebWorker } from '../../utils/environment/$isRunningInWebWorker'; -import { OpenAiAssistantExecutionTools } from './OpenAiAssistantExecutionTools'; -import type { OpenAiAssistantExecutionToolsOptions } from './OpenAiAssistantExecutionToolsOptions'; +import type { LlmExecutionToolsConstructor } from "../../execution/LlmExecutionToolsConstructor"; +import { $isRunningInBrowser } from "../../utils/environment/$isRunningInBrowser"; +import { $isRunningInWebWorker } from "../../utils/environment/$isRunningInWebWorker"; +import { OpenAiAssistantExecutionTools } from "./OpenAiAssistantExecutionTools"; +import type { OpenAiAssistantExecutionToolsOptions } from "./OpenAiAssistantExecutionToolsOptions"; /** * Execution Tools for calling OpenAI API @@ -10,19 +10,24 @@ import type { OpenAiAssistantExecutionToolsOptions } from './OpenAiAssistantExec * @public exported from `@promptbook/openai` */ export const createOpenAiAssistantExecutionTools = Object.assign( - (options: OpenAiAssistantExecutionToolsOptions): OpenAiAssistantExecutionTools => { - // TODO: [🧠][main] !!4 If browser, auto add `dangerouslyAllowBrowser` + ( + options: OpenAiAssistantExecutionToolsOptions, + ): OpenAiAssistantExecutionTools => { + // TODO: [🧠][main] !!4 If browser, auto add `dangerouslyAllowBrowser` - if (($isRunningInBrowser() || $isRunningInWebWorker()) && !options.dangerouslyAllowBrowser) { - options = { ...options, dangerouslyAllowBrowser: true }; - } + if ( + ($isRunningInBrowser() || $isRunningInWebWorker()) && + !options.dangerouslyAllowBrowser + ) { + options = { ...options, dangerouslyAllowBrowser: true }; + } - return new OpenAiAssistantExecutionTools(options); - }, - { - packageName: '@promptbook/openai', - className: 'OpenAiAssistantExecutionTools', - }, + return new OpenAiAssistantExecutionTools(options); + }, + { + packageName: "@promptbook/openai", + className: "OpenAiAssistantExecutionTools", + }, ) satisfies LlmExecutionToolsConstructor; /** diff --git a/src/llm-providers/openai/createOpenAiExecutionTools.ts b/src/llm-providers/openai/createOpenAiExecutionTools.ts index 081c67b42e..b08a5895f9 100644 --- a/src/llm-providers/openai/createOpenAiExecutionTools.ts +++ b/src/llm-providers/openai/createOpenAiExecutionTools.ts @@ -1,8 +1,8 @@ -import type { LlmExecutionToolsConstructor } from '../../execution/LlmExecutionToolsConstructor'; -import { $isRunningInBrowser } from '../../utils/environment/$isRunningInBrowser'; -import { $isRunningInWebWorker } from '../../utils/environment/$isRunningInWebWorker'; -import { OpenAiExecutionTools } from './OpenAiExecutionTools'; -import type { OpenAiExecutionToolsOptions } from './OpenAiExecutionToolsOptions'; +import type { LlmExecutionToolsConstructor } from "../../execution/LlmExecutionToolsConstructor"; +import { $isRunningInBrowser } from "../../utils/environment/$isRunningInBrowser"; +import { $isRunningInWebWorker } from "../../utils/environment/$isRunningInWebWorker"; +import { OpenAiExecutionTools } from "./OpenAiExecutionTools"; +import type { OpenAiExecutionToolsOptions } from "./OpenAiExecutionToolsOptions"; /** * Execution Tools for calling OpenAI API @@ -10,19 +10,22 @@ import type { OpenAiExecutionToolsOptions } from './OpenAiExecutionToolsOptions' * @public exported from `@promptbook/openai` */ export const createOpenAiExecutionTools = Object.assign( - (options: OpenAiExecutionToolsOptions): OpenAiExecutionTools => { - // TODO: [🧠][main] !!4 If browser, auto add `dangerouslyAllowBrowser` + (options: OpenAiExecutionToolsOptions): OpenAiExecutionTools => { + // TODO: [🧠][main] !!4 If browser, auto add `dangerouslyAllowBrowser` - if (($isRunningInBrowser() || $isRunningInWebWorker()) && !options.dangerouslyAllowBrowser) { - options = { ...options, dangerouslyAllowBrowser: true }; - } + if ( + ($isRunningInBrowser() || $isRunningInWebWorker()) && + !options.dangerouslyAllowBrowser + ) { + options = { ...options, dangerouslyAllowBrowser: true }; + } - return new OpenAiExecutionTools(options); - }, - { - packageName: '@promptbook/openai', - className: 'OpenAiExecutionTools', - }, + return new OpenAiExecutionTools(options); + }, + { + packageName: "@promptbook/openai", + className: "OpenAiExecutionTools", + }, ) satisfies LlmExecutionToolsConstructor; /** diff --git a/src/llm-providers/openai/openai-models.ts b/src/llm-providers/openai/openai-models.ts index 1db1feb06b..a22539f0f3 100644 --- a/src/llm-providers/openai/openai-models.ts +++ b/src/llm-providers/openai/openai-models.ts @@ -1,7 +1,7 @@ -import type { AvailableModel } from '../../execution/AvailableModel'; -import type { number_usd } from '../../types/typeAliases'; -import { exportJson } from '../../utils/serialization/exportJson'; -import { computeUsage } from './computeUsage'; +import type { AvailableModel } from "../../execution/AvailableModel"; +import type { number_usd } from "../../types/typeAliases"; +import { exportJson } from "../../utils/serialization/exportJson"; +import { computeUsage } from "./computeUsage"; /** * List of available OpenAI models with pricing @@ -13,99 +13,99 @@ import { computeUsage } from './computeUsage'; * @public exported from `@promptbook/openai` */ export const OPENAI_MODELS: ReadonlyArray< - AvailableModel & { - pricing?: { - readonly prompt: number_usd; - readonly output: number_usd; - }; - } + AvailableModel & { + pricing?: { + readonly prompt: number_usd; + readonly output: number_usd; + }; + } > = exportJson({ - name: 'OPENAI_MODELS', - value: [ - /*/ + name: "OPENAI_MODELS", + value: [ + /*/ { modelTitle: 'dall-e-3', modelName: 'dall-e-3', }, /**/ - /*/ + /*/ { modelTitle: 'whisper-1', modelName: 'whisper-1', }, /**/ - /**/ - { - modelVariant: 'COMPLETION', - modelTitle: 'davinci-002', - modelName: 'davinci-002', - pricing: { - prompt: computeUsage(`$2.00 / 1M tokens`), // <- not sure - output: computeUsage(`$2.00 / 1M tokens`), // <- not sure - }, - }, - /**/ + /**/ + { + modelVariant: "COMPLETION", + modelTitle: "davinci-002", + modelName: "davinci-002", + pricing: { + prompt: computeUsage(`$2.00 / 1M tokens`), // <- not sure + output: computeUsage(`$2.00 / 1M tokens`), // <- not sure + }, + }, + /**/ - /*/ + /*/ { modelTitle: 'dall-e-2', modelName: 'dall-e-2', }, /**/ - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-3.5-turbo-16k', - modelName: 'gpt-3.5-turbo-16k', - pricing: { - prompt: computeUsage(`$3.00 / 1M tokens`), - output: computeUsage(`$4.00 / 1M tokens`), - }, - }, - /**/ - - /*/ + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-3.5-turbo-16k", + modelName: "gpt-3.5-turbo-16k", + pricing: { + prompt: computeUsage(`$3.00 / 1M tokens`), + output: computeUsage(`$4.00 / 1M tokens`), + }, + }, + /**/ + + /*/ { modelTitle: 'tts-1-hd-1106', modelName: 'tts-1-hd-1106', }, /**/ - /*/ + /*/ { modelTitle: 'tts-1-hd', modelName: 'tts-1-hd', }, /**/ - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-4', - modelName: 'gpt-4', - pricing: { - prompt: computeUsage(`$30.00 / 1M tokens`), - output: computeUsage(`$60.00 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-4-32k', - modelName: 'gpt-4-32k', - pricing: { - prompt: computeUsage(`$60.00 / 1M tokens`), - output: computeUsage(`$120.00 / 1M tokens`), - }, - }, - /**/ - - /*/ + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-4", + modelName: "gpt-4", + pricing: { + prompt: computeUsage(`$30.00 / 1M tokens`), + output: computeUsage(`$60.00 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-4-32k", + modelName: "gpt-4-32k", + pricing: { + prompt: computeUsage(`$60.00 / 1M tokens`), + output: computeUsage(`$120.00 / 1M tokens`), + }, + }, + /**/ + + /*/ { modelVariant: 'CHAT', modelTitle: 'gpt-4-0613', @@ -117,216 +117,216 @@ export const OPENAI_MODELS: ReadonlyArray< }, /**/ - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-4-turbo-2024-04-09', - modelName: 'gpt-4-turbo-2024-04-09', - pricing: { - prompt: computeUsage(`$10.00 / 1M tokens`), - output: computeUsage(`$30.00 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-3.5-turbo-1106', - modelName: 'gpt-3.5-turbo-1106', - pricing: { - prompt: computeUsage(`$1.00 / 1M tokens`), - output: computeUsage(`$2.00 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-4-turbo', - modelName: 'gpt-4-turbo', - pricing: { - prompt: computeUsage(`$10.00 / 1M tokens`), - output: computeUsage(`$30.00 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'COMPLETION', - modelTitle: 'gpt-3.5-turbo-instruct-0914', - modelName: 'gpt-3.5-turbo-instruct-0914', - pricing: { - prompt: computeUsage(`$1.50 / 1M tokens`), // <- For gpt-3.5-turbo-instruct - output: computeUsage(`$2.00 / 1M tokens`), // <- For gpt-3.5-turbo-instruct - }, - }, - /**/ - - /**/ - { - modelVariant: 'COMPLETION', - modelTitle: 'gpt-3.5-turbo-instruct', - modelName: 'gpt-3.5-turbo-instruct', - pricing: { - prompt: computeUsage(`$1.50 / 1M tokens`), - output: computeUsage(`$2.00 / 1M tokens`), - }, - }, - /**/ - - /*/ + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-4-turbo-2024-04-09", + modelName: "gpt-4-turbo-2024-04-09", + pricing: { + prompt: computeUsage(`$10.00 / 1M tokens`), + output: computeUsage(`$30.00 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-3.5-turbo-1106", + modelName: "gpt-3.5-turbo-1106", + pricing: { + prompt: computeUsage(`$1.00 / 1M tokens`), + output: computeUsage(`$2.00 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-4-turbo", + modelName: "gpt-4-turbo", + pricing: { + prompt: computeUsage(`$10.00 / 1M tokens`), + output: computeUsage(`$30.00 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "COMPLETION", + modelTitle: "gpt-3.5-turbo-instruct-0914", + modelName: "gpt-3.5-turbo-instruct-0914", + pricing: { + prompt: computeUsage(`$1.50 / 1M tokens`), // <- For gpt-3.5-turbo-instruct + output: computeUsage(`$2.00 / 1M tokens`), // <- For gpt-3.5-turbo-instruct + }, + }, + /**/ + + /**/ + { + modelVariant: "COMPLETION", + modelTitle: "gpt-3.5-turbo-instruct", + modelName: "gpt-3.5-turbo-instruct", + pricing: { + prompt: computeUsage(`$1.50 / 1M tokens`), + output: computeUsage(`$2.00 / 1M tokens`), + }, + }, + /**/ + + /*/ { modelTitle: 'tts-1', modelName: 'tts-1', }, /**/ - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-3.5-turbo', - modelName: 'gpt-3.5-turbo', - pricing: { - prompt: computeUsage(`$3.00 / 1M tokens`), // <- Not sure, refer to gpt-3.5-turbo in Fine-tuning models - output: computeUsage(`$6.00 / 1M tokens`), // <- Not sure, refer to gpt-3.5-turbo in Fine-tuning models - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-3.5-turbo-0301', - modelName: 'gpt-3.5-turbo-0301', - pricing: { - prompt: computeUsage(`$1.50 / 1M tokens`), - output: computeUsage(`$2.00 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'COMPLETION', - modelTitle: 'babbage-002', - modelName: 'babbage-002', - pricing: { - prompt: computeUsage(`$0.40 / 1M tokens`), // <- Not sure - output: computeUsage(`$0.40 / 1M tokens`), // <- Not sure - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-4-1106-preview', - modelName: 'gpt-4-1106-preview', - pricing: { - prompt: computeUsage(`$10.00 / 1M tokens`), - output: computeUsage(`$30.00 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-4-0125-preview', - modelName: 'gpt-4-0125-preview', - pricing: { - prompt: computeUsage(`$10.00 / 1M tokens`), - output: computeUsage(`$30.00 / 1M tokens`), - }, - }, - /**/ - - /*/ + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-3.5-turbo", + modelName: "gpt-3.5-turbo", + pricing: { + prompt: computeUsage(`$3.00 / 1M tokens`), // <- Not sure, refer to gpt-3.5-turbo in Fine-tuning models + output: computeUsage(`$6.00 / 1M tokens`), // <- Not sure, refer to gpt-3.5-turbo in Fine-tuning models + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-3.5-turbo-0301", + modelName: "gpt-3.5-turbo-0301", + pricing: { + prompt: computeUsage(`$1.50 / 1M tokens`), + output: computeUsage(`$2.00 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "COMPLETION", + modelTitle: "babbage-002", + modelName: "babbage-002", + pricing: { + prompt: computeUsage(`$0.40 / 1M tokens`), // <- Not sure + output: computeUsage(`$0.40 / 1M tokens`), // <- Not sure + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-4-1106-preview", + modelName: "gpt-4-1106-preview", + pricing: { + prompt: computeUsage(`$10.00 / 1M tokens`), + output: computeUsage(`$30.00 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-4-0125-preview", + modelName: "gpt-4-0125-preview", + pricing: { + prompt: computeUsage(`$10.00 / 1M tokens`), + output: computeUsage(`$30.00 / 1M tokens`), + }, + }, + /**/ + + /*/ { modelTitle: 'tts-1-1106', modelName: 'tts-1-1106', }, /**/ - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-3.5-turbo-0125', - modelName: 'gpt-3.5-turbo-0125', - pricing: { - prompt: computeUsage(`$0.50 / 1M tokens`), - output: computeUsage(`$1.50 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-4-turbo-preview', - modelName: 'gpt-4-turbo-preview', - pricing: { - prompt: computeUsage(`$10.00 / 1M tokens`), // <- Not sure, just for gpt-4-turbo - output: computeUsage(`$30.00 / 1M tokens`), // <- Not sure, just for gpt-4-turbo - }, - }, - /**/ - - /**/ - { - modelVariant: 'EMBEDDING', - modelTitle: 'text-embedding-3-large', - modelName: 'text-embedding-3-large', - pricing: { - prompt: computeUsage(`$0.13 / 1M tokens`), - // TODO: [🏏] Leverage the batch API @see https://platform.openai.com/docs/guides/batch - output: 0, // <- Note: [🆖] In Embedding models you dont pay for output - }, - }, - /**/ - - /**/ - { - modelVariant: 'EMBEDDING', - modelTitle: 'text-embedding-3-small', - modelName: 'text-embedding-3-small', - pricing: { - prompt: computeUsage(`$0.02 / 1M tokens`), - // TODO: [🏏] Leverage the batch API @see https://platform.openai.com/docs/guides/batch - output: 0, // <- Note: [🆖] In Embedding models you dont pay for output - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-3.5-turbo-0613', - modelName: 'gpt-3.5-turbo-0613', - pricing: { - prompt: computeUsage(`$1.50 / 1M tokens`), - output: computeUsage(`$2.00 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'EMBEDDING', - modelTitle: 'text-embedding-ada-002', - modelName: 'text-embedding-ada-002', - pricing: { - prompt: computeUsage(`$0.1 / 1M tokens`), // <- Not sure, taken from https://tokescompare.io/product/openai-text-embedding-ada-002/, no official pricing - // TODO: [🏏] Leverage the batch API @see https://platform.openai.com/docs/guides/batch - output: 0, // <- Note: [🆖] In Embedding models you dont pay for output - }, - }, - /**/ - - /*/ + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-3.5-turbo-0125", + modelName: "gpt-3.5-turbo-0125", + pricing: { + prompt: computeUsage(`$0.50 / 1M tokens`), + output: computeUsage(`$1.50 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-4-turbo-preview", + modelName: "gpt-4-turbo-preview", + pricing: { + prompt: computeUsage(`$10.00 / 1M tokens`), // <- Not sure, just for gpt-4-turbo + output: computeUsage(`$30.00 / 1M tokens`), // <- Not sure, just for gpt-4-turbo + }, + }, + /**/ + + /**/ + { + modelVariant: "EMBEDDING", + modelTitle: "text-embedding-3-large", + modelName: "text-embedding-3-large", + pricing: { + prompt: computeUsage(`$0.13 / 1M tokens`), + // TODO: [🏏] Leverage the batch API @see https://platform.openai.com/docs/guides/batch + output: 0, // <- Note: [🆖] In Embedding models you dont pay for output + }, + }, + /**/ + + /**/ + { + modelVariant: "EMBEDDING", + modelTitle: "text-embedding-3-small", + modelName: "text-embedding-3-small", + pricing: { + prompt: computeUsage(`$0.02 / 1M tokens`), + // TODO: [🏏] Leverage the batch API @see https://platform.openai.com/docs/guides/batch + output: 0, // <- Note: [🆖] In Embedding models you dont pay for output + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-3.5-turbo-0613", + modelName: "gpt-3.5-turbo-0613", + pricing: { + prompt: computeUsage(`$1.50 / 1M tokens`), + output: computeUsage(`$2.00 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "EMBEDDING", + modelTitle: "text-embedding-ada-002", + modelName: "text-embedding-ada-002", + pricing: { + prompt: computeUsage(`$0.1 / 1M tokens`), // <- Not sure, taken from https://tokescompare.io/product/openai-text-embedding-ada-002/, no official pricing + // TODO: [🏏] Leverage the batch API @see https://platform.openai.com/docs/guides/batch + output: 0, // <- Note: [🆖] In Embedding models you dont pay for output + }, + }, + /**/ + + /*/ { modelVariant: 'CHAT', modelTitle: 'gpt-4-1106-vision-preview', @@ -334,7 +334,7 @@ export const OPENAI_MODELS: ReadonlyArray< }, /**/ - /*/ + /*/ { modelVariant: 'CHAT', modelTitle: 'gpt-4-vision-preview', @@ -346,118 +346,118 @@ export const OPENAI_MODELS: ReadonlyArray< }, /**/ - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-4o-2024-05-13', - modelName: 'gpt-4o-2024-05-13', - pricing: { - prompt: computeUsage(`$5.00 / 1M tokens`), - output: computeUsage(`$15.00 / 1M tokens`), - }, - //TODO: [main] !!3 Add gpt-4o-mini-2024-07-18 and all others to be up to date - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-4o', - modelName: 'gpt-4o', - pricing: { - prompt: computeUsage(`$5.00 / 1M tokens`), - output: computeUsage(`$15.00 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'o1-preview', - modelName: 'o1-preview', - pricing: { - prompt: computeUsage(`$15.00 / 1M tokens`), - output: computeUsage(`$60.00 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'o1-preview-2024-09-12', - modelName: 'o1-preview-2024-09-12', - // <- TODO: [💩] Some better system to organize theese date suffixes and versions - pricing: { - prompt: computeUsage(`$15.00 / 1M tokens`), - output: computeUsage(`$60.00 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'o1-mini', - modelName: 'o1-mini', - pricing: { - prompt: computeUsage(`$3.00 / 1M tokens`), - output: computeUsage(`$12.00 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'o1', - modelName: 'o1', - pricing: { - prompt: computeUsage(`$3.00 / 1M tokens`), - output: computeUsage(`$12.00 / 1M tokens`), - // <- TODO: !! Unsure, check the pricing - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'o3-mini', - modelName: 'o3-mini', - pricing: { - prompt: computeUsage(`$3.00 / 1M tokens`), - output: computeUsage(`$12.00 / 1M tokens`), - // <- TODO: !! Unsure, check the pricing - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'o1-mini-2024-09-12', - modelName: 'o1-mini-2024-09-12', - pricing: { - prompt: computeUsage(`$3.00 / 1M tokens`), - output: computeUsage(`$12.00 / 1M tokens`), - }, - }, - /**/ - - /**/ - { - modelVariant: 'CHAT', - modelTitle: 'gpt-3.5-turbo-16k-0613', - modelName: 'gpt-3.5-turbo-16k-0613', - pricing: { - prompt: computeUsage(`$3.00 / 1M tokens`), - output: computeUsage(`$4.00 / 1M tokens`), - }, - }, - /**/ - ], + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-4o-2024-05-13", + modelName: "gpt-4o-2024-05-13", + pricing: { + prompt: computeUsage(`$5.00 / 1M tokens`), + output: computeUsage(`$15.00 / 1M tokens`), + }, + //TODO: [main] !!3 Add gpt-4o-mini-2024-07-18 and all others to be up to date + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-4o", + modelName: "gpt-4o", + pricing: { + prompt: computeUsage(`$5.00 / 1M tokens`), + output: computeUsage(`$15.00 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "o1-preview", + modelName: "o1-preview", + pricing: { + prompt: computeUsage(`$15.00 / 1M tokens`), + output: computeUsage(`$60.00 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "o1-preview-2024-09-12", + modelName: "o1-preview-2024-09-12", + // <- TODO: [💩] Some better system to organize theese date suffixes and versions + pricing: { + prompt: computeUsage(`$15.00 / 1M tokens`), + output: computeUsage(`$60.00 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "o1-mini", + modelName: "o1-mini", + pricing: { + prompt: computeUsage(`$3.00 / 1M tokens`), + output: computeUsage(`$12.00 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "o1", + modelName: "o1", + pricing: { + prompt: computeUsage(`$3.00 / 1M tokens`), + output: computeUsage(`$12.00 / 1M tokens`), + // <- TODO: !! Unsure, check the pricing + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "o3-mini", + modelName: "o3-mini", + pricing: { + prompt: computeUsage(`$3.00 / 1M tokens`), + output: computeUsage(`$12.00 / 1M tokens`), + // <- TODO: !! Unsure, check the pricing + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "o1-mini-2024-09-12", + modelName: "o1-mini-2024-09-12", + pricing: { + prompt: computeUsage(`$3.00 / 1M tokens`), + output: computeUsage(`$12.00 / 1M tokens`), + }, + }, + /**/ + + /**/ + { + modelVariant: "CHAT", + modelTitle: "gpt-3.5-turbo-16k-0613", + modelName: "gpt-3.5-turbo-16k-0613", + pricing: { + prompt: computeUsage(`$3.00 / 1M tokens`), + output: computeUsage(`$4.00 / 1M tokens`), + }, + }, + /**/ + ], }); /** diff --git a/src/llm-providers/openai/playground/playground.ts b/src/llm-providers/openai/playground/playground.ts index 1a9b08477f..bfa631b6d6 100644 --- a/src/llm-providers/openai/playground/playground.ts +++ b/src/llm-providers/openai/playground/playground.ts @@ -1,65 +1,65 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { embeddingVectorToString } from '../../../execution/embeddingVectorToString'; -import { usageToHuman } from '../../../execution/utils/usageToHuman'; -import type { Prompt } from '../../../types/Prompt'; -import { keepUnused } from '../../../utils/organization/keepUnused'; -import { OpenAiAssistantExecutionTools } from '../OpenAiAssistantExecutionTools'; -import { OpenAiExecutionTools } from '../OpenAiExecutionTools'; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { embeddingVectorToString } from "../../../execution/embeddingVectorToString"; +import { usageToHuman } from "../../../execution/utils/usageToHuman"; +import type { Prompt } from "../../../types/Prompt"; +import { keepUnused } from "../../../utils/organization/keepUnused"; +import { OpenAiAssistantExecutionTools } from "../OpenAiAssistantExecutionTools"; +import { OpenAiExecutionTools } from "../OpenAiExecutionTools"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 OpenAI Playground`); - - // Do here stuff you want to test - //========================================> - - const openAiExecutionTools = new OpenAiExecutionTools( - // <- TODO: [🧱] Implement in a functional (not new Class) way - { - isVerbose: true, - userId: 'playground', - apiKey: process.env.OPENAI_API_KEY!, - }, - ); - - const openAiAssistantExecutionTools = new OpenAiAssistantExecutionTools( - // <- TODO: [🧱] Implement in a functional (not new Class) way - { - isVerbose: true, - userId: 'playground', - apiKey: process.env.OPENAI_API_KEY!, - assistantId: 'asst_CJCZzFCbBL0f2D4OWMXVTdBB', - // <- Note: This is not a private information, just ID of the assistant which is accessible only with correct API key - }, - ); - - keepUnused(openAiExecutionTools); - keepUnused(openAiAssistantExecutionTools); - keepUnused(embeddingVectorToString); - keepUnused(usageToHuman); - keepUnused(); - - /*/ + console.info(`🧸 OpenAI Playground`); + + // Do here stuff you want to test + //========================================> + + const openAiExecutionTools = new OpenAiExecutionTools( + // <- TODO: [🧱] Implement in a functional (not new Class) way + { + isVerbose: true, + userId: "playground", + apiKey: process.env.OPENAI_API_KEY!, + }, + ); + + const openAiAssistantExecutionTools = new OpenAiAssistantExecutionTools( + // <- TODO: [🧱] Implement in a functional (not new Class) way + { + isVerbose: true, + userId: "playground", + apiKey: process.env.OPENAI_API_KEY!, + assistantId: "asst_CJCZzFCbBL0f2D4OWMXVTdBB", + // <- Note: This is not a private information, just ID of the assistant which is accessible only with correct API key + }, + ); + + keepUnused(openAiExecutionTools); + keepUnused(openAiAssistantExecutionTools); + keepUnused(embeddingVectorToString); + keepUnused(usageToHuman); + keepUnused(); + + /*/ const models = await openAiExecutionTools.listModels(); console.info({ models }); /**/ - /*/ + /*/ const completionPrompt = { title: 'Hello', parameters: {}, @@ -74,7 +74,7 @@ async function playground() { console.info(chalk.green(completionPrompt.content + completionPromptResult.content)); /**/ - /*/ + /*/ const chatPrompt = { title: 'Promptbook speech', parameters: {}, @@ -92,11 +92,11 @@ async function playground() { console.info(colors.bgGreen(' Chat: ') + colors.green(chatPromptResult.content)); /**/ - /*/ + /*/ // TODO: Test Translations in playground /**/ - /*/ + /*/ const prompt = { title: 'Hello', parameters: {}, @@ -113,32 +113,35 @@ async function playground() { console.info(chalk.bgGreen(' Embedding: ') + chalk.green(embeddingVectorToString(promptResult.content))); /**/ - /**/ - const chatPrompt = { - title: 'Promptbook speech', - parameters: {}, - content: `Write me speech about Promptbook and how it can help me to build the most beautiful chatbot and change the world`, - modelRequirements: { - modelVariant: 'CHAT', - // TODO: [👨‍👨‍👧‍👧] systemMessage: 'You are an assistant who only speaks in rhymes.', - // TODO: [👨‍👨‍👧‍👧] temperature: 1.5, - }, - - /* + /**/ + const chatPrompt = { + title: "Promptbook speech", + parameters: {}, + content: `Write me speech about Promptbook and how it can help me to build the most beautiful chatbot and change the world`, + modelRequirements: { + modelVariant: "CHAT", + // TODO: [👨‍👨‍👧‍👧] systemMessage: 'You are an assistant who only speaks in rhymes.', + // TODO: [👨‍👨‍👧‍👧] temperature: 1.5, + }, + + /* [🗯] replyingTo: { } */ - } as const satisfies Prompt; - const chatPromptResult = await openAiAssistantExecutionTools.callChatModel(chatPrompt); - console.info({ chatPromptResult }); - console.info(colors.cyan(usageToHuman(chatPromptResult.usage))); - console.info(colors.bgBlue(' User: ') + colors.blue(chatPrompt.content)); - console.info(colors.bgGreen(' Assistant: ') + colors.green(chatPromptResult.content)); - /**/ - - /*/ + } as const satisfies Prompt; + const chatPromptResult = + await openAiAssistantExecutionTools.callChatModel(chatPrompt); + console.info({ chatPromptResult }); + console.info(colors.cyan(usageToHuman(chatPromptResult.usage))); + console.info(colors.bgBlue(" User: ") + colors.blue(chatPrompt.content)); + console.info( + colors.bgGreen(" Assistant: ") + colors.green(chatPromptResult.content), + ); + /**/ + + /*/ const openai = await openAiExecutionTools.getClient(); const stream = openai.beta.threads.createAndRunStream({ stream: true, @@ -172,11 +175,11 @@ async function playground() { /**/ - /*/ + /*/ // <- Note: [🤖] Test here new model variant if needed /**/ - //========================================/ + //========================================/ } /** diff --git a/src/llm-providers/openai/playground/tsconfig.json b/src/llm-providers/openai/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/llm-providers/openai/playground/tsconfig.json +++ b/src/llm-providers/openai/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/llm-providers/openai/register-configuration.ts b/src/llm-providers/openai/register-configuration.ts index 74f6e679db..ee2e41f59e 100644 --- a/src/llm-providers/openai/register-configuration.ts +++ b/src/llm-providers/openai/register-configuration.ts @@ -1,8 +1,8 @@ -import type { string_name } from '../../types/typeAliases'; -import type { Registration } from '../../utils/$Register'; -import { keepUnused } from '../../utils/organization/keepUnused'; -import { $llmToolsMetadataRegister } from '../_common/register/$llmToolsMetadataRegister'; -import type { LlmToolsConfiguration } from '../_common/register/LlmToolsConfiguration'; +import type { string_name } from "../../types/typeAliases"; +import type { Registration } from "../../utils/$Register"; +import { keepUnused } from "../../utils/organization/keepUnused"; +import { $llmToolsMetadataRegister } from "../_common/register/$llmToolsMetadataRegister"; +import type { LlmToolsConfiguration } from "../_common/register/LlmToolsConfiguration"; /** * Registration of LLM provider metadata @@ -13,39 +13,42 @@ import type { LlmToolsConfiguration } from '../_common/register/LlmToolsConfigur * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _OpenAiMetadataRegistration: Registration = $llmToolsMetadataRegister.register({ - title: 'Open AI', - packageName: '@promptbook/openai', - className: 'OpenAiExecutionTools', - envVariables: ['OPENAI_API_KEY'], +export const _OpenAiMetadataRegistration: Registration = + $llmToolsMetadataRegister.register({ + title: "Open AI", + packageName: "@promptbook/openai", + className: "OpenAiExecutionTools", + envVariables: ["OPENAI_API_KEY"], - getBoilerplateConfiguration(): LlmToolsConfiguration[number] { - return { - title: 'Open AI (boilerplate)', - packageName: '@promptbook/openai', - className: 'OpenAiExecutionTools', - options: { - apiKey: 'sk-', - }, - }; - }, + getBoilerplateConfiguration(): LlmToolsConfiguration[number] { + return { + title: "Open AI (boilerplate)", + packageName: "@promptbook/openai", + className: "OpenAiExecutionTools", + options: { + apiKey: "sk-", + }, + }; + }, - createConfigurationFromEnv(env: Record): LlmToolsConfiguration[number] | null { - // Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically - if (typeof env.OPENAI_API_KEY === 'string') { - return { - title: 'Open AI (from env)', - packageName: '@promptbook/openai', - className: 'OpenAiExecutionTools', - options: { - apiKey: env.OPENAI_API_KEY!, - }, - }; - } + createConfigurationFromEnv( + env: Record, + ): LlmToolsConfiguration[number] | null { + // Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically + if (typeof env.OPENAI_API_KEY === "string") { + return { + title: "Open AI (from env)", + packageName: "@promptbook/openai", + className: "OpenAiExecutionTools", + options: { + apiKey: env.OPENAI_API_KEY!, + }, + }; + } - return null; - }, -}); + return null; + }, + }); /** * @@@ registration1 of default configuration for Open AI @@ -56,30 +59,33 @@ export const _OpenAiMetadataRegistration: Registration = $llmToolsMetadataRegist * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _OpenAiAssistantMetadataRegistration = $llmToolsMetadataRegister.register({ - title: 'Open AI Assistant', - packageName: '@promptbook/openai', - className: 'OpenAiAssistantExecutionTools', - envVariables: null, - // <- TODO: ['OPENAI_API_KEY', 'OPENAI_ASSISTANT_ID'] +export const _OpenAiAssistantMetadataRegistration = + $llmToolsMetadataRegister.register({ + title: "Open AI Assistant", + packageName: "@promptbook/openai", + className: "OpenAiAssistantExecutionTools", + envVariables: null, + // <- TODO: ['OPENAI_API_KEY', 'OPENAI_ASSISTANT_ID'] - getBoilerplateConfiguration(): LlmToolsConfiguration[number] { - return { - title: 'Open AI Assistant (boilerplate)', - packageName: '@promptbook/openai', - className: 'OpenAiAssistantExecutionTools', - options: { - apiKey: 'sk-', - assistantId: 'asst_', - }, - }; - }, + getBoilerplateConfiguration(): LlmToolsConfiguration[number] { + return { + title: "Open AI Assistant (boilerplate)", + packageName: "@promptbook/openai", + className: "OpenAiAssistantExecutionTools", + options: { + apiKey: "sk-", + assistantId: "asst_", + }, + }; + }, - createConfigurationFromEnv(env: Record): LlmToolsConfiguration[number] | null { - // TODO: Maybe auto-configure (multiple) assistants from env variables - keepUnused(env); - return null; - /* + createConfigurationFromEnv( + env: Record, + ): LlmToolsConfiguration[number] | null { + // TODO: Maybe auto-configure (multiple) assistants from env variables + keepUnused(env); + return null; + /* if (typeof env.OPENAI_API_KEY === 'string' || typeof env.OPENAI_XXX === 'string') { return { title: 'Open AI Assistant (from env)', @@ -94,8 +100,8 @@ export const _OpenAiAssistantMetadataRegistration = $llmToolsMetadataRegister.re return null; */ - }, -}); + }, + }); /** * Note: [💞] Ignore a discrepancy between file name and entity name diff --git a/src/llm-providers/openai/register-constructor.ts b/src/llm-providers/openai/register-constructor.ts index aa887a7a9d..5778552a3d 100644 --- a/src/llm-providers/openai/register-constructor.ts +++ b/src/llm-providers/openai/register-constructor.ts @@ -1,7 +1,7 @@ -import type { Registration } from '../../utils/$Register'; -import { $llmToolsRegister } from '../_common/register/$llmToolsRegister'; -import { createOpenAiAssistantExecutionTools } from './createOpenAiAssistantExecutionTools'; -import { createOpenAiExecutionTools } from './createOpenAiExecutionTools'; +import type { Registration } from "../../utils/$Register"; +import { $llmToolsRegister } from "../_common/register/$llmToolsRegister"; +import { createOpenAiAssistantExecutionTools } from "./createOpenAiAssistantExecutionTools"; +import { createOpenAiExecutionTools } from "./createOpenAiExecutionTools"; /** * Registration of LLM provider @@ -12,7 +12,9 @@ import { createOpenAiExecutionTools } from './createOpenAiExecutionTools'; * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _OpenAiRegistration: Registration = $llmToolsRegister.register(createOpenAiExecutionTools); +export const _OpenAiRegistration: Registration = $llmToolsRegister.register( + createOpenAiExecutionTools, +); /** * @@@ registration2 @@ -23,7 +25,9 @@ export const _OpenAiRegistration: Registration = $llmToolsRegister.register(crea * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _OpenAiAssistantRegistration = $llmToolsRegister.register(createOpenAiAssistantExecutionTools); +export const _OpenAiAssistantRegistration = $llmToolsRegister.register( + createOpenAiAssistantExecutionTools, +); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/llm-providers/remote/RemoteLlmExecutionTools.ts b/src/llm-providers/remote/RemoteLlmExecutionTools.ts index 0c9697844a..3f8e532ddc 100644 --- a/src/llm-providers/remote/RemoteLlmExecutionTools.ts +++ b/src/llm-providers/remote/RemoteLlmExecutionTools.ts @@ -1,26 +1,26 @@ -import { deserializeError } from '../../errors/utils/deserializeError'; -import type { AvailableModel } from '../../execution/AvailableModel'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import type { ChatPromptResult } from '../../execution/PromptResult'; -import type { CompletionPromptResult } from '../../execution/PromptResult'; -import type { EmbeddingPromptResult } from '../../execution/PromptResult'; -import type { PromptResult } from '../../execution/PromptResult'; -import { createRemoteClient } from '../../remote-server/createRemoteClient'; -import type { PromptbookServer_Error } from '../../remote-server/socket-types/_common/PromptbookServer_Error'; -import type { PromptbookServer_ListModels_Request } from '../../remote-server/socket-types/listModels/PromptbookServer_ListModels_Request'; -import type { PromptbookServer_ListModels_Response } from '../../remote-server/socket-types/listModels/PromptbookServer_ListModels_Response'; -import type { PromptbookServer_Prompt_Request } from '../../remote-server/socket-types/prompt/PromptbookServer_Prompt_Request'; -import type { PromptbookServer_Prompt_Response } from '../../remote-server/socket-types/prompt/PromptbookServer_Prompt_Response'; -import type { RemoteClientOptions } from '../../remote-server/types/RemoteClientOptions'; -import type { ChatPrompt } from '../../types/Prompt'; -import type { CompletionPrompt } from '../../types/Prompt'; -import type { EmbeddingPrompt } from '../../types/Prompt'; -import type { Prompt } from '../../types/Prompt'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_title } from '../../types/typeAliases'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import type { really_any } from '../../utils/organization/really_any'; +import { deserializeError } from "../../errors/utils/deserializeError"; +import type { AvailableModel } from "../../execution/AvailableModel"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import type { ChatPromptResult } from "../../execution/PromptResult"; +import type { CompletionPromptResult } from "../../execution/PromptResult"; +import type { EmbeddingPromptResult } from "../../execution/PromptResult"; +import type { PromptResult } from "../../execution/PromptResult"; +import { createRemoteClient } from "../../remote-server/createRemoteClient"; +import type { PromptbookServer_Error } from "../../remote-server/socket-types/_common/PromptbookServer_Error"; +import type { PromptbookServer_ListModels_Request } from "../../remote-server/socket-types/listModels/PromptbookServer_ListModels_Request"; +import type { PromptbookServer_ListModels_Response } from "../../remote-server/socket-types/listModels/PromptbookServer_ListModels_Response"; +import type { PromptbookServer_Prompt_Request } from "../../remote-server/socket-types/prompt/PromptbookServer_Prompt_Request"; +import type { PromptbookServer_Prompt_Response } from "../../remote-server/socket-types/prompt/PromptbookServer_Prompt_Response"; +import type { RemoteClientOptions } from "../../remote-server/types/RemoteClientOptions"; +import type { ChatPrompt } from "../../types/Prompt"; +import type { CompletionPrompt } from "../../types/Prompt"; +import type { EmbeddingPrompt } from "../../types/Prompt"; +import type { Prompt } from "../../types/Prompt"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_title } from "../../types/typeAliases"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import type { really_any } from "../../utils/organization/really_any"; keepTypeImported>(); keepTypeImported>(); @@ -34,121 +34,143 @@ keepTypeImported>(); * @see https://github.com/webgptorg/promptbook#remote-server * @public exported from `@promptbook/remote-client` */ -export class RemoteLlmExecutionTools implements LlmExecutionTools { - /* <- TODO: [🍚] `, Destroyable` */ - public constructor(protected readonly options: RemoteClientOptions) {} - - public get title(): string_title & string_markdown_text { - // TODO: [🧠] Maybe fetch title+description from the remote server (as well as if model methods are defined) - return 'Remote server'; - } - - public get description(): string_markdown { - return 'Use all models by your remote server'; - } - - /** - * Check the configuration of all execution tools - */ - public async checkConfiguration(): Promise { - const socket = await createRemoteClient(this.options); - socket.disconnect(); - - // TODO: [main] !!3 Check version of the remote server and compatibility - // TODO: [🎍] Send checkConfiguration - } - - /** - * List all available models that can be used - */ - public async listModels(): Promise> { - // TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io - const socket = await createRemoteClient(this.options); - - socket.emit( - 'listModels-request', - { - identification: this.options.identification, - } satisfies PromptbookServer_ListModels_Request /* <- Note: [🤛] */, - ); - - const promptResult = await new Promise>((resolve, reject) => { - socket.on('listModels-response', (response: PromptbookServer_ListModels_Response) => { - resolve(response.models); - socket.disconnect(); - }); - socket.on('error', (error: PromptbookServer_Error) => { - reject(deserializeError(error)); - socket.disconnect(); - }); - }); - - socket.disconnect(); - - return promptResult; - } - - /** - * Calls remote proxy server to use a chat model - */ - public callChatModel(prompt: ChatPrompt): Promise { - if (this.options.isVerbose) { - console.info(`🖋 Remote callChatModel call`); - } - return /* not await */ this.callCommonModel(prompt) as Promise; - } - - /** - * Calls remote proxy server to use a completion model - */ - public callCompletionModel(prompt: CompletionPrompt): Promise { - if (this.options.isVerbose) { - console.info(`💬 Remote callCompletionModel call`); - } - return /* not await */ this.callCommonModel(prompt) as Promise; - } - - /** - * Calls remote proxy server to use a embedding model - */ - public callEmbeddingModel(prompt: EmbeddingPrompt): Promise { - if (this.options.isVerbose) { - console.info(`💬 Remote callEmbeddingModel call`); - } - return /* not await */ this.callCommonModel(prompt) as Promise; - } - - // <- Note: [🤖] callXxxModel - - /** - * Calls remote proxy server to use both completion or chat model - */ - private async callCommonModel(prompt: Prompt): Promise { - const socket = await createRemoteClient(this.options); - - socket.emit( - 'prompt-request', - { - identification: this.options.identification, - prompt, - } satisfies PromptbookServer_Prompt_Request /* <- Note: [🤛] */, - ); - - const promptResult = await new Promise((resolve, reject) => { - socket.on('prompt-response', (response: PromptbookServer_Prompt_Response) => { - resolve(response.promptResult); - socket.disconnect(); - }); - socket.on('error', (error: PromptbookServer_Error) => { - reject(deserializeError(error)); - socket.disconnect(); - }); - }); - - socket.disconnect(); - - return promptResult; - } +export class RemoteLlmExecutionTools + implements LlmExecutionTools +{ + /* <- TODO: [🍚] `, Destroyable` */ + public constructor( + protected readonly options: RemoteClientOptions, + ) {} + + public get title(): string_title & string_markdown_text { + // TODO: [🧠] Maybe fetch title+description from the remote server (as well as if model methods are defined) + return "Remote server"; + } + + public get description(): string_markdown { + return "Use all models by your remote server"; + } + + /** + * Check the configuration of all execution tools + */ + public async checkConfiguration(): Promise { + const socket = await createRemoteClient(this.options); + socket.disconnect(); + + // TODO: [main] !!3 Check version of the remote server and compatibility + // TODO: [🎍] Send checkConfiguration + } + + /** + * List all available models that can be used + */ + public async listModels(): Promise> { + // TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io + const socket = await createRemoteClient(this.options); + + socket.emit( + "listModels-request", + { + identification: this.options.identification, + } satisfies PromptbookServer_ListModels_Request /* <- Note: [🤛] */, + ); + + const promptResult = await new Promise>( + (resolve, reject) => { + socket.on( + "listModels-response", + (response: PromptbookServer_ListModels_Response) => { + resolve(response.models); + socket.disconnect(); + }, + ); + socket.on("error", (error: PromptbookServer_Error) => { + reject(deserializeError(error)); + socket.disconnect(); + }); + }, + ); + + socket.disconnect(); + + return promptResult; + } + + /** + * Calls remote proxy server to use a chat model + */ + public callChatModel(prompt: ChatPrompt): Promise { + if (this.options.isVerbose) { + console.info(`🖋 Remote callChatModel call`); + } + return /* not await */ this.callCommonModel( + prompt, + ) as Promise; + } + + /** + * Calls remote proxy server to use a completion model + */ + public callCompletionModel( + prompt: CompletionPrompt, + ): Promise { + if (this.options.isVerbose) { + console.info(`💬 Remote callCompletionModel call`); + } + return /* not await */ this.callCommonModel( + prompt, + ) as Promise; + } + + /** + * Calls remote proxy server to use a embedding model + */ + public callEmbeddingModel( + prompt: EmbeddingPrompt, + ): Promise { + if (this.options.isVerbose) { + console.info(`💬 Remote callEmbeddingModel call`); + } + return /* not await */ this.callCommonModel( + prompt, + ) as Promise; + } + + // <- Note: [🤖] callXxxModel + + /** + * Calls remote proxy server to use both completion or chat model + */ + private async callCommonModel(prompt: Prompt): Promise { + const socket = await createRemoteClient(this.options); + + socket.emit( + "prompt-request", + { + identification: this.options.identification, + prompt, + } satisfies PromptbookServer_Prompt_Request /* <- Note: [🤛] */, + ); + + const promptResult = await new Promise((resolve, reject) => { + socket.on( + "prompt-response", + (response: PromptbookServer_Prompt_Response) => { + resolve(response.promptResult); + socket.disconnect(); + }, + ); + socket.on("error", (error: PromptbookServer_Error) => { + reject(deserializeError(error)); + socket.disconnect(); + }); + }); + + socket.disconnect(); + + return promptResult; + } } /** diff --git a/src/llm-providers/remote/playground/playground.ts b/src/llm-providers/remote/playground/playground.ts index 9cda697bb6..cb377bdc20 100644 --- a/src/llm-providers/remote/playground/playground.ts +++ b/src/llm-providers/remote/playground/playground.ts @@ -1,115 +1,137 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { forEver, forTime } from 'waitasecond'; -import { createCollectionFromDirectory } from '../../../collection/constructors/createCollectionFromDirectory'; -import { startRemoteServer } from '../../../remote-server/startRemoteServer'; -import { OpenAiExecutionTools } from '../../openai/OpenAiExecutionTools'; -import '../../openai/register-constructor'; -import { RemoteLlmExecutionTools } from '../RemoteLlmExecutionTools'; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { forEver, forTime } from "waitasecond"; +import { createCollectionFromDirectory } from "../../../collection/constructors/createCollectionFromDirectory"; +import { startRemoteServer } from "../../../remote-server/startRemoteServer"; +import { OpenAiExecutionTools } from "../../openai/OpenAiExecutionTools"; +import "../../openai/register-constructor"; +import { RemoteLlmExecutionTools } from "../RemoteLlmExecutionTools"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 Remote server playground`); - - // Do here stuff you want to test - //========================================> - - console.info(colors.bgCyan('Playground:'), colors.bgWhite(`Starting remote server`)); - startRemoteServer({ - rootPath: '/promptbook', - port: 4460, - isVerbose: true, - isAnonymousModeAllowed: true, - isApplicationModeAllowed: true, - collection: await createCollectionFromDirectory( - './examples/pipelines/', - {}, - { - isRecursive: false, - }, - ), - createLlmExecutionTools(options) { - const { appId, userId, customOptions } = options; - - console.info(colors.bgCyan('Playground:'), { appId, userId, customOptions }); - return new OpenAiExecutionTools( - // <- TODO: [🧱] Implement in a functional (not new Class) way - { - isVerbose: true, - apiKey: process.env.OPENAI_API_KEY!, - userId: userId, - }, - ); - }, - }); - - for (const mode of ['anonymous', 'collection'] as const) { - await forTime(500); - console.info(colors.bgCyan('Playground:'), colors.bgWhite(`Creating RemoteLlmExecutionTools (${mode} mode) `)); - - const remoteUrl = 'http://localhost:4460'; - const path = '/promptbook'; - - const tools = new RemoteLlmExecutionTools( - mode === 'anonymous' - ? { - remoteUrl, - path, - identification: { - isAnonymous: true, - userId: 'playground', - llmToolsConfiguration: [ - { - title: 'OpenAI', - packageName: '@promptbook/openai', - className: 'OpenAiExecutionTools', - options: { - apiKey: process.env.OPENAI_API_KEY!, - }, - }, - ], - }, - } - : { - remoteUrl, - path, - identification: { - isAnonymous: false, - appId: 'playground', - userId: 'playground', - }, - }, - ); - - await forTime(500); - console.info(colors.bgCyan('Playground:'), colors.bgWhite(`Checking configuration...`)); - await tools.checkConfiguration(); - console.info(colors.bgCyan('Playground:'), colors.bgGreen(`Configuration checked!`)); - - await forTime(500); - console.info(colors.bgCyan('Playground:'), colors.bgWhite(`Listing models...`)); - const models = await tools.listModels(); - console.info({ models }); - console.info(colors.bgCyan('Playground:'), colors.bgGreen(`Models listed!`)); - } - - await forEver(); - - //========================================/ + console.info(`🧸 Remote server playground`); + + // Do here stuff you want to test + //========================================> + + console.info( + colors.bgCyan("Playground:"), + colors.bgWhite(`Starting remote server`), + ); + startRemoteServer({ + rootPath: "/promptbook", + port: 4460, + isVerbose: true, + isAnonymousModeAllowed: true, + isApplicationModeAllowed: true, + collection: await createCollectionFromDirectory( + "./examples/pipelines/", + {}, + { + isRecursive: false, + }, + ), + createLlmExecutionTools(options) { + const { appId, userId, customOptions } = options; + + console.info(colors.bgCyan("Playground:"), { + appId, + userId, + customOptions, + }); + return new OpenAiExecutionTools( + // <- TODO: [🧱] Implement in a functional (not new Class) way + { + isVerbose: true, + apiKey: process.env.OPENAI_API_KEY!, + userId: userId, + }, + ); + }, + }); + + for (const mode of ["anonymous", "collection"] as const) { + await forTime(500); + console.info( + colors.bgCyan("Playground:"), + colors.bgWhite(`Creating RemoteLlmExecutionTools (${mode} mode) `), + ); + + const remoteUrl = "http://localhost:4460"; + const path = "/promptbook"; + + const tools = new RemoteLlmExecutionTools( + mode === "anonymous" + ? { + remoteUrl, + path, + identification: { + isAnonymous: true, + userId: "playground", + llmToolsConfiguration: [ + { + title: "OpenAI", + packageName: "@promptbook/openai", + className: "OpenAiExecutionTools", + options: { + apiKey: process.env.OPENAI_API_KEY!, + }, + }, + ], + }, + } + : { + remoteUrl, + path, + identification: { + isAnonymous: false, + appId: "playground", + userId: "playground", + }, + }, + ); + + await forTime(500); + console.info( + colors.bgCyan("Playground:"), + colors.bgWhite(`Checking configuration...`), + ); + await tools.checkConfiguration(); + console.info( + colors.bgCyan("Playground:"), + colors.bgGreen(`Configuration checked!`), + ); + + await forTime(500); + console.info( + colors.bgCyan("Playground:"), + colors.bgWhite(`Listing models...`), + ); + const models = await tools.listModels(); + console.info({ models }); + console.info( + colors.bgCyan("Playground:"), + colors.bgGreen(`Models listed!`), + ); + } + + await forEver(); + + //========================================/ } /** diff --git a/src/llm-providers/remote/playground/tsconfig.json b/src/llm-providers/remote/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/llm-providers/remote/playground/tsconfig.json +++ b/src/llm-providers/remote/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/llm-providers/vercel/VercelExecutionToolsOptions.ts b/src/llm-providers/vercel/VercelExecutionToolsOptions.ts index bbad6c93dd..c3e179d58f 100644 --- a/src/llm-providers/vercel/VercelExecutionToolsOptions.ts +++ b/src/llm-providers/vercel/VercelExecutionToolsOptions.ts @@ -1,7 +1,7 @@ -import type { AvailableModel } from '../../execution/AvailableModel'; -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import type { VercelProvider } from './VercelProvider'; +import type { AvailableModel } from "../../execution/AvailableModel"; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import type { VercelProvider } from "./VercelProvider"; /** * Options for `createExecutionToolsFromVercelProvider` @@ -9,22 +9,24 @@ import type { VercelProvider } from './VercelProvider'; * @public exported from `@promptbook/google` */ export type VercelExecutionToolsOptions = CommonToolsOptions & - Pick & { - /** - * Vercel provider for the execution tools - */ - readonly vercelProvider: VercelProvider; + Pick & { + /** + * Vercel provider for the execution tools + */ + readonly vercelProvider: VercelProvider; - /** - * List of available models for given Vercel provider - */ - readonly availableModels: ReadonlyArray; - // ^^^^ - // TODO: This is stupid, because good design would be to have list of models as a part of the Vercel provider itself - // and not to pass it as a separate parameter + /** + * List of available models for given Vercel provider + */ + readonly availableModels: ReadonlyArray; + // ^^^^ + // TODO: This is stupid, because good design would be to have list of models as a part of the Vercel provider itself + // and not to pass it as a separate parameter - /** - * Additional settings for chat models when calling `vercelProvider.chat('model-name', settings)` - */ - readonly additionalChatSettings?: Partial[1]>; - }; + /** + * Additional settings for chat models when calling `vercelProvider.chat('model-name', settings)` + */ + readonly additionalChatSettings?: Partial< + Parameters[1] + >; + }; diff --git a/src/llm-providers/vercel/VercelProvider.ts b/src/llm-providers/vercel/VercelProvider.ts index e52778ea71..7633c153c4 100644 --- a/src/llm-providers/vercel/VercelProvider.ts +++ b/src/llm-providers/vercel/VercelProvider.ts @@ -1,12 +1,14 @@ -import type { createGoogleGenerativeAI } from '@ai-sdk/google'; // <- TODO: This shoud be installed just as dev dependency in the `@promptbook/vercel` package, because it is only used as a type -import type { createOpenAI } from '@ai-sdk/openai'; // <- TODO: This shoud be installed just as dev dependency in the `@promptbook/vercel` package, because it is only used as a type +import type { createGoogleGenerativeAI } from "@ai-sdk/google"; // <- TODO: This shoud be installed just as dev dependency in the `@promptbook/vercel` package, because it is only used as a type +import type { createOpenAI } from "@ai-sdk/openai"; // <- TODO: This shoud be installed just as dev dependency in the `@promptbook/vercel` package, because it is only used as a type /** * This is common interface for all v1 Vercel providers * * @public exported from `@promptbook/vercel` */ -export type VercelProvider = ReturnType | ReturnType; +export type VercelProvider = + | ReturnType + | ReturnType; /** * ^^^^ diff --git a/src/llm-providers/vercel/createExecutionToolsFromVercelProvider.ts b/src/llm-providers/vercel/createExecutionToolsFromVercelProvider.ts index ee144e97f2..6ded65d855 100644 --- a/src/llm-providers/vercel/createExecutionToolsFromVercelProvider.ts +++ b/src/llm-providers/vercel/createExecutionToolsFromVercelProvider.ts @@ -1,66 +1,76 @@ -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import spaceTrim from 'spacetrim'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import type { ChatPromptResult } from '../../execution/PromptResult'; -import { UNCERTAIN_USAGE } from '../../execution/utils/usage-constants'; -import type { Prompt } from '../../types/Prompt'; -import type { string_date_iso8601 } from '../../types/typeAliases'; -import { $getCurrentDate } from '../../utils/$getCurrentDate'; -import { templateParameters } from '../../utils/parameters/templateParameters'; -import { asSerializable } from '../../utils/serialization/asSerializable'; -import { exportJson } from '../../utils/serialization/exportJson'; -import type { VercelExecutionToolsOptions } from './VercelExecutionToolsOptions'; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import spaceTrim from "spacetrim"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import type { ChatPromptResult } from "../../execution/PromptResult"; +import { UNCERTAIN_USAGE } from "../../execution/utils/usage-constants"; +import type { Prompt } from "../../types/Prompt"; +import type { string_date_iso8601 } from "../../types/typeAliases"; +import { $getCurrentDate } from "../../utils/$getCurrentDate"; +import { templateParameters } from "../../utils/parameters/templateParameters"; +import { asSerializable } from "../../utils/serialization/asSerializable"; +import { exportJson } from "../../utils/serialization/exportJson"; +import type { VercelExecutionToolsOptions } from "./VercelExecutionToolsOptions"; /** * Adapter which creates Promptbook execution tools from Vercel provider * * @public exported from `@promptbook/vercel` */ -export function createExecutionToolsFromVercelProvider(options: VercelExecutionToolsOptions): LlmExecutionTools { - let { title, description } = options; - const { vercelProvider, availableModels, userId, additionalChatSettings = {} } = options; - - if (!/Vercel/i.test(title)) { - title = `${title} (through Vercel)`; - } /* not else */ - - if (description === undefined) { - description = `Implementation of ${title} through Vercel`; - } /* not else */ - - if (!/Vercel/i.test(description)) { - description = `${description} (through Vercel)`; - } /* not else */ - - return { - title, - description, - checkConfiguration() { - // Note: There is no way how to check configuration of Vercel provider - return Promise.resolve(); - }, - - async listModels() { - return availableModels; - }, - - async callChatModel( - prompt: Pick, - ): Promise { - const { content, parameters, modelRequirements } = prompt; - - if (modelRequirements.modelVariant !== 'CHAT') { - throw new PipelineExecutionError('Use callChatModel only for CHAT variant'); - } - - const modelName = - modelRequirements.modelName || - availableModels.find(({ modelVariant }) => modelVariant === 'CHAT')?.modelName; - - if (!modelName) { - throw new PipelineExecutionError( - spaceTrim(` +export function createExecutionToolsFromVercelProvider( + options: VercelExecutionToolsOptions, +): LlmExecutionTools { + let { title, description } = options; + const { + vercelProvider, + availableModels, + userId, + additionalChatSettings = {}, + } = options; + + if (!/Vercel/i.test(title)) { + title = `${title} (through Vercel)`; + } /* not else */ + + if (description === undefined) { + description = `Implementation of ${title} through Vercel`; + } /* not else */ + + if (!/Vercel/i.test(description)) { + description = `${description} (through Vercel)`; + } /* not else */ + + return { + title, + description, + checkConfiguration() { + // Note: There is no way how to check configuration of Vercel provider + return Promise.resolve(); + }, + + async listModels() { + return availableModels; + }, + + async callChatModel( + prompt: Pick, + ): Promise { + const { content, parameters, modelRequirements } = prompt; + + if (modelRequirements.modelVariant !== "CHAT") { + throw new PipelineExecutionError( + "Use callChatModel only for CHAT variant", + ); + } + + const modelName = + modelRequirements.modelName || + availableModels.find(({ modelVariant }) => modelVariant === "CHAT") + ?.modelName; + + if (!modelName) { + throw new PipelineExecutionError( + spaceTrim(` Can not determine which model to use. You need to provide at least one of: @@ -68,93 +78,103 @@ export function createExecutionToolsFromVercelProvider(options: VercelExecutionT 2) In \`prompt.modelRequirements\`, provide \`modelName\` with the name of the model to use `), - ); - } - - const model = await vercelProvider.chat(modelName, { - user: userId?.toString() || undefined, - ...additionalChatSettings, - }); - - const rawPromptContent = templateParameters(content, { ...parameters, modelName }); - const rawRequest: Parameters[0] = { - // <- TODO: [☂] - inputFormat: 'messages', - mode: { - type: 'regular', - tools: [ - /* <- TODO: Pass the tools */ - ], - }, - prompt: [ - ...(modelRequirements.systemMessage === undefined - ? [] - : ([ - { - role: 'system', - content: modelRequirements.systemMessage, - }, - ] as const)), - { - role: 'user', - content: [ - { - type: 'text', - text: rawPromptContent, - }, - ], - }, - ], - }; - - const start: string_date_iso8601 = $getCurrentDate(); - - if (options.isVerbose) { - console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4)); - } - const rawResponse = await (async () => await model.doGenerate(rawRequest))().catch((error) => { - // <- Note: This weird structure is here to catch errors in both sync and async `doGenerate` - if (options.isVerbose) { - console.info(colors.bgRed('error'), error); - } - throw error; - }); - - await model.doGenerate(rawRequest); - - if (options.isVerbose) { - console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4)); - } - - if (rawResponse.text === undefined) { - throw new PipelineExecutionError('No response message'); - } - - const complete: string_date_iso8601 = $getCurrentDate(); - - /* + ); + } + + const model = await vercelProvider.chat(modelName, { + user: userId?.toString() || undefined, + ...additionalChatSettings, + }); + + const rawPromptContent = templateParameters(content, { + ...parameters, + modelName, + }); + const rawRequest: Parameters[0] = { + // <- TODO: [☂] + inputFormat: "messages", + mode: { + type: "regular", + tools: [ + /* <- TODO: Pass the tools */ + ], + }, + prompt: [ + ...(modelRequirements.systemMessage === undefined + ? [] + : ([ + { + role: "system", + content: modelRequirements.systemMessage, + }, + ] as const)), + { + role: "user", + content: [ + { + type: "text", + text: rawPromptContent, + }, + ], + }, + ], + }; + + const start: string_date_iso8601 = $getCurrentDate(); + + if (options.isVerbose) { + console.info( + colors.bgWhite("rawRequest"), + JSON.stringify(rawRequest, null, 4), + ); + } + const rawResponse = await (async () => + await model.doGenerate(rawRequest))().catch((error) => { + // <- Note: This weird structure is here to catch errors in both sync and async `doGenerate` + if (options.isVerbose) { + console.info(colors.bgRed("error"), error); + } + throw error; + }); + + await model.doGenerate(rawRequest); + + if (options.isVerbose) { + console.info( + colors.bgWhite("rawResponse"), + JSON.stringify(rawResponse, null, 4), + ); + } + + if (rawResponse.text === undefined) { + throw new PipelineExecutionError("No response message"); + } + + const complete: string_date_iso8601 = $getCurrentDate(); + + /* TODO: [🕘] Usage count const usage = computeOpenAiUsage(content || '', resultContent || '', rawResponse); */ - const usage = UNCERTAIN_USAGE; - - return exportJson({ - name: 'promptResult', - message: `Result of \`createExecutionToolsFromVercelProvider.callChatModel\``, - value: { - content: rawResponse.text, - modelName, - timing: { - start, - complete, - }, - usage, - rawPromptContent, - rawRequest, - rawResponse: asSerializable(rawResponse), - // <- [🗯] - }, - }); - }, - }; + const usage = UNCERTAIN_USAGE; + + return exportJson({ + name: "promptResult", + message: `Result of \`createExecutionToolsFromVercelProvider.callChatModel\``, + value: { + content: rawResponse.text, + modelName, + timing: { + start, + complete, + }, + usage, + rawPromptContent, + rawRequest, + rawResponse: asSerializable(rawResponse), + // <- [🗯] + }, + }); + }, + }; } diff --git a/src/llm-providers/vercel/playground/playground.ts b/src/llm-providers/vercel/playground/playground.ts index 36f03f6e25..65b2528104 100644 --- a/src/llm-providers/vercel/playground/playground.ts +++ b/src/llm-providers/vercel/playground/playground.ts @@ -1,79 +1,84 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import { createOpenAI } from '@ai-sdk/openai'; -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { embeddingVectorToString } from '../../../execution/embeddingVectorToString'; -import { usageToHuman } from '../../../execution/utils/usageToHuman'; -import type { Prompt } from '../../../types/Prompt'; -import { keepUnused } from '../../../utils/organization/keepUnused'; -import { createExecutionToolsFromVercelProvider } from '../createExecutionToolsFromVercelProvider'; +import { createOpenAI } from "@ai-sdk/openai"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { embeddingVectorToString } from "../../../execution/embeddingVectorToString"; +import { usageToHuman } from "../../../execution/utils/usageToHuman"; +import type { Prompt } from "../../../types/Prompt"; +import { keepUnused } from "../../../utils/organization/keepUnused"; +import { createExecutionToolsFromVercelProvider } from "../createExecutionToolsFromVercelProvider"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 Vercel Playground`); - - // Do here stuff you want to test - //========================================> - - const openaiVercelProvider = createOpenAI({ - apiKey: process.env.OPENAI_API_KEY, - // custom settings, e.g. - compatibility: 'strict', // strict mode, enable when using the OpenAI API - }); - - const openaiPromptbookExecutionTools = createExecutionToolsFromVercelProvider({ - title: 'OpenAI', - vercelProvider: openaiVercelProvider, - availableModels: [ - { - modelName: 'gpt-3.5-turbo', - modelVariant: 'CHAT', - }, - ], - }); - - keepUnused(openaiPromptbookExecutionTools); - keepUnused(embeddingVectorToString); - keepUnused(usageToHuman); - keepUnused(); - - /*/ + console.info(`🧸 Vercel Playground`); + + // Do here stuff you want to test + //========================================> + + const openaiVercelProvider = createOpenAI({ + apiKey: process.env.OPENAI_API_KEY, + // custom settings, e.g. + compatibility: "strict", // strict mode, enable when using the OpenAI API + }); + + const openaiPromptbookExecutionTools = createExecutionToolsFromVercelProvider( + { + title: "OpenAI", + vercelProvider: openaiVercelProvider, + availableModels: [ + { + modelName: "gpt-3.5-turbo", + modelVariant: "CHAT", + }, + ], + }, + ); + + keepUnused(openaiPromptbookExecutionTools); + keepUnused(embeddingVectorToString); + keepUnused(usageToHuman); + keepUnused(); + + /*/ const models = await openaiPromptbookExecutionTools.listModels(); console.info({ models }); /**/ - /**/ - const chatPrompt = { - title: 'Promptbook speech', - parameters: {}, - content: `Write me speech about Promptbook and how it can help me to build the most beautiful chatbot and change the world`, - modelRequirements: { - modelVariant: 'CHAT', - systemMessage: 'You are an assistant who only speaks in rhymes.', - temperature: 1.5, - }, - } as const satisfies Prompt; - const chatPromptResult = await openaiPromptbookExecutionTools.callChatModel!(chatPrompt); - console.info({ chatPromptResult }); - console.info(colors.cyan(usageToHuman(chatPromptResult.usage))); - console.info(colors.bgBlue(' User: ') + colors.blue(chatPrompt.content)); - console.info(colors.bgGreen(' Chat: ') + colors.green(chatPromptResult.content)); - /**/ - - /*/ + /**/ + const chatPrompt = { + title: "Promptbook speech", + parameters: {}, + content: `Write me speech about Promptbook and how it can help me to build the most beautiful chatbot and change the world`, + modelRequirements: { + modelVariant: "CHAT", + systemMessage: "You are an assistant who only speaks in rhymes.", + temperature: 1.5, + }, + } as const satisfies Prompt; + const chatPromptResult = + await openaiPromptbookExecutionTools.callChatModel!(chatPrompt); + console.info({ chatPromptResult }); + console.info(colors.cyan(usageToHuman(chatPromptResult.usage))); + console.info(colors.bgBlue(" User: ") + colors.blue(chatPrompt.content)); + console.info( + colors.bgGreen(" Chat: ") + colors.green(chatPromptResult.content), + ); + /**/ + + /*/ const completionPrompt = { title: 'Hello', parameters: {}, @@ -88,11 +93,11 @@ async function playground() { console.info(chalk.green(completionPrompt.content + completionPromptResult.content)); /**/ - /*/ + /*/ // TODO: Test Translations in playground /**/ - /*/ + /*/ const prompt = { title: 'Hello', parameters: {}, @@ -109,11 +114,11 @@ async function playground() { console.info(chalk.bgGreen(' Embedding: ') + chalk.green(embeddingVectorToString(promptResult.content))); /**/ - /*/ + /*/ // <- Note: [🤖] Test here new model variant if needed /**/ - //========================================/ + //========================================/ } /** diff --git a/src/llm-providers/vercel/playground/tsconfig.json b/src/llm-providers/vercel/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/llm-providers/vercel/playground/tsconfig.json +++ b/src/llm-providers/vercel/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/other/templates/getBookTemplates.ts b/src/other/templates/getBookTemplates.ts index 8a1efa8492..28fed81cc6 100644 --- a/src/other/templates/getBookTemplates.ts +++ b/src/other/templates/getBookTemplates.ts @@ -1,9 +1,9 @@ -import { SetRequired } from 'type-fest'; -import type { string_formfactor_name } from '../../formfactors/_common/string_formfactor_name'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { NonEmptyReadonlyArray } from '../../types/NonEmptyArray'; -import type { string_pipeline_url } from '../../types/typeAliases'; -import { getTemplatesPipelineCollection } from './getTemplatesPipelineCollection'; +import type { SetRequired } from "type-fest"; +import type { string_formfactor_name } from "../../formfactors/_common/string_formfactor_name"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { NonEmptyReadonlyArray } from "../../types/NonEmptyArray"; +import type { string_pipeline_url } from "../../types/typeAliases"; +import { getTemplatesPipelineCollection } from "./getTemplatesPipelineCollection"; /** * @@@ @@ -11,7 +11,8 @@ import { getTemplatesPipelineCollection } from './getTemplatesPipelineCollection * @singleton * @private internal cache of `getBookTemplate` */ -export let pipelines: Array> | null = null; +export let pipelines: Array> | null = + null; /** * Get template for new book @@ -21,24 +22,31 @@ export let pipelines: Array> | null = n * @public exported from `@promptbook/templates` */ export function getBookTemplates( - formfactorName?: string_formfactor_name, -): NonEmptyReadonlyArray> { - if (pipelines === null) { - const collection = getTemplatesPipelineCollection(); - const pipelineUrls = collection.listPipelines() as ReadonlyArray; // <- Note: [0] Function `listPipelines` is sync because `templatesPipelineCollection` is `SimplePipelineCollection` - pipelines = pipelineUrls.map( - (pipelineUrl) => collection?.getPipelineByUrl(pipelineUrl) as SetRequired, - // <- Note: [0] - ); - } + formfactorName?: string_formfactor_name, +): NonEmptyReadonlyArray> { + if (pipelines === null) { + const collection = getTemplatesPipelineCollection(); + const pipelineUrls = + collection.listPipelines() as ReadonlyArray; // <- Note: [0] Function `listPipelines` is sync because `templatesPipelineCollection` is `SimplePipelineCollection` + pipelines = pipelineUrls.map( + (pipelineUrl) => + collection?.getPipelineByUrl(pipelineUrl) as SetRequired< + PipelineJson, + "pipelineUrl" + >, + // <- Note: [0] + ); + } - if (formfactorName === undefined) { - return pipelines as NonEmptyReadonlyArray>; - } else { - return pipelines.filter((pipeline) => pipeline.formfactorName === formfactorName) as NonEmptyReadonlyArray< - SetRequired - >; - } + if (formfactorName === undefined) { + return pipelines as NonEmptyReadonlyArray< + SetRequired + >; + } else { + return pipelines.filter( + (pipeline) => pipeline.formfactorName === formfactorName, + ) as NonEmptyReadonlyArray>; + } } /** diff --git a/src/personas/preparePersona.ts b/src/personas/preparePersona.ts index 617c2583fe..fb5cdb28be 100644 --- a/src/personas/preparePersona.ts +++ b/src/personas/preparePersona.ts @@ -1,16 +1,16 @@ -import PipelineCollection from '../../books/index.json'; -import { createCollectionFromJson } from '../collection/constructors/createCollectionFromJson'; -import { DEFAULT_IS_VERBOSE } from '../config'; -import { MissingToolsError } from '../errors/MissingToolsError'; -import { createPipelineExecutor } from '../execution/createPipelineExecutor/00-createPipelineExecutor'; -import type { ExecutionTools } from '../execution/ExecutionTools'; -import { joinLlmExecutionTools } from '../llm-providers/multiple/joinLlmExecutionTools'; -import type { PersonaPreparedJson } from '../pipeline/PipelineJson/PersonaJson'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import type { PrepareAndScrapeOptions } from '../prepare/PrepareAndScrapeOptions'; -import type { string_persona_description } from '../types/typeAliases'; -import { arrayableToArray } from '../utils/arrayableToArray'; -import type { TODO_any } from '../utils/organization/TODO_any'; +import PipelineCollection from "../../books/index.json"; +import { createCollectionFromJson } from "../collection/constructors/createCollectionFromJson"; +import { DEFAULT_IS_VERBOSE } from "../config"; +import { MissingToolsError } from "../errors/MissingToolsError"; +import type { ExecutionTools } from "../execution/ExecutionTools"; +import { createPipelineExecutor } from "../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { joinLlmExecutionTools } from "../llm-providers/multiple/joinLlmExecutionTools"; +import type { PersonaPreparedJson } from "../pipeline/PipelineJson/PersonaJson"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import type { PrepareAndScrapeOptions } from "../prepare/PrepareAndScrapeOptions"; +import type { string_persona_description } from "../types/typeAliases"; +import { arrayableToArray } from "../utils/arrayableToArray"; +import type { TODO_any } from "../utils/organization/TODO_any"; /** * Prepares the persona for the pipeline @@ -19,53 +19,61 @@ import type { TODO_any } from '../utils/organization/TODO_any'; * @public exported from `@promptbook/core` */ export async function preparePersona( - personaDescription: string_persona_description, - tools: Pick, - options: PrepareAndScrapeOptions, -): Promise { - const { isVerbose = DEFAULT_IS_VERBOSE } = options; + personaDescription: string_persona_description, + tools: Pick, + options: PrepareAndScrapeOptions, +): Promise { + const { isVerbose = DEFAULT_IS_VERBOSE } = options; - if (tools === undefined || tools.llm === undefined) { - throw new MissingToolsError('LLM tools are required for preparing persona'); - } + if (tools === undefined || tools.llm === undefined) { + throw new MissingToolsError("LLM tools are required for preparing persona"); + } - // TODO: [🌼] In future use `ptbk make` and made getPipelineCollection - const collection = createCollectionFromJson(...(PipelineCollection as TODO_any as ReadonlyArray)); + // TODO: [🌼] In future use `ptbk make` and made getPipelineCollection + const collection = createCollectionFromJson( + ...(PipelineCollection as TODO_any as ReadonlyArray), + ); - const preparePersonaExecutor = createPipelineExecutor({ - pipeline: await collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.book'), - tools, - }); + const preparePersonaExecutor = createPipelineExecutor({ + pipeline: await collection.getPipelineByUrl( + "https://promptbook.studio/promptbook/prepare-persona.book", + ), + tools, + }); - // TODO: [🚐] Make arrayable LLMs -> single LLM DRY - const _llms = arrayableToArray(tools.llm); - const llmTools = _llms.length === 1 ? _llms[0]! : joinLlmExecutionTools(..._llms); + // TODO: [🚐] Make arrayable LLMs -> single LLM DRY + const _llms = arrayableToArray(tools.llm); + const llmTools = + _llms.length === 1 ? _llms[0]! : joinLlmExecutionTools(..._llms); - const availableModels = await llmTools.listModels(); - const availableModelNames = availableModels - .filter(({ modelVariant }) => modelVariant === 'CHAT') - .map(({ modelName }) => modelName) - .join(','); + const availableModels = await llmTools.listModels(); + const availableModelNames = availableModels + .filter(({ modelVariant }) => modelVariant === "CHAT") + .map(({ modelName }) => modelName) + .join(","); - const result = await preparePersonaExecutor({ availableModelNames, personaDescription }).asPromise(); + const result = await preparePersonaExecutor({ + availableModelNames, + personaDescription, + }).asPromise(); - const { outputParameters } = result; - const { modelRequirements: modelRequirementsRaw } = outputParameters; + const { outputParameters } = result; + const { modelRequirements: modelRequirementsRaw } = outputParameters; - const modelRequirements = JSON.parse(modelRequirementsRaw!); + const modelRequirements = JSON.parse(modelRequirementsRaw!); - if (isVerbose) { - console.info(`PERSONA ${personaDescription}`, modelRequirements); - } + if (isVerbose) { + console.info(`PERSONA ${personaDescription}`, modelRequirements); + } - const { modelName, systemMessage, temperature } = modelRequirements; + const { modelName, systemMessage, temperature } = modelRequirements; - return { - modelVariant: 'CHAT', - modelName, - systemMessage, - temperature, - }; + return { + modelVariant: "CHAT", + modelName, + systemMessage, + temperature, + }; } /** diff --git a/src/pipeline/PipelineInterface/PipelineInterface.ts b/src/pipeline/PipelineInterface/PipelineInterface.ts index 7abc790eec..d273ac7f46 100644 --- a/src/pipeline/PipelineInterface/PipelineInterface.ts +++ b/src/pipeline/PipelineInterface/PipelineInterface.ts @@ -1,5 +1,5 @@ -import type { InputParameterJson } from '../PipelineJson/ParameterJson'; -import type { OutputParameterJson } from '../PipelineJson/ParameterJson'; +import type { InputParameterJson } from "../PipelineJson/ParameterJson"; +import type { OutputParameterJson } from "../PipelineJson/ParameterJson"; /** * @@@ @@ -8,17 +8,17 @@ import type { OutputParameterJson } from '../PipelineJson/ParameterJson'; * @see https://github.com/webgptorg/promptbook/discussions/171 */ export type PipelineInterface = { - /** - * @@@ - * - * Note: Sorted alphabetically - */ - readonly inputParameters: ReadonlyArray; + /** + * @@@ + * + * Note: Sorted alphabetically + */ + readonly inputParameters: ReadonlyArray; - /** - * @@@ - * - * Note: Sorted alphabetically - */ - readonly outputParameters: ReadonlyArray; + /** + * @@@ + * + * Note: Sorted alphabetically + */ + readonly outputParameters: ReadonlyArray; }; diff --git a/src/pipeline/PipelineInterface/constants.ts b/src/pipeline/PipelineInterface/constants.ts index 912e0cfffe..a816751a44 100644 --- a/src/pipeline/PipelineInterface/constants.ts +++ b/src/pipeline/PipelineInterface/constants.ts @@ -1,4 +1,4 @@ -import type { PipelineInterface } from './PipelineInterface'; +import type { PipelineInterface } from "./PipelineInterface"; /** * @@@ @@ -8,8 +8,8 @@ import type { PipelineInterface } from './PipelineInterface'; * @public exported from `@promptbook/core` */ export const GENERIC_PIPELINE_INTERFACE = { - inputParameters: [], - outputParameters: [], + inputParameters: [], + outputParameters: [], } as const satisfies PipelineInterface; /** diff --git a/src/pipeline/PipelineInterface/getPipelineInterface.ts b/src/pipeline/PipelineInterface/getPipelineInterface.ts index 7cdfec95d1..f8cf955dd4 100644 --- a/src/pipeline/PipelineInterface/getPipelineInterface.ts +++ b/src/pipeline/PipelineInterface/getPipelineInterface.ts @@ -1,9 +1,9 @@ -import type { WritableDeep } from 'type-fest'; -import type { TODO_remove_as } from '../../utils/organization/TODO_remove_as'; -import { deepClone } from '../../utils/serialization/deepClone'; -import { exportJson } from '../../utils/serialization/exportJson'; -import type { PipelineJson } from '../PipelineJson/PipelineJson'; -import type { PipelineInterface } from './PipelineInterface'; +import type { WritableDeep } from "type-fest"; +import type { TODO_remove_as } from "../../utils/organization/TODO_remove_as"; +import { deepClone } from "../../utils/serialization/deepClone"; +import { exportJson } from "../../utils/serialization/exportJson"; +import type { PipelineJson } from "../PipelineJson/PipelineJson"; +import type { PipelineInterface } from "./PipelineInterface"; /** * @@@ @@ -13,38 +13,44 @@ import type { PipelineInterface } from './PipelineInterface'; * * @public exported from `@promptbook/core` */ -export function getPipelineInterface(pipeline: PipelineJson): PipelineInterface { - const pipelineInterface: WritableDeep = { - inputParameters: [], - outputParameters: [], - }; +export function getPipelineInterface( + pipeline: PipelineJson, +): PipelineInterface { + const pipelineInterface: WritableDeep = { + inputParameters: [], + outputParameters: [], + }; - for (const parameter of pipeline.parameters) { - const { isInput, isOutput } = parameter; + for (const parameter of pipeline.parameters) { + const { isInput, isOutput } = parameter; - if (isInput) { - pipelineInterface.inputParameters.push( - deepClone(parameter), - // <- Note: Clone to prevent mutation when `$deepFreeze` is called at the end - ); - } + if (isInput) { + pipelineInterface.inputParameters.push( + deepClone(parameter), + // <- Note: Clone to prevent mutation when `$deepFreeze` is called at the end + ); + } - if (isOutput) { - pipelineInterface.outputParameters.push( - deepClone(parameter), - // <- Note: Clone to prevent mutation when `$deepFreeze` is called at the end - ); - } - } + if (isOutput) { + pipelineInterface.outputParameters.push( + deepClone(parameter), + // <- Note: Clone to prevent mutation when `$deepFreeze` is called at the end + ); + } + } - for (const key of ['inputParameters', 'outputParameters'] as Array) { - pipelineInterface[key].sort(({ name: name1 }, { name: name2 }) => name1.localeCompare(name2)); - } + for (const key of ["inputParameters", "outputParameters"] as Array< + keyof PipelineInterface + >) { + pipelineInterface[key].sort(({ name: name1 }, { name: name2 }) => + name1.localeCompare(name2), + ); + } - return exportJson({ - name: `pipelineInterface`, - message: `Result of \`getPipelineInterface\``, - order: ['inputParameters', 'outputParameters'], - value: pipelineInterface, - }) as TODO_remove_as; + return exportJson({ + name: `pipelineInterface`, + message: `Result of \`getPipelineInterface\``, + order: ["inputParameters", "outputParameters"], + value: pipelineInterface, + }) as TODO_remove_as; } diff --git a/src/pipeline/PipelineInterface/isPipelineImplementingInterface.ts b/src/pipeline/PipelineInterface/isPipelineImplementingInterface.ts index 486293a6d7..f1b0e018fa 100644 --- a/src/pipeline/PipelineInterface/isPipelineImplementingInterface.ts +++ b/src/pipeline/PipelineInterface/isPipelineImplementingInterface.ts @@ -1,7 +1,7 @@ -import type { PipelineJson } from '../PipelineJson/PipelineJson'; -import { getPipelineInterface } from './getPipelineInterface'; -import { isPipelineInterfacesEqual } from './isPipelineInterfacesEqual'; -import type { PipelineInterface } from './PipelineInterface'; +import type { PipelineJson } from "../PipelineJson/PipelineJson"; +import type { PipelineInterface } from "./PipelineInterface"; +import { getPipelineInterface } from "./getPipelineInterface"; +import { isPipelineInterfacesEqual } from "./isPipelineInterfacesEqual"; /** * @@@ @@ -9,15 +9,15 @@ import type { PipelineInterface } from './PipelineInterface'; * @see https://github.com/webgptorg/promptbook/discussions/171 */ export type IsPipelineImplementingInterfaceOptions = { - /** - * @@@ - */ - pipeline: PipelineJson; + /** + * @@@ + */ + pipeline: PipelineJson; - /** - * @@@ - */ - pipelineInterface: PipelineInterface; + /** + * @@@ + */ + pipelineInterface: PipelineInterface; }; /** @@ -28,8 +28,13 @@ export type IsPipelineImplementingInterfaceOptions = { * * @public exported from `@promptbook/core` */ -export function isPipelineImplementingInterface(options: IsPipelineImplementingInterfaceOptions): boolean { - const { pipeline, pipelineInterface } = options; +export function isPipelineImplementingInterface( + options: IsPipelineImplementingInterfaceOptions, +): boolean { + const { pipeline, pipelineInterface } = options; - return isPipelineInterfacesEqual(getPipelineInterface(pipeline), pipelineInterface); + return isPipelineInterfacesEqual( + getPipelineInterface(pipeline), + pipelineInterface, + ); } diff --git a/src/pipeline/PipelineInterface/isPipelineInterfacesEqual.ts b/src/pipeline/PipelineInterface/isPipelineInterfacesEqual.ts index de2d26067c..652d09a5cc 100644 --- a/src/pipeline/PipelineInterface/isPipelineInterfacesEqual.ts +++ b/src/pipeline/PipelineInterface/isPipelineInterfacesEqual.ts @@ -1,5 +1,5 @@ -import type { really_any } from '../../utils/organization/really_any'; -import type { PipelineInterface } from './PipelineInterface'; +import type { really_any } from "../../utils/organization/really_any"; +import type { PipelineInterface } from "./PipelineInterface"; /** * @@@ @@ -10,35 +10,40 @@ import type { PipelineInterface } from './PipelineInterface'; * @public exported from `@promptbook/core` */ export function isPipelineInterfacesEqual( - pipelineInterface1: PipelineInterface, - pipelineInterface2: PipelineInterface, + pipelineInterface1: PipelineInterface, + pipelineInterface2: PipelineInterface, ): boolean { - for (const whichParameters of ['inputParameters', 'outputParameters'] as const) { - const parameters1 = pipelineInterface1[whichParameters] as really_any; // <- Note: `isPipelineInterfacesEqual` is just temporary solution, no need to fix this - const parameters2 = pipelineInterface2[whichParameters] as really_any; - - if (parameters1.length !== parameters2.length) { - return false; - } - - for (const parameter of parameters1) { - const matchingParameter = parameters2.find(({ name }: really_any) => name === parameter.name); - - if (!matchingParameter) { - return false; - } - - // Note: Do not compare description, it is not relevant for compatibility - - if (matchingParameter.isInput !== parameter.isInput) { - return false; - } - - if (matchingParameter.isOutput !== parameter.isOutput) { - return false; - } - } - } - - return true; + for (const whichParameters of [ + "inputParameters", + "outputParameters", + ] as const) { + const parameters1 = pipelineInterface1[whichParameters] as really_any; // <- Note: `isPipelineInterfacesEqual` is just temporary solution, no need to fix this + const parameters2 = pipelineInterface2[whichParameters] as really_any; + + if (parameters1.length !== parameters2.length) { + return false; + } + + for (const parameter of parameters1) { + const matchingParameter = parameters2.find( + ({ name }: really_any) => name === parameter.name, + ); + + if (!matchingParameter) { + return false; + } + + // Note: Do not compare description, it is not relevant for compatibility + + if (matchingParameter.isInput !== parameter.isInput) { + return false; + } + + if (matchingParameter.isOutput !== parameter.isOutput) { + return false; + } + } + } + + return true; } diff --git a/src/pipeline/PipelineJson/CommonTaskJson.ts b/src/pipeline/PipelineJson/CommonTaskJson.ts index 338e50b09f..5f9ef7ebde 100644 --- a/src/pipeline/PipelineJson/CommonTaskJson.ts +++ b/src/pipeline/PipelineJson/CommonTaskJson.ts @@ -1,114 +1,120 @@ -import type { ForeachJson } from '../../commands/FOREACH/ForeachJson'; -import type { FormatCommand } from '../../commands/FORMAT/FormatCommand'; -import type { SectionType } from '../../types/SectionType'; -import type { string_javascript } from '../../types/typeAliases'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_name } from '../../types/typeAliases'; -import type { string_parameter_name } from '../../types/typeAliases'; -import type { string_postprocessing_function_name } from '../../types/typeAliases'; -import type { string_prompt } from '../../types/typeAliases'; -import type { string_template } from '../../types/typeAliases'; -import type { Expectations } from './Expectations'; +import type { ForeachJson } from "../../commands/FOREACH/ForeachJson"; +import type { FormatCommand } from "../../commands/FORMAT/FormatCommand"; +import type { SectionType } from "../../types/SectionType"; +import type { string_javascript } from "../../types/typeAliases"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_name } from "../../types/typeAliases"; +import type { string_parameter_name } from "../../types/typeAliases"; +import type { string_postprocessing_function_name } from "../../types/typeAliases"; +import type { string_prompt } from "../../types/typeAliases"; +import type { string_template } from "../../types/typeAliases"; +import type { Expectations } from "./Expectations"; /** * Common properties of all tasks */ export type CommonTaskJson = { - /** - * Name of the task - * - It must be unique across the pipeline - * - It should start uppercase and can contain letters and numbers - * - The pipelineUrl together with hash and name are used to identify the task in the pipeline - */ - readonly name: string_name; + /** + * Name of the task + * - It must be unique across the pipeline + * - It should start uppercase and can contain letters and numbers + * - The pipelineUrl together with hash and name are used to identify the task in the pipeline + */ + readonly name: string_name; - /** - * Title of the task - * It can use simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure - */ - readonly title: string; + /** + * Title of the task + * It can use simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure + */ + readonly title: string; - /** - * Description of the task - * It can use multiple paragraphs of simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure - */ - readonly description?: string_markdown_text; + /** + * Description of the task + * It can use multiple paragraphs of simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure + */ + readonly description?: string_markdown_text; - /** - * List of parameter names that are used in the task and must be defined before the task is executed - * - * Note: Joker is one of the dependent parameters - */ - readonly dependentParameterNames: Array; - // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems + /** + * List of parameter names that are used in the task and must be defined before the task is executed + * + * Note: Joker is one of the dependent parameters + */ + readonly dependentParameterNames: Array; + // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems - /** - * If theese parameters meet the expectations requirements, they are used instead of executing this task - * - * @see https://github.com/webgptorg/promptbook/discussions/66 - */ - readonly jokerParameterNames?: Array; - // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems + /** + * If theese parameters meet the expectations requirements, they are used instead of executing this task + * + * @see https://github.com/webgptorg/promptbook/discussions/66 + */ + readonly jokerParameterNames?: Array; + // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems - /** - * @@@ - */ - readonly foreach?: ForeachJson; + /** + * @@@ + */ + readonly foreach?: ForeachJson; - /** - * Type of the execution - * This determines if the task is send to LLM, user or some scripting evaluation - */ - readonly taskType: SectionType; + /** + * Type of the execution + * This determines if the task is send to LLM, user or some scripting evaluation + */ + readonly taskType: SectionType; - /** - * Content of the task with {placeholders} for parameters - * - * @@@ content vs preparedContent - */ - readonly content: (string_prompt | string_javascript | string_markdown) & string_template; + /** + * Content of the task with {placeholders} for parameters + * + * @@@ content vs preparedContent + */ + readonly content: (string_prompt | string_javascript | string_markdown) & + string_template; - /** - * @@@ Content of the task with {placeholders} for parameters - * - * @@@ content vs preparedContent - * - * @default "{content}" - */ - readonly preparedContent?: (string_prompt | string_javascript | string_markdown) & string_template; + /** + * @@@ Content of the task with {placeholders} for parameters + * + * @@@ content vs preparedContent + * + * @default "{content}" + */ + readonly preparedContent?: ( + | string_prompt + | string_javascript + | string_markdown + ) & + string_template; - /** - * List of postprocessing steps that are executed after the task - * - * @see https://github.com/webgptorg/promptbook/discussions/31 - */ - readonly postprocessingFunctionNames?: Array; - // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems + /** + * List of postprocessing steps that are executed after the task + * + * @see https://github.com/webgptorg/promptbook/discussions/31 + */ + readonly postprocessingFunctionNames?: Array; + // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems - /** - * Expect this amount of each unit in the answer - * - * For example 5 words, 3 sentences, 2 paragraphs, ... - * - * Note: Expectations are performed after all postprocessing steps - * @see https://github.com/webgptorg/promptbook/discussions/30 - */ - readonly expectations?: Expectations; + /** + * Expect this amount of each unit in the answer + * + * For example 5 words, 3 sentences, 2 paragraphs, ... + * + * Note: Expectations are performed after all postprocessing steps + * @see https://github.com/webgptorg/promptbook/discussions/30 + */ + readonly expectations?: Expectations; - /** - * Expect this format of the answer - * - * Note: Expectations are performed after all postprocessing steps - * @see https://github.com/webgptorg/promptbook/discussions/30 - * @deprecated [💝] - */ - readonly format?: FormatCommand['format']; + /** + * Expect this format of the answer + * + * Note: Expectations are performed after all postprocessing steps + * @see https://github.com/webgptorg/promptbook/discussions/30 + * @deprecated [💝] + */ + readonly format?: FormatCommand["format"]; - /** - * Name of the parameter that is the result of the task - */ - readonly resultingParameterName: string_name; + /** + * Name of the parameter that is the result of the task + */ + readonly resultingParameterName: string_name; }; /** diff --git a/src/pipeline/PipelineJson/DialogTaskJson.ts b/src/pipeline/PipelineJson/DialogTaskJson.ts index e371d1090f..6032cce8b9 100644 --- a/src/pipeline/PipelineJson/DialogTaskJson.ts +++ b/src/pipeline/PipelineJson/DialogTaskJson.ts @@ -1,4 +1,4 @@ -import type { CommonTaskJson } from './CommonTaskJson'; +import type { CommonTaskJson } from "./CommonTaskJson"; /** * Task for prompt to user @@ -7,7 +7,7 @@ import type { CommonTaskJson } from './CommonTaskJson'; * @see https://github.com/webgptorg/promptbook/discussions/76 */ export type DialogTaskJson = CommonTaskJson & { - readonly taskType: 'DIALOG_TASK'; + readonly taskType: "DIALOG_TASK"; }; /** diff --git a/src/pipeline/PipelineJson/Expectations.ts b/src/pipeline/PipelineJson/Expectations.ts index 911c49f0cd..0711260909 100644 --- a/src/pipeline/PipelineJson/Expectations.ts +++ b/src/pipeline/PipelineJson/Expectations.ts @@ -1,6 +1,6 @@ -import type { TupleToUnion } from 'type-fest'; -import type { number_integer } from '../../types/typeAliases'; -import type { number_positive } from '../../types/typeAliases'; +import type { TupleToUnion } from "type-fest"; +import type { number_integer } from "../../types/typeAliases"; +import type { number_positive } from "../../types/typeAliases"; /** * Expect this amount of each unit in the answer @@ -12,7 +12,10 @@ import type { number_positive } from '../../types/typeAliases'; * @see https://github.com/webgptorg/promptbook/discussions/30 */ export type Expectations = Partial< - Record, { readonly min?: ExpectationAmount; readonly max?: ExpectationAmount }> + Record< + Lowercase, + { readonly min?: ExpectationAmount; readonly max?: ExpectationAmount } + > >; /** @@ -29,7 +32,14 @@ export type ExpectationUnit = TupleToUnion; * @see https://github.com/webgptorg/promptbook/discussions/30 * @public exported from `@promptbook/core` */ -export const EXPECTATION_UNITS = ['CHARACTERS', 'WORDS', 'SENTENCES', 'LINES', 'PARAGRAPHS', 'PAGES'] as const; +export const EXPECTATION_UNITS = [ + "CHARACTERS", + "WORDS", + "SENTENCES", + "LINES", + "PARAGRAPHS", + "PAGES", +] as const; /** * Amount of text measurement diff --git a/src/pipeline/PipelineJson/KnowledgePieceJson.ts b/src/pipeline/PipelineJson/KnowledgePieceJson.ts index 2d31bc1ded..4d9c20021b 100644 --- a/src/pipeline/PipelineJson/KnowledgePieceJson.ts +++ b/src/pipeline/PipelineJson/KnowledgePieceJson.ts @@ -1,11 +1,11 @@ -import type { EmbeddingVector } from '../../execution/EmbeddingVector'; -import type { number_id } from '../../types/typeAliases'; -import type { number_linecol_number } from '../../types/typeAliases'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_model_name } from '../../types/typeAliases'; -import type { string_name } from '../../types/typeAliases'; -import type { string_keyword } from '../../utils/normalization/IKeywords'; +import type { EmbeddingVector } from "../../execution/EmbeddingVector"; +import type { number_id } from "../../types/typeAliases"; +import type { number_linecol_number } from "../../types/typeAliases"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_model_name } from "../../types/typeAliases"; +import type { string_name } from "../../types/typeAliases"; +import type { string_keyword } from "../../utils/normalization/IKeywords"; /** * Defines one piece of knowledge in the pipeline @@ -17,65 +17,65 @@ import type { string_keyword } from '../../utils/normalization/IKeywords'; * @see https://github.com/webgptorg/promptbook/discussions/41 */ export type KnowledgePiecePreparedJson = { - /** - * Unique name of the knowledge piece based on the title - */ - readonly name?: string_name; + /** + * Unique name of the knowledge piece based on the title + */ + readonly name?: string_name; - /** - * Short title for the information - */ - readonly title?: string_markdown_text; + /** + * Short title for the information + */ + readonly title?: string_markdown_text; - /** - * The information in markdown format - */ - readonly content?: string_markdown; + /** + * The information in markdown format + */ + readonly content?: string_markdown; - /** - * List of sources where the information comes from - */ - readonly sources: ReadonlyArray<{ - /** - * Identifier of the source - */ - readonly name: string_name; + /** + * List of sources where the information comes from + */ + readonly sources: ReadonlyArray<{ + /** + * Identifier of the source + */ + readonly name: string_name; - /** - * Line number - */ - readonly line?: number_linecol_number; + /** + * Line number + */ + readonly line?: number_linecol_number; - /** - * Column number - */ - readonly column?: number_linecol_number; - }>; + /** + * Column number + */ + readonly column?: number_linecol_number; + }>; - /** - * List of keywords that are associated with the knowledge piece - */ - readonly keywords: ReadonlyArray; + /** + * List of keywords that are associated with the knowledge piece + */ + readonly keywords: ReadonlyArray; - /** - * List of models embeddings that are associated with the knowledge piece - */ - readonly index: ReadonlyArray<{ - /** - * Model name which generated the embedding - */ - readonly modelName: string_model_name; + /** + * List of models embeddings that are associated with the knowledge piece + */ + readonly index: ReadonlyArray<{ + /** + * Model name which generated the embedding + */ + readonly modelName: string_model_name; - /** - * Embedding vector of the knowledge piece - */ - readonly position: EmbeddingVector; - }>; + /** + * Embedding vector of the knowledge piece + */ + readonly position: EmbeddingVector; + }>; - /** - * List of preparation ids that were used to prepare this knowledge piece - */ - readonly preparationIds: ReadonlyArray; + /** + * List of preparation ids that were used to prepare this knowledge piece + */ + readonly preparationIds: ReadonlyArray; }; /** diff --git a/src/pipeline/PipelineJson/KnowledgeSourceJson.ts b/src/pipeline/PipelineJson/KnowledgeSourceJson.ts index 2e3dfb5a81..435d43e6d7 100644 --- a/src/pipeline/PipelineJson/KnowledgeSourceJson.ts +++ b/src/pipeline/PipelineJson/KnowledgeSourceJson.ts @@ -1,6 +1,6 @@ -import type { number_id } from '../../types/typeAliases'; -import type { string_knowledge_source_content } from '../../types/typeAliases'; -import type { string_name } from '../../types/typeAliases'; +import type { number_id } from "../../types/typeAliases"; +import type { string_knowledge_source_content } from "../../types/typeAliases"; +import type { string_name } from "../../types/typeAliases"; /** * Defines one source of knowledge in the pipeline @@ -11,22 +11,22 @@ import type { string_name } from '../../types/typeAliases'; * @see https://github.com/webgptorg/promptbook/discussions/41 */ export type KnowledgeSourceJson = { - /** - * Unique identifier of the knowledge source - */ - readonly name: string_name; + /** + * Unique identifier of the knowledge source + */ + readonly name: string_name; - /** - * Source of one knowledge - * - * It can be a link, a relative path to file or direct text or combination of those - * - * For example `"https://pavolhejny.com/"` - * For example `"./pavol-hejny-cv.pdf"` - * For example `"Pavol Hejný has web https://pavolhejny.com/"` - * For example `"Pavol Hejný is web developer and creator of Promptbook and Collboard"` - */ - readonly knowledgeSourceContent: string_knowledge_source_content; + /** + * Source of one knowledge + * + * It can be a link, a relative path to file or direct text or combination of those + * + * For example `"https://pavolhejny.com/"` + * For example `"./pavol-hejny-cv.pdf"` + * For example `"Pavol Hejný has web https://pavolhejny.com/"` + * For example `"Pavol Hejný is web developer and creator of Promptbook and Collboard"` + */ + readonly knowledgeSourceContent: string_knowledge_source_content; }; /** @@ -37,10 +37,10 @@ export type KnowledgeSourceJson = { * @see https://github.com/webgptorg/promptbook/discussions/41 */ export type KnowledgeSourcePreparedJson = KnowledgeSourceJson & { - /** - * List of preparation ids that were used to prepare this knowledge source to knowledge pieces - */ - readonly preparationIds: ReadonlyArray; + /** + * List of preparation ids that were used to prepare this knowledge source to knowledge pieces + */ + readonly preparationIds: ReadonlyArray; }; /** diff --git a/src/pipeline/PipelineJson/ParameterJson.ts b/src/pipeline/PipelineJson/ParameterJson.ts index 224fcc6c2c..046a7dc728 100644 --- a/src/pipeline/PipelineJson/ParameterJson.ts +++ b/src/pipeline/PipelineJson/ParameterJson.ts @@ -1,13 +1,16 @@ -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_parameter_name } from '../../types/typeAliases'; -import type { string_parameter_value } from '../../types/typeAliases'; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_parameter_name } from "../../types/typeAliases"; +import type { string_parameter_value } from "../../types/typeAliases"; /** * Describes one parameter of the pipeline * * Note: [🚉] This is fully serializable as JSON */ -export type ParameterJson = InputParameterJson | IntermediateParameterJson | OutputParameterJson; +export type ParameterJson = + | InputParameterJson + | IntermediateParameterJson + | OutputParameterJson; /** * Describes input parameter of the pipeline @@ -17,15 +20,15 @@ export type ParameterJson = InputParameterJson | IntermediateParameterJson | Out * Note: [🚉] This is fully serializable as JSON */ export type InputParameterJson = CommonParameterJson & { - /** - * The parameter is input of the pipeline - */ - readonly isInput: true; + /** + * The parameter is input of the pipeline + */ + readonly isInput: true; - /** - * The parameter is NOT output of the pipeline - */ - readonly isOutput: false; + /** + * The parameter is NOT output of the pipeline + */ + readonly isOutput: false; }; /** @@ -36,15 +39,15 @@ export type InputParameterJson = CommonParameterJson & { * Note: [🚉] This is fully serializable as JSON */ export type IntermediateParameterJson = CommonParameterJson & { - /** - * The parameter is NOT input of the pipeline - */ - readonly isInput: false; + /** + * The parameter is NOT input of the pipeline + */ + readonly isInput: false; - /** - * The parameter is NOT output of the pipeline - */ - readonly isOutput: false; + /** + * The parameter is NOT output of the pipeline + */ + readonly isOutput: false; }; /** @@ -55,15 +58,15 @@ export type IntermediateParameterJson = CommonParameterJson & { * Note: [🚉] This is fully serializable as JSON */ export type OutputParameterJson = CommonParameterJson & { - /** - * The parameter is NOT input of the pipeline - */ - readonly isInput: false; + /** + * The parameter is NOT input of the pipeline + */ + readonly isInput: false; - /** - * The parameter is output of the pipeline - */ - readonly isOutput: true; + /** + * The parameter is output of the pipeline + */ + readonly isOutput: true; }; /** @@ -72,35 +75,35 @@ export type OutputParameterJson = CommonParameterJson & { * Note: [🚉] This is fully serializable as JSON */ export type CommonParameterJson = { - /** - * Name of the parameter - * - It must be unique across the pipeline - * - It should start lowercase and contain letters and numbers - */ - readonly name: string_parameter_name; + /** + * Name of the parameter + * - It must be unique across the pipeline + * - It should start lowercase and contain letters and numbers + */ + readonly name: string_parameter_name; - /** - * The parameter is input of the pipeline - */ - readonly isInput: boolean; + /** + * The parameter is input of the pipeline + */ + readonly isInput: boolean; - /** - * The parameter is output of the pipeline - */ - readonly isOutput: boolean; + /** + * The parameter is output of the pipeline + */ + readonly isOutput: boolean; - /** - * Description of the parameter - * - It can use simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure - */ - readonly description?: string_markdown_text; + /** + * Description of the parameter + * - It can use simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure + */ + readonly description?: string_markdown_text; - /** - * Example values of the parameter - * Note: This values won't be actually used as some default values, but they are just for better understanding of the parameter - */ - readonly exampleValues?: Array; - // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems + /** + * Example values of the parameter + * Note: This values won't be actually used as some default values, but they are just for better understanding of the parameter + */ + readonly exampleValues?: Array; + // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems }; /** diff --git a/src/pipeline/PipelineJson/PersonaJson.ts b/src/pipeline/PipelineJson/PersonaJson.ts index 9f701818e1..ef0bf0cf9a 100644 --- a/src/pipeline/PipelineJson/PersonaJson.ts +++ b/src/pipeline/PipelineJson/PersonaJson.ts @@ -1,7 +1,7 @@ -import type { ChatModelRequirements } from '../../types/ModelRequirements'; -import type { number_id } from '../../types/typeAliases'; -import type { string_name } from '../../types/typeAliases'; -import type { string_persona_description } from '../../types/typeAliases'; +import type { ChatModelRequirements } from "../../types/ModelRequirements"; +import type { number_id } from "../../types/typeAliases"; +import type { string_name } from "../../types/typeAliases"; +import type { string_persona_description } from "../../types/typeAliases"; /** * Defines a persona in the pipeline @@ -11,17 +11,17 @@ import type { string_persona_description } from '../../types/typeAliases'; * @see https://github.com/webgptorg/promptbook/discussions/22 */ export type PersonaJson = { - /** - * Name of the persona - * - It must be unique across the pipeline - * - It should start uppercase and can contain letters and numbers - */ - readonly name: string_name; + /** + * Name of the persona + * - It must be unique across the pipeline + * - It should start uppercase and can contain letters and numbers + */ + readonly name: string_name; - /** - * Description of persona - */ - readonly description: string_persona_description; + /** + * Description of persona + */ + readonly description: string_persona_description; }; /** @@ -32,17 +32,17 @@ export type PersonaJson = { * @see https://github.com/webgptorg/promptbook/discussions/22 */ export type PersonaPreparedJson = PersonaJson & { - /** - * Model requirements for the persona - * - * Note: The model must be CHAT variant to be usable through persona - */ - readonly modelRequirements: ChatModelRequirements; + /** + * Model requirements for the persona + * + * Note: The model must be CHAT variant to be usable through persona + */ + readonly modelRequirements: ChatModelRequirements; - /** - * List of preparation ids that were used to prepare this persona - */ - readonly preparationIds: ReadonlyArray; + /** + * List of preparation ids that were used to prepare this persona + */ + readonly preparationIds: ReadonlyArray; }; /** diff --git a/src/pipeline/PipelineJson/PipelineJson.ts b/src/pipeline/PipelineJson/PipelineJson.ts index cd4a7834eb..06ebe496dd 100644 --- a/src/pipeline/PipelineJson/PipelineJson.ts +++ b/src/pipeline/PipelineJson/PipelineJson.ts @@ -1,19 +1,19 @@ -import type { string_formfactor_name } from '../../formfactors/_common/string_formfactor_name'; -import type { ModelRequirements } from '../../types/ModelRequirements'; -import type { string_filename } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { string_pipeline_url } from '../../types/typeAliases'; -import type { string_relative_filename } from '../../types/typeAliases'; -import type { string_semantic_version } from '../../types/typeAliases'; -import type { PipelineString } from '../PipelineString'; -import type { KnowledgePiecePreparedJson } from './KnowledgePieceJson'; -import type { KnowledgeSourceJson } from './KnowledgeSourceJson'; -import type { KnowledgeSourcePreparedJson } from './KnowledgeSourceJson'; -import type { ParameterJson } from './ParameterJson'; -import type { PersonaJson } from './PersonaJson'; -import type { PersonaPreparedJson } from './PersonaJson'; -import type { PreparationJson } from './PreparationJson'; -import type { TaskJson } from './TaskJson'; +import type { string_formfactor_name } from "../../formfactors/_common/string_formfactor_name"; +import type { ModelRequirements } from "../../types/ModelRequirements"; +import type { string_filename } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { string_pipeline_url } from "../../types/typeAliases"; +import type { string_relative_filename } from "../../types/typeAliases"; +import type { string_semantic_version } from "../../types/typeAliases"; +import type { PipelineString } from "../PipelineString"; +import type { KnowledgePiecePreparedJson } from "./KnowledgePieceJson"; +import type { KnowledgeSourceJson } from "./KnowledgeSourceJson"; +import type { KnowledgeSourcePreparedJson } from "./KnowledgeSourceJson"; +import type { ParameterJson } from "./ParameterJson"; +import type { PersonaJson } from "./PersonaJson"; +import type { PersonaPreparedJson } from "./PersonaJson"; +import type { PreparationJson } from "./PreparationJson"; +import type { TaskJson } from "./TaskJson"; /** * Promptbook is the **core concept of this package**. @@ -24,106 +24,112 @@ import type { TaskJson } from './TaskJson'; * @see @@@ https://github.com/webgptorg/promptbook#promptbook */ export type PipelineJson = { - /* + /* TODO: [💼] > readonly type: 'PIPELINE'; + make type test for this */ - /** - * Unique identifier of the pipeline - * - * Note: It must be unique across all pipeline collections - * Note: It must use HTTPs URL - * Tip: You can do versioning in the URL - * For example: https://promptbook.studio/webgpt/write-website-content-cs.book@1.0.0 - * Warning: Do not hash part of the URL, hash part is used for identification of the task in the pipeline - */ - readonly pipelineUrl?: string_pipeline_url; - - /** - * Internal helper for tracking the source `.book` file of the pipeline - */ - readonly sourceFile?: string_filename; - - /** - * Title of the promptbook - * -It can use simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure - */ - readonly title: string_markdown_text; - - /** - * Version of the Book language - * - * @see https://github.com/webgptorg/book - */ - readonly bookVersion?: string_semantic_version; - - /** - * Description of the promptbook - * It can use multiple paragraphs of simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure - */ - readonly description?: string_markdown_text; - - /** - * Name of the formfactor that the pipeline is shaped for - */ - readonly formfactorName: string_formfactor_name; - - /** - * Set of variables that are used across the pipeline - */ - readonly parameters: Array; - // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems - - /** - * Default model requirements for the model for all `tasks` - */ - readonly defaultModelRequirements?: Partial; - - /** - * Sequence of tasks that are chained together to form a pipeline - */ - readonly tasks: Array; - // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems - - /** - * Set of information that are used as external knowledge in the pipeline - * - * @see https://github.com/webgptorg/promptbook/discussions/41 - */ - readonly knowledgeSources: Array; - // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems - - /** - * Set of information that are used as external knowledge in the pipeline - * - * @see https://github.com/webgptorg/promptbook/discussions/41 - */ - readonly knowledgePieces: Array; - // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems - - /** - * List of prepared virtual personas that are used in the pipeline - * - * @see https://github.com/webgptorg/promptbook/discussions/22 - */ - readonly personas: Array; - // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems - - /** - * List of prepared virtual personas that are used in the pipeline - * - * @see https://github.com/webgptorg/promptbook/discussions/78 - */ - readonly preparations: Array; - // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems - - /** - * Backup of the original book source - */ - readonly sources: ReadonlyArray<{ type: 'BOOK'; path: string_relative_filename | null; content: PipelineString }>; + /** + * Unique identifier of the pipeline + * + * Note: It must be unique across all pipeline collections + * Note: It must use HTTPs URL + * Tip: You can do versioning in the URL + * For example: https://promptbook.studio/webgpt/write-website-content-cs.book@1.0.0 + * Warning: Do not hash part of the URL, hash part is used for identification of the task in the pipeline + */ + readonly pipelineUrl?: string_pipeline_url; + + /** + * Internal helper for tracking the source `.book` file of the pipeline + */ + readonly sourceFile?: string_filename; + + /** + * Title of the promptbook + * -It can use simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure + */ + readonly title: string_markdown_text; + + /** + * Version of the Book language + * + * @see https://github.com/webgptorg/book + */ + readonly bookVersion?: string_semantic_version; + + /** + * Description of the promptbook + * It can use multiple paragraphs of simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure + */ + readonly description?: string_markdown_text; + + /** + * Name of the formfactor that the pipeline is shaped for + */ + readonly formfactorName: string_formfactor_name; + + /** + * Set of variables that are used across the pipeline + */ + readonly parameters: Array; + // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems + + /** + * Default model requirements for the model for all `tasks` + */ + readonly defaultModelRequirements?: Partial; + + /** + * Sequence of tasks that are chained together to form a pipeline + */ + readonly tasks: Array; + // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems + + /** + * Set of information that are used as external knowledge in the pipeline + * + * @see https://github.com/webgptorg/promptbook/discussions/41 + */ + readonly knowledgeSources: Array< + KnowledgeSourceJson | KnowledgeSourcePreparedJson + >; + // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems + + /** + * Set of information that are used as external knowledge in the pipeline + * + * @see https://github.com/webgptorg/promptbook/discussions/41 + */ + readonly knowledgePieces: Array; + // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems + + /** + * List of prepared virtual personas that are used in the pipeline + * + * @see https://github.com/webgptorg/promptbook/discussions/22 + */ + readonly personas: Array; + // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems + + /** + * List of prepared virtual personas that are used in the pipeline + * + * @see https://github.com/webgptorg/promptbook/discussions/78 + */ + readonly preparations: Array; + // <- TODO: [🪓] This should really be `ReadonlyArray`, but it causes problems + + /** + * Backup of the original book source + */ + readonly sources: ReadonlyArray<{ + type: "BOOK"; + path: string_relative_filename | null; + content: PipelineString; + }>; }; /** diff --git a/src/pipeline/PipelineJson/PreparationJson.ts b/src/pipeline/PipelineJson/PreparationJson.ts index b07700fa5f..260711e5f7 100644 --- a/src/pipeline/PipelineJson/PreparationJson.ts +++ b/src/pipeline/PipelineJson/PreparationJson.ts @@ -1,14 +1,14 @@ -import type { PromptResultUsage } from '../../execution/PromptResultUsage'; -import type { number_id } from '../../types/typeAliases'; -import type { string_promptbook_version } from '../../version'; +import type { PromptResultUsage } from "../../execution/PromptResultUsage"; +import type { number_id } from "../../types/typeAliases"; +import type { string_promptbook_version } from "../../version"; export type PreparationJson = { - /** - * Incremental ID of the preparation - */ - readonly id: number_id; + /** + * Incremental ID of the preparation + */ + readonly id: number_id; - /* + /* TODO: [🍥] > /** > * Date and time of the preparation @@ -16,15 +16,15 @@ export type PreparationJson = { > readonly date: string_date_iso8601; */ - /** - * Version of the promptbook used for the preparation - */ - readonly promptbookVersion: string_promptbook_version; + /** + * Version of the promptbook used for the preparation + */ + readonly promptbookVersion: string_promptbook_version; - /** - * Usage of the prompt execution - */ - readonly usage: PromptResultUsage; + /** + * Usage of the prompt execution + */ + readonly usage: PromptResultUsage; }; /** diff --git a/src/pipeline/PipelineJson/PromptTaskJson.ts b/src/pipeline/PipelineJson/PromptTaskJson.ts index b6b0a4aefb..c4997c708d 100644 --- a/src/pipeline/PipelineJson/PromptTaskJson.ts +++ b/src/pipeline/PipelineJson/PromptTaskJson.ts @@ -1,6 +1,6 @@ -import type { ModelRequirements } from '../../types/ModelRequirements'; -import type { string_name } from '../../types/typeAliases'; -import type { CommonTaskJson } from './CommonTaskJson'; +import type { ModelRequirements } from "../../types/ModelRequirements"; +import type { string_name } from "../../types/typeAliases"; +import type { CommonTaskJson } from "./CommonTaskJson"; /** * Task for prompt to LLM @@ -8,18 +8,18 @@ import type { CommonTaskJson } from './CommonTaskJson'; * Note: [🚉] This is fully serializable as JSON */ export type PromptTaskJson = CommonTaskJson & { - readonly taskType: 'PROMPT_TASK'; + readonly taskType: "PROMPT_TASK"; - /** - * Name of the persona who will be responding to this prompt - */ - readonly personaName?: string_name; + /** + * Name of the persona who will be responding to this prompt + */ + readonly personaName?: string_name; - /** - * Requirements for the model - * - This is required only for taskType PROMPT_TASK - */ - readonly modelRequirements?: Partial; + /** + * Requirements for the model + * - This is required only for taskType PROMPT_TASK + */ + readonly modelRequirements?: Partial; }; /** diff --git a/src/pipeline/PipelineJson/ScriptTaskJson.ts b/src/pipeline/PipelineJson/ScriptTaskJson.ts index 4eef54723e..2b79ad352c 100644 --- a/src/pipeline/PipelineJson/ScriptTaskJson.ts +++ b/src/pipeline/PipelineJson/ScriptTaskJson.ts @@ -1,5 +1,5 @@ -import type { ScriptLanguage } from '../../types/ScriptLanguage'; -import type { CommonTaskJson } from './CommonTaskJson'; +import type { ScriptLanguage } from "../../types/ScriptLanguage"; +import type { CommonTaskJson } from "./CommonTaskJson"; /** * Task for script execution @@ -8,14 +8,14 @@ import type { CommonTaskJson } from './CommonTaskJson'; * @see https://github.com/webgptorg/promptbook/discussions/77 */ export type ScriptTaskJson = CommonTaskJson & { - readonly taskType: 'SCRIPT_TASK'; + readonly taskType: "SCRIPT_TASK"; - /** - * Language of the script - * - This is required only for taskType SCRIPT - * - */ - readonly contentLanguage?: ScriptLanguage; + /** + * Language of the script + * - This is required only for taskType SCRIPT + * + */ + readonly contentLanguage?: ScriptLanguage; }; /** diff --git a/src/pipeline/PipelineJson/SimpleTaskJson.ts b/src/pipeline/PipelineJson/SimpleTaskJson.ts index d40472c944..010b53e939 100644 --- a/src/pipeline/PipelineJson/SimpleTaskJson.ts +++ b/src/pipeline/PipelineJson/SimpleTaskJson.ts @@ -1,4 +1,4 @@ -import type { CommonTaskJson } from './CommonTaskJson'; +import type { CommonTaskJson } from "./CommonTaskJson"; /** * Task for simple concatenation of strings @@ -7,7 +7,7 @@ import type { CommonTaskJson } from './CommonTaskJson'; * @see https://github.com/webgptorg/promptbook/discussions/17 */ export type SimpleTaskJson = CommonTaskJson & { - readonly taskType: 'SIMPLE_TASK'; + readonly taskType: "SIMPLE_TASK"; }; /** diff --git a/src/pipeline/PipelineJson/TaskJson.ts b/src/pipeline/PipelineJson/TaskJson.ts index 18587b85f5..609b4b9dfc 100644 --- a/src/pipeline/PipelineJson/TaskJson.ts +++ b/src/pipeline/PipelineJson/TaskJson.ts @@ -1,13 +1,19 @@ -import type { ___or___ } from '../../utils/organization/___or___'; -import type { DialogTaskJson } from './DialogTaskJson'; -import type { PromptTaskJson } from './PromptTaskJson'; -import type { ScriptTaskJson } from './ScriptTaskJson'; -import type { SimpleTaskJson } from './SimpleTaskJson'; +import type { ___or___ } from "../../utils/organization/___or___"; +import type { DialogTaskJson } from "./DialogTaskJson"; +import type { PromptTaskJson } from "./PromptTaskJson"; +import type { ScriptTaskJson } from "./ScriptTaskJson"; +import type { SimpleTaskJson } from "./SimpleTaskJson"; /** * Describes one (prompt) task in the pipeline * * Note: [🚉] This is fully serializable as JSON */ -export type TaskJson = PromptTaskJson | SimpleTaskJson | ScriptTaskJson | DialogTaskJson | ___or___ | ___or___; +export type TaskJson = + | PromptTaskJson + | SimpleTaskJson + | ScriptTaskJson + | DialogTaskJson + | ___or___ + | ___or___; // <- | [🅱] + Add the file with this (execution) block type diff --git a/src/pipeline/PipelineString.ts b/src/pipeline/PipelineString.ts index 27f76c96d6..d2bf0dbaf3 100644 --- a/src/pipeline/PipelineString.ts +++ b/src/pipeline/PipelineString.ts @@ -5,5 +5,5 @@ * @see @@@ https://github.com/webgptorg/promptbook#promptbook */ export type PipelineString = string & { - readonly _type: 'Promptbook' /* <- TODO: [🏟] What is the best shape of the additional object in branded types */; + readonly _type: "Promptbook" /* <- TODO: [🏟] What is the best shape of the additional object in branded types */; }; diff --git a/src/pipeline/book-notation.ts b/src/pipeline/book-notation.ts index 04cd6d65d7..8e800736da 100644 --- a/src/pipeline/book-notation.ts +++ b/src/pipeline/book-notation.ts @@ -1,7 +1,7 @@ -import spaceTrim from 'spacetrim'; -import type { PipelineString } from './PipelineString'; -import { isValidPipelineString } from './isValidPipelineString'; -import { prompt } from './prompt-notation'; +import spaceTrim from "spacetrim"; +import type { PipelineString } from "./PipelineString"; +import { isValidPipelineString } from "./isValidPipelineString"; +import { prompt } from "./prompt-notation"; /** * Tag function for notating a pipeline with a book\`...\ notation as template literal @@ -16,23 +16,26 @@ import { prompt } from './prompt-notation'; * @returns the pipeline string * @public exported from `@promptbook/core` */ -export function book(strings: TemplateStringsArray, ...values: Array): PipelineString { - const pipelineString = prompt(strings, ...values); +export function book( + strings: TemplateStringsArray, + ...values: Array +): PipelineString { + const pipelineString = prompt(strings, ...values); - if (!isValidPipelineString(pipelineString)) { - // TODO: Make the CustomError for this - throw new Error( - spaceTrim(` + if (!isValidPipelineString(pipelineString)) { + // TODO: Make the CustomError for this + throw new Error( + spaceTrim(` The string is not a valid pipeline string book\` ${pipelineString} \` `), - ); - } + ); + } - return pipelineString; + return pipelineString; } /** diff --git a/src/pipeline/isValidPipelineString.test.ts b/src/pipeline/isValidPipelineString.test.ts index f674017aa8..b68263fcb8 100644 --- a/src/pipeline/isValidPipelineString.test.ts +++ b/src/pipeline/isValidPipelineString.test.ts @@ -1,20 +1,20 @@ -import { describe, expect, it } from '@jest/globals'; -import { book } from './book-notation'; -import { isValidPipelineString } from './isValidPipelineString'; +import { describe, expect, it } from "@jest/globals"; +import { book } from "./book-notation"; +import { isValidPipelineString } from "./isValidPipelineString"; -describe('how `isValidPipelineString` works', () => { - it('should work with foo', () => - expect( - isValidPipelineString( - book` +describe("how `isValidPipelineString` works", () => { + it("should work with foo", () => + expect( + isValidPipelineString( + book` # Book Write a joke about the {topic} -> {joke} `, - ), - ).toBe(true)); + ), + ).toBe(true)); }); /** diff --git a/src/pipeline/isValidPipelineString.ts b/src/pipeline/isValidPipelineString.ts index 90eae4c978..613b4b02e3 100644 --- a/src/pipeline/isValidPipelineString.ts +++ b/src/pipeline/isValidPipelineString.ts @@ -1,5 +1,5 @@ -import type { PipelineString } from './PipelineString'; -import { validatePipelineString } from './validatePipelineString'; +import type { PipelineString } from "./PipelineString"; +import { validatePipelineString } from "./validatePipelineString"; /** * Function `isValidPipelineString` will validate the if the string is a valid pipeline string @@ -9,17 +9,19 @@ import { validatePipelineString } from './validatePipelineString'; * @returns {boolean} if the string is a valid pipeline string * @public exported from `@promptbook/core` */ -export function isValidPipelineString(pipelineString: string): pipelineString is PipelineString { - try { - validatePipelineString(pipelineString); - return true; - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } +export function isValidPipelineString( + pipelineString: string, +): pipelineString is PipelineString { + try { + validatePipelineString(pipelineString); + return true; + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } - return false; - } + return false; + } } /** diff --git a/src/pipeline/prompt-notation.test.ts b/src/pipeline/prompt-notation.test.ts index 22d3bf0f7f..90a0e3ee55 100644 --- a/src/pipeline/prompt-notation.test.ts +++ b/src/pipeline/prompt-notation.test.ts @@ -1,87 +1,87 @@ -import { describe, expect, it } from '@jest/globals'; -import spaceTrim from 'spacetrim'; -import { prompt } from './prompt-notation'; +import { describe, expect, it } from "@jest/globals"; +import spaceTrim from "spacetrim"; +import { prompt } from "./prompt-notation"; -describe('how prompt tag function works', () => { - it('should work with simple prompt', () => - expect( - prompt` +describe("how prompt tag function works", () => { + it("should work with simple prompt", () => + expect( + prompt` You are a biochemistry expert - Explain the process of ATP synthesis in mitochondria `, - ).toBe( - spaceTrim(` + ).toBe( + spaceTrim(` You are a biochemistry expert - Explain the process of ATP synthesis in mitochondria `), - )); + )); - it('should work with interpolated string template', () => - expect( - prompt` + it("should work with interpolated string template", () => + expect( + prompt` You are a chemistry expert - - Explain the chemical bonds in ${'water molecules'} + - Explain the chemical bonds in ${"water molecules"} `, - ).toBe( - spaceTrim(` + ).toBe( + spaceTrim(` You are a chemistry expert - Explain the chemical bonds in water molecules `), - )); + )); - it('should work with multiline interpolated string template', () => - expect( - prompt` + it("should work with multiline interpolated string template", () => + expect( + prompt` You are a molecular biologist - ${'DNA replication\nTranscription'} + ${"DNA replication\nTranscription"} `, - ).toBe( - spaceTrim(` + ).toBe( + spaceTrim(` You are a molecular biologist DNA replication Transcription `), - )); + )); - it('should separate data and instructions', () => - expect( - prompt` + it("should separate data and instructions", () => + expect( + prompt` You are a geneticist - - ${'Dominant alleles\nRecessive alleles'} + - ${"Dominant alleles\nRecessive alleles"} `, - ).toBe( - spaceTrim(` + ).toBe( + spaceTrim(` You are a geneticist - Dominant alleles - Recessive alleles `), - )); + )); - it('should keep non-javascript parameters', () => - expect( - prompt` + it("should keep non-javascript parameters", () => + expect( + prompt` You are a geneticist - - ${'Dominant alleles\nRecessive alleles'} + - ${"Dominant alleles\nRecessive alleles"} - {more} `, - ).toBe( - spaceTrim(` + ).toBe( + spaceTrim(` You are a geneticist - Dominant alleles - Recessive alleles - {more} `), - )); + )); }); /** diff --git a/src/pipeline/prompt-notation.ts b/src/pipeline/prompt-notation.ts index 9775b13701..5c2cd419bf 100644 --- a/src/pipeline/prompt-notation.ts +++ b/src/pipeline/prompt-notation.ts @@ -1,9 +1,9 @@ -import spaceTrim from 'spacetrim'; -import { REPLACING_NONCE } from '../constants'; -import { PipelineExecutionError } from '../errors/PipelineExecutionError'; -import { UnexpectedError } from '../errors/UnexpectedError'; -import type { string_prompt } from '../types/typeAliases'; -import { templateParameters } from '../utils/parameters/templateParameters'; +import spaceTrim from "spacetrim"; +import { REPLACING_NONCE } from "../constants"; +import { PipelineExecutionError } from "../errors/PipelineExecutionError"; +import { UnexpectedError } from "../errors/UnexpectedError"; +import type { string_prompt } from "../types/typeAliases"; +import { templateParameters } from "../utils/parameters/templateParameters"; /** * Tag function for notating a prompt as template literal @@ -18,58 +18,74 @@ import { templateParameters } from '../utils/parameters/templateParameters'; * @returns the prompt string * @public exported from `@promptbook/utils` */ -export function prompt(strings: TemplateStringsArray, ...values: Array): string_prompt { - if (values.length === 0) { - return spaceTrim(strings.join('')); - } +export function prompt( + strings: TemplateStringsArray, + ...values: Array +): string_prompt { + if (values.length === 0) { + return spaceTrim(strings.join("")); + } - const stringsWithHiddenParameters = strings.map((stringsItem) => - // TODO: [0] DRY - stringsItem.split('{').join(`${REPLACING_NONCE}beginbracket`).split('}').join(`${REPLACING_NONCE}endbracket`), - ); + const stringsWithHiddenParameters = strings.map((stringsItem) => + // TODO: [0] DRY + stringsItem + .split("{") + .join(`${REPLACING_NONCE}beginbracket`) + .split("}") + .join(`${REPLACING_NONCE}endbracket`), + ); - const placeholderParameterNames = values.map((value, i) => `${REPLACING_NONCE}${i}`); - const parameters = Object.fromEntries(values.map((value, i) => [placeholderParameterNames[i], value])); + const placeholderParameterNames = values.map( + (value, i) => `${REPLACING_NONCE}${i}`, + ); + const parameters = Object.fromEntries( + values.map((value, i) => [placeholderParameterNames[i], value]), + ); - // Combine strings and values - let pipelineString = stringsWithHiddenParameters.reduce( - (result, stringsItem, i) => - placeholderParameterNames[i] === undefined - ? `${result}${stringsItem}` - : `${result}${stringsItem}{${placeholderParameterNames[i]}}`, - '', - ); + // Combine strings and values + let pipelineString = stringsWithHiddenParameters.reduce( + (result, stringsItem, i) => + placeholderParameterNames[i] === undefined + ? `${result}${stringsItem}` + : `${result}${stringsItem}{${placeholderParameterNames[i]}}`, + "", + ); - pipelineString = spaceTrim(pipelineString); + pipelineString = spaceTrim(pipelineString); - try { - pipelineString = templateParameters(pipelineString, parameters); - } catch (error) { - if (!(error instanceof PipelineExecutionError)) { - throw error; - } + try { + pipelineString = templateParameters(pipelineString, parameters); + } catch (error) { + if (!(error instanceof PipelineExecutionError)) { + throw error; + } - console.error({ pipelineString, parameters, placeholderParameterNames, error }); - throw new UnexpectedError( - spaceTrim( - (block) => ` + console.error({ + pipelineString, + parameters, + placeholderParameterNames, + error, + }); + throw new UnexpectedError( + spaceTrim( + (block) => ` Internal error in prompt template literal ${block(JSON.stringify({ strings, values }, null, 4))}} `, - ), - ); - } + ), + ); + } - // TODO: [0] DRY - pipelineString = pipelineString - .split(`${REPLACING_NONCE}beginbracket`) - .join('{') - .split(`${REPLACING_NONCE}endbracket`) - .join('}'); + // TODO: [0] DRY + pipelineString = pipelineString + .split(`${REPLACING_NONCE}beginbracket`) + .join("{") + .split(`${REPLACING_NONCE}endbracket`) + .join("}"); - return pipelineString; + return pipelineString; } /** diff --git a/src/pipeline/validatePipelineString.ts b/src/pipeline/validatePipelineString.ts index 2126dc76a3..7d2a427e31 100644 --- a/src/pipeline/validatePipelineString.ts +++ b/src/pipeline/validatePipelineString.ts @@ -1,9 +1,9 @@ -import { isValidEmail } from '../utils/validators/email/isValidEmail'; -import { isValidFilePath } from '../utils/validators/filePath/isValidFilePath'; -import { isValidUrl } from '../utils/validators/url/isValidUrl'; -import { ParseError } from '../errors/ParseError'; -import { isValidJsonString } from '../formats/json/utils/isValidJsonString'; -import type { PipelineString } from './PipelineString'; +import { ParseError } from "../errors/ParseError"; +import { isValidJsonString } from "../formats/json/utils/isValidJsonString"; +import { isValidEmail } from "../utils/validators/email/isValidEmail"; +import { isValidFilePath } from "../utils/validators/filePath/isValidFilePath"; +import { isValidUrl } from "../utils/validators/url/isValidUrl"; +import type { PipelineString } from "./PipelineString"; /** * Function `validatePipelineString` will validate the if the string is a valid pipeline string @@ -15,19 +15,25 @@ import type { PipelineString } from './PipelineString'; * @public exported from `@promptbook/core` */ export function validatePipelineString(pipelineString: string): PipelineString { - if (isValidJsonString(pipelineString)) { - throw new ParseError('Expected a book, but got a JSON string'); - } else if (isValidUrl(pipelineString)) { - throw new ParseError(`Expected a book, but got just the URL "${pipelineString}"`); - } else if (isValidFilePath(pipelineString)) { - throw new ParseError(`Expected a book, but got just the file path "${pipelineString}"`); - } else if (isValidEmail(pipelineString)) { - throw new ParseError(`Expected a book, but got just the email "${pipelineString}"`); - } + if (isValidJsonString(pipelineString)) { + throw new ParseError("Expected a book, but got a JSON string"); + } else if (isValidUrl(pipelineString)) { + throw new ParseError( + `Expected a book, but got just the URL "${pipelineString}"`, + ); + } else if (isValidFilePath(pipelineString)) { + throw new ParseError( + `Expected a book, but got just the file path "${pipelineString}"`, + ); + } else if (isValidEmail(pipelineString)) { + throw new ParseError( + `Expected a book, but got just the email "${pipelineString}"`, + ); + } - // <- TODO: Implement the validation + add tests when the pipeline logic considered as invalid + // <- TODO: Implement the validation + add tests when the pipeline logic considered as invalid - return pipelineString as PipelineString; + return pipelineString as PipelineString; } /** diff --git a/src/postprocessing/utils/extractBlock.ts b/src/postprocessing/utils/extractBlock.ts index f6041582c1..58cb213cbc 100644 --- a/src/postprocessing/utils/extractBlock.ts +++ b/src/postprocessing/utils/extractBlock.ts @@ -1,5 +1,5 @@ -import type { string_markdown } from '../../types/typeAliases'; -import { extractOneBlockFromMarkdown } from '../../utils/markdown/extractOneBlockFromMarkdown'; +import type { string_markdown } from "../../types/typeAliases"; +import { extractOneBlockFromMarkdown } from "../../utils/markdown/extractOneBlockFromMarkdown"; /** * Extracts code block from markdown. @@ -16,7 +16,7 @@ import { extractOneBlockFromMarkdown } from '../../utils/markdown/extractOneBloc * @throws {ParseError} if there is not exactly one code block in the markdown */ export function extractBlock(markdown: string_markdown): string { - const { content } = extractOneBlockFromMarkdown(markdown); + const { content } = extractOneBlockFromMarkdown(markdown); - return content; + return content; } diff --git a/src/postprocessing/utils/extractJsonBlock.test.ts b/src/postprocessing/utils/extractJsonBlock.test.ts index 90af270354..b917ad95e6 100644 --- a/src/postprocessing/utils/extractJsonBlock.test.ts +++ b/src/postprocessing/utils/extractJsonBlock.test.ts @@ -1,85 +1,87 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { extractJsonBlock } from './extractJsonBlock'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { extractJsonBlock } from "./extractJsonBlock"; -describe('how `extractJsonBlock` works', () => { - it('return one JSON block', () => - expect( - JSON.parse( - extractJsonBlock( - spaceTrim(` +describe("how `extractJsonBlock` works", () => { + it("return one JSON block", () => + expect( + JSON.parse( + extractJsonBlock( + spaceTrim(` Here is your JSON block: \`\`\`json {"foo": "bar"} \`\`\` `), - ), - ), - ).toEqual({ foo: 'bar' })); + ), + ), + ).toEqual({ foo: "bar" })); - it('does not matter on declared type, just the validity of the block', () => - expect( - JSON.parse( - extractJsonBlock( - spaceTrim(` + it("does not matter on declared type, just the validity of the block", () => + expect( + JSON.parse( + extractJsonBlock( + spaceTrim(` Here is your JSON block: \`\`\`foooo {"foo": "bar"} \`\`\` `), - ), - ), - ).toEqual({ foo: 'bar' })); + ), + ), + ).toEqual({ foo: "bar" })); - it('if given string is a valid JSON as it is, it just returns it', () => - expect( - JSON.parse( - extractJsonBlock( - spaceTrim(` + it("if given string is a valid JSON as it is, it just returns it", () => + expect( + JSON.parse( + extractJsonBlock( + spaceTrim(` {"foo": "bar"} `), - ), - ), - ).toEqual({ foo: 'bar' })); + ), + ), + ).toEqual({ foo: "bar" })); - it('does not matter on formatting', () => - expect( - JSON.parse( - extractJsonBlock(` + it("does not matter on formatting", () => + expect( + JSON.parse( + extractJsonBlock(` { "foo" : "bar" } `), - ), - ).toEqual({ foo: 'bar' })); + ), + ).toEqual({ foo: "bar" })); - it('throw error on empty string', () => - expect(() => extractJsonBlock('')).toThrowError(/There is no valid JSON block in the markdown/i)); + it("throw error on empty string", () => + expect(() => extractJsonBlock("")).toThrowError( + /There is no valid JSON block in the markdown/i, + )); - it('throw error on no block', () => - expect(() => extractJsonBlock('There is no block here')).toThrowError( - /There is no valid JSON block in the markdown/i, - )); + it("throw error on no block", () => + expect(() => extractJsonBlock("There is no block here")).toThrowError( + /There is no valid JSON block in the markdown/i, + )); - it('throw error on invalid json block', () => - expect(() => - extractJsonBlock( - spaceTrim(` + it("throw error on invalid json block", () => + expect(() => + extractJsonBlock( + spaceTrim(` Here is your JSON block: \`\`\`json {"foo": "bar} \`\`\` `), - ), - ).toThrowError(/There is no valid JSON block in the markdown/i)); + ), + ).toThrowError(/There is no valid JSON block in the markdown/i)); - it('throws error on multiple valid JSON blocks', () => - expect(() => - extractJsonBlock( - spaceTrim(` + it("throws error on multiple valid JSON blocks", () => + expect(() => + extractJsonBlock( + spaceTrim(` Here is your JSON block: \`\`\`foooo @@ -92,6 +94,6 @@ describe('how `extractJsonBlock` works', () => { {"fooo": "baar"} \`\`\` `), - ), - ).toThrowError(/There are multiple JSON code blocks in the markdown/i)); + ), + ).toThrowError(/There are multiple JSON code blocks in the markdown/i)); }); diff --git a/src/postprocessing/utils/extractJsonBlock.ts b/src/postprocessing/utils/extractJsonBlock.ts index 2d73db0eb9..a2614ff119 100644 --- a/src/postprocessing/utils/extractJsonBlock.ts +++ b/src/postprocessing/utils/extractJsonBlock.ts @@ -1,8 +1,8 @@ -import { isValidJsonString } from '../../formats/json/utils/isValidJsonString'; -import type { string_json } from '../../types/typeAliases'; -import type { string_markdown } from '../../types/typeAliases'; -import { extractAllBlocksFromMarkdown } from '../../utils/markdown/extractAllBlocksFromMarkdown'; -import type { really_unknown } from '../../utils/organization/really_unknown'; +import { isValidJsonString } from "../../formats/json/utils/isValidJsonString"; +import type { string_json } from "../../types/typeAliases"; +import type { string_markdown } from "../../types/typeAliases"; +import { extractAllBlocksFromMarkdown } from "../../utils/markdown/extractAllBlocksFromMarkdown"; +import type { really_unknown } from "../../utils/organization/really_unknown"; /** * Extracts extracts exactly one valid JSON code block @@ -21,27 +21,29 @@ import type { really_unknown } from '../../utils/organization/really_unknown'; * @public exported from `@promptbook/markdown-utils` * @throws {ParseError} if there is no valid JSON block in the markdown */ -export function extractJsonBlock(markdown: string_markdown): string_json { - if (isValidJsonString(markdown)) { - return markdown as string_json; - } +export function extractJsonBlock( + markdown: string_markdown, +): string_json { + if (isValidJsonString(markdown)) { + return markdown as string_json; + } - const codeBlocks = extractAllBlocksFromMarkdown(markdown); + const codeBlocks = extractAllBlocksFromMarkdown(markdown); - const jsonBlocks = codeBlocks.filter( - ({ content }) => isValidJsonString(content), - // <- Note: It is not important if marked as JSON `language === 'json'` BUT if it is VALID JSON - ); + const jsonBlocks = codeBlocks.filter( + ({ content }) => isValidJsonString(content), + // <- Note: It is not important if marked as JSON `language === 'json'` BUT if it is VALID JSON + ); - if (jsonBlocks.length === 0) { - throw new Error('There is no valid JSON block in the markdown'); - } + if (jsonBlocks.length === 0) { + throw new Error("There is no valid JSON block in the markdown"); + } - if (jsonBlocks.length > 1) { - throw new Error('There are multiple JSON code blocks in the markdown'); - } + if (jsonBlocks.length > 1) { + throw new Error("There are multiple JSON code blocks in the markdown"); + } - return jsonBlocks[0]!.content as string_json; + return jsonBlocks[0]!.content as string_json; } /** diff --git a/src/prepare/PrepareAndScrapeOptions.ts b/src/prepare/PrepareAndScrapeOptions.ts index 399a8409c2..0d3746e898 100644 --- a/src/prepare/PrepareAndScrapeOptions.ts +++ b/src/prepare/PrepareAndScrapeOptions.ts @@ -1,56 +1,56 @@ -import type { IntermediateFilesStrategy } from '../types/IntermediateFilesStrategy'; -import type { string_dirname } from '../types/typeAliases'; +import type { IntermediateFilesStrategy } from "../types/IntermediateFilesStrategy"; +import type { string_dirname } from "../types/typeAliases"; /** * Options for preparation of the pipeline */ export type PrepareAndScrapeOptions = { - /** - * Path to the root folder of the pipeline - * - * Note: When the pipeline is not created from files, it is `null` - * Note: This folder must exist (=it is not created recursively) - * - * @default null or `process.cwd()` when created via `$provide...` function in node - */ - readonly rootDirname?: string_dirname | null; + /** + * Path to the root folder of the pipeline + * + * Note: When the pipeline is not created from files, it is `null` + * Note: This folder must exist (=it is not created recursively) + * + * @default null or `process.cwd()` when created via `$provide...` function in node + */ + readonly rootDirname?: string_dirname | null; - /** - * Path to the cache folder - * - * Note: When the folder does not exist, it is created recursively - * - * @default DEFAULT_SCRAPE_CACHE_DIRNAME - */ - readonly cacheDirname?: string_dirname; + /** + * Path to the cache folder + * + * Note: When the folder does not exist, it is created recursively + * + * @default DEFAULT_SCRAPE_CACHE_DIRNAME + */ + readonly cacheDirname?: string_dirname; - /** - * Strategy for caching the intermediate results for knowledge source - * - * @default DEFAULT_INTERMEDIATE_FILES_STRATEGY - */ - readonly intermediateFilesStrategy?: IntermediateFilesStrategy; + /** + * Strategy for caching the intermediate results for knowledge source + * + * @default DEFAULT_INTERMEDIATE_FILES_STRATEGY + */ + readonly intermediateFilesStrategy?: IntermediateFilesStrategy; - /** - * Maximum number of tasks running in parallel - * - * @default DEFAULT_MAX_PARALLEL_COUNT - */ - readonly maxParallelCount?: number; + /** + * Maximum number of tasks running in parallel + * + * @default DEFAULT_MAX_PARALLEL_COUNT + */ + readonly maxParallelCount?: number; - /** - * If true, the missing software is automatically installed - * - * @default DEFAULT_IS_AUTO_INSTALLED - */ - readonly isAutoInstalled?: boolean; + /** + * If true, the missing software is automatically installed + * + * @default DEFAULT_IS_AUTO_INSTALLED + */ + readonly isAutoInstalled?: boolean; - /** - * If true, the preparation logs additional information - * - * @default DEFAULT_IS_VERBOSE - */ - readonly isVerbose?: boolean; + /** + * If true, the preparation logs additional information + * + * @default DEFAULT_IS_VERBOSE + */ + readonly isVerbose?: boolean; }; /** diff --git a/src/prepare/isPipelinePrepared.test.ts b/src/prepare/isPipelinePrepared.test.ts index b4f1734c85..38e7ad73b7 100644 --- a/src/prepare/isPipelinePrepared.test.ts +++ b/src/prepare/isPipelinePrepared.test.ts @@ -1,28 +1,48 @@ -import { describe, expect, it } from '@jest/globals'; -import { parsePipeline } from '../conversion/parsePipeline'; -import { importPipelineJson } from '../conversion/validation/_importPipeline'; -import { importPipelineWithoutPreparation } from '../conversion/validation/_importPipeline'; -import { isPipelinePrepared } from './isPipelinePrepared'; -import { unpreparePipeline } from './unpreparePipeline'; +import { describe, expect, it } from "@jest/globals"; +import { parsePipeline } from "../conversion/parsePipeline"; +import { importPipelineJson } from "../conversion/validation/_importPipeline"; +import { importPipelineWithoutPreparation } from "../conversion/validation/_importPipeline"; +import { isPipelinePrepared } from "./isPipelinePrepared"; +import { unpreparePipeline } from "./unpreparePipeline"; -describe('how isPipelinePrepared works', () => { - it('should tell that pipeline is prepared', async () => { - expect(isPipelinePrepared(await importPipelineJson('25-simple-knowledge.bookc'))).toBe(true); - expect(isPipelinePrepared(await importPipelineJson('01-simple.bookc'))).toBe(true); - }); +describe("how isPipelinePrepared works", () => { + it("should tell that pipeline is prepared", async () => { + expect( + isPipelinePrepared(await importPipelineJson("25-simple-knowledge.bookc")), + ).toBe(true); + expect( + isPipelinePrepared(await importPipelineJson("01-simple.bookc")), + ).toBe(true); + }); - it('should tell that simple pipeline is always prepared', async () => { - expect(isPipelinePrepared(unpreparePipeline(await importPipelineJson('01-simple.bookc')))).toBe(true); - expect(isPipelinePrepared(parsePipeline(await importPipelineWithoutPreparation('01-simple.book')))).toBe(true); - // Note: [🍫] - }); + it("should tell that simple pipeline is always prepared", async () => { + expect( + isPipelinePrepared( + unpreparePipeline(await importPipelineJson("01-simple.bookc")), + ), + ).toBe(true); + expect( + isPipelinePrepared( + parsePipeline(await importPipelineWithoutPreparation("01-simple.book")), + ), + ).toBe(true); + // Note: [🍫] + }); - it('should tell that pipeline is NOT prepared', async () => { - expect(isPipelinePrepared(unpreparePipeline(await importPipelineJson('25-simple-knowledge.bookc')))).toBe( - false, - ); - expect( - isPipelinePrepared(parsePipeline(await importPipelineWithoutPreparation('25-simple-knowledge.book'))), - ).toBe(false); - }); + it("should tell that pipeline is NOT prepared", async () => { + expect( + isPipelinePrepared( + unpreparePipeline( + await importPipelineJson("25-simple-knowledge.bookc"), + ), + ), + ).toBe(false); + expect( + isPipelinePrepared( + parsePipeline( + await importPipelineWithoutPreparation("25-simple-knowledge.book"), + ), + ), + ).toBe(false); + }); }); diff --git a/src/prepare/isPipelinePrepared.ts b/src/prepare/isPipelinePrepared.ts index c65b848128..ed78a518f8 100644 --- a/src/prepare/isPipelinePrepared.ts +++ b/src/prepare/isPipelinePrepared.ts @@ -1,7 +1,7 @@ -import { DEFAULT_BOOK_TITLE } from '../config'; -import type { KnowledgeSourcePreparedJson } from '../pipeline/PipelineJson/KnowledgeSourceJson'; -import type { PersonaPreparedJson } from '../pipeline/PipelineJson/PersonaJson'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; +import { DEFAULT_BOOK_TITLE } from "../config"; +import type { KnowledgeSourcePreparedJson } from "../pipeline/PipelineJson/KnowledgeSourceJson"; +import type { PersonaPreparedJson } from "../pipeline/PipelineJson/PersonaJson"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; /** * Determine if the pipeline is fully prepared @@ -11,33 +11,44 @@ import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; * @public exported from `@promptbook/core` */ export function isPipelinePrepared(pipeline: PipelineJson): boolean { - // Note: Ignoring `pipeline.preparations` @@@ - // Note: Ignoring `pipeline.knowledgePieces` @@@ + // Note: Ignoring `pipeline.preparations` @@@ + // Note: Ignoring `pipeline.knowledgePieces` @@@ - if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) { - return false; - } + if ( + pipeline.title === undefined || + pipeline.title === "" || + pipeline.title === DEFAULT_BOOK_TITLE + ) { + return false; + } - if (!pipeline.personas.every((persona) => (persona as PersonaPreparedJson).modelRequirements !== undefined)) { - return false; - } + if ( + !pipeline.personas.every( + (persona) => + (persona as PersonaPreparedJson).modelRequirements !== undefined, + ) + ) { + return false; + } - if ( - !pipeline.knowledgeSources.every( - (knowledgeSource) => (knowledgeSource as KnowledgeSourcePreparedJson).preparationIds !== undefined, - ) - ) { - return false; - } + if ( + !pipeline.knowledgeSources.every( + (knowledgeSource) => + (knowledgeSource as KnowledgeSourcePreparedJson).preparationIds !== + undefined, + ) + ) { + return false; + } - /* + /* TODO: [🧠][🍫] `tasks` can not be determined if they are fully prepared SO ignoring them > if (!pipeline.tasks.every(({ preparedContent }) => preparedContent === undefined)) { > return false; > } */ - return true; + return true; } /** diff --git a/src/prepare/preparePipeline.ts b/src/prepare/preparePipeline.ts index 39b38f6e55..11f178ce73 100644 --- a/src/prepare/preparePipeline.ts +++ b/src/prepare/preparePipeline.ts @@ -1,29 +1,29 @@ -import type { Writable } from 'type-fest'; -import PipelineCollection from '../../books/index.json'; -import { createCollectionFromJson } from '../collection/constructors/createCollectionFromJson'; -import { DEFAULT_BOOK_TITLE } from '../config'; -import { DEFAULT_IS_VERBOSE } from '../config'; -import { DEFAULT_MAX_PARALLEL_COUNT } from '../config'; -import { ORDER_OF_PIPELINE_JSON } from '../constants'; -import { MissingToolsError } from '../errors/MissingToolsError'; -import { createPipelineExecutor } from '../execution/createPipelineExecutor/00-createPipelineExecutor'; -import type { ExecutionTools } from '../execution/ExecutionTools'; -import { forEachAsync } from '../execution/utils/forEachAsync'; -import { ZERO_USAGE } from '../execution/utils/usage-constants'; -import { countTotalUsage } from '../llm-providers/_common/utils/count-total-usage/countTotalUsage'; -import { joinLlmExecutionTools } from '../llm-providers/multiple/joinLlmExecutionTools'; -import { preparePersona } from '../personas/preparePersona'; -import type { PersonaPreparedJson } from '../pipeline/PipelineJson/PersonaJson'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import type { PreparationJson } from '../pipeline/PipelineJson/PreparationJson'; -import { prepareKnowledgePieces } from '../scrapers/_common/prepareKnowledgePieces'; -import { arrayableToArray } from '../utils/arrayableToArray'; -import type { TODO_any } from '../utils/organization/TODO_any'; -import { exportJson } from '../utils/serialization/exportJson'; -import { PROMPTBOOK_ENGINE_VERSION } from '../version'; -import { isPipelinePrepared } from './isPipelinePrepared'; -import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions'; -import { prepareTasks } from './prepareTasks'; +import type { Writable } from "type-fest"; +import PipelineCollection from "../../books/index.json"; +import { createCollectionFromJson } from "../collection/constructors/createCollectionFromJson"; +import { DEFAULT_BOOK_TITLE } from "../config"; +import { DEFAULT_IS_VERBOSE } from "../config"; +import { DEFAULT_MAX_PARALLEL_COUNT } from "../config"; +import { ORDER_OF_PIPELINE_JSON } from "../constants"; +import { MissingToolsError } from "../errors/MissingToolsError"; +import type { ExecutionTools } from "../execution/ExecutionTools"; +import { createPipelineExecutor } from "../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { forEachAsync } from "../execution/utils/forEachAsync"; +import { ZERO_USAGE } from "../execution/utils/usage-constants"; +import { countTotalUsage } from "../llm-providers/_common/utils/count-total-usage/countTotalUsage"; +import { joinLlmExecutionTools } from "../llm-providers/multiple/joinLlmExecutionTools"; +import { preparePersona } from "../personas/preparePersona"; +import type { PersonaPreparedJson } from "../pipeline/PipelineJson/PersonaJson"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import type { PreparationJson } from "../pipeline/PipelineJson/PreparationJson"; +import { prepareKnowledgePieces } from "../scrapers/_common/prepareKnowledgePieces"; +import { arrayableToArray } from "../utils/arrayableToArray"; +import type { TODO_any } from "../utils/organization/TODO_any"; +import { exportJson } from "../utils/serialization/exportJson"; +import { PROMPTBOOK_ENGINE_VERSION } from "../version"; +import type { PrepareAndScrapeOptions } from "./PrepareAndScrapeOptions"; +import { isPipelinePrepared } from "./isPipelinePrepared"; +import { prepareTasks } from "./prepareTasks"; /** * Prepare pipeline locally @@ -36,183 +36,201 @@ import { prepareTasks } from './prepareTasks'; * @public exported from `@promptbook/core` */ export async function preparePipeline( - pipeline: PipelineJson, - tools: Pick, - options: PrepareAndScrapeOptions, + pipeline: PipelineJson, + tools: Pick, + options: PrepareAndScrapeOptions, ): Promise { - if (isPipelinePrepared(pipeline)) { - return pipeline; - } - - const { rootDirname, maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, isVerbose = DEFAULT_IS_VERBOSE } = options; - const { - parameters, - tasks, - /* + if (isPipelinePrepared(pipeline)) { + return pipeline; + } + + const { + rootDirname, + maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, + isVerbose = DEFAULT_IS_VERBOSE, + } = options; + const { + parameters, + tasks, + /* <- TODO: [🧠][🪑] `promptbookVersion` */ - knowledgeSources /* + knowledgeSources /* <- TODO: [🧊] `knowledgePieces` */, - personas /* + personas /* <- TODO: [🧊] `preparations` */, - sources, - } = pipeline; + sources, + } = pipeline; - if (tools === undefined || tools.llm === undefined) { - throw new MissingToolsError('LLM tools are required for preparing the pipeline'); - } + if (tools === undefined || tools.llm === undefined) { + throw new MissingToolsError( + "LLM tools are required for preparing the pipeline", + ); + } - // TODO: [🚐] Make arrayable LLMs -> single LLM DRY - const _llms = arrayableToArray(tools.llm); - const llmTools = _llms.length === 1 ? _llms[0]! : joinLlmExecutionTools(..._llms); + // TODO: [🚐] Make arrayable LLMs -> single LLM DRY + const _llms = arrayableToArray(tools.llm); + const llmTools = + _llms.length === 1 ? _llms[0]! : joinLlmExecutionTools(..._llms); - const llmToolsWithUsage = countTotalUsage(llmTools); - // <- TODO: [🌯] + const llmToolsWithUsage = countTotalUsage(llmTools); + // <- TODO: [🌯] - /* + /* TODO: [🧠][🪑][🔃] Should this be done or not if (promptbookVersion !== PROMPTBOOK_ENGINE_VERSION) { throw new VersionMismatchError(`Can not prepare the pipeline`, promptbookVersion); } */ - // TODO: [🔃][main] If the pipeline was prepared with different version or different set of models, prepare it once again - - // ----- ID ----- - const currentPreparation: Writable = { - id: 1, // <- TODO: [🧊] Make incremental - // TODO: [🍥]> date: $getCurrentDate(), - promptbookVersion: PROMPTBOOK_ENGINE_VERSION, - usage: ZERO_USAGE, - }; - - const preparations: ReadonlyArray = [ - // ...preparations - // <- TODO: [🧊] - currentPreparation, - ]; - // ----- /ID ----- - - // ----- Title preparation ----- - - let title = pipeline.title; - if (title === undefined || title === '' || title === DEFAULT_BOOK_TITLE) { - // TODO: [🌼] In future use `ptbk make` and made getPipelineCollection - const collection = createCollectionFromJson(...(PipelineCollection as TODO_any as ReadonlyArray)); - - const prepareTitleExecutor = createPipelineExecutor({ - pipeline: await collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-title.book'), - tools, - }); - - const result = await prepareTitleExecutor({ - book: sources.map(({ content }) => content).join('\n\n'), - }).asPromise(); - - const { outputParameters } = result; - const { title: titleRaw } = outputParameters; - - if (isVerbose) { - console.info(`The title is "${titleRaw}"`); - } - - title = titleRaw || DEFAULT_BOOK_TITLE; - } - // ----- /Title preparation ----- - - // ----- Personas preparation ----- - // TODO: Extract to similar function as `prepareTasks` - // TODO: [🖌][🧠] Implement some `mapAsync` function - const preparedPersonas: Array = new Array(personas.length); - await forEachAsync( - personas, - { maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, - async (persona, index) => { - const modelRequirements = await preparePersona( - persona.description, - { ...tools, llm: llmToolsWithUsage }, - { - rootDirname, - maxParallelCount /* <- TODO: [🪂] */, - isVerbose, - }, - ); - - const preparedPersona: PersonaPreparedJson = { - ...persona, - modelRequirements, - preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id], - // <- TODO: [🍙] Make some standard order of json properties - }; - - preparedPersonas[index] = preparedPersona; - }, - ); - // ----- /Personas preparation ----- - - // ----- Knowledge preparation ----- - // TODO: Extract to similar function as `prepareTasks` - const knowledgeSourcesPrepared = knowledgeSources.map((source) => ({ - ...source, - preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id], - })); - - const partialknowledgePiecesPrepared = await prepareKnowledgePieces( - knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, - { ...tools, llm: llmToolsWithUsage }, - { - ...options, - rootDirname, - maxParallelCount /* <- TODO: [🪂] */, - isVerbose, - }, - ); - - const knowledgePiecesPrepared = partialknowledgePiecesPrepared.map((piece) => ({ - ...piece, - preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id], - // <- TODO: [🍙] Make some standard order of json properties - })); - // ----- /Knowledge preparation ----- - - // ----- Tasks preparation ----- - const { tasksPrepared /* TODO: parameters: parametersPrepared*/ } = await prepareTasks( - { - parameters, - tasks, - knowledgePiecesCount: knowledgePiecesPrepared.length, - }, - { ...tools, llm: llmToolsWithUsage }, - { - rootDirname, - maxParallelCount /* <- TODO: [🪂] */, - isVerbose, - }, - ); - // ----- /Tasks preparation ----- - - // TODO: [😂] Use here all `AsyncHighLevelAbstraction` - - // Note: Count total usage - currentPreparation.usage = llmToolsWithUsage.getTotalUsage(); - - return exportJson({ - name: 'pipelineJson', - message: `Result of \`preparePipeline\``, - order: ORDER_OF_PIPELINE_JSON, - value: { - ...pipeline, - // <- TODO: Probbably deeply clone the pipeline because `$exportJson` freezes the subobjects - title, - knowledgeSources: knowledgeSourcesPrepared, - knowledgePieces: knowledgePiecesPrepared, - tasks: [...tasksPrepared], - // <- TODO: [🪓] Here should be no need for spreading new array, just ` tasks: tasksPrepared` - personas: preparedPersonas, - preparations: [...preparations], - // <- TODO: [🪓] Here should be no need for spreading new array, just `preparations` - }, - }); + // TODO: [🔃][main] If the pipeline was prepared with different version or different set of models, prepare it once again + + // ----- ID ----- + const currentPreparation: Writable = { + id: 1, // <- TODO: [🧊] Make incremental + // TODO: [🍥]> date: $getCurrentDate(), + promptbookVersion: PROMPTBOOK_ENGINE_VERSION, + usage: ZERO_USAGE, + }; + + const preparations: ReadonlyArray = [ + // ...preparations + // <- TODO: [🧊] + currentPreparation, + ]; + // ----- /ID ----- + + // ----- Title preparation ----- + + let title = pipeline.title; + if (title === undefined || title === "" || title === DEFAULT_BOOK_TITLE) { + // TODO: [🌼] In future use `ptbk make` and made getPipelineCollection + const collection = createCollectionFromJson( + ...(PipelineCollection as TODO_any as ReadonlyArray), + ); + + const prepareTitleExecutor = createPipelineExecutor({ + pipeline: await collection.getPipelineByUrl( + "https://promptbook.studio/promptbook/prepare-title.book", + ), + tools, + }); + + const result = await prepareTitleExecutor({ + book: sources.map(({ content }) => content).join("\n\n"), + }).asPromise(); + + const { outputParameters } = result; + const { title: titleRaw } = outputParameters; + + if (isVerbose) { + console.info(`The title is "${titleRaw}"`); + } + + title = titleRaw || DEFAULT_BOOK_TITLE; + } + // ----- /Title preparation ----- + + // ----- Personas preparation ----- + // TODO: Extract to similar function as `prepareTasks` + // TODO: [🖌][🧠] Implement some `mapAsync` function + const preparedPersonas: Array = new Array( + personas.length, + ); + await forEachAsync( + personas, + { + maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */, + }, + async (persona, index) => { + const modelRequirements = await preparePersona( + persona.description, + { ...tools, llm: llmToolsWithUsage }, + { + rootDirname, + maxParallelCount /* <- TODO: [🪂] */, + isVerbose, + }, + ); + + const preparedPersona: PersonaPreparedJson = { + ...persona, + modelRequirements, + preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id], + // <- TODO: [🍙] Make some standard order of json properties + }; + + preparedPersonas[index] = preparedPersona; + }, + ); + // ----- /Personas preparation ----- + + // ----- Knowledge preparation ----- + // TODO: Extract to similar function as `prepareTasks` + const knowledgeSourcesPrepared = knowledgeSources.map((source) => ({ + ...source, + preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id], + })); + + const partialknowledgePiecesPrepared = await prepareKnowledgePieces( + knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, + { ...tools, llm: llmToolsWithUsage }, + { + ...options, + rootDirname, + maxParallelCount /* <- TODO: [🪂] */, + isVerbose, + }, + ); + + const knowledgePiecesPrepared = partialknowledgePiecesPrepared.map( + (piece) => ({ + ...piece, + preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id], + // <- TODO: [🍙] Make some standard order of json properties + }), + ); + // ----- /Knowledge preparation ----- + + // ----- Tasks preparation ----- + const { tasksPrepared /* TODO: parameters: parametersPrepared*/ } = + await prepareTasks( + { + parameters, + tasks, + knowledgePiecesCount: knowledgePiecesPrepared.length, + }, + { ...tools, llm: llmToolsWithUsage }, + { + rootDirname, + maxParallelCount /* <- TODO: [🪂] */, + isVerbose, + }, + ); + // ----- /Tasks preparation ----- + + // TODO: [😂] Use here all `AsyncHighLevelAbstraction` + + // Note: Count total usage + currentPreparation.usage = llmToolsWithUsage.getTotalUsage(); + + return exportJson({ + name: "pipelineJson", + message: `Result of \`preparePipeline\``, + order: ORDER_OF_PIPELINE_JSON, + value: { + ...pipeline, + // <- TODO: Probbably deeply clone the pipeline because `$exportJson` freezes the subobjects + title, + knowledgeSources: knowledgeSourcesPrepared, + knowledgePieces: knowledgePiecesPrepared, + tasks: [...tasksPrepared], + // <- TODO: [🪓] Here should be no need for spreading new array, just ` tasks: tasksPrepared` + personas: preparedPersonas, + preparations: [...preparations], + // <- TODO: [🪓] Here should be no need for spreading new array, just `preparations` + }, + }); } /** diff --git a/src/prepare/preparePipelineOnRemoteServer.ts b/src/prepare/preparePipelineOnRemoteServer.ts index 940ac287f8..172bb2f829 100644 --- a/src/prepare/preparePipelineOnRemoteServer.ts +++ b/src/prepare/preparePipelineOnRemoteServer.ts @@ -1,10 +1,10 @@ -import { deserializeError } from '../errors/utils/deserializeError'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import { createRemoteClient } from '../remote-server/createRemoteClient'; -import type { PromptbookServer_Error } from '../remote-server/socket-types/_common/PromptbookServer_Error'; -import type { PromptbookServer_PreparePipeline_Request } from '../remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Request'; -import type { PromptbookServer_PreparePipeline_Response } from '../remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Response'; -import type { RemoteClientOptions } from '../remote-server/types/RemoteClientOptions'; +import { deserializeError } from "../errors/utils/deserializeError"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import { createRemoteClient } from "../remote-server/createRemoteClient"; +import type { PromptbookServer_Error } from "../remote-server/socket-types/_common/PromptbookServer_Error"; +import type { PromptbookServer_PreparePipeline_Request } from "../remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Request"; +import type { PromptbookServer_PreparePipeline_Response } from "../remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Response"; +import type { RemoteClientOptions } from "../remote-server/types/RemoteClientOptions"; /** * Prepare pipeline on remote server @@ -18,34 +18,39 @@ import type { RemoteClientOptions } from '../remote-server/types/RemoteClientOpt * @public exported from `@promptbook/remote-client` */ export async function preparePipelineOnRemoteServer( - pipeline: PipelineJson, - options: RemoteClientOptions, + pipeline: PipelineJson, + options: RemoteClientOptions, ): Promise { - const socket = await createRemoteClient(options); + const socket = await createRemoteClient(options); - socket.emit( - 'preparePipeline-request', - { - identification: options.identification, - pipeline, - } satisfies PromptbookServer_PreparePipeline_Request /* <- Note: [🤛] */, - ); + socket.emit( + "preparePipeline-request", + { + identification: options.identification, + pipeline, + } satisfies PromptbookServer_PreparePipeline_Request /* <- Note: [🤛] */, + ); - const preparedPipeline = await new Promise((resolve, reject) => { - socket.on('preparePipeline-response', (response: PromptbookServer_PreparePipeline_Response) => { - resolve(response.preparedPipeline); - socket.disconnect(); - }); - socket.on('error', (error: PromptbookServer_Error) => { - reject(deserializeError(error)); - socket.disconnect(); - }); - }); + const preparedPipeline = await new Promise( + (resolve, reject) => { + socket.on( + "preparePipeline-response", + (response: PromptbookServer_PreparePipeline_Response) => { + resolve(response.preparedPipeline); + socket.disconnect(); + }, + ); + socket.on("error", (error: PromptbookServer_Error) => { + reject(deserializeError(error)); + socket.disconnect(); + }); + }, + ); - socket.disconnect(); + socket.disconnect(); - // TODO: [🧠] Maybe do $exportJson - return preparedPipeline; + // TODO: [🧠] Maybe do $exportJson + return preparedPipeline; } /** diff --git a/src/prepare/prepareTasks.ts b/src/prepare/prepareTasks.ts index f4bc4dd9c7..9b671f68e1 100644 --- a/src/prepare/prepareTasks.ts +++ b/src/prepare/prepareTasks.ts @@ -1,24 +1,24 @@ -import { spaceTrim } from 'spacetrim'; -import { DEFAULT_MAX_PARALLEL_COUNT } from '../config'; -import type { ExecutionTools } from '../execution/ExecutionTools'; -import { forEachAsync } from '../execution/utils/forEachAsync'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import type { TaskJson } from '../pipeline/PipelineJson/TaskJson'; -import { TODO_USE } from '../utils/organization/TODO_USE'; -import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions'; +import { spaceTrim } from "spacetrim"; +import { DEFAULT_MAX_PARALLEL_COUNT } from "../config"; +import type { ExecutionTools } from "../execution/ExecutionTools"; +import { forEachAsync } from "../execution/utils/forEachAsync"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import type { TaskJson } from "../pipeline/PipelineJson/TaskJson"; +import { TODO_USE } from "../utils/organization/TODO_USE"; +import type { PrepareAndScrapeOptions } from "./PrepareAndScrapeOptions"; -type PrepareTaskInput = Pick & { - /** - * @@@ - */ - readonly knowledgePiecesCount: number; +type PrepareTaskInput = Pick & { + /** + * @@@ + */ + readonly knowledgePiecesCount: number; }; type PreparedTasks = { - /** - * @@@ Sequence of tasks that are chained together to form a pipeline - */ - readonly tasksPrepared: ReadonlyArray; + /** + * @@@ Sequence of tasks that are chained together to form a pipeline + */ + readonly tasksPrepared: ReadonlyArray; }; /** @@ -27,58 +27,62 @@ type PreparedTasks = { * @public exported from `@promptbook/core` */ export async function prepareTasks( - pipeline: PrepareTaskInput, - tools: Pick, - options: PrepareAndScrapeOptions, + pipeline: PrepareTaskInput, + tools: Pick, + options: PrepareAndScrapeOptions, ): Promise { - const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options; - const { tasks, parameters, knowledgePiecesCount } = pipeline; + const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options; + const { tasks, parameters, knowledgePiecesCount } = pipeline; - // TODO: [main] Apply examples to each task (if missing and is for the task defined) - TODO_USE(parameters); + // TODO: [main] Apply examples to each task (if missing and is for the task defined) + TODO_USE(parameters); - // TODO: [🖌][🧠] Implement some `mapAsync` function - const tasksPrepared: Array = new Array(tasks.length); - await forEachAsync( - tasks, - { maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, - async (task, index) => { - let { /* preparedContent <- TODO: Maybe use [🧊] */ dependentParameterNames } = task; - let preparedContent: string | undefined = undefined; + // TODO: [🖌][🧠] Implement some `mapAsync` function + const tasksPrepared: Array = new Array(tasks.length); + await forEachAsync( + tasks, + { + maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */, + }, + async (task, index) => { + let { + /* preparedContent <- TODO: Maybe use [🧊] */ dependentParameterNames, + } = task; + let preparedContent: string | undefined = undefined; - if ( - task.taskType === 'PROMPT_TASK' && - knowledgePiecesCount > 0 && - !dependentParameterNames.includes('knowledge') - ) { - preparedContent = spaceTrim(` + if ( + task.taskType === "PROMPT_TASK" && + knowledgePiecesCount > 0 && + !dependentParameterNames.includes("knowledge") + ) { + preparedContent = spaceTrim(` {content} ## Knowledge {knowledge} `); - // <- TODO: [🧠][🧻] Cutomize shape/language/formatting of the addition to the prompt + // <- TODO: [🧠][🧻] Cutomize shape/language/formatting of the addition to the prompt - dependentParameterNames = [ - ...dependentParameterNames, - 'knowledge', - // <- [🏷] There is the reverse process to remove {knowledge} from `dependentParameterNames` - ]; - } + dependentParameterNames = [ + ...dependentParameterNames, + "knowledge", + // <- [🏷] There is the reverse process to remove {knowledge} from `dependentParameterNames` + ]; + } - const preparedTask: TaskJson = { - ...task, - dependentParameterNames, - preparedContent, - // <- TODO: [🍙] Make some standard order of json properties - }; + const preparedTask: TaskJson = { + ...task, + dependentParameterNames, + preparedContent, + // <- TODO: [🍙] Make some standard order of json properties + }; - tasksPrepared[index] = preparedTask; - }, - ); + tasksPrepared[index] = preparedTask; + }, + ); - return { tasksPrepared }; + return { tasksPrepared }; } /** diff --git a/src/prepare/unpreparePipeline.ts b/src/prepare/unpreparePipeline.ts index 80f5e3cc22..8df2129178 100644 --- a/src/prepare/unpreparePipeline.ts +++ b/src/prepare/unpreparePipeline.ts @@ -1,7 +1,7 @@ -import { ORDER_OF_PIPELINE_JSON } from '../constants'; -import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson'; -import { extractParameterNames } from '../utils/parameters/extractParameterNames'; -import { exportJson } from '../utils/serialization/exportJson'; +import { ORDER_OF_PIPELINE_JSON } from "../constants"; +import type { PipelineJson } from "../pipeline/PipelineJson/PipelineJson"; +import { extractParameterNames } from "../utils/parameters/extractParameterNames"; +import { exportJson } from "../utils/serialization/exportJson"; /** * Unprepare just strips the preparation data of the pipeline @@ -10,39 +10,46 @@ import { exportJson } from '../utils/serialization/exportJson'; * @public exported from `@promptbook/core` */ export function unpreparePipeline(pipeline: PipelineJson): PipelineJson { - let { personas, knowledgeSources, tasks } = pipeline; - - personas = personas.map((persona) => ({ ...persona, modelRequirements: undefined, preparationIds: undefined })); - knowledgeSources = knowledgeSources.map((knowledgeSource) => ({ ...knowledgeSource, preparationIds: undefined })); - tasks = tasks.map((task) => { - let { dependentParameterNames } = task; - - const parameterNames = extractParameterNames(task.preparedContent || ''); - - dependentParameterNames = dependentParameterNames.filter( - (dependentParameterName) => !parameterNames.has(dependentParameterName), - // <- [🏷] This is the reverse process to remove {knowledge} from `dependentParameterNames` - ); - - const taskUnprepared = { ...task, dependentParameterNames }; - delete taskUnprepared.preparedContent; - - return taskUnprepared; - }); - - return exportJson({ - name: 'pipelineJson', - message: `Result of \`unpreparePipeline\``, - order: ORDER_OF_PIPELINE_JSON, - value: { - ...pipeline, - tasks, - knowledgeSources, - knowledgePieces: [], - personas, - preparations: [], - }, - }); + let { personas, knowledgeSources, tasks } = pipeline; + + personas = personas.map((persona) => ({ + ...persona, + modelRequirements: undefined, + preparationIds: undefined, + })); + knowledgeSources = knowledgeSources.map((knowledgeSource) => ({ + ...knowledgeSource, + preparationIds: undefined, + })); + tasks = tasks.map((task) => { + let { dependentParameterNames } = task; + + const parameterNames = extractParameterNames(task.preparedContent || ""); + + dependentParameterNames = dependentParameterNames.filter( + (dependentParameterName) => !parameterNames.has(dependentParameterName), + // <- [🏷] This is the reverse process to remove {knowledge} from `dependentParameterNames` + ); + + const taskUnprepared = { ...task, dependentParameterNames }; + delete taskUnprepared.preparedContent; + + return taskUnprepared; + }); + + return exportJson({ + name: "pipelineJson", + message: `Result of \`unpreparePipeline\``, + order: ORDER_OF_PIPELINE_JSON, + value: { + ...pipeline, + tasks, + knowledgeSources, + knowledgePieces: [], + personas, + preparations: [], + }, + }); } /** diff --git a/src/remote-server/createRemoteClient.ts b/src/remote-server/createRemoteClient.ts index 82f3fe5a19..b33cfa18d7 100644 --- a/src/remote-server/createRemoteClient.ts +++ b/src/remote-server/createRemoteClient.ts @@ -1,8 +1,8 @@ -import type { Socket } from 'socket.io-client'; -import { io } from 'socket.io-client'; -import { CONNECTION_RETRIES_LIMIT } from '../config'; -import { CONNECTION_TIMEOUT_MS } from '../config'; -import type { RemoteClientOptions } from './types/RemoteClientOptions'; +import type { Socket } from "socket.io-client"; +import { io } from "socket.io-client"; +import { CONNECTION_RETRIES_LIMIT } from "../config"; +import { CONNECTION_TIMEOUT_MS } from "../config"; +import type { RemoteClientOptions } from "./types/RemoteClientOptions"; /** * Creates a connection to the remote proxy server. @@ -12,29 +12,31 @@ import type { RemoteClientOptions } from './types/RemoteClientOptions'; * @private internal utility function */ export async function createRemoteClient( - options: RemoteClientOptions, + options: RemoteClientOptions, ): Promise { - const { remoteUrl, path } = options; + const { remoteUrl, path } = options; - return new Promise((resolve, reject) => { - const socket = io(remoteUrl, { - retries: CONNECTION_RETRIES_LIMIT, - timeout: CONNECTION_TIMEOUT_MS, - path, - // path: `${this.remoteUrl.pathname}/socket.io`, - transports: [/*'websocket', <- TODO: [🌬] Make websocket transport work */ 'polling'], - }); + return new Promise((resolve, reject) => { + const socket = io(remoteUrl, { + retries: CONNECTION_RETRIES_LIMIT, + timeout: CONNECTION_TIMEOUT_MS, + path, + // path: `${this.remoteUrl.pathname}/socket.io`, + transports: [ + /*'websocket', <- TODO: [🌬] Make websocket transport work */ "polling", + ], + }); - // console.log('Connecting to', this.options.remoteUrl.href, { socket }); + // console.log('Connecting to', this.options.remoteUrl.href, { socket }); - socket.on('connect', () => { - resolve(socket); - }); + socket.on("connect", () => { + resolve(socket); + }); - // TODO: [💩] Better timeout handling + // TODO: [💩] Better timeout handling - setTimeout(() => { - reject(new Error(`Timeout while connecting to ${remoteUrl}`)); - }, CONNECTION_TIMEOUT_MS); - }); + setTimeout(() => { + reject(new Error(`Timeout while connecting to ${remoteUrl}`)); + }, CONNECTION_TIMEOUT_MS); + }); } diff --git a/src/remote-server/other/index.umd.js b/src/remote-server/other/index.umd.js index f96078aaf5..f460da5eea 100644 --- a/src/remote-server/other/index.umd.js +++ b/src/remote-server/other/index.umd.js @@ -2,41 +2,119 @@ // /node_modules/@promptbook/remote-server/umd/index.umd.js // TODO: [🚋] There is a problem with the remote server handling errors and sending them back to the client -(function (global, factory) { - typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('http'), require('socket.io'), require('spacetrim'), require('fs/promises'), require('child_process'), require('util'), require('path'), require('prettier'), require('prettier/parser-html'), require('waitasecond'), require('papaparse'), require('crypto-js'), require('crypto-js/enc-hex'), require('mime-types')) : - typeof define === 'function' && define.amd ? define(['exports', 'colors', 'http', 'socket.io', 'spacetrim', 'fs/promises', 'child_process', 'util', 'path', 'prettier', 'prettier/parser-html', 'waitasecond', 'papaparse', 'crypto-js', 'crypto-js/enc-hex', 'mime-types'], factory) : - (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-remote-server"] = {}, global.colors, global.http, global.socket_io, global.spaceTrim, global.promises, global.child_process, global.util, global.path, global.prettier, global.parserHtml, global.waitasecond, global.papaparse, global.cryptoJs, global.hexEncoder, global.mimeTypes)); -})(this, (function (exports, colors, http, socket_io, spaceTrim, promises, child_process, util, path, prettier, parserHtml, waitasecond, papaparse, cryptoJs, hexEncoder, mimeTypes) { 'use strict'; - - function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; } - - var colors__default = /*#__PURE__*/_interopDefaultLegacy(colors); - var http__default = /*#__PURE__*/_interopDefaultLegacy(http); - var spaceTrim__default = /*#__PURE__*/_interopDefaultLegacy(spaceTrim); - var parserHtml__default = /*#__PURE__*/_interopDefaultLegacy(parserHtml); - var hexEncoder__default = /*#__PURE__*/_interopDefaultLegacy(hexEncoder); - - // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten - /** - * The version of the Book language - * - * @generated - * @see https://github.com/webgptorg/book - */ - var BOOK_LANGUAGE_VERSION = '1.0.0'; - /** - * The version of the Promptbook engine - * - * @generated - * @see https://github.com/webgptorg/promptbook - */ - var PROMPTBOOK_ENGINE_VERSION = '0.83.0'; - /** - * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine - * Note: [💞] Ignore a discrepancy between file name and entity name - */ - - /*! ***************************************************************************** +((global, factory) => { + typeof exports === "object" && typeof module !== "undefined" + ? factory( + exports, + require("colors"), + require("http"), + require("socket.io"), + require("spacetrim"), + require("fs/promises"), + require("child_process"), + require("util"), + require("path"), + require("prettier"), + require("prettier/parser-html"), + require("waitasecond"), + require("papaparse"), + require("crypto-js"), + require("crypto-js/enc-hex"), + require("mime-types"), + ) + : typeof define === "function" && define.amd + ? define( + [ + "exports", + "colors", + "http", + "socket.io", + "spacetrim", + "fs/promises", + "child_process", + "util", + "path", + "prettier", + "prettier/parser-html", + "waitasecond", + "papaparse", + "crypto-js", + "crypto-js/enc-hex", + "mime-types", + ], + factory, + ) + : ((global = + typeof globalThis !== "undefined" ? globalThis : global || self), + factory( + (global["promptbook-remote-server"] = {}), + global.colors, + global.http, + global.socket_io, + global.spaceTrim, + global.promises, + global.child_process, + global.util, + global.path, + global.prettier, + global.parserHtml, + global.waitasecond, + global.papaparse, + global.cryptoJs, + global.hexEncoder, + global.mimeTypes, + )); +})( + this, + ( + exports, + colors, + http, + socket_io, + spaceTrim, + promises, + child_process, + util, + path, + prettier, + parserHtml, + waitasecond, + papaparse, + cryptoJs, + hexEncoder, + mimeTypes, + ) => { + function _interopDefaultLegacy(e) { + return e && typeof e === "object" && "default" in e ? e : { default: e }; + } + + var colors__default = /*#__PURE__*/ _interopDefaultLegacy(colors); + var http__default = /*#__PURE__*/ _interopDefaultLegacy(http); + var spaceTrim__default = /*#__PURE__*/ _interopDefaultLegacy(spaceTrim); + var parserHtml__default = /*#__PURE__*/ _interopDefaultLegacy(parserHtml); + var hexEncoder__default = /*#__PURE__*/ _interopDefaultLegacy(hexEncoder); + + // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten + /** + * The version of the Book language + * + * @generated + * @see https://github.com/webgptorg/book + */ + var BOOK_LANGUAGE_VERSION = "1.0.0"; + /** + * The version of the Promptbook engine + * + * @generated + * @see https://github.com/webgptorg/promptbook + */ + var PROMPTBOOK_ENGINE_VERSION = "0.83.0"; + /** + * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine + * Note: [💞] Ignore a discrepancy between file name and entity name + */ + + /*! ***************************************************************************** Copyright (c) Microsoft Corporation. Permission to use, copy, modify, and/or distribute this software for any @@ -50,800 +128,1004 @@ OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ***************************************************************************** */ - /* global Reflect, Promise */ - - var extendStatics = function(d, b) { - extendStatics = Object.setPrototypeOf || - ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || - function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; - return extendStatics(d, b); - }; - - function __extends(d, b) { - if (typeof b !== "function" && b !== null) - throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); - extendStatics(d, b); - function __() { this.constructor = d; } - d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); - } - - var __assign = function() { - __assign = Object.assign || function __assign(t) { - for (var s, i = 1, n = arguments.length; i < n; i++) { - s = arguments[i]; - for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p]; - } - return t; - }; - return __assign.apply(this, arguments); - }; - - function __awaiter(thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); - } - - function __generator(thisArg, body) { - var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; - return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; - function verb(n) { return function (v) { return step([n, v]); }; } - function step(op) { - if (f) throw new TypeError("Generator is already executing."); - while (_) try { - if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; - if (y = 0, t) op = [op[0] & 2, t.value]; - switch (op[0]) { - case 0: case 1: t = op; break; - case 4: _.label++; return { value: op[1], done: false }; - case 5: _.label++; y = op[1]; op = [0]; continue; - case 7: op = _.ops.pop(); _.trys.pop(); continue; - default: - if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } - if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } - if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } - if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } - if (t[2]) _.ops.pop(); - _.trys.pop(); continue; - } - op = body.call(thisArg, _); - } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } - if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; - } - } - - function __values(o) { - var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0; - if (m) return m.call(o); - if (o && typeof o.length === "number") return { - next: function () { - if (o && i >= o.length) o = void 0; - return { value: o && o[i++], done: !o }; - } - }; - throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined."); - } - - function __read(o, n) { - var m = typeof Symbol === "function" && o[Symbol.iterator]; - if (!m) return o; - var i = m.call(o), r, ar = [], e; - try { - while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value); - } - catch (error) { e = { error: error }; } - finally { - try { - if (r && !r.done && (m = i["return"])) m.call(i); - } - finally { if (e) throw e.error; } - } - return ar; - } - - function __spreadArray(to, from, pack) { - if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) { - if (ar || !(i in from)) { - if (!ar) ar = Array.prototype.slice.call(from, 0, i); - ar[i] = from[i]; - } - } - return to.concat(ar || Array.prototype.slice.call(from)); - } - - /** - * Returns the same value that is passed as argument. - * No side effects. - * - * Note: It can be usefull for: - * - * 1) Leveling indentation - * 2) Putting always-true or always-false conditions without getting eslint errors - * - * @param value any values - * @returns the same values - * @private within the repository - */ - function just(value) { - if (value === undefined) { - return undefined; - } - return value; - } - - /** - * Name for the Promptbook - * - * TODO: [🗽] Unite branding and make single place for it - * - * @public exported from `@promptbook/core` - */ - var NAME = "Promptbook"; - /** - * Email of the responsible person - * - * @public exported from `@promptbook/core` - */ - var ADMIN_EMAIL = 'me@pavolhejny.com'; - /** - * Name of the responsible person for the Promptbook on GitHub - * - * @public exported from `@promptbook/core` - */ - var ADMIN_GITHUB_NAME = 'hejny'; - /** - * When the title is not provided, the default title is used - * - * @public exported from `@promptbook/core` - */ - var DEFAULT_BOOK_TITLE = "\u2728 Untitled Book"; - // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`" - /** - * The maximum number of iterations for a loops - * - * @private within the repository - too low-level in comparison with other `MAX_...` - */ - var LOOP_LIMIT = 1000; - /** - * Strings to represent various values in the context of parameter values - * - * @public exported from `@promptbook/utils` - */ - var VALUE_STRINGS = { - empty: '(nothing; empty string)', - null: '(no value; null)', - undefined: '(unknown value; undefined)', - nan: '(not a number; NaN)', - infinity: '(infinity; ∞)', - negativeInfinity: '(negative infinity; -∞)', - unserializable: '(unserializable value)', - }; - /** - * Small number limit - * - * @public exported from `@promptbook/utils` - */ - var SMALL_NUMBER = 0.001; - /** - * Short time interval to prevent race conditions in milliseconds - * - * @private within the repository - too low-level in comparison with other `MAX_...` - */ - var IMMEDIATE_TIME = 10; - /** - * Strategy for caching the intermediate results for knowledge sources - * - * @public exported from `@promptbook/core` - */ - var DEFAULT_INTERMEDIATE_FILES_STRATEGY = 'HIDE_AND_KEEP'; - // <- TODO: [😡] Change to 'VISIBLE' - /** - * The maximum number of (LLM) tasks running in parallel - * - * @public exported from `@promptbook/core` - */ - var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️] - /** - * The maximum number of attempts to execute LLM task before giving up - * - * @public exported from `@promptbook/core` - */ - var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️] - /** - * Where to store the scrape cache - * - * Note: When the folder does not exist, it is created recursively - * - * @public exported from `@promptbook/core` - */ - var DEFAULT_SCRAPE_CACHE_DIRNAME = './.promptbook/scrape-cache'; - // <- TODO: [🧜‍♂️] - /** - * @@@ - * - * @public exported from `@promptbook/core` - */ - var DEFAULT_CSV_SETTINGS = Object.freeze({ - delimiter: ',', - quoteChar: '"', - newline: '\n', - skipEmptyLines: true, - }); - /** - * @@@ - * - * @public exported from `@promptbook/core` - */ - var DEFAULT_IS_VERBOSE = false; - /** - * @@@ - * - * @public exported from `@promptbook/core` - */ - var DEFAULT_IS_AUTO_INSTALLED = false; - /** - * @@@ - * - * @private within the repository - */ - var IS_PIPELINE_LOGIC_VALIDATED = just( - /**/ - // Note: In normal situations, we check the pipeline logic: - true); - /** - * Note: [💞] Ignore a discrepancy between file name and entity name - * TODO: [🧠][🧜‍♂️] Maybe join remoteUrl and path into single value - */ - - /** - * This error type indicates that you try to use a feature that is not available in the current environment - * - * @public exported from `@promptbook/core` - */ - var EnvironmentMismatchError = /** @class */ (function (_super) { - __extends(EnvironmentMismatchError, _super); - function EnvironmentMismatchError(message) { - var _this = _super.call(this, message) || this; - _this.name = 'EnvironmentMismatchError'; - Object.setPrototypeOf(_this, EnvironmentMismatchError.prototype); - return _this; - } - return EnvironmentMismatchError; - }(Error)); - - /** - * Detects if the code is running in a Node.js environment - * - * Note: `$` is used to indicate that this function is not a pure function - it looks at the global object to determine the environment - * - * @public exported from `@promptbook/utils` - */ - var $isRunningInNode = new Function("\n try {\n return this === global;\n } catch (e) {\n return false;\n }\n"); - - /** - * @@@ - * - * @public exported from `@promptbook/node` - */ - function $provideFilesystemForNode(options) { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment'); - } - (options || {}).isVerbose; - return { - stat: promises.stat, - access: promises.access, - constants: promises.constants, - readFile: promises.readFile, - writeFile: promises.writeFile, - readdir: promises.readdir, - }; - } - /** - * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment - */ - - /** - * This error type indicates that some part of the code is not implemented yet - * - * @public exported from `@promptbook/core` - */ - var NotYetImplementedError = /** @class */ (function (_super) { - __extends(NotYetImplementedError, _super); - function NotYetImplementedError(message) { - var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this; - _this.name = 'NotYetImplementedError'; - Object.setPrototypeOf(_this, NotYetImplementedError.prototype); - return _this; - } - return NotYetImplementedError; - }(Error)); - - /** - * Make error report URL for the given error - * - * @private private within the repository - */ - function getErrorReportUrl(error) { - var report = { - title: "\uD83D\uDC1C Error report from ".concat(NAME), - body: spaceTrim__default["default"](function (block) { return "\n\n\n `".concat(error.name || 'Error', "` has occurred in the [").concat(NAME, "], please look into it @").concat(ADMIN_GITHUB_NAME, ".\n\n ```\n ").concat(block(error.message || '(no error message)'), "\n ```\n\n\n ## More info:\n\n - **Promptbook engine version:** ").concat(PROMPTBOOK_ENGINE_VERSION, "\n - **Book language version:** ").concat(BOOK_LANGUAGE_VERSION, "\n - **Time:** ").concat(new Date().toISOString(), "\n\n
\n Stack trace:\n\n ## Stack trace:\n\n ```stacktrace\n ").concat(block(error.stack || '(empty)'), "\n ```\n
\n\n "); }), - }; - var reportUrl = new URL("https://github.com/webgptorg/promptbook/issues/new"); - reportUrl.searchParams.set('labels', 'bug'); - reportUrl.searchParams.set('assignees', ADMIN_GITHUB_NAME); - reportUrl.searchParams.set('title', report.title); - reportUrl.searchParams.set('body', report.body); - return reportUrl; - } - - /** - * This error type indicates that the error should not happen and its last check before crashing with some other error - * - * @public exported from `@promptbook/core` - */ - var UnexpectedError = /** @class */ (function (_super) { - __extends(UnexpectedError, _super); - function UnexpectedError(message) { - var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This error should not happen.\n It's probbably a bug in the pipeline collection\n\n Please report issue:\n ").concat(block(getErrorReportUrl(new Error(message)).href), "\n\n Or contact us on ").concat(ADMIN_EMAIL, "\n\n "); })) || this; - _this.name = 'UnexpectedError'; - Object.setPrototypeOf(_this, UnexpectedError.prototype); - return _this; - } - return UnexpectedError; - }(Error)); - - /** - * @@@ - * - * Note: `$` is used to indicate that this function is not a pure function - it access global scope - * - * @private internal function of `$Register` - */ - function $getGlobalScope() { - return Function('return this')(); - } - - /** - * @@@ - * - * @param text @@@ - * @returns @@@ - * @example 'HELLO_WORLD' - * @example 'I_LOVE_PROMPTBOOK' - * @public exported from `@promptbook/utils` - */ - function normalizeTo_SCREAMING_CASE(text) { - var e_1, _a; - var charType; - var lastCharType = 'OTHER'; - var normalizedName = ''; - try { - for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) { - var char = text_1_1.value; - var normalizedChar = void 0; - if (/^[a-z]$/.test(char)) { - charType = 'LOWERCASE'; - normalizedChar = char.toUpperCase(); - } - else if (/^[A-Z]$/.test(char)) { - charType = 'UPPERCASE'; - normalizedChar = char; - } - else if (/^[0-9]$/.test(char)) { - charType = 'NUMBER'; - normalizedChar = char; - } - else { - charType = 'OTHER'; - normalizedChar = '_'; - } - if (charType !== lastCharType && - !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') && - !(lastCharType === 'NUMBER') && - !(charType === 'NUMBER')) { - normalizedName += '_'; - } - normalizedName += normalizedChar; - lastCharType = charType; - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1); - } - finally { if (e_1) throw e_1.error; } - } - normalizedName = normalizedName.replace(/_+/g, '_'); - normalizedName = normalizedName.replace(/_?\/_?/g, '/'); - normalizedName = normalizedName.replace(/^_/, ''); - normalizedName = normalizedName.replace(/_$/, ''); - return normalizedName; - } - /** - * TODO: Tests - * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule'); - * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu'); - * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj'); - * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj'); - * TODO: [🌺] Use some intermediate util splitWords - */ - - /** - * @@@ - * - * @param text @@@ - * @returns @@@ - * @example 'hello_world' - * @example 'i_love_promptbook' - * @public exported from `@promptbook/utils` - */ - function normalizeTo_snake_case(text) { - return normalizeTo_SCREAMING_CASE(text).toLowerCase(); - } - - /** - * Register is @@@ - * - * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope. - * - * @private internal utility, exported are only signleton instances of this class - */ - var $Register = /** @class */ (function () { - function $Register(registerName) { - this.registerName = registerName; - var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName)); - var globalScope = $getGlobalScope(); - if (globalScope[storageName] === undefined) { - globalScope[storageName] = []; - } - else if (!Array.isArray(globalScope[storageName])) { - throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName])); - } - this.storage = globalScope[storageName]; - } - $Register.prototype.list = function () { - // <- TODO: ReadonlyDeep> - return this.storage; - }; - $Register.prototype.register = function (registered) { - var packageName = registered.packageName, className = registered.className; - var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; }); - var existingRegistration = this.storage[existingRegistrationIndex]; - if (!existingRegistration) { - this.storage.push(registered); - } - else { - this.storage[existingRegistrationIndex] = registered; - } - return { - registerName: this.registerName, - packageName: packageName, - className: className, - get isDestroyed() { - return false; - }, - destroy: function () { - throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook")); - }, - }; - }; - return $Register; - }()); - - /** - * @@@ - * - * Note: `$` is used to indicate that this interacts with the global scope - * @singleton Only one instance of each register is created per build, but thare can be more @@@ - * @public exported from `@promptbook/core` - */ - var $scrapersRegister = new $Register('scraper_constructors'); - /** - * TODO: [®] DRY Register logic - */ - - /** - * @@@ - * - * 1) @@@ - * 2) @@@ - * - * @public exported from `@promptbook/node` - */ - function $provideScrapersForNode(tools, options) { - return __awaiter(this, void 0, void 0, function () { - var _a, scrapers, _d, _e, scraperFactory, scraper, e_1_1; - var e_1, _f; - return __generator(this, function (_g) { - switch (_g.label) { - case 0: - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment'); - } - _a = options || {}, _a.isAutoInstalled, _a.isVerbose; - scrapers = []; - _g.label = 1; - case 1: - _g.trys.push([1, 6, 7, 8]); - _d = __values($scrapersRegister.list()), _e = _d.next(); - _g.label = 2; - case 2: - if (!!_e.done) return [3 /*break*/, 5]; - scraperFactory = _e.value; - return [4 /*yield*/, scraperFactory(tools, options || {})]; - case 3: - scraper = _g.sent(); - scrapers.push(scraper); - _g.label = 4; - case 4: - _e = _d.next(); - return [3 /*break*/, 2]; - case 5: return [3 /*break*/, 8]; - case 6: - e_1_1 = _g.sent(); - e_1 = { error: e_1_1 }; - return [3 /*break*/, 8]; - case 7: - try { - if (_e && !_e.done && (_f = _d.return)) _f.call(_d); - } - finally { if (e_1) throw e_1.error; } - return [7 /*endfinally*/]; - case 8: return [2 /*return*/, scrapers]; - } - }); - }); - } - /** - * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment - */ - - /** - * This error indicates errors during the execution of the pipeline - * - * @public exported from `@promptbook/core` - */ - var PipelineExecutionError = /** @class */ (function (_super) { - __extends(PipelineExecutionError, _super); - function PipelineExecutionError(message) { - var _this = _super.call(this, message) || this; - _this.name = 'PipelineExecutionError'; - Object.setPrototypeOf(_this, PipelineExecutionError.prototype); - return _this; - } - return PipelineExecutionError; - }(Error)); - - /** - * This error indicates problems parsing the format value - * - * For example, when the format value is not a valid JSON or CSV - * This is not thrown directly but in extended classes - * - * @public exported from `@promptbook/core` - */ - var AbstractFormatError = /** @class */ (function (_super) { - __extends(AbstractFormatError, _super); - // Note: To allow instanceof do not put here error `name` - // public readonly name = 'AbstractFormatError'; - function AbstractFormatError(message) { - var _this = _super.call(this, message) || this; - Object.setPrototypeOf(_this, AbstractFormatError.prototype); - return _this; - } - return AbstractFormatError; - }(Error)); - - /** - * This error indicates problem with parsing of CSV - * - * @public exported from `@promptbook/core` - */ - var CsvFormatError = /** @class */ (function (_super) { - __extends(CsvFormatError, _super); - function CsvFormatError(message) { - var _this = _super.call(this, message) || this; - _this.name = 'CsvFormatError'; - Object.setPrototypeOf(_this, CsvFormatError.prototype); - return _this; - } - return CsvFormatError; - }(AbstractFormatError)); - - /** - * This error indicates that the pipeline collection cannot be propperly loaded - * - * @public exported from `@promptbook/core` - */ - var CollectionError = /** @class */ (function (_super) { - __extends(CollectionError, _super); - function CollectionError(message) { - var _this = _super.call(this, message) || this; - _this.name = 'CollectionError'; - Object.setPrototypeOf(_this, CollectionError.prototype); - return _this; - } - return CollectionError; - }(Error)); - - /** - * This error occurs when some expectation is not met in the execution of the pipeline - * - * @public exported from `@promptbook/core` - * Note: Do not throw this error, its reserved for `checkExpectations` and `createPipelineExecutor` and public ONLY to be serializable through remote server - * Note: Always thrown in `checkExpectations` and catched in `createPipelineExecutor` and rethrown as `PipelineExecutionError` - * Note: This is a kindof subtype of PipelineExecutionError - */ - var ExpectError = /** @class */ (function (_super) { - __extends(ExpectError, _super); - function ExpectError(message) { - var _this = _super.call(this, message) || this; - _this.name = 'ExpectError'; - Object.setPrototypeOf(_this, ExpectError.prototype); - return _this; - } - return ExpectError; - }(Error)); - - /** - * This error indicates that the promptbook can not retrieve knowledge from external sources - * - * @public exported from `@promptbook/core` - */ - var KnowledgeScrapeError = /** @class */ (function (_super) { - __extends(KnowledgeScrapeError, _super); - function KnowledgeScrapeError(message) { - var _this = _super.call(this, message) || this; - _this.name = 'KnowledgeScrapeError'; - Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype); - return _this; - } - return KnowledgeScrapeError; - }(Error)); - - /** - * This error type indicates that some limit was reached - * - * @public exported from `@promptbook/core` - */ - var LimitReachedError = /** @class */ (function (_super) { - __extends(LimitReachedError, _super); - function LimitReachedError(message) { - var _this = _super.call(this, message) || this; - _this.name = 'LimitReachedError'; - Object.setPrototypeOf(_this, LimitReachedError.prototype); - return _this; - } - return LimitReachedError; - }(Error)); - - /** - * This error type indicates that some tools are missing for pipeline execution or preparation - * - * @public exported from `@promptbook/core` - */ - var MissingToolsError = /** @class */ (function (_super) { - __extends(MissingToolsError, _super); - function MissingToolsError(message) { - var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: You have probbably forgot to provide some tools for pipeline execution or preparation\n\n "); })) || this; - _this.name = 'MissingToolsError'; - Object.setPrototypeOf(_this, MissingToolsError.prototype); - return _this; - } - return MissingToolsError; - }(Error)); - - /** - * This error indicates that promptbook not found in the collection - * - * @public exported from `@promptbook/core` - */ - var NotFoundError = /** @class */ (function (_super) { - __extends(NotFoundError, _super); - function NotFoundError(message) { - var _this = _super.call(this, message) || this; - _this.name = 'NotFoundError'; - Object.setPrototypeOf(_this, NotFoundError.prototype); - return _this; - } - return NotFoundError; - }(Error)); - - /** - * This error indicates that the promptbook in a markdown format cannot be parsed into a valid promptbook object - * - * @public exported from `@promptbook/core` - */ - var ParseError = /** @class */ (function (_super) { - __extends(ParseError, _super); - function ParseError(message) { - var _this = _super.call(this, message) || this; - _this.name = 'ParseError'; - Object.setPrototypeOf(_this, ParseError.prototype); - return _this; - } - return ParseError; - }(Error)); - /** - * TODO: Maybe split `ParseError` and `ApplyError` - */ - - /** - * This error indicates that the promptbook object has valid syntax (=can be parsed) but contains logical errors (like circular dependencies) - * - * @public exported from `@promptbook/core` - */ - var PipelineLogicError = /** @class */ (function (_super) { - __extends(PipelineLogicError, _super); - function PipelineLogicError(message) { - var _this = _super.call(this, message) || this; - _this.name = 'PipelineLogicError'; - Object.setPrototypeOf(_this, PipelineLogicError.prototype); - return _this; - } - return PipelineLogicError; - }(Error)); - - /** - * This error indicates errors in referencing promptbooks between each other - * - * @public exported from `@promptbook/core` - */ - var PipelineUrlError = /** @class */ (function (_super) { - __extends(PipelineUrlError, _super); - function PipelineUrlError(message) { - var _this = _super.call(this, message) || this; - _this.name = 'PipelineUrlError'; - Object.setPrototypeOf(_this, PipelineUrlError.prototype); - return _this; - } - return PipelineUrlError; - }(Error)); - - /** - * Index of all custom errors - * - * @public exported from `@promptbook/core` - */ - var PROMPTBOOK_ERRORS = { - AbstractFormatError: AbstractFormatError, - CsvFormatError: CsvFormatError, - CollectionError: CollectionError, - EnvironmentMismatchError: EnvironmentMismatchError, - ExpectError: ExpectError, - KnowledgeScrapeError: KnowledgeScrapeError, - LimitReachedError: LimitReachedError, - MissingToolsError: MissingToolsError, - NotFoundError: NotFoundError, - NotYetImplementedError: NotYetImplementedError, - ParseError: ParseError, - PipelineExecutionError: PipelineExecutionError, - PipelineLogicError: PipelineLogicError, - PipelineUrlError: PipelineUrlError, - UnexpectedError: UnexpectedError, - // TODO: [🪑]> VersionMismatchError, - }; - /** - * Index of all javascript errors - * - * @private for internal usage - */ - var COMMON_JAVASCRIPT_ERRORS = { - Error: Error, - EvalError: EvalError, - RangeError: RangeError, - ReferenceError: ReferenceError, - SyntaxError: SyntaxError, - TypeError: TypeError, - URIError: URIError, - AggregateError: AggregateError, - /* + /* global Reflect, Promise */ + + var extendStatics = (d, b) => { + extendStatics = + Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && + ((d, b) => { + d.__proto__ = b; + })) || + ((d, b) => { + for (var p in b) + if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; + }); + return extendStatics(d, b); + }; + + function __extends(d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError( + "Class extends value " + String(b) + " is not a constructor or null", + ); + extendStatics(d, b); + function __() { + this.constructor = d; + } + d.prototype = + b === null + ? Object.create(b) + : ((__.prototype = b.prototype), new __()); + } + + var __assign = function () { + __assign = + Object.assign || + function __assign(t) { + for (var s, i = 1, n = arguments.length; i < n; i++) { + s = arguments[i]; + for (var p in s) + if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p]; + } + return t; + }; + return __assign.apply(this, arguments); + }; + + function __awaiter(thisArg, _arguments, P, generator) { + function adopt(value) { + return value instanceof P + ? value + : new P((resolve) => { + resolve(value); + }); + } + return new (P || (P = Promise))((resolve, reject) => { + function fulfilled(value) { + try { + step(generator.next(value)); + } catch (e) { + reject(e); + } + } + function rejected(value) { + try { + step(generator["throw"](value)); + } catch (e) { + reject(e); + } + } + function step(result) { + result.done + ? resolve(result.value) + : adopt(result.value).then(fulfilled, rejected); + } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); + } + + function __generator(thisArg, body) { + var _ = { + label: 0, + sent: () => { + if (t[0] & 1) throw t[1]; + return t[1]; + }, + trys: [], + ops: [], + }, + f, + y, + t, + g; + return ( + (g = { next: verb(0), throw: verb(1), return: verb(2) }), + typeof Symbol === "function" && + (g[Symbol.iterator] = function () { + return this; + }), + g + ); + function verb(n) { + return (v) => step([n, v]); + } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (_) + try { + if ( + ((f = 1), + y && + (t = + op[0] & 2 + ? y["return"] + : op[0] + ? y["throw"] || ((t = y["return"]) && t.call(y), 0) + : y.next) && + !(t = t.call(y, op[1])).done) + ) + return t; + if (((y = 0), t)) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: + case 1: + t = op; + break; + case 4: + _.label++; + return { value: op[1], done: false }; + case 5: + _.label++; + y = op[1]; + op = [0]; + continue; + case 7: + op = _.ops.pop(); + _.trys.pop(); + continue; + default: + if ( + !((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && + (op[0] === 6 || op[0] === 2) + ) { + _ = 0; + continue; + } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { + _.label = op[1]; + break; + } + if (op[0] === 6 && _.label < t[1]) { + _.label = t[1]; + t = op; + break; + } + if (t && _.label < t[2]) { + _.label = t[2]; + _.ops.push(op); + break; + } + if (t[2]) _.ops.pop(); + _.trys.pop(); + continue; + } + op = body.call(thisArg, _); + } catch (e) { + op = [6, e]; + y = 0; + } finally { + f = t = 0; + } + if (op[0] & 5) throw op[1]; + return { value: op[0] ? op[1] : void 0, done: true }; + } + } + + function __values(o) { + var s = typeof Symbol === "function" && Symbol.iterator, + m = s && o[s], + i = 0; + if (m) return m.call(o); + if (o && typeof o.length === "number") + return { + next: () => { + if (o && i >= o.length) o = void 0; + return { value: o && o[i++], done: !o }; + }, + }; + throw new TypeError( + s ? "Object is not iterable." : "Symbol.iterator is not defined.", + ); + } + + function __read(o, n) { + var m = typeof Symbol === "function" && o[Symbol.iterator]; + if (!m) return o; + var i = m.call(o), + r, + ar = [], + e; + try { + while ((n === void 0 || n-- > 0) && !(r = i.next()).done) + ar.push(r.value); + } catch (error) { + e = { error: error }; + } finally { + try { + if (r && !r.done && (m = i["return"])) m.call(i); + } finally { + if (e) throw e.error; + } + } + return ar; + } + + function __spreadArray(to, from, pack) { + if (pack || arguments.length === 2) + for (var i = 0, l = from.length, ar; i < l; i++) { + if (ar || !(i in from)) { + if (!ar) ar = Array.prototype.slice.call(from, 0, i); + ar[i] = from[i]; + } + } + return to.concat(ar || Array.prototype.slice.call(from)); + } + + /** + * Returns the same value that is passed as argument. + * No side effects. + * + * Note: It can be usefull for: + * + * 1) Leveling indentation + * 2) Putting always-true or always-false conditions without getting eslint errors + * + * @param value any values + * @returns the same values + * @private within the repository + */ + function just(value) { + if (value === undefined) { + return undefined; + } + return value; + } + + /** + * Name for the Promptbook + * + * TODO: [🗽] Unite branding and make single place for it + * + * @public exported from `@promptbook/core` + */ + var NAME = "Promptbook"; + /** + * Email of the responsible person + * + * @public exported from `@promptbook/core` + */ + var ADMIN_EMAIL = "me@pavolhejny.com"; + /** + * Name of the responsible person for the Promptbook on GitHub + * + * @public exported from `@promptbook/core` + */ + var ADMIN_GITHUB_NAME = "hejny"; + /** + * When the title is not provided, the default title is used + * + * @public exported from `@promptbook/core` + */ + var DEFAULT_BOOK_TITLE = "\u2728 Untitled Book"; + // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`" + /** + * The maximum number of iterations for a loops + * + * @private within the repository - too low-level in comparison with other `MAX_...` + */ + var LOOP_LIMIT = 1000; + /** + * Strings to represent various values in the context of parameter values + * + * @public exported from `@promptbook/utils` + */ + var VALUE_STRINGS = { + empty: "(nothing; empty string)", + null: "(no value; null)", + undefined: "(unknown value; undefined)", + nan: "(not a number; NaN)", + infinity: "(infinity; ∞)", + negativeInfinity: "(negative infinity; -∞)", + unserializable: "(unserializable value)", + }; + /** + * Small number limit + * + * @public exported from `@promptbook/utils` + */ + var SMALL_NUMBER = 0.001; + /** + * Short time interval to prevent race conditions in milliseconds + * + * @private within the repository - too low-level in comparison with other `MAX_...` + */ + var IMMEDIATE_TIME = 10; + /** + * Strategy for caching the intermediate results for knowledge sources + * + * @public exported from `@promptbook/core` + */ + var DEFAULT_INTERMEDIATE_FILES_STRATEGY = "HIDE_AND_KEEP"; + // <- TODO: [😡] Change to 'VISIBLE' + /** + * The maximum number of (LLM) tasks running in parallel + * + * @public exported from `@promptbook/core` + */ + var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️] + /** + * The maximum number of attempts to execute LLM task before giving up + * + * @public exported from `@promptbook/core` + */ + var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️] + /** + * Where to store the scrape cache + * + * Note: When the folder does not exist, it is created recursively + * + * @public exported from `@promptbook/core` + */ + var DEFAULT_SCRAPE_CACHE_DIRNAME = "./.promptbook/scrape-cache"; + // <- TODO: [🧜‍♂️] + /** + * @@@ + * + * @public exported from `@promptbook/core` + */ + var DEFAULT_CSV_SETTINGS = Object.freeze({ + delimiter: ",", + quoteChar: '"', + newline: "\n", + skipEmptyLines: true, + }); + /** + * @@@ + * + * @public exported from `@promptbook/core` + */ + var DEFAULT_IS_VERBOSE = false; + /** + * @@@ + * + * @public exported from `@promptbook/core` + */ + var DEFAULT_IS_AUTO_INSTALLED = false; + /** + * @@@ + * + * @private within the repository + */ + var IS_PIPELINE_LOGIC_VALIDATED = just( + /**/ + // Note: In normal situations, we check the pipeline logic: + true, + ); + /** + * Note: [💞] Ignore a discrepancy between file name and entity name + * TODO: [🧠][🧜‍♂️] Maybe join remoteUrl and path into single value + */ + + /** + * This error type indicates that you try to use a feature that is not available in the current environment + * + * @public exported from `@promptbook/core` + */ + var EnvironmentMismatchError = /** @class */ ((_super) => { + __extends(EnvironmentMismatchError, _super); + function EnvironmentMismatchError(message) { + var _this = _super.call(this, message) || this; + _this.name = "EnvironmentMismatchError"; + Object.setPrototypeOf(_this, EnvironmentMismatchError.prototype); + return _this; + } + return EnvironmentMismatchError; + })(Error); + + /** + * Detects if the code is running in a Node.js environment + * + * Note: `$` is used to indicate that this function is not a pure function - it looks at the global object to determine the environment + * + * @public exported from `@promptbook/utils` + */ + var $isRunningInNode = new Function( + "\n try {\n return this === global;\n } catch (e) {\n return false;\n }\n", + ); + + /** + * @@@ + * + * @public exported from `@promptbook/node` + */ + function $provideFilesystemForNode(options) { + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Function `$provideFilesystemForNode` works only in Node.js environment", + ); + } + (options || {}).isVerbose; + return { + stat: promises.stat, + access: promises.access, + constants: promises.constants, + readFile: promises.readFile, + writeFile: promises.writeFile, + readdir: promises.readdir, + }; + } + /** + * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment + */ + + /** + * This error type indicates that some part of the code is not implemented yet + * + * @public exported from `@promptbook/core` + */ + var NotYetImplementedError = /** @class */ ((_super) => { + __extends(NotYetImplementedError, _super); + function NotYetImplementedError(message) { + var _this = + _super.call( + this, + spaceTrim.spaceTrim((block) => + "\n ".concat( + block(message), + "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n ", + ), + ), + ) || this; + _this.name = "NotYetImplementedError"; + Object.setPrototypeOf(_this, NotYetImplementedError.prototype); + return _this; + } + return NotYetImplementedError; + })(Error); + + /** + * Make error report URL for the given error + * + * @private private within the repository + */ + function getErrorReportUrl(error) { + var report = { + title: "\uD83D\uDC1C Error report from ".concat(NAME), + body: spaceTrim__default["default"]((block) => + "\n\n\n `" + .concat(error.name || "Error", "` has occurred in the [") + .concat(NAME, "], please look into it @") + .concat(ADMIN_GITHUB_NAME, ".\n\n ```\n ") + .concat( + block(error.message || "(no error message)"), + "\n ```\n\n\n ## More info:\n\n - **Promptbook engine version:** ", + ) + .concat( + PROMPTBOOK_ENGINE_VERSION, + "\n - **Book language version:** ", + ) + .concat(BOOK_LANGUAGE_VERSION, "\n - **Time:** ") + .concat( + new Date().toISOString(), + "\n\n
\n Stack trace:\n\n ## Stack trace:\n\n ```stacktrace\n ", + ) + .concat( + block(error.stack || "(empty)"), + "\n ```\n
\n\n ", + ), + ), + }; + var reportUrl = new URL( + "https://github.com/webgptorg/promptbook/issues/new", + ); + reportUrl.searchParams.set("labels", "bug"); + reportUrl.searchParams.set("assignees", ADMIN_GITHUB_NAME); + reportUrl.searchParams.set("title", report.title); + reportUrl.searchParams.set("body", report.body); + return reportUrl; + } + + /** + * This error type indicates that the error should not happen and its last check before crashing with some other error + * + * @public exported from `@promptbook/core` + */ + var UnexpectedError = /** @class */ ((_super) => { + __extends(UnexpectedError, _super); + function UnexpectedError(message) { + var _this = + _super.call( + this, + spaceTrim.spaceTrim((block) => + "\n " + .concat( + block(message), + "\n\n Note: This error should not happen.\n It's probbably a bug in the pipeline collection\n\n Please report issue:\n ", + ) + .concat( + block(getErrorReportUrl(new Error(message)).href), + "\n\n Or contact us on ", + ) + .concat(ADMIN_EMAIL, "\n\n "), + ), + ) || this; + _this.name = "UnexpectedError"; + Object.setPrototypeOf(_this, UnexpectedError.prototype); + return _this; + } + return UnexpectedError; + })(Error); + + /** + * @@@ + * + * Note: `$` is used to indicate that this function is not a pure function - it access global scope + * + * @private internal function of `$Register` + */ + function $getGlobalScope() { + return Function("return this")(); + } + + /** + * @@@ + * + * @param text @@@ + * @returns @@@ + * @example 'HELLO_WORLD' + * @example 'I_LOVE_PROMPTBOOK' + * @public exported from `@promptbook/utils` + */ + function normalizeTo_SCREAMING_CASE(text) { + var e_1, _a; + var charType; + var lastCharType = "OTHER"; + var normalizedName = ""; + try { + for ( + var text_1 = __values(text), text_1_1 = text_1.next(); + !text_1_1.done; + text_1_1 = text_1.next() + ) { + var char = text_1_1.value; + var normalizedChar = void 0; + if (/^[a-z]$/.test(char)) { + charType = "LOWERCASE"; + normalizedChar = char.toUpperCase(); + } else if (/^[A-Z]$/.test(char)) { + charType = "UPPERCASE"; + normalizedChar = char; + } else if (/^[0-9]$/.test(char)) { + charType = "NUMBER"; + normalizedChar = char; + } else { + charType = "OTHER"; + normalizedChar = "_"; + } + if ( + charType !== lastCharType && + !(lastCharType === "UPPERCASE" && charType === "LOWERCASE") && + !(lastCharType === "NUMBER") && + !(charType === "NUMBER") + ) { + normalizedName += "_"; + } + normalizedName += normalizedChar; + lastCharType = charType; + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (text_1_1 && !text_1_1.done && (_a = text_1.return)) + _a.call(text_1); + } finally { + if (e_1) throw e_1.error; + } + } + normalizedName = normalizedName.replace(/_+/g, "_"); + normalizedName = normalizedName.replace(/_?\/_?/g, "/"); + normalizedName = normalizedName.replace(/^_/, ""); + normalizedName = normalizedName.replace(/_$/, ""); + return normalizedName; + } + /** + * TODO: Tests + * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule'); + * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu'); + * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj'); + * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj'); + * TODO: [🌺] Use some intermediate util splitWords + */ + + /** + * @@@ + * + * @param text @@@ + * @returns @@@ + * @example 'hello_world' + * @example 'i_love_promptbook' + * @public exported from `@promptbook/utils` + */ + function normalizeTo_snake_case(text) { + return normalizeTo_SCREAMING_CASE(text).toLowerCase(); + } + + /** + * Register is @@@ + * + * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope. + * + * @private internal utility, exported are only signleton instances of this class + */ + var $Register = /** @class */ (() => { + function $Register(registerName) { + this.registerName = registerName; + var storageName = "_promptbook_".concat( + normalizeTo_snake_case(registerName), + ); + var globalScope = $getGlobalScope(); + if (globalScope[storageName] === undefined) { + globalScope[storageName] = []; + } else if (!Array.isArray(globalScope[storageName])) { + throw new UnexpectedError( + "Expected (global) " + .concat(storageName, " to be an array, but got ") + .concat(typeof globalScope[storageName]), + ); + } + this.storage = globalScope[storageName]; + } + $Register.prototype.list = function () { + // <- TODO: ReadonlyDeep> + return this.storage; + }; + $Register.prototype.register = function (registered) { + var packageName = registered.packageName, + className = registered.className; + var existingRegistrationIndex = this.storage.findIndex( + (item) => + item.packageName === packageName && item.className === className, + ); + var existingRegistration = this.storage[existingRegistrationIndex]; + if (!existingRegistration) { + this.storage.push(registered); + } else { + this.storage[existingRegistrationIndex] = registered; + } + return { + registerName: this.registerName, + packageName: packageName, + className: className, + get isDestroyed() { + return false; + }, + destroy: function () { + throw new NotYetImplementedError( + "Registration to ".concat( + this.registerName, + " is permanent in this version of Promptbook", + ), + ); + }, + }; + }; + return $Register; + })(); + + /** + * @@@ + * + * Note: `$` is used to indicate that this interacts with the global scope + * @singleton Only one instance of each register is created per build, but thare can be more @@@ + * @public exported from `@promptbook/core` + */ + var $scrapersRegister = new $Register("scraper_constructors"); + /** + * TODO: [®] DRY Register logic + */ + + /** + * @@@ + * + * 1) @@@ + * 2) @@@ + * + * @public exported from `@promptbook/node` + */ + function $provideScrapersForNode(tools, options) { + return __awaiter(this, void 0, void 0, function () { + var _a, scrapers, _d, _e, scraperFactory, scraper, e_1_1; + var e_1, _f; + return __generator(this, (_g) => { + switch (_g.label) { + case 0: + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Function `$getScrapersForNode` works only in Node.js environment", + ); + } + (_a = options || {}), _a.isAutoInstalled, _a.isVerbose; + scrapers = []; + _g.label = 1; + case 1: + _g.trys.push([1, 6, 7, 8]); + (_d = __values($scrapersRegister.list())), (_e = _d.next()); + _g.label = 2; + case 2: + if (!!_e.done) return [3 /*break*/, 5]; + scraperFactory = _e.value; + return [4 /*yield*/, scraperFactory(tools, options || {})]; + case 3: + scraper = _g.sent(); + scrapers.push(scraper); + _g.label = 4; + case 4: + _e = _d.next(); + return [3 /*break*/, 2]; + case 5: + return [3 /*break*/, 8]; + case 6: + e_1_1 = _g.sent(); + e_1 = { error: e_1_1 }; + return [3 /*break*/, 8]; + case 7: + try { + if (_e && !_e.done && (_f = _d.return)) _f.call(_d); + } finally { + if (e_1) throw e_1.error; + } + return [7 /*endfinally*/]; + case 8: + return [2 /*return*/, scrapers]; + } + }); + }); + } + /** + * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment + */ + + /** + * This error indicates errors during the execution of the pipeline + * + * @public exported from `@promptbook/core` + */ + var PipelineExecutionError = /** @class */ ((_super) => { + __extends(PipelineExecutionError, _super); + function PipelineExecutionError(message) { + var _this = _super.call(this, message) || this; + _this.name = "PipelineExecutionError"; + Object.setPrototypeOf(_this, PipelineExecutionError.prototype); + return _this; + } + return PipelineExecutionError; + })(Error); + + /** + * This error indicates problems parsing the format value + * + * For example, when the format value is not a valid JSON or CSV + * This is not thrown directly but in extended classes + * + * @public exported from `@promptbook/core` + */ + var AbstractFormatError = /** @class */ ((_super) => { + __extends(AbstractFormatError, _super); + // Note: To allow instanceof do not put here error `name` + // public readonly name = 'AbstractFormatError'; + function AbstractFormatError(message) { + var _this = _super.call(this, message) || this; + Object.setPrototypeOf(_this, AbstractFormatError.prototype); + return _this; + } + return AbstractFormatError; + })(Error); + + /** + * This error indicates problem with parsing of CSV + * + * @public exported from `@promptbook/core` + */ + var CsvFormatError = /** @class */ ((_super) => { + __extends(CsvFormatError, _super); + function CsvFormatError(message) { + var _this = _super.call(this, message) || this; + _this.name = "CsvFormatError"; + Object.setPrototypeOf(_this, CsvFormatError.prototype); + return _this; + } + return CsvFormatError; + })(AbstractFormatError); + + /** + * This error indicates that the pipeline collection cannot be propperly loaded + * + * @public exported from `@promptbook/core` + */ + var CollectionError = /** @class */ ((_super) => { + __extends(CollectionError, _super); + function CollectionError(message) { + var _this = _super.call(this, message) || this; + _this.name = "CollectionError"; + Object.setPrototypeOf(_this, CollectionError.prototype); + return _this; + } + return CollectionError; + })(Error); + + /** + * This error occurs when some expectation is not met in the execution of the pipeline + * + * @public exported from `@promptbook/core` + * Note: Do not throw this error, its reserved for `checkExpectations` and `createPipelineExecutor` and public ONLY to be serializable through remote server + * Note: Always thrown in `checkExpectations` and catched in `createPipelineExecutor` and rethrown as `PipelineExecutionError` + * Note: This is a kindof subtype of PipelineExecutionError + */ + var ExpectError = /** @class */ ((_super) => { + __extends(ExpectError, _super); + function ExpectError(message) { + var _this = _super.call(this, message) || this; + _this.name = "ExpectError"; + Object.setPrototypeOf(_this, ExpectError.prototype); + return _this; + } + return ExpectError; + })(Error); + + /** + * This error indicates that the promptbook can not retrieve knowledge from external sources + * + * @public exported from `@promptbook/core` + */ + var KnowledgeScrapeError = /** @class */ ((_super) => { + __extends(KnowledgeScrapeError, _super); + function KnowledgeScrapeError(message) { + var _this = _super.call(this, message) || this; + _this.name = "KnowledgeScrapeError"; + Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype); + return _this; + } + return KnowledgeScrapeError; + })(Error); + + /** + * This error type indicates that some limit was reached + * + * @public exported from `@promptbook/core` + */ + var LimitReachedError = /** @class */ ((_super) => { + __extends(LimitReachedError, _super); + function LimitReachedError(message) { + var _this = _super.call(this, message) || this; + _this.name = "LimitReachedError"; + Object.setPrototypeOf(_this, LimitReachedError.prototype); + return _this; + } + return LimitReachedError; + })(Error); + + /** + * This error type indicates that some tools are missing for pipeline execution or preparation + * + * @public exported from `@promptbook/core` + */ + var MissingToolsError = /** @class */ ((_super) => { + __extends(MissingToolsError, _super); + function MissingToolsError(message) { + var _this = + _super.call( + this, + spaceTrim.spaceTrim((block) => + "\n ".concat( + block(message), + "\n\n Note: You have probbably forgot to provide some tools for pipeline execution or preparation\n\n ", + ), + ), + ) || this; + _this.name = "MissingToolsError"; + Object.setPrototypeOf(_this, MissingToolsError.prototype); + return _this; + } + return MissingToolsError; + })(Error); + + /** + * This error indicates that promptbook not found in the collection + * + * @public exported from `@promptbook/core` + */ + var NotFoundError = /** @class */ ((_super) => { + __extends(NotFoundError, _super); + function NotFoundError(message) { + var _this = _super.call(this, message) || this; + _this.name = "NotFoundError"; + Object.setPrototypeOf(_this, NotFoundError.prototype); + return _this; + } + return NotFoundError; + })(Error); + + /** + * This error indicates that the promptbook in a markdown format cannot be parsed into a valid promptbook object + * + * @public exported from `@promptbook/core` + */ + var ParseError = /** @class */ ((_super) => { + __extends(ParseError, _super); + function ParseError(message) { + var _this = _super.call(this, message) || this; + _this.name = "ParseError"; + Object.setPrototypeOf(_this, ParseError.prototype); + return _this; + } + return ParseError; + })(Error); + /** + * TODO: Maybe split `ParseError` and `ApplyError` + */ + + /** + * This error indicates that the promptbook object has valid syntax (=can be parsed) but contains logical errors (like circular dependencies) + * + * @public exported from `@promptbook/core` + */ + var PipelineLogicError = /** @class */ ((_super) => { + __extends(PipelineLogicError, _super); + function PipelineLogicError(message) { + var _this = _super.call(this, message) || this; + _this.name = "PipelineLogicError"; + Object.setPrototypeOf(_this, PipelineLogicError.prototype); + return _this; + } + return PipelineLogicError; + })(Error); + + /** + * This error indicates errors in referencing promptbooks between each other + * + * @public exported from `@promptbook/core` + */ + var PipelineUrlError = /** @class */ ((_super) => { + __extends(PipelineUrlError, _super); + function PipelineUrlError(message) { + var _this = _super.call(this, message) || this; + _this.name = "PipelineUrlError"; + Object.setPrototypeOf(_this, PipelineUrlError.prototype); + return _this; + } + return PipelineUrlError; + })(Error); + + /** + * Index of all custom errors + * + * @public exported from `@promptbook/core` + */ + var PROMPTBOOK_ERRORS = { + AbstractFormatError: AbstractFormatError, + CsvFormatError: CsvFormatError, + CollectionError: CollectionError, + EnvironmentMismatchError: EnvironmentMismatchError, + ExpectError: ExpectError, + KnowledgeScrapeError: KnowledgeScrapeError, + LimitReachedError: LimitReachedError, + MissingToolsError: MissingToolsError, + NotFoundError: NotFoundError, + NotYetImplementedError: NotYetImplementedError, + ParseError: ParseError, + PipelineExecutionError: PipelineExecutionError, + PipelineLogicError: PipelineLogicError, + PipelineUrlError: PipelineUrlError, + UnexpectedError: UnexpectedError, + // TODO: [🪑]> VersionMismatchError, + }; + /** + * Index of all javascript errors + * + * @private for internal usage + */ + var COMMON_JAVASCRIPT_ERRORS = { + Error: Error, + EvalError: EvalError, + RangeError: RangeError, + ReferenceError: ReferenceError, + SyntaxError: SyntaxError, + TypeError: TypeError, + URIError: URIError, + AggregateError: AggregateError, + /* Note: Not widely supported > InternalError, > ModuleError, @@ -851,39 +1133,51 @@ > WebAssemblyCompileError, > WebAssemblyRuntimeError, */ - }; - /** - * Index of all errors - * - * @private for internal usage - */ - var ALL_ERRORS = __assign(__assign({}, PROMPTBOOK_ERRORS), COMMON_JAVASCRIPT_ERRORS); - /** - * Note: [💞] Ignore a discrepancy between file name and entity name - */ - - /** - * Serializes an error into a [🚉] JSON-serializable object - * - * @public exported from `@promptbook/utils` - */ - function serializeError(error) { - var name = error.name, message = error.message, stack = error.stack; - if (!Object.keys(ALL_ERRORS).includes(name)) { - console.error(spaceTrim__default["default"](function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); })); - } - - -console.log('-------'); -console.log(stack); -console.log('-------'); - - - return { - name: name, - message: message, - stack: '---', - /* + }; + /** + * Index of all errors + * + * @private for internal usage + */ + var ALL_ERRORS = __assign( + __assign({}, PROMPTBOOK_ERRORS), + COMMON_JAVASCRIPT_ERRORS, + ); + /** + * Note: [💞] Ignore a discrepancy between file name and entity name + */ + + /** + * Serializes an error into a [🚉] JSON-serializable object + * + * @public exported from `@promptbook/utils` + */ + function serializeError(error) { + var name = error.name, + message = error.message, + stack = error.stack; + if (!Object.keys(ALL_ERRORS).includes(name)) { + console.error( + spaceTrim__default["default"]((block) => + '\n \n Cannot serialize error with name "' + .concat(name, '"\n\n ') + .concat( + block(stack || message), + "\n \n ", + ), + ), + ); + } + + console.log("-------"); + console.log(stack); + console.log("-------"); + + return { + name: name, + message: message, + stack: "---", + /* stack: ` at new NotYetImplementedError (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\src\\errors\\NotYetImplementedError.ts:11:9) at PdfScraper. (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\src\\scrapers\\pdf\\PdfScraper.ts:68:15) @@ -897,572 +1191,695 @@ at C:\\Users\\me\\work\\webgpt\\promptbook-studio\\src\\scrapers\\_common\\prepa at step (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\node_modules\\node_modules\\tslib\\tslib.es6.js:102:23) `, */ - //message: message, - //stack: stack, - }; - } - - // Note: We want to use the `exec` as async function - var exec$1 = util.promisify(child_process.exec); - /** - * @@@ - * - * @private within the repository - */ - function locateAppOnLinux(_a) { - var appName = _a.appName, linuxWhich = _a.linuxWhich; - return __awaiter(this, void 0, void 0, function () { - var _b, stderr, stdout, error_1; - return __generator(this, function (_c) { - switch (_c.label) { - case 0: - _c.trys.push([0, 2, , 3]); - return [4 /*yield*/, exec$1("which ".concat(linuxWhich))]; - case 1: - _b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout; - if (!stderr && stdout) { - return [2 /*return*/, stdout.trim()]; - } - throw new Error("Can not locate app ".concat(appName, " on Linux.\n ").concat(stderr)); - case 2: - error_1 = _c.sent(); - if (!(error_1 instanceof Error)) {console.log('!(error instanceof Error)') - throw error_1; - } - return [2 /*return*/, null]; - case 3: return [2 /*return*/]; - } - }); - }); - } - /** - * TODO: [🧠][♿] Maybe export through `@promptbook/node` - * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment - */ - - /** - * Checks if the file is executable - * - * @private within the repository - */ - function isExecutable(path, fs) { - return __awaiter(this, void 0, void 0, function () { - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - _a.trys.push([0, 2, , 3]); - return [4 /*yield*/, fs.access(path, fs.constants.X_OK)]; - case 1: - _a.sent(); - return [2 /*return*/, true]; - case 2: - _a.sent(); - return [2 /*return*/, false]; - case 3: return [2 /*return*/]; - } - }); - }); - } - /** - * Note: Not [~🟢~] because it is not directly dependent on `fs - * TODO: [🖇] What about symlinks? - */ - - // Note: Module `userhome` has no types available, so it is imported using `require` - // @see https://stackoverflow.com/questions/37000981/how-to-import-node-module-in-typescript-without-type-definitions - // eslint-disable-next-line @typescript-eslint/no-var-requires - var userhome = require('userhome'); - // Note: We want to use the `exec` as async function - var exec = util.promisify(child_process.exec); - /** - * @@@ - * - * @private within the repository - */ - function locateAppOnMacOs(_a) { - var appName = _a.appName, macOsName = _a.macOsName; - return __awaiter(this, void 0, void 0, function () { - var toExec, regPath, altPath, _b, stderr, stdout, error_1; - return __generator(this, function (_c) { - switch (_c.label) { - case 0: - _c.trys.push([0, 6, , 7]); - toExec = "/Contents/MacOS/".concat(macOsName); - regPath = "/Applications/".concat(macOsName, ".app") + toExec; - altPath = userhome(regPath.slice(1)); - return [4 /*yield*/, isExecutable(regPath, $provideFilesystemForNode())]; - case 1: - if (!_c.sent()) return [3 /*break*/, 2]; - return [2 /*return*/, regPath]; - case 2: return [4 /*yield*/, isExecutable(altPath, $provideFilesystemForNode())]; - case 3: - if (_c.sent()) { - return [2 /*return*/, altPath]; - } - _c.label = 4; - case 4: return [4 /*yield*/, exec("mdfind 'kMDItemDisplayName == \"".concat(macOsName, "\" && kMDItemKind == Application'"))]; - case 5: - _b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout; - if (!stderr && stdout) { - return [2 /*return*/, stdout.trim() + toExec]; - } - throw new Error("Can not locate app ".concat(appName, " on macOS.\n ").concat(stderr)); - case 6: - error_1 = _c.sent(); - if (!(error_1 instanceof Error)) {console.log('!(error instanceof Error)') - throw error_1; - } - return [2 /*return*/, null]; - case 7: return [2 /*return*/]; - } - }); - }); - } - /** - * TODO: [🧠][♿] Maybe export through `@promptbook/node` - * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment - */ - - /** - * @@@ - * - * @private within the repository - */ - function locateAppOnWindows(_a) { - var appName = _a.appName, windowsSuffix = _a.windowsSuffix; - return __awaiter(this, void 0, void 0, function () { - var prefixes, prefixes_1, prefixes_1_1, prefix, path$1, e_1_1, error_1; - var e_1, _b; - return __generator(this, function (_c) { - switch (_c.label) { - case 0: - _c.trys.push([0, 9, , 10]); - prefixes = [ - process.env.LOCALAPPDATA, - path.join(process.env.LOCALAPPDATA || '', 'Programs'), - process.env.PROGRAMFILES, - process.env['PROGRAMFILES(X86)'], - ]; - _c.label = 1; - case 1: - _c.trys.push([1, 6, 7, 8]); - prefixes_1 = __values(prefixes), prefixes_1_1 = prefixes_1.next(); - _c.label = 2; - case 2: - if (!!prefixes_1_1.done) return [3 /*break*/, 5]; - prefix = prefixes_1_1.value; - path$1 = prefix + windowsSuffix; - return [4 /*yield*/, isExecutable(path$1, $provideFilesystemForNode())]; - case 3: - if (_c.sent()) { - return [2 /*return*/, path$1]; - } - _c.label = 4; - case 4: - prefixes_1_1 = prefixes_1.next(); - return [3 /*break*/, 2]; - case 5: return [3 /*break*/, 8]; - case 6: - e_1_1 = _c.sent(); - e_1 = { error: e_1_1 }; - return [3 /*break*/, 8]; - case 7: - try { - if (prefixes_1_1 && !prefixes_1_1.done && (_b = prefixes_1.return)) _b.call(prefixes_1); - } - finally { if (e_1) throw e_1.error; } - return [7 /*endfinally*/]; - case 8: throw new Error("Can not locate app ".concat(appName, " on Windows.")); - case 9: - error_1 = _c.sent(); - if (!(error_1 instanceof Error)) {console.log('!(error instanceof Error)') - throw error_1; - } - return [2 /*return*/, null]; - case 10: return [2 /*return*/]; - } - }); - }); - } - /** - * TODO: [🧠][♿] Maybe export through `@promptbook/node` - * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment - */ - - /** - * Locates an application on the system - * - * @private within the repository - */ - function locateApp(options) { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError('Locating apps works only in Node.js environment'); - } - var appName = options.appName, linuxWhich = options.linuxWhich, windowsSuffix = options.windowsSuffix, macOsName = options.macOsName; - if (process.platform === 'win32') { - if (windowsSuffix) { - return locateAppOnWindows({ appName: appName, windowsSuffix: windowsSuffix }); - } - else { - throw new Error("".concat(appName, " is not available on Windows.")); - } - } - else if (process.platform === 'darwin') { - if (macOsName) { - return locateAppOnMacOs({ appName: appName, macOsName: macOsName }); - } - else { - throw new Error("".concat(appName, " is not available on macOS.")); - } - } - else { - if (linuxWhich) { - return locateAppOnLinux({ appName: appName, linuxWhich: linuxWhich }); - } - else { - throw new Error("".concat(appName, " is not available on Linux.")); - } - } - } - /** - * TODO: [🧠][♿] Maybe export through `@promptbook/node` - * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment - */ - - /** - * @@@ - * - * @private within the repository - */ - function locateLibreoffice() { - return locateApp({ - appName: 'Libreoffice', - linuxWhich: 'libreoffice', - windowsSuffix: '\\LibreOffice\\program\\soffice.exe', - macOsName: 'LibreOffice', - }); - } - /** - * TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/legacy-documents` - * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment - */ - - /** - * @@@ - * - * @private within the repository - */ - function locatePandoc() { - return locateApp({ - appName: 'Pandoc', - linuxWhich: 'pandoc', - windowsSuffix: '\\Pandoc\\pandoc.exe', - macOsName: 'Pandoc', - }); - } - /** - * TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/documents` - * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment - */ - - /** - * @@@ - * - * @public exported from `@promptbook/node` - */ - function $provideExecutablesForNode(options) { - return __awaiter(this, void 0, void 0, function () { - var _a; - var _d; - return __generator(this, function (_e) { - switch (_e.label) { - case 0: - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment'); - } - _a = options || {}, _a.isAutoInstalled, _a.isVerbose; - _d = {}; - return [4 /*yield*/, locatePandoc()]; - case 1: - _d.pandocPath = (_e.sent()) || undefined; - return [4 /*yield*/, locateLibreoffice()]; - case 2: return [2 /*return*/, (_d.libreOfficePath = (_e.sent()) || undefined, - _d)]; - } - }); - }); - } - /** - * TODO: [🧠] Allow to override the executables without need to call `locatePandoc` / `locateLibreoffice` in case of provided - * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment - */ - - /** - * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally. - * - * Note: Internal utility of `joinLlmExecutionTools` but exposed type - * @public exported from `@promptbook/core` - */ - var MultipleLlmExecutionTools = /** @class */ (function () { - /** - * Gets array of execution tools in order of priority - */ - function MultipleLlmExecutionTools() { - var llmExecutionTools = []; - for (var _i = 0; _i < arguments.length; _i++) { - llmExecutionTools[_i] = arguments[_i]; - } - this.llmExecutionTools = llmExecutionTools; - } - Object.defineProperty(MultipleLlmExecutionTools.prototype, "title", { - get: function () { - return 'Multiple LLM Providers'; - }, - enumerable: false, - configurable: true - }); - Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", { - get: function () { - return this.llmExecutionTools.map(function (_a, index) { - var title = _a.title; - return "".concat(index + 1, ") `").concat(title, "`"); - }).join('\n'); - }, - enumerable: false, - configurable: true - }); - /** - * Check the configuration of all execution tools - */ - MultipleLlmExecutionTools.prototype.checkConfiguration = function () { - return __awaiter(this, void 0, void 0, function () { - var _a, _b, llmExecutionTools, e_1_1; - var e_1, _c; - return __generator(this, function (_d) { - switch (_d.label) { - case 0: - _d.trys.push([0, 5, 6, 7]); - _a = __values(this.llmExecutionTools), _b = _a.next(); - _d.label = 1; - case 1: - if (!!_b.done) return [3 /*break*/, 4]; - llmExecutionTools = _b.value; - return [4 /*yield*/, llmExecutionTools.checkConfiguration()]; - case 2: - _d.sent(); - _d.label = 3; - case 3: - _b = _a.next(); - return [3 /*break*/, 1]; - case 4: return [3 /*break*/, 7]; - case 5: - e_1_1 = _d.sent(); - e_1 = { error: e_1_1 }; - return [3 /*break*/, 7]; - case 6: - try { - if (_b && !_b.done && (_c = _a.return)) _c.call(_a); - } - finally { if (e_1) throw e_1.error; } - return [7 /*endfinally*/]; - case 7: return [2 /*return*/]; - } - }); - }); - }; - /** - * List all available models that can be used - * This lists is a combination of all available models from all execution tools - */ - MultipleLlmExecutionTools.prototype.listModels = function () { - return __awaiter(this, void 0, void 0, function () { - var availableModels, _a, _b, llmExecutionTools, models, e_2_1; - var e_2, _c; - return __generator(this, function (_d) { - switch (_d.label) { - case 0: - availableModels = []; - _d.label = 1; - case 1: - _d.trys.push([1, 6, 7, 8]); - _a = __values(this.llmExecutionTools), _b = _a.next(); - _d.label = 2; - case 2: - if (!!_b.done) return [3 /*break*/, 5]; - llmExecutionTools = _b.value; - return [4 /*yield*/, llmExecutionTools.listModels()]; - case 3: - models = _d.sent(); - availableModels.push.apply(availableModels, __spreadArray([], __read(models), false)); - _d.label = 4; - case 4: - _b = _a.next(); - return [3 /*break*/, 2]; - case 5: return [3 /*break*/, 8]; - case 6: - e_2_1 = _d.sent(); - e_2 = { error: e_2_1 }; - return [3 /*break*/, 8]; - case 7: - try { - if (_b && !_b.done && (_c = _a.return)) _c.call(_a); - } - finally { if (e_2) throw e_2.error; } - return [7 /*endfinally*/]; - case 8: return [2 /*return*/, availableModels]; - } - }); - }); - }; - /** - * Calls the best available chat model - */ - MultipleLlmExecutionTools.prototype.callChatModel = function (prompt) { - return this.callCommonModel(prompt); - }; - /** - * Calls the best available completion model - */ - MultipleLlmExecutionTools.prototype.callCompletionModel = function (prompt) { - return this.callCommonModel(prompt); - }; - /** - * Calls the best available embedding model - */ - MultipleLlmExecutionTools.prototype.callEmbeddingModel = function (prompt) { - return this.callCommonModel(prompt); - }; - // <- Note: [🤖] - /** - * Calls the best available model - * - * Note: This should be private or protected but is public to be usable with duck typing - */ - MultipleLlmExecutionTools.prototype.callCommonModel = function (prompt) { - return __awaiter(this, void 0, void 0, function () { - var errors, _a, _b, llmExecutionTools, _c, error_1, e_3_1; - var e_3, _d; - var _this = this; - return __generator(this, function (_e) { - switch (_e.label) { - case 0: - errors = []; - _e.label = 1; - case 1: - _e.trys.push([1, 15, 16, 17]); - _a = __values(this.llmExecutionTools), _b = _a.next(); - _e.label = 2; - case 2: - if (!!_b.done) return [3 /*break*/, 14]; - llmExecutionTools = _b.value; - _e.label = 3; - case 3: - _e.trys.push([3, 12, , 13]); - _c = prompt.modelRequirements.modelVariant; - switch (_c) { - case 'CHAT': return [3 /*break*/, 4]; - case 'COMPLETION': return [3 /*break*/, 6]; - case 'EMBEDDING': return [3 /*break*/, 8]; - } - return [3 /*break*/, 10]; - case 4: - if (llmExecutionTools.callChatModel === undefined) { - return [3 /*break*/, 13]; - } - return [4 /*yield*/, llmExecutionTools.callChatModel(prompt)]; - case 5: return [2 /*return*/, _e.sent()]; - case 6: - if (llmExecutionTools.callCompletionModel === undefined) { - return [3 /*break*/, 13]; - } - return [4 /*yield*/, llmExecutionTools.callCompletionModel(prompt)]; - case 7: return [2 /*return*/, _e.sent()]; - case 8: - if (llmExecutionTools.callEmbeddingModel === undefined) { - return [3 /*break*/, 13]; - } - return [4 /*yield*/, llmExecutionTools.callEmbeddingModel(prompt)]; - case 9: return [2 /*return*/, _e.sent()]; - case 10: throw new UnexpectedError("Unknown model variant \"".concat(prompt.modelRequirements.modelVariant, "\"")); - case 11: return [3 /*break*/, 13]; - case 12: - error_1 = _e.sent(); - if (!(error_1 instanceof Error) || error_1 instanceof UnexpectedError) { - throw error_1; - } - errors.push(error_1); - return [3 /*break*/, 13]; - case 13: - _b = _a.next(); - return [3 /*break*/, 2]; - case 14: return [3 /*break*/, 17]; - case 15: - e_3_1 = _e.sent(); - e_3 = { error: e_3_1 }; - return [3 /*break*/, 17]; - case 16: - try { - if (_b && !_b.done && (_d = _a.return)) _d.call(_a); - } - finally { if (e_3) throw e_3.error; } - return [7 /*endfinally*/]; - case 17: - if (errors.length === 1) { - throw errors[0]; - } - else if (errors.length > 1) { - throw new PipelineExecutionError( - // TODO: Tell which execution tools failed like - // 1) OpenAI throw PipelineExecutionError: Parameter `{knowledge}` is not defined - // 2) AnthropicClaude throw PipelineExecutionError: Parameter `{knowledge}` is not defined - // 3) ... - spaceTrim__default["default"](function (block) { return "\n All execution tools failed:\n\n ".concat(block(errors - .map(function (error, i) { return "".concat(i + 1, ") **").concat(error.name || 'Error', ":** ").concat(error.message); }) - .join('\n')), "\n\n "); })); - } - else if (this.llmExecutionTools.length === 0) { - throw new PipelineExecutionError("You have not provided any `LlmExecutionTools`"); - } - else { - throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n You have not provided any `LlmExecutionTools` that support model variant \"".concat(prompt.modelRequirements.modelVariant, "\"\n\n Available `LlmExecutionTools`:\n ").concat(block(_this.description), "\n\n "); })); - } - } - }); - }); - }; - return MultipleLlmExecutionTools; - }()); - /** - * TODO: [🧠][🎛] Aggregating multiple models - have result not only from one first available model BUT all of them - * TODO: [🏖] If no llmTools have for example not defined `callCompletionModel` this will still return object with defined `callCompletionModel` which just throws `PipelineExecutionError`, make it undefined instead - * Look how `countTotalUsage` (and `cacheLlmTools`) implements it - */ - - /** - * Joins multiple LLM Execution Tools into one - * - * @returns {LlmExecutionTools} Single wrapper for multiple LlmExecutionTools - * - * 0) If there is no LlmExecutionTools, it warns and returns valid but empty LlmExecutionTools - * 1) If there is only one LlmExecutionTools, it returns it wrapped in a proxy object - * 2) If there are multiple LlmExecutionTools, first will be used first, second will be used if the first hasn`t defined model variant or fails, etc. - * 3) When all LlmExecutionTools fail, it throws an error with a list of all errors merged into one - * - * - * Tip: You don't have to use this function directly, just pass an array of LlmExecutionTools to the `ExecutionTools` - * - * @public exported from `@promptbook/core` - */ - function joinLlmExecutionTools() { - var llmExecutionTools = []; - for (var _i = 0; _i < arguments.length; _i++) { - llmExecutionTools[_i] = arguments[_i]; - } - if (llmExecutionTools.length === 0) { - var warningMessage = spaceTrim__default["default"]("\n You have not provided any `LlmExecutionTools`\n This means that you won't be able to execute any prompts that require large language models like GPT-4 or Anthropic's Claude.\n\n Technically, it's not an error, but it's probably not what you want because it does not make sense to use Promptbook without language models.\n "); - // TODO: [🟥] Detect browser / node and make it colorfull - console.warn(warningMessage); - /* + //message: message, + //stack: stack, + }; + } + + // Note: We want to use the `exec` as async function + var exec$1 = util.promisify(child_process.exec); + /** + * @@@ + * + * @private within the repository + */ + function locateAppOnLinux(_a) { + var appName = _a.appName, + linuxWhich = _a.linuxWhich; + return __awaiter(this, void 0, void 0, function () { + var _b, stderr, stdout, error_1; + return __generator(this, (_c) => { + switch (_c.label) { + case 0: + _c.trys.push([0, 2, , 3]); + return [4 /*yield*/, exec$1("which ".concat(linuxWhich))]; + case 1: + (_b = _c.sent()), (stderr = _b.stderr), (stdout = _b.stdout); + if (!stderr && stdout) { + return [2 /*return*/, stdout.trim()]; + } + throw new Error( + "Can not locate app " + .concat(appName, " on Linux.\n ") + .concat(stderr), + ); + case 2: + error_1 = _c.sent(); + if (!(error_1 instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error_1; + } + return [2 /*return*/, null]; + case 3: + return [2 /*return*/]; + } + }); + }); + } + /** + * TODO: [🧠][♿] Maybe export through `@promptbook/node` + * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment + */ + + /** + * Checks if the file is executable + * + * @private within the repository + */ + function isExecutable(path, fs) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + _a.trys.push([0, 2, , 3]); + return [4 /*yield*/, fs.access(path, fs.constants.X_OK)]; + case 1: + _a.sent(); + return [2 /*return*/, true]; + case 2: + _a.sent(); + return [2 /*return*/, false]; + case 3: + return [2 /*return*/]; + } + }); + }); + } + /** + * Note: Not [~🟢~] because it is not directly dependent on `fs + * TODO: [🖇] What about symlinks? + */ + + // Note: Module `userhome` has no types available, so it is imported using `require` + // @see https://stackoverflow.com/questions/37000981/how-to-import-node-module-in-typescript-without-type-definitions + // eslint-disable-next-line @typescript-eslint/no-var-requires + var userhome = require("userhome"); + // Note: We want to use the `exec` as async function + var exec = util.promisify(child_process.exec); + /** + * @@@ + * + * @private within the repository + */ + function locateAppOnMacOs(_a) { + var appName = _a.appName, + macOsName = _a.macOsName; + return __awaiter(this, void 0, void 0, function () { + var toExec, regPath, altPath, _b, stderr, stdout, error_1; + return __generator(this, (_c) => { + switch (_c.label) { + case 0: + _c.trys.push([0, 6, , 7]); + toExec = "/Contents/MacOS/".concat(macOsName); + regPath = "/Applications/".concat(macOsName, ".app") + toExec; + altPath = userhome(regPath.slice(1)); + return [ + 4 /*yield*/, + isExecutable(regPath, $provideFilesystemForNode()), + ]; + case 1: + if (!_c.sent()) return [3 /*break*/, 2]; + return [2 /*return*/, regPath]; + case 2: + return [ + 4 /*yield*/, + isExecutable(altPath, $provideFilesystemForNode()), + ]; + case 3: + if (_c.sent()) { + return [2 /*return*/, altPath]; + } + _c.label = 4; + case 4: + return [ + 4 /*yield*/, + exec( + "mdfind 'kMDItemDisplayName == \"".concat( + macOsName, + "\" && kMDItemKind == Application'", + ), + ), + ]; + case 5: + (_b = _c.sent()), (stderr = _b.stderr), (stdout = _b.stdout); + if (!stderr && stdout) { + return [2 /*return*/, stdout.trim() + toExec]; + } + throw new Error( + "Can not locate app " + .concat(appName, " on macOS.\n ") + .concat(stderr), + ); + case 6: + error_1 = _c.sent(); + if (!(error_1 instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error_1; + } + return [2 /*return*/, null]; + case 7: + return [2 /*return*/]; + } + }); + }); + } + /** + * TODO: [🧠][♿] Maybe export through `@promptbook/node` + * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment + */ + + /** + * @@@ + * + * @private within the repository + */ + function locateAppOnWindows(_a) { + var appName = _a.appName, + windowsSuffix = _a.windowsSuffix; + return __awaiter(this, void 0, void 0, function () { + var prefixes, prefixes_1, prefixes_1_1, prefix, path$1, e_1_1, error_1; + var e_1, _b; + return __generator(this, (_c) => { + switch (_c.label) { + case 0: + _c.trys.push([0, 9, , 10]); + prefixes = [ + process.env.LOCALAPPDATA, + path.join(process.env.LOCALAPPDATA || "", "Programs"), + process.env.PROGRAMFILES, + process.env["PROGRAMFILES(X86)"], + ]; + _c.label = 1; + case 1: + _c.trys.push([1, 6, 7, 8]); + (prefixes_1 = __values(prefixes)), + (prefixes_1_1 = prefixes_1.next()); + _c.label = 2; + case 2: + if (!!prefixes_1_1.done) return [3 /*break*/, 5]; + prefix = prefixes_1_1.value; + path$1 = prefix + windowsSuffix; + return [ + 4 /*yield*/, + isExecutable(path$1, $provideFilesystemForNode()), + ]; + case 3: + if (_c.sent()) { + return [2 /*return*/, path$1]; + } + _c.label = 4; + case 4: + prefixes_1_1 = prefixes_1.next(); + return [3 /*break*/, 2]; + case 5: + return [3 /*break*/, 8]; + case 6: + e_1_1 = _c.sent(); + e_1 = { error: e_1_1 }; + return [3 /*break*/, 8]; + case 7: + try { + if ( + prefixes_1_1 && + !prefixes_1_1.done && + (_b = prefixes_1.return) + ) + _b.call(prefixes_1); + } finally { + if (e_1) throw e_1.error; + } + return [7 /*endfinally*/]; + case 8: + throw new Error( + "Can not locate app ".concat(appName, " on Windows."), + ); + case 9: + error_1 = _c.sent(); + if (!(error_1 instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error_1; + } + return [2 /*return*/, null]; + case 10: + return [2 /*return*/]; + } + }); + }); + } + /** + * TODO: [🧠][♿] Maybe export through `@promptbook/node` + * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment + */ + + /** + * Locates an application on the system + * + * @private within the repository + */ + function locateApp(options) { + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Locating apps works only in Node.js environment", + ); + } + var appName = options.appName, + linuxWhich = options.linuxWhich, + windowsSuffix = options.windowsSuffix, + macOsName = options.macOsName; + if (process.platform === "win32") { + if (windowsSuffix) { + return locateAppOnWindows({ + appName: appName, + windowsSuffix: windowsSuffix, + }); + } else { + throw new Error("".concat(appName, " is not available on Windows.")); + } + } else if (process.platform === "darwin") { + if (macOsName) { + return locateAppOnMacOs({ appName: appName, macOsName: macOsName }); + } else { + throw new Error("".concat(appName, " is not available on macOS.")); + } + } else { + if (linuxWhich) { + return locateAppOnLinux({ appName: appName, linuxWhich: linuxWhich }); + } else { + throw new Error("".concat(appName, " is not available on Linux.")); + } + } + } + /** + * TODO: [🧠][♿] Maybe export through `@promptbook/node` + * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment + */ + + /** + * @@@ + * + * @private within the repository + */ + function locateLibreoffice() { + return locateApp({ + appName: "Libreoffice", + linuxWhich: "libreoffice", + windowsSuffix: "\\LibreOffice\\program\\soffice.exe", + macOsName: "LibreOffice", + }); + } + /** + * TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/legacy-documents` + * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment + */ + + /** + * @@@ + * + * @private within the repository + */ + function locatePandoc() { + return locateApp({ + appName: "Pandoc", + linuxWhich: "pandoc", + windowsSuffix: "\\Pandoc\\pandoc.exe", + macOsName: "Pandoc", + }); + } + /** + * TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/documents` + * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment + */ + + /** + * @@@ + * + * @public exported from `@promptbook/node` + */ + function $provideExecutablesForNode(options) { + return __awaiter(this, void 0, void 0, function () { + var _a; + var _d; + return __generator(this, (_e) => { + switch (_e.label) { + case 0: + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Function `$getScrapersForNode` works only in Node.js environment", + ); + } + (_a = options || {}), _a.isAutoInstalled, _a.isVerbose; + _d = {}; + return [4 /*yield*/, locatePandoc()]; + case 1: + _d.pandocPath = _e.sent() || undefined; + return [4 /*yield*/, locateLibreoffice()]; + case 2: + return [ + 2 /*return*/, + ((_d.libreOfficePath = _e.sent() || undefined), _d), + ]; + } + }); + }); + } + /** + * TODO: [🧠] Allow to override the executables without need to call `locatePandoc` / `locateLibreoffice` in case of provided + * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment + */ + + /** + * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally. + * + * Note: Internal utility of `joinLlmExecutionTools` but exposed type + * @public exported from `@promptbook/core` + */ + var MultipleLlmExecutionTools = /** @class */ (() => { + /** + * Gets array of execution tools in order of priority + */ + function MultipleLlmExecutionTools() { + var llmExecutionTools = []; + for (var _i = 0; _i < arguments.length; _i++) { + llmExecutionTools[_i] = arguments[_i]; + } + this.llmExecutionTools = llmExecutionTools; + } + Object.defineProperty(MultipleLlmExecutionTools.prototype, "title", { + get: () => "Multiple LLM Providers", + enumerable: false, + configurable: true, + }); + Object.defineProperty( + MultipleLlmExecutionTools.prototype, + "description", + { + get: function () { + return this.llmExecutionTools + .map((_a, index) => { + var title = _a.title; + return "".concat(index + 1, ") `").concat(title, "`"); + }) + .join("\n"); + }, + enumerable: false, + configurable: true, + }, + ); + /** + * Check the configuration of all execution tools + */ + MultipleLlmExecutionTools.prototype.checkConfiguration = function () { + return __awaiter(this, void 0, void 0, function () { + var _a, _b, llmExecutionTools, e_1_1; + var e_1, _c; + return __generator(this, function (_d) { + switch (_d.label) { + case 0: + _d.trys.push([0, 5, 6, 7]); + (_a = __values(this.llmExecutionTools)), (_b = _a.next()); + _d.label = 1; + case 1: + if (!!_b.done) return [3 /*break*/, 4]; + llmExecutionTools = _b.value; + return [4 /*yield*/, llmExecutionTools.checkConfiguration()]; + case 2: + _d.sent(); + _d.label = 3; + case 3: + _b = _a.next(); + return [3 /*break*/, 1]; + case 4: + return [3 /*break*/, 7]; + case 5: + e_1_1 = _d.sent(); + e_1 = { error: e_1_1 }; + return [3 /*break*/, 7]; + case 6: + try { + if (_b && !_b.done && (_c = _a.return)) _c.call(_a); + } finally { + if (e_1) throw e_1.error; + } + return [7 /*endfinally*/]; + case 7: + return [2 /*return*/]; + } + }); + }); + }; + /** + * List all available models that can be used + * This lists is a combination of all available models from all execution tools + */ + MultipleLlmExecutionTools.prototype.listModels = function () { + return __awaiter(this, void 0, void 0, function () { + var availableModels, _a, _b, llmExecutionTools, models, e_2_1; + var e_2, _c; + return __generator(this, function (_d) { + switch (_d.label) { + case 0: + availableModels = []; + _d.label = 1; + case 1: + _d.trys.push([1, 6, 7, 8]); + (_a = __values(this.llmExecutionTools)), (_b = _a.next()); + _d.label = 2; + case 2: + if (!!_b.done) return [3 /*break*/, 5]; + llmExecutionTools = _b.value; + return [4 /*yield*/, llmExecutionTools.listModels()]; + case 3: + models = _d.sent(); + availableModels.push.apply( + availableModels, + __spreadArray([], __read(models), false), + ); + _d.label = 4; + case 4: + _b = _a.next(); + return [3 /*break*/, 2]; + case 5: + return [3 /*break*/, 8]; + case 6: + e_2_1 = _d.sent(); + e_2 = { error: e_2_1 }; + return [3 /*break*/, 8]; + case 7: + try { + if (_b && !_b.done && (_c = _a.return)) _c.call(_a); + } finally { + if (e_2) throw e_2.error; + } + return [7 /*endfinally*/]; + case 8: + return [2 /*return*/, availableModels]; + } + }); + }); + }; + /** + * Calls the best available chat model + */ + MultipleLlmExecutionTools.prototype.callChatModel = function (prompt) { + return this.callCommonModel(prompt); + }; + /** + * Calls the best available completion model + */ + MultipleLlmExecutionTools.prototype.callCompletionModel = function ( + prompt, + ) { + return this.callCommonModel(prompt); + }; + /** + * Calls the best available embedding model + */ + MultipleLlmExecutionTools.prototype.callEmbeddingModel = function ( + prompt, + ) { + return this.callCommonModel(prompt); + }; + // <- Note: [🤖] + /** + * Calls the best available model + * + * Note: This should be private or protected but is public to be usable with duck typing + */ + MultipleLlmExecutionTools.prototype.callCommonModel = function (prompt) { + return __awaiter(this, void 0, void 0, function () { + var errors, _a, _b, llmExecutionTools, _c, error_1, e_3_1; + var e_3, _d; + var _this = this; + return __generator(this, function (_e) { + switch (_e.label) { + case 0: + errors = []; + _e.label = 1; + case 1: + _e.trys.push([1, 15, 16, 17]); + (_a = __values(this.llmExecutionTools)), (_b = _a.next()); + _e.label = 2; + case 2: + if (!!_b.done) return [3 /*break*/, 14]; + llmExecutionTools = _b.value; + _e.label = 3; + case 3: + _e.trys.push([3, 12, , 13]); + _c = prompt.modelRequirements.modelVariant; + switch (_c) { + case "CHAT": + return [3 /*break*/, 4]; + case "COMPLETION": + return [3 /*break*/, 6]; + case "EMBEDDING": + return [3 /*break*/, 8]; + } + return [3 /*break*/, 10]; + case 4: + if (llmExecutionTools.callChatModel === undefined) { + return [3 /*break*/, 13]; + } + return [4 /*yield*/, llmExecutionTools.callChatModel(prompt)]; + case 5: + return [2 /*return*/, _e.sent()]; + case 6: + if (llmExecutionTools.callCompletionModel === undefined) { + return [3 /*break*/, 13]; + } + return [ + 4 /*yield*/, + llmExecutionTools.callCompletionModel(prompt), + ]; + case 7: + return [2 /*return*/, _e.sent()]; + case 8: + if (llmExecutionTools.callEmbeddingModel === undefined) { + return [3 /*break*/, 13]; + } + return [ + 4 /*yield*/, + llmExecutionTools.callEmbeddingModel(prompt), + ]; + case 9: + return [2 /*return*/, _e.sent()]; + case 10: + throw new UnexpectedError( + 'Unknown model variant "'.concat( + prompt.modelRequirements.modelVariant, + '"', + ), + ); + case 11: + return [3 /*break*/, 13]; + case 12: + error_1 = _e.sent(); + if ( + !(error_1 instanceof Error) || + error_1 instanceof UnexpectedError + ) { + throw error_1; + } + errors.push(error_1); + return [3 /*break*/, 13]; + case 13: + _b = _a.next(); + return [3 /*break*/, 2]; + case 14: + return [3 /*break*/, 17]; + case 15: + e_3_1 = _e.sent(); + e_3 = { error: e_3_1 }; + return [3 /*break*/, 17]; + case 16: + try { + if (_b && !_b.done && (_d = _a.return)) _d.call(_a); + } finally { + if (e_3) throw e_3.error; + } + return [7 /*endfinally*/]; + case 17: + if (errors.length === 1) { + throw errors[0]; + } else if (errors.length > 1) { + throw new PipelineExecutionError( + // TODO: Tell which execution tools failed like + // 1) OpenAI throw PipelineExecutionError: Parameter `{knowledge}` is not defined + // 2) AnthropicClaude throw PipelineExecutionError: Parameter `{knowledge}` is not defined + // 3) ... + spaceTrim__default["default"]((block) => + "\n All execution tools failed:\n\n ".concat( + block( + errors + .map((error, i) => + "" + .concat(i + 1, ") **") + .concat(error.name || "Error", ":** ") + .concat(error.message), + ) + .join("\n"), + ), + "\n\n ", + ), + ), + ); + } else if (this.llmExecutionTools.length === 0) { + throw new PipelineExecutionError( + "You have not provided any `LlmExecutionTools`", + ); + } else { + throw new PipelineExecutionError( + spaceTrim__default["default"]((block) => + '\n You have not provided any `LlmExecutionTools` that support model variant "' + .concat( + prompt.modelRequirements.modelVariant, + '"\n\n Available `LlmExecutionTools`:\n ', + ) + .concat( + block(_this.description), + "\n\n ", + ), + ), + ); + } + } + }); + }); + }; + return MultipleLlmExecutionTools; + })(); + /** + * TODO: [🧠][🎛] Aggregating multiple models - have result not only from one first available model BUT all of them + * TODO: [🏖] If no llmTools have for example not defined `callCompletionModel` this will still return object with defined `callCompletionModel` which just throws `PipelineExecutionError`, make it undefined instead + * Look how `countTotalUsage` (and `cacheLlmTools`) implements it + */ + + /** + * Joins multiple LLM Execution Tools into one + * + * @returns {LlmExecutionTools} Single wrapper for multiple LlmExecutionTools + * + * 0) If there is no LlmExecutionTools, it warns and returns valid but empty LlmExecutionTools + * 1) If there is only one LlmExecutionTools, it returns it wrapped in a proxy object + * 2) If there are multiple LlmExecutionTools, first will be used first, second will be used if the first hasn`t defined model variant or fails, etc. + * 3) When all LlmExecutionTools fail, it throws an error with a list of all errors merged into one + * + * + * Tip: You don't have to use this function directly, just pass an array of LlmExecutionTools to the `ExecutionTools` + * + * @public exported from `@promptbook/core` + */ + function joinLlmExecutionTools() { + var llmExecutionTools = []; + for (var _i = 0; _i < arguments.length; _i++) { + llmExecutionTools[_i] = arguments[_i]; + } + if (llmExecutionTools.length === 0) { + var warningMessage = spaceTrim__default["default"]( + "\n You have not provided any `LlmExecutionTools`\n This means that you won't be able to execute any prompts that require large language models like GPT-4 or Anthropic's Claude.\n\n Technically, it's not an error, but it's probably not what you want because it does not make sense to use Promptbook without language models.\n ", + ); + // TODO: [🟥] Detect browser / node and make it colorfull + console.warn(warningMessage); + /* return { async listModels() { // TODO: [🟥] Detect browser / node and make it colorfull @@ -1482,654 +1899,1185 @@ at step (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\node_modules\\node_modu }, }; */ - } - return new (MultipleLlmExecutionTools.bind.apply(MultipleLlmExecutionTools, __spreadArray([void 0], __read(llmExecutionTools), false)))(); - } - /** - * TODO: [👷‍♂️] @@@ Manual about construction of llmTools - */ - - /** - * @@@ - * - * Note: `$` is used to indicate that this interacts with the global scope - * @singleton Only one instance of each register is created per build, but thare can be more @@@ - * @public exported from `@promptbook/core` - */ - var $llmToolsRegister = new $Register('llm_execution_tools_constructors'); - /** - * TODO: [®] DRY Register logic - */ - - /** - * @@@ - * - * Note: `$` is used to indicate that this interacts with the global scope - * @singleton Only one instance of each register is created per build, but thare can be more @@@ - * @public exported from `@promptbook/core` - */ - var $llmToolsMetadataRegister = new $Register('llm_tools_metadata'); - /** - * TODO: [®] DRY Register logic - */ - - /** - * Creates a message with all registered LLM tools - * - * Note: This function is used to create a (error) message when there is no constructor for some LLM provider - * - * @private internal function of `createLlmToolsFromConfiguration` and `$provideLlmToolsFromEnv` - */ - function $registeredLlmToolsMessage() { - var e_1, _a, e_2, _b; - var env; - if ($isRunningInNode()) { - env = process.env; - // <- TODO: [⚛] Some DRY way how to get to `process.env` and pass it into functions - ACRY search for `env` - } - else { - env = {}; - } - /** - * Mixes registered LLM tools from $llmToolsMetadataRegister and $llmToolsRegister - */ - var all = []; - var _loop_1 = function (title, packageName, className, envVariables) { - if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) { - return "continue"; - } - all.push({ title: title, packageName: packageName, className: className, envVariables: envVariables }); - }; - try { - for (var _c = __values($llmToolsMetadataRegister.list()), _d = _c.next(); !_d.done; _d = _c.next()) { - var _e = _d.value, title = _e.title, packageName = _e.packageName, className = _e.className, envVariables = _e.envVariables; - _loop_1(title, packageName, className, envVariables); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_d && !_d.done && (_a = _c.return)) _a.call(_c); - } - finally { if (e_1) throw e_1.error; } - } - var _loop_2 = function (packageName, className) { - if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) { - return "continue"; - } - all.push({ packageName: packageName, className: className }); - }; - try { - for (var _f = __values($llmToolsRegister.list()), _g = _f.next(); !_g.done; _g = _f.next()) { - var _h = _g.value, packageName = _h.packageName, className = _h.className; - _loop_2(packageName, className); - } - } - catch (e_2_1) { e_2 = { error: e_2_1 }; } - finally { - try { - if (_g && !_g.done && (_b = _f.return)) _b.call(_f); - } - finally { if (e_2) throw e_2.error; } - } - var metadata = all.map(function (metadata) { - var _a, _b; - var isMetadataAviailable = $llmToolsMetadataRegister - .list() - .find(function (_a) { - var packageName = _a.packageName, className = _a.className; - return metadata.packageName === packageName && metadata.className === className; - }); - var isInstalled = $llmToolsRegister - .list() - .find(function (_a) { - var packageName = _a.packageName, className = _a.className; - return metadata.packageName === packageName && metadata.className === className; - }); - var isFullyConfigured = ((_a = metadata.envVariables) === null || _a === void 0 ? void 0 : _a.every(function (envVariableName) { return env[envVariableName] !== undefined; })) || false; - var isPartiallyConfigured = ((_b = metadata.envVariables) === null || _b === void 0 ? void 0 : _b.some(function (envVariableName) { return env[envVariableName] !== undefined; })) || false; - // <- Note: [🗨] - return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isFullyConfigured: isFullyConfigured, isPartiallyConfigured: isPartiallyConfigured }); - }); - if (metadata.length === 0) { - return "No LLM providers are available."; - } - return spaceTrim__default["default"](function (block) { return "\n Relevant environment variables:\n ".concat(block(Object.keys(env) - .filter(function (envVariableName) { - return metadata.some(function (_a) { - var envVariables = _a.envVariables; - return envVariables === null || envVariables === void 0 ? void 0 : envVariables.includes(envVariableName); - }); - }) - .map(function (envVariableName) { return "- `".concat(envVariableName, "`"); }) - .join('\n')), "\n\n Available LLM providers are:\n ").concat(block(metadata - .map(function (_a, i) { - var title = _a.title, packageName = _a.packageName, className = _a.className, envVariables = _a.envVariables, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, isFullyConfigured = _a.isFullyConfigured, isPartiallyConfigured = _a.isPartiallyConfigured; - var morePieces = []; - if (just(false)) ; - else if (!isMetadataAviailable && !isInstalled) { - // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found - morePieces.push("Not installed and no metadata, looks like a unexpected behavior"); - } - else if (isMetadataAviailable && !isInstalled) { - // TODO: [�][�] - morePieces.push("Not installed"); - } - else if (!isMetadataAviailable && isInstalled) { - morePieces.push("No metadata but installed, looks like a unexpected behavior"); - } - else if (isMetadataAviailable && isInstalled) { - morePieces.push("Installed"); - } - else { - morePieces.push("unknown state, looks like a unexpected behavior"); - } /* not else */ - if (isFullyConfigured) { - morePieces.push("Configured"); - } - else if (isPartiallyConfigured) { - morePieces.push("Partially confugured, missing ".concat(envVariables === null || envVariables === void 0 ? void 0 : envVariables.filter(function (envVariable) { return env[envVariable] === undefined; }).join(' + '))); - } - else { - if (envVariables !== null) { - morePieces.push("Not configured, to configure set env ".concat(envVariables === null || envVariables === void 0 ? void 0 : envVariables.join(' + '))); - } - else { - morePieces.push("Not configured"); // <- Note: Can not be configured via environment variables - } - } - var providerMessage = spaceTrim__default["default"]("\n ".concat(i + 1, ") **").concat(title, "** `").concat(className, "` from `").concat(packageName, "`\n ").concat(morePieces.join('; '), "\n ")); - if ($isRunningInNode) { - if (isInstalled && isFullyConfigured) { - providerMessage = colors__default["default"].green(providerMessage); - } - else if (isInstalled && isPartiallyConfigured) { - providerMessage = colors__default["default"].yellow(providerMessage); - } - else { - providerMessage = colors__default["default"].gray(providerMessage); - } - } - return providerMessage; - }) - .join('\n')), "\n "); }); - } - /** - * TODO: [®] DRY Register logic - * TODO: [🧠][⚛] Maybe pass env as argument - */ - - /** - * @@@ - * - * Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools` - * - * @returns @@@ - * @public exported from `@promptbook/core` - */ - function createLlmToolsFromConfiguration(configuration, options) { - if (options === void 0) { options = {}; } - var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId; - var llmTools = configuration.map(function (llmConfiguration) { - var registeredItem = $llmToolsRegister - .list() - .find(function (_a) { - var packageName = _a.packageName, className = _a.className; - return llmConfiguration.packageName === packageName && llmConfiguration.className === className; - }); - if (registeredItem === undefined) { - throw new Error(spaceTrim__default["default"](function (block) { return "\n There is no constructor for LLM provider `".concat(llmConfiguration.className, "` from `").concat(llmConfiguration.packageName, "`\n\n You have probably forgotten install and import the provider package.\n To fix this issue, you can:\n\n Install:\n\n > npm install ").concat(llmConfiguration.packageName, "\n\n And import:\n\n > import '").concat(llmConfiguration.packageName, "';\n\n\n ").concat(block($registeredLlmToolsMessage()), "\n "); })); - } - return registeredItem(__assign({ isVerbose: isVerbose, userId: userId }, llmConfiguration.options)); - }); - return joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(llmTools), false)); - } - /** - * TODO: [🎌] Together with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient - * TODO: [🧠][🎌] Dynamically install required providers - * TODO: @@@ write discussion about this - wizzard - * TODO: [🧠][🍛] Which name is better `createLlmToolsFromConfig` or `createLlmToolsFromConfiguration`? - * TODO: [🧠] Is there some meaningfull way how to test this util - * TODO: This should be maybe not under `_common` but under `utils` - * TODO: [®] DRY Register logic - */ - - var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book.md",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge from Markdown\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book`\n- INPUT PARAMETER `{knowledgeContent}` Markdown document content\n- OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n"}],sourceFile:"./books/prepare-knowledge-from-markdown.book.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book.md",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Keywords\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n"}],sourceFile:"./books/prepare-knowledge-keywords.book.md"},{title:"Prepare Knowledge-piece Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book.md",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge-piece Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n- EXPECT MIN 1 WORD\n- EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-knowledge-title.book.md"},{title:"Prepare Persona",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book.md",formfactorName:"GENERIC",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Persona\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book`\n- INPUT PARAMETER `{availableModelNames}` List of available model names separated by comma (,)\n- INPUT PARAMETER `{personaDescription}` Description of the persona\n- OUTPUT PARAMETER `{modelRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n- FORMAT JSON\n\n```markdown\nYou are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n\\`\\`\\`json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n\\`\\`\\`\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelRequirements}`\n"}],sourceFile:"./books/prepare-persona.book.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-title.book.md",formfactorName:"GENERIC",parameters:[{name:"book",description:"The book to prepare the title for",isInput:true,isOutput:false},{name:"title",description:"Best title for the book",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-title",title:"Make title",content:"Make best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"✍ Convert Knowledge-piece to title\" but \"✍ Title\"_\n\n## The workflow\n\n> {book}",resultingParameterName:"title",expectations:{words:{min:1,max:8},lines:{min:1,max:1}},dependentParameterNames:["book"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book`\n- INPUT PARAMETER `{book}` The book to prepare the title for\n- OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n- EXPECT MIN 1 Word\n- EXPECT MAX 8 Words\n- EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"✍ Convert Knowledge-piece to title\" but \"✍ Title\"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-title.book.md"}]; - - /** - * Function isValidJsonString will tell you if the string is valid JSON or not - * - * @public exported from `@promptbook/utils` - */ - function isValidJsonString(value /* <- [👨‍⚖️] */) { - try { - JSON.parse(value); - return true; - } - catch (error) { - if (!(error instanceof Error)) {console.log('!(error instanceof Error)') - throw error; - } - if (error.message.includes('Unexpected token')) { - return false; - } - return false; - } - } - - /** - * Function `validatePipelineString` will validate the if the string is a valid pipeline string - * It does not check if the string is fully logically correct, but if it is a string that can be a pipeline string or the string looks completely different. - * - * @param {string} pipelineString the candidate for a pipeline string - * @returns {PipelineString} the same string as input, but validated as valid - * @throws {ParseError} if the string is not a valid pipeline string - * @public exported from `@promptbook/core` - */ - function validatePipelineString(pipelineString) { - if (isValidJsonString(pipelineString)) { - throw new ParseError('Expected a book, but got a JSON string'); - } - // <- TODO: Implement the validation + add tests when the pipeline logic considered as invalid - return pipelineString; - } - /** - * TODO: [🧠][🈴] Where is the best location for this file - */ - - /** - * Prettify the html code - * - * @param content raw html code - * @returns formatted html code - * @private withing the package because of HUGE size of prettier dependency - */ - function prettifyMarkdown(content) { - try { - return prettier.format(content, { - parser: 'markdown', - plugins: [parserHtml__default["default"]], - // TODO: DRY - make some import or auto-copy of .prettierrc - endOfLine: 'lf', - tabWidth: 4, - singleQuote: true, - trailingComma: 'all', - arrowParens: 'always', - printWidth: 120, - htmlWhitespaceSensitivity: 'ignore', - jsxBracketSameLine: false, - bracketSpacing: true, - }); - } - catch (error) { - // TODO: [🟥] Detect browser / node and make it colorfull - console.error('There was an error with prettifying the markdown, using the original as the fallback', { - error: error, - html: content, - }); - return content; - } - } - - /** - * Makes first letter of a string uppercase - * - * @public exported from `@promptbook/utils` - */ - function capitalize(word) { - return word.substring(0, 1).toUpperCase() + word.substring(1); - } - - /** - * Converts promptbook in JSON format to string format - * - * @deprecated TODO: [🥍][🧠] Backup original files in `PipelineJson` same as in Promptbook.studio - * @param pipelineJson Promptbook in JSON format (.bookc) - * @returns Promptbook in string format (.book.md) - * @public exported from `@promptbook/core` - */ - function pipelineJsonToString(pipelineJson) { - var e_1, _a, e_2, _b, e_3, _c, e_4, _d, e_5, _e, e_6, _f; - var title = pipelineJson.title, pipelineUrl = pipelineJson.pipelineUrl, bookVersion = pipelineJson.bookVersion, description = pipelineJson.description, parameters = pipelineJson.parameters, tasks = pipelineJson.tasks; - var pipelineString = "# ".concat(title); - if (description) { - pipelineString += '\n\n'; - pipelineString += description; - } - var commands = []; - if (pipelineUrl) { - commands.push("PIPELINE URL ".concat(pipelineUrl)); - } - if (bookVersion !== "undefined") { - commands.push("BOOK VERSION ".concat(bookVersion)); - } - // TODO: [main] !!5 This increases size of the bundle and is probbably not necessary - pipelineString = prettifyMarkdown(pipelineString); - try { - for (var _g = __values(parameters.filter(function (_a) { - var isInput = _a.isInput; - return isInput; - })), _h = _g.next(); !_h.done; _h = _g.next()) { - var parameter = _h.value; - commands.push("INPUT PARAMETER ".concat(taskParameterJsonToString(parameter))); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_h && !_h.done && (_a = _g.return)) _a.call(_g); - } - finally { if (e_1) throw e_1.error; } - } - try { - for (var _j = __values(parameters.filter(function (_a) { - var isOutput = _a.isOutput; - return isOutput; - })), _k = _j.next(); !_k.done; _k = _j.next()) { - var parameter = _k.value; - commands.push("OUTPUT PARAMETER ".concat(taskParameterJsonToString(parameter))); - } - } - catch (e_2_1) { e_2 = { error: e_2_1 }; } - finally { - try { - if (_k && !_k.done && (_b = _j.return)) _b.call(_j); - } - finally { if (e_2) throw e_2.error; } - } - pipelineString += '\n\n'; - pipelineString += commands.map(function (command) { return "- ".concat(command); }).join('\n'); - try { - for (var tasks_1 = __values(tasks), tasks_1_1 = tasks_1.next(); !tasks_1_1.done; tasks_1_1 = tasks_1.next()) { - var task = tasks_1_1.value; - var - /* Note: Not using:> name, */ - title_1 = task.title, description_1 = task.description, - /* Note: dependentParameterNames, */ - jokers = task.jokerParameterNames, taskType = task.taskType, content = task.content, postprocessing = task.postprocessingFunctionNames, expectations = task.expectations, format = task.format, resultingParameterName = task.resultingParameterName; - pipelineString += '\n\n'; - pipelineString += "## ".concat(title_1); - if (description_1) { - pipelineString += '\n\n'; - pipelineString += description_1; - } - var commands_1 = []; - var contentLanguage = 'text'; - if (taskType === 'PROMPT_TASK') { - var modelRequirements = task.modelRequirements; - var _l = modelRequirements || {}, modelName = _l.modelName, modelVariant = _l.modelVariant; - // Note: Do nothing, it is default - // commands.push(`PROMPT`); - if (modelVariant) { - commands_1.push("MODEL VARIANT ".concat(capitalize(modelVariant))); - } - if (modelName) { - commands_1.push("MODEL NAME `".concat(modelName, "`")); - } - } - else if (taskType === 'SIMPLE_TASK') { - commands_1.push("SIMPLE TEMPLATE"); - // Note: Nothing special here - } - else if (taskType === 'SCRIPT_TASK') { - commands_1.push("SCRIPT"); - if (task.contentLanguage) { - contentLanguage = task.contentLanguage; - } - else { - contentLanguage = ''; - } - } - else if (taskType === 'DIALOG_TASK') { - commands_1.push("DIALOG"); - // Note: Nothing special here - } // <- }else if([🅱] - if (jokers) { - try { - for (var jokers_1 = (e_4 = void 0, __values(jokers)), jokers_1_1 = jokers_1.next(); !jokers_1_1.done; jokers_1_1 = jokers_1.next()) { - var joker = jokers_1_1.value; - commands_1.push("JOKER {".concat(joker, "}")); - } - } - catch (e_4_1) { e_4 = { error: e_4_1 }; } - finally { - try { - if (jokers_1_1 && !jokers_1_1.done && (_d = jokers_1.return)) _d.call(jokers_1); - } - finally { if (e_4) throw e_4.error; } - } - } /* not else */ - if (postprocessing) { - try { - for (var postprocessing_1 = (e_5 = void 0, __values(postprocessing)), postprocessing_1_1 = postprocessing_1.next(); !postprocessing_1_1.done; postprocessing_1_1 = postprocessing_1.next()) { - var postprocessingFunctionName = postprocessing_1_1.value; - commands_1.push("POSTPROCESSING `".concat(postprocessingFunctionName, "`")); - } - } - catch (e_5_1) { e_5 = { error: e_5_1 }; } - finally { - try { - if (postprocessing_1_1 && !postprocessing_1_1.done && (_e = postprocessing_1.return)) _e.call(postprocessing_1); - } - finally { if (e_5) throw e_5.error; } - } - } /* not else */ - if (expectations) { - try { - for (var _m = (e_6 = void 0, __values(Object.entries(expectations))), _o = _m.next(); !_o.done; _o = _m.next()) { - var _p = __read(_o.value, 2), unit = _p[0], _q = _p[1], min = _q.min, max = _q.max; - if (min === max) { - commands_1.push("EXPECT EXACTLY ".concat(min, " ").concat(capitalize(unit + (min > 1 ? 's' : '')))); - } - else { - if (min !== undefined) { - commands_1.push("EXPECT MIN ".concat(min, " ").concat(capitalize(unit + (min > 1 ? 's' : '')))); - } /* not else */ - if (max !== undefined) { - commands_1.push("EXPECT MAX ".concat(max, " ").concat(capitalize(unit + (max > 1 ? 's' : '')))); - } - } - } - } - catch (e_6_1) { e_6 = { error: e_6_1 }; } - finally { - try { - if (_o && !_o.done && (_f = _m.return)) _f.call(_m); - } - finally { if (e_6) throw e_6.error; } - } - } /* not else */ - if (format) { - if (format === 'JSON') { - // TODO: @deprecated remove - commands_1.push("FORMAT JSON"); - } - } /* not else */ - pipelineString += '\n\n'; - pipelineString += commands_1.map(function (command) { return "- ".concat(command); }).join('\n'); - pipelineString += '\n\n'; - pipelineString += '```' + contentLanguage; - pipelineString += '\n'; - pipelineString += spaceTrim__default["default"](content); - // <- TODO: [main] !!3 Escape - // <- TODO: [🧠] Some clear strategy how to spaceTrim the blocks - pipelineString += '\n'; - pipelineString += '```'; - pipelineString += '\n\n'; - pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: [main] !!3 If the parameter here has description, add it and use taskParameterJsonToString - } - } - catch (e_3_1) { e_3 = { error: e_3_1 }; } - finally { - try { - if (tasks_1_1 && !tasks_1_1.done && (_c = tasks_1.return)) _c.call(tasks_1); - } - finally { if (e_3) throw e_3.error; } - } - return validatePipelineString(pipelineString); - } - /** - * @private internal utility of `pipelineJsonToString` - */ - function taskParameterJsonToString(taskParameterJson) { - var name = taskParameterJson.name, description = taskParameterJson.description; - var parameterString = "{".concat(name, "}"); - if (description) { - parameterString = "".concat(parameterString, " ").concat(description); - } - return parameterString; - } - /** - * TODO: [🛋] Implement new features and commands into `pipelineJsonToString` + `taskParameterJsonToString` , use `stringifyCommand` - * TODO: [🧠] Is there a way to auto-detect missing features in pipelineJsonToString - * TODO: [🏛] Maybe make some markdown builder - * TODO: [🏛] Escape all - * TODO: [🧠] Should be in generated .book file GENERATOR_WARNING - */ - - /** - * Orders JSON object by keys - * - * @returns The same type of object as the input re-ordered - * @public exported from `@promptbook/utils` - */ - function orderJson(options) { - var value = options.value, order = options.order; - var orderedValue = __assign(__assign({}, (order === undefined ? {} : Object.fromEntries(order.map(function (key) { return [key, undefined]; })))), value); - return orderedValue; - } - - /** - * Freezes the given object and all its nested objects recursively - * - * Note: `$` is used to indicate that this function is not a pure function - it mutates given object - * Note: This function mutates the object and returns the original (but mutated-deep-freezed) object - * - * @returns The same object as the input, but deeply frozen - * @public exported from `@promptbook/utils` - */ - function $deepFreeze(objectValue) { - var e_1, _a; - if (Array.isArray(objectValue)) { - return Object.freeze(objectValue.map(function (item) { return $deepFreeze(item); })); - } - var propertyNames = Object.getOwnPropertyNames(objectValue); - try { - for (var propertyNames_1 = __values(propertyNames), propertyNames_1_1 = propertyNames_1.next(); !propertyNames_1_1.done; propertyNames_1_1 = propertyNames_1.next()) { - var propertyName = propertyNames_1_1.value; - var value = objectValue[propertyName]; - if (value && typeof value === 'object') { - $deepFreeze(value); - } - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (propertyNames_1_1 && !propertyNames_1_1.done && (_a = propertyNames_1.return)) _a.call(propertyNames_1); - } - finally { if (e_1) throw e_1.error; } - } - Object.freeze(objectValue); - return objectValue; - } - /** - * TODO: [🧠] Is there a way how to meaningfully test this utility - */ - - /** - * Checks if the value is [🚉] serializable as JSON - * If not, throws an UnexpectedError with a rich error message and tracking - * - * - Almost all primitives are serializable BUT: - * - `undefined` is not serializable - * - `NaN` is not serializable - * - Objects and arrays are serializable if all their properties are serializable - * - Functions are not serializable - * - Circular references are not serializable - * - `Date` objects are not serializable - * - `Map` and `Set` objects are not serializable - * - `RegExp` objects are not serializable - * - `Error` objects are not serializable - * - `Symbol` objects are not serializable - * - And much more... - * - * @throws UnexpectedError if the value is not serializable as JSON - * @public exported from `@promptbook/utils` - */ - function checkSerializableAsJson(options) { - var e_1, _a; - var value = options.value, name = options.name, message = options.message; - if (value === undefined) { - throw new UnexpectedError("".concat(name, " is undefined")); - } - else if (value === null) { - return; - } - else if (typeof value === 'boolean') { - return; - } - else if (typeof value === 'number' && !isNaN(value)) { - return; - } - else if (typeof value === 'string') { - return; - } - else if (typeof value === 'symbol') { - throw new UnexpectedError("".concat(name, " is symbol")); - } - else if (typeof value === 'function') { - throw new UnexpectedError("".concat(name, " is function")); - } - else if (typeof value === 'object' && Array.isArray(value)) { - for (var i = 0; i < value.length; i++) { - checkSerializableAsJson({ name: "".concat(name, "[").concat(i, "]"), value: value[i], message: message }); - } - } - else if (typeof value === 'object') { - if (value instanceof Date) { - throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n `".concat(name, "` is Date\n\n Use `string_date_iso8601` instead\n\n Additional message for `").concat(name, "`:\n ").concat(block(message || '(nothing)'), "\n "); })); - } - else if (value instanceof Map) { - throw new UnexpectedError("".concat(name, " is Map")); - } - else if (value instanceof Set) { - throw new UnexpectedError("".concat(name, " is Set")); - } - else if (value instanceof RegExp) { - throw new UnexpectedError("".concat(name, " is RegExp")); - } - else if (value instanceof Error) { - throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n `".concat(name, "` is unserialized Error\n\n Use function `serializeError`\n\n Additional message for `").concat(name, "`:\n ").concat(block(message || '(nothing)'), "\n\n "); })); - } - else { - try { - for (var _b = __values(Object.entries(value)), _c = _b.next(); !_c.done; _c = _b.next()) { - var _d = __read(_c.value, 2), subName = _d[0], subValue = _d[1]; - if (subValue === undefined) { - // Note: undefined in object is serializable - it is just omited - continue; - } - checkSerializableAsJson({ name: "".concat(name, ".").concat(subName), value: subValue, message: message }); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_c && !_c.done && (_a = _b.return)) _a.call(_b); - } - finally { if (e_1) throw e_1.error; } - } - try { - JSON.stringify(value); // <- TODO: [0] - } - catch (error) { - if (!(error instanceof Error)) {console.log('!(error instanceof Error)') - throw error; - } - throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n `".concat(name, "` is not serializable\n\n ").concat(block(error.stack || error.message), "\n\n Additional message for `").concat(name, "`:\n ").concat(block(message || '(nothing)'), "\n "); })); - } - /* + } + return new (MultipleLlmExecutionTools.bind.apply( + MultipleLlmExecutionTools, + __spreadArray([void 0], __read(llmExecutionTools), false), + ))(); + } + /** + * TODO: [👷‍♂️] @@@ Manual about construction of llmTools + */ + + /** + * @@@ + * + * Note: `$` is used to indicate that this interacts with the global scope + * @singleton Only one instance of each register is created per build, but thare can be more @@@ + * @public exported from `@promptbook/core` + */ + var $llmToolsRegister = new $Register("llm_execution_tools_constructors"); + /** + * TODO: [®] DRY Register logic + */ + + /** + * @@@ + * + * Note: `$` is used to indicate that this interacts with the global scope + * @singleton Only one instance of each register is created per build, but thare can be more @@@ + * @public exported from `@promptbook/core` + */ + var $llmToolsMetadataRegister = new $Register("llm_tools_metadata"); + /** + * TODO: [®] DRY Register logic + */ + + /** + * Creates a message with all registered LLM tools + * + * Note: This function is used to create a (error) message when there is no constructor for some LLM provider + * + * @private internal function of `createLlmToolsFromConfiguration` and `$provideLlmToolsFromEnv` + */ + function $registeredLlmToolsMessage() { + var e_1, _a, e_2, _b; + var env; + if ($isRunningInNode()) { + env = process.env; + // <- TODO: [⚛] Some DRY way how to get to `process.env` and pass it into functions - ACRY search for `env` + } else { + env = {}; + } + /** + * Mixes registered LLM tools from $llmToolsMetadataRegister and $llmToolsRegister + */ + var all = []; + var _loop_1 = (title, packageName, className, envVariables) => { + if ( + all.some( + (item) => + item.packageName === packageName && item.className === className, + ) + ) { + return "continue"; + } + all.push({ + title: title, + packageName: packageName, + className: className, + envVariables: envVariables, + }); + }; + try { + for ( + var _c = __values($llmToolsMetadataRegister.list()), _d = _c.next(); + !_d.done; + _d = _c.next() + ) { + var _e = _d.value, + title = _e.title, + packageName = _e.packageName, + className = _e.className, + envVariables = _e.envVariables; + _loop_1(title, packageName, className, envVariables); + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_d && !_d.done && (_a = _c.return)) _a.call(_c); + } finally { + if (e_1) throw e_1.error; + } + } + var _loop_2 = (packageName, className) => { + if ( + all.some( + (item) => + item.packageName === packageName && item.className === className, + ) + ) { + return "continue"; + } + all.push({ packageName: packageName, className: className }); + }; + try { + for ( + var _f = __values($llmToolsRegister.list()), _g = _f.next(); + !_g.done; + _g = _f.next() + ) { + var _h = _g.value, + packageName = _h.packageName, + className = _h.className; + _loop_2(packageName, className); + } + } catch (e_2_1) { + e_2 = { error: e_2_1 }; + } finally { + try { + if (_g && !_g.done && (_b = _f.return)) _b.call(_f); + } finally { + if (e_2) throw e_2.error; + } + } + var metadata = all.map((metadata) => { + var _a, _b; + var isMetadataAviailable = $llmToolsMetadataRegister + .list() + .find((_a) => { + var packageName = _a.packageName, + className = _a.className; + return ( + metadata.packageName === packageName && + metadata.className === className + ); + }); + var isInstalled = $llmToolsRegister.list().find((_a) => { + var packageName = _a.packageName, + className = _a.className; + return ( + metadata.packageName === packageName && + metadata.className === className + ); + }); + var isFullyConfigured = + ((_a = metadata.envVariables) === null || _a === void 0 + ? void 0 + : _a.every( + (envVariableName) => env[envVariableName] !== undefined, + )) || false; + var isPartiallyConfigured = + ((_b = metadata.envVariables) === null || _b === void 0 + ? void 0 + : _b.some( + (envVariableName) => env[envVariableName] !== undefined, + )) || false; + // <- Note: [🗨] + return __assign(__assign({}, metadata), { + isMetadataAviailable: isMetadataAviailable, + isInstalled: isInstalled, + isFullyConfigured: isFullyConfigured, + isPartiallyConfigured: isPartiallyConfigured, + }); + }); + if (metadata.length === 0) { + return "No LLM providers are available."; + } + return spaceTrim__default["default"]((block) => + "\n Relevant environment variables:\n " + .concat( + block( + Object.keys(env) + .filter((envVariableName) => + metadata.some((_a) => { + var envVariables = _a.envVariables; + return envVariables === null || envVariables === void 0 + ? void 0 + : envVariables.includes(envVariableName); + }), + ) + .map((envVariableName) => "- `".concat(envVariableName, "`")) + .join("\n"), + ), + "\n\n Available LLM providers are:\n ", + ) + .concat( + block( + metadata + .map((_a, i) => { + var title = _a.title, + packageName = _a.packageName, + className = _a.className, + envVariables = _a.envVariables, + isMetadataAviailable = _a.isMetadataAviailable, + isInstalled = _a.isInstalled, + isFullyConfigured = _a.isFullyConfigured, + isPartiallyConfigured = _a.isPartiallyConfigured; + var morePieces = []; + if (just(false)); + else if (!isMetadataAviailable && !isInstalled) { + // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found + morePieces.push( + "Not installed and no metadata, looks like a unexpected behavior", + ); + } else if (isMetadataAviailable && !isInstalled) { + // TODO: [�][�] + morePieces.push("Not installed"); + } else if (!isMetadataAviailable && isInstalled) { + morePieces.push( + "No metadata but installed, looks like a unexpected behavior", + ); + } else if (isMetadataAviailable && isInstalled) { + morePieces.push("Installed"); + } else { + morePieces.push( + "unknown state, looks like a unexpected behavior", + ); + } /* not else */ + if (isFullyConfigured) { + morePieces.push("Configured"); + } else if (isPartiallyConfigured) { + morePieces.push( + "Partially confugured, missing ".concat( + envVariables === null || envVariables === void 0 + ? void 0 + : envVariables + .filter( + (envVariable) => env[envVariable] === undefined, + ) + .join(" + "), + ), + ); + } else { + if (envVariables !== null) { + morePieces.push( + "Not configured, to configure set env ".concat( + envVariables === null || envVariables === void 0 + ? void 0 + : envVariables.join(" + "), + ), + ); + } else { + morePieces.push("Not configured"); // <- Note: Can not be configured via environment variables + } + } + var providerMessage = spaceTrim__default["default"]( + "\n " + .concat(i + 1, ") **") + .concat(title, "** `") + .concat(className, "` from `") + .concat( + packageName, + "`\n ", + ) + .concat( + morePieces.join("; "), + "\n ", + ), + ); + if ($isRunningInNode) { + if (isInstalled && isFullyConfigured) { + providerMessage = + colors__default["default"].green(providerMessage); + } else if (isInstalled && isPartiallyConfigured) { + providerMessage = + colors__default["default"].yellow(providerMessage); + } else { + providerMessage = + colors__default["default"].gray(providerMessage); + } + } + return providerMessage; + }) + .join("\n"), + ), + "\n ", + ), + ); + } + /** + * TODO: [®] DRY Register logic + * TODO: [🧠][⚛] Maybe pass env as argument + */ + + /** + * @@@ + * + * Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools` + * + * @returns @@@ + * @public exported from `@promptbook/core` + */ + function createLlmToolsFromConfiguration(configuration, options) { + if (options === void 0) { + options = {}; + } + var _a = options.isVerbose, + isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, + userId = options.userId; + var llmTools = configuration.map((llmConfiguration) => { + var registeredItem = $llmToolsRegister.list().find((_a) => { + var packageName = _a.packageName, + className = _a.className; + return ( + llmConfiguration.packageName === packageName && + llmConfiguration.className === className + ); + }); + if (registeredItem === undefined) { + throw new Error( + spaceTrim__default["default"]((block) => + "\n There is no constructor for LLM provider `" + .concat(llmConfiguration.className, "` from `") + .concat( + llmConfiguration.packageName, + "`\n\n You have probably forgotten install and import the provider package.\n To fix this issue, you can:\n\n Install:\n\n > npm install ", + ) + .concat( + llmConfiguration.packageName, + "\n\n And import:\n\n > import '", + ) + .concat( + llmConfiguration.packageName, + "';\n\n\n ", + ) + .concat( + block($registeredLlmToolsMessage()), + "\n ", + ), + ), + ); + } + return registeredItem( + __assign( + { isVerbose: isVerbose, userId: userId }, + llmConfiguration.options, + ), + ); + }); + return joinLlmExecutionTools.apply( + void 0, + __spreadArray([], __read(llmTools), false), + ); + } + /** + * TODO: [🎌] Together with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient + * TODO: [🧠][🎌] Dynamically install required providers + * TODO: @@@ write discussion about this - wizzard + * TODO: [🧠][🍛] Which name is better `createLlmToolsFromConfig` or `createLlmToolsFromConfiguration`? + * TODO: [🧠] Is there some meaningfull way how to test this util + * TODO: This should be maybe not under `_common` but under `utils` + * TODO: [®] DRY Register logic + */ + + var PipelineCollection = [ + { + title: "Prepare Knowledge from Markdown", + pipelineUrl: + "https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book.md", + formfactorName: "GENERIC", + parameters: [ + { + name: "knowledgeContent", + description: "Markdown document content", + isInput: true, + isOutput: false, + }, + { + name: "knowledgePieces", + description: "The knowledge JSON object", + isInput: false, + isOutput: true, + }, + ], + tasks: [ + { + taskType: "PROMPT_TASK", + name: "knowledge", + title: "Knowledge", + content: + "You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}", + resultingParameterName: "knowledgePieces", + dependentParameterNames: ["knowledgeContent"], + }, + ], + personas: [], + preparations: [], + knowledgeSources: [], + knowledgePieces: [], + sources: [ + { + type: "BOOK", + path: null, + content: + "# Prepare Knowledge from Markdown\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book`\n- INPUT PARAMETER `{knowledgeContent}` Markdown document content\n- OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n", + }, + ], + sourceFile: "./books/prepare-knowledge-from-markdown.book.md", + }, + { + title: "Prepare Keywords", + pipelineUrl: + "https://promptbook.studio/promptbook/prepare-knowledge-keywords.book.md", + formfactorName: "GENERIC", + parameters: [ + { + name: "knowledgePieceContent", + description: "The content", + isInput: true, + isOutput: false, + }, + { + name: "keywords", + description: "Keywords separated by comma", + isInput: false, + isOutput: true, + }, + ], + tasks: [ + { + taskType: "PROMPT_TASK", + name: "knowledge", + title: "Knowledge", + content: + "You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}", + resultingParameterName: "keywords", + dependentParameterNames: ["knowledgePieceContent"], + }, + ], + personas: [], + preparations: [], + knowledgeSources: [], + knowledgePieces: [], + sources: [ + { + type: "BOOK", + path: null, + content: + "# Prepare Keywords\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n", + }, + ], + sourceFile: "./books/prepare-knowledge-keywords.book.md", + }, + { + title: "Prepare Knowledge-piece Title", + pipelineUrl: + "https://promptbook.studio/promptbook/prepare-knowledge-title.book.md", + formfactorName: "GENERIC", + parameters: [ + { + name: "knowledgePieceContent", + description: "The content", + isInput: true, + isOutput: false, + }, + { + name: "title", + description: "The title of the document", + isInput: false, + isOutput: true, + }, + ], + tasks: [ + { + taskType: "PROMPT_TASK", + name: "knowledge", + title: "Knowledge", + content: + "You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}", + resultingParameterName: "title", + expectations: { words: { min: 1, max: 8 } }, + dependentParameterNames: ["knowledgePieceContent"], + }, + ], + personas: [], + preparations: [], + knowledgeSources: [], + knowledgePieces: [], + sources: [ + { + type: "BOOK", + path: null, + content: + "# Prepare Knowledge-piece Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n- EXPECT MIN 1 WORD\n- EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n", + }, + ], + sourceFile: "./books/prepare-knowledge-title.book.md", + }, + { + title: "Prepare Persona", + pipelineUrl: + "https://promptbook.studio/promptbook/prepare-persona.book.md", + formfactorName: "GENERIC", + parameters: [ + { + name: "availableModelNames", + description: "List of available model names separated by comma (,)", + isInput: true, + isOutput: false, + }, + { + name: "personaDescription", + description: "Description of the persona", + isInput: true, + isOutput: false, + }, + { + name: "modelRequirements", + description: "Specific requirements for the model", + isInput: false, + isOutput: true, + }, + ], + tasks: [ + { + taskType: "PROMPT_TASK", + name: "make-model-requirements", + title: "Make modelRequirements", + content: + 'You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n```json\n{\n"modelName": "gpt-4o",\n"systemMessage": "You are experienced AI engineer and helpfull assistant.",\n"temperature": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model\'s behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}', + resultingParameterName: "modelRequirements", + format: "JSON", + dependentParameterNames: [ + "availableModelNames", + "personaDescription", + ], + }, + ], + personas: [], + preparations: [], + knowledgeSources: [], + knowledgePieces: [], + sources: [ + { + type: "BOOK", + path: null, + content: + '# Prepare Persona\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book`\n- INPUT PARAMETER `{availableModelNames}` List of available model names separated by comma (,)\n- INPUT PARAMETER `{personaDescription}` Description of the persona\n- OUTPUT PARAMETER `{modelRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n- FORMAT JSON\n\n```markdown\nYou are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n\\`\\`\\`json\n{\n"modelName": "gpt-4o",\n"systemMessage": "You are experienced AI engineer and helpfull assistant.",\n"temperature": 0.7\n}\n\\`\\`\\`\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model\'s behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelRequirements}`\n', + }, + ], + sourceFile: "./books/prepare-persona.book.md", + }, + { + title: "Prepare Title", + pipelineUrl: + "https://promptbook.studio/promptbook/prepare-title.book.md", + formfactorName: "GENERIC", + parameters: [ + { + name: "book", + description: "The book to prepare the title for", + isInput: true, + isOutput: false, + }, + { + name: "title", + description: "Best title for the book", + isInput: false, + isOutput: true, + }, + ], + tasks: [ + { + taskType: "PROMPT_TASK", + name: "make-title", + title: "Make title", + content: + 'Make best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not "✍ Convert Knowledge-piece to title" but "✍ Title"_\n\n## The workflow\n\n> {book}', + resultingParameterName: "title", + expectations: { + words: { min: 1, max: 8 }, + lines: { min: 1, max: 1 }, + }, + dependentParameterNames: ["book"], + }, + ], + personas: [], + preparations: [], + knowledgeSources: [], + knowledgePieces: [], + sources: [ + { + type: "BOOK", + path: null, + content: + '# Prepare Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book`\n- INPUT PARAMETER `{book}` The book to prepare the title for\n- OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n- EXPECT MIN 1 Word\n- EXPECT MAX 8 Words\n- EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not "✍ Convert Knowledge-piece to title" but "✍ Title"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n', + }, + ], + sourceFile: "./books/prepare-title.book.md", + }, + ]; + + /** + * Function isValidJsonString will tell you if the string is valid JSON or not + * + * @public exported from `@promptbook/utils` + */ + function isValidJsonString(value /* <- [👨‍⚖️] */) { + try { + JSON.parse(value); + return true; + } catch (error) { + if (!(error instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error; + } + if (error.message.includes("Unexpected token")) { + return false; + } + return false; + } + } + + /** + * Function `validatePipelineString` will validate the if the string is a valid pipeline string + * It does not check if the string is fully logically correct, but if it is a string that can be a pipeline string or the string looks completely different. + * + * @param {string} pipelineString the candidate for a pipeline string + * @returns {PipelineString} the same string as input, but validated as valid + * @throws {ParseError} if the string is not a valid pipeline string + * @public exported from `@promptbook/core` + */ + function validatePipelineString(pipelineString) { + if (isValidJsonString(pipelineString)) { + throw new ParseError("Expected a book, but got a JSON string"); + } + // <- TODO: Implement the validation + add tests when the pipeline logic considered as invalid + return pipelineString; + } + /** + * TODO: [🧠][🈴] Where is the best location for this file + */ + + /** + * Prettify the html code + * + * @param content raw html code + * @returns formatted html code + * @private withing the package because of HUGE size of prettier dependency + */ + function prettifyMarkdown(content) { + try { + return prettier.format(content, { + parser: "markdown", + plugins: [parserHtml__default["default"]], + // TODO: DRY - make some import or auto-copy of .prettierrc + endOfLine: "lf", + tabWidth: 4, + singleQuote: true, + trailingComma: "all", + arrowParens: "always", + printWidth: 120, + htmlWhitespaceSensitivity: "ignore", + jsxBracketSameLine: false, + bracketSpacing: true, + }); + } catch (error) { + // TODO: [🟥] Detect browser / node and make it colorfull + console.error( + "There was an error with prettifying the markdown, using the original as the fallback", + { + error: error, + html: content, + }, + ); + return content; + } + } + + /** + * Makes first letter of a string uppercase + * + * @public exported from `@promptbook/utils` + */ + function capitalize(word) { + return word.substring(0, 1).toUpperCase() + word.substring(1); + } + + /** + * Converts promptbook in JSON format to string format + * + * @deprecated TODO: [🥍][🧠] Backup original files in `PipelineJson` same as in Promptbook.studio + * @param pipelineJson Promptbook in JSON format (.bookc) + * @returns Promptbook in string format (.book.md) + * @public exported from `@promptbook/core` + */ + function pipelineJsonToString(pipelineJson) { + var e_1, _a, e_2, _b, e_3, _c, e_4, _d, e_5, _e, e_6, _f; + var title = pipelineJson.title, + pipelineUrl = pipelineJson.pipelineUrl, + bookVersion = pipelineJson.bookVersion, + description = pipelineJson.description, + parameters = pipelineJson.parameters, + tasks = pipelineJson.tasks; + var pipelineString = "# ".concat(title); + if (description) { + pipelineString += "\n\n"; + pipelineString += description; + } + var commands = []; + if (pipelineUrl) { + commands.push("PIPELINE URL ".concat(pipelineUrl)); + } + if (bookVersion !== "undefined") { + commands.push("BOOK VERSION ".concat(bookVersion)); + } + // TODO: [main] !!5 This increases size of the bundle and is probbably not necessary + pipelineString = prettifyMarkdown(pipelineString); + try { + for ( + var _g = __values( + parameters.filter((_a) => { + var isInput = _a.isInput; + return isInput; + }), + ), + _h = _g.next(); + !_h.done; + _h = _g.next() + ) { + var parameter = _h.value; + commands.push( + "INPUT PARAMETER ".concat(taskParameterJsonToString(parameter)), + ); + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_h && !_h.done && (_a = _g.return)) _a.call(_g); + } finally { + if (e_1) throw e_1.error; + } + } + try { + for ( + var _j = __values( + parameters.filter((_a) => { + var isOutput = _a.isOutput; + return isOutput; + }), + ), + _k = _j.next(); + !_k.done; + _k = _j.next() + ) { + var parameter = _k.value; + commands.push( + "OUTPUT PARAMETER ".concat(taskParameterJsonToString(parameter)), + ); + } + } catch (e_2_1) { + e_2 = { error: e_2_1 }; + } finally { + try { + if (_k && !_k.done && (_b = _j.return)) _b.call(_j); + } finally { + if (e_2) throw e_2.error; + } + } + pipelineString += "\n\n"; + pipelineString += commands + .map((command) => "- ".concat(command)) + .join("\n"); + try { + for ( + var tasks_1 = __values(tasks), tasks_1_1 = tasks_1.next(); + !tasks_1_1.done; + tasks_1_1 = tasks_1.next() + ) { + var task = tasks_1_1.value; + var /* Note: Not using:> name, */ + title_1 = task.title, + description_1 = task.description, + /* Note: dependentParameterNames, */ + jokers = task.jokerParameterNames, + taskType = task.taskType, + content = task.content, + postprocessing = task.postprocessingFunctionNames, + expectations = task.expectations, + format = task.format, + resultingParameterName = task.resultingParameterName; + pipelineString += "\n\n"; + pipelineString += "## ".concat(title_1); + if (description_1) { + pipelineString += "\n\n"; + pipelineString += description_1; + } + var commands_1 = []; + var contentLanguage = "text"; + if (taskType === "PROMPT_TASK") { + var modelRequirements = task.modelRequirements; + var _l = modelRequirements || {}, + modelName = _l.modelName, + modelVariant = _l.modelVariant; + // Note: Do nothing, it is default + // commands.push(`PROMPT`); + if (modelVariant) { + commands_1.push( + "MODEL VARIANT ".concat(capitalize(modelVariant)), + ); + } + if (modelName) { + commands_1.push("MODEL NAME `".concat(modelName, "`")); + } + } else if (taskType === "SIMPLE_TASK") { + commands_1.push("SIMPLE TEMPLATE"); + // Note: Nothing special here + } else if (taskType === "SCRIPT_TASK") { + commands_1.push("SCRIPT"); + if (task.contentLanguage) { + contentLanguage = task.contentLanguage; + } else { + contentLanguage = ""; + } + } else if (taskType === "DIALOG_TASK") { + commands_1.push("DIALOG"); + // Note: Nothing special here + } // <- }else if([🅱] + if (jokers) { + try { + for ( + var jokers_1 = ((e_4 = void 0), __values(jokers)), + jokers_1_1 = jokers_1.next(); + !jokers_1_1.done; + jokers_1_1 = jokers_1.next() + ) { + var joker = jokers_1_1.value; + commands_1.push("JOKER {".concat(joker, "}")); + } + } catch (e_4_1) { + e_4 = { error: e_4_1 }; + } finally { + try { + if (jokers_1_1 && !jokers_1_1.done && (_d = jokers_1.return)) + _d.call(jokers_1); + } finally { + if (e_4) throw e_4.error; + } + } + } /* not else */ + if (postprocessing) { + try { + for ( + var postprocessing_1 = + ((e_5 = void 0), __values(postprocessing)), + postprocessing_1_1 = postprocessing_1.next(); + !postprocessing_1_1.done; + postprocessing_1_1 = postprocessing_1.next() + ) { + var postprocessingFunctionName = postprocessing_1_1.value; + commands_1.push( + "POSTPROCESSING `".concat(postprocessingFunctionName, "`"), + ); + } + } catch (e_5_1) { + e_5 = { error: e_5_1 }; + } finally { + try { + if ( + postprocessing_1_1 && + !postprocessing_1_1.done && + (_e = postprocessing_1.return) + ) + _e.call(postprocessing_1); + } finally { + if (e_5) throw e_5.error; + } + } + } /* not else */ + if (expectations) { + try { + for ( + var _m = + ((e_6 = void 0), __values(Object.entries(expectations))), + _o = _m.next(); + !_o.done; + _o = _m.next() + ) { + var _p = __read(_o.value, 2), + unit = _p[0], + _q = _p[1], + min = _q.min, + max = _q.max; + if (min === max) { + commands_1.push( + "EXPECT EXACTLY " + .concat(min, " ") + .concat(capitalize(unit + (min > 1 ? "s" : ""))), + ); + } else { + if (min !== undefined) { + commands_1.push( + "EXPECT MIN " + .concat(min, " ") + .concat(capitalize(unit + (min > 1 ? "s" : ""))), + ); + } /* not else */ + if (max !== undefined) { + commands_1.push( + "EXPECT MAX " + .concat(max, " ") + .concat(capitalize(unit + (max > 1 ? "s" : ""))), + ); + } + } + } + } catch (e_6_1) { + e_6 = { error: e_6_1 }; + } finally { + try { + if (_o && !_o.done && (_f = _m.return)) _f.call(_m); + } finally { + if (e_6) throw e_6.error; + } + } + } /* not else */ + if (format) { + if (format === "JSON") { + // TODO: @deprecated remove + commands_1.push("FORMAT JSON"); + } + } /* not else */ + pipelineString += "\n\n"; + pipelineString += commands_1 + .map((command) => "- ".concat(command)) + .join("\n"); + pipelineString += "\n\n"; + pipelineString += "```" + contentLanguage; + pipelineString += "\n"; + pipelineString += spaceTrim__default["default"](content); + // <- TODO: [main] !!3 Escape + // <- TODO: [🧠] Some clear strategy how to spaceTrim the blocks + pipelineString += "\n"; + pipelineString += "```"; + pipelineString += "\n\n"; + pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: [main] !!3 If the parameter here has description, add it and use taskParameterJsonToString + } + } catch (e_3_1) { + e_3 = { error: e_3_1 }; + } finally { + try { + if (tasks_1_1 && !tasks_1_1.done && (_c = tasks_1.return)) + _c.call(tasks_1); + } finally { + if (e_3) throw e_3.error; + } + } + return validatePipelineString(pipelineString); + } + /** + * @private internal utility of `pipelineJsonToString` + */ + function taskParameterJsonToString(taskParameterJson) { + var name = taskParameterJson.name, + description = taskParameterJson.description; + var parameterString = "{".concat(name, "}"); + if (description) { + parameterString = "".concat(parameterString, " ").concat(description); + } + return parameterString; + } + /** + * TODO: [🛋] Implement new features and commands into `pipelineJsonToString` + `taskParameterJsonToString` , use `stringifyCommand` + * TODO: [🧠] Is there a way to auto-detect missing features in pipelineJsonToString + * TODO: [🏛] Maybe make some markdown builder + * TODO: [🏛] Escape all + * TODO: [🧠] Should be in generated .book file GENERATOR_WARNING + */ + + /** + * Orders JSON object by keys + * + * @returns The same type of object as the input re-ordered + * @public exported from `@promptbook/utils` + */ + function orderJson(options) { + var value = options.value, + order = options.order; + var orderedValue = __assign( + __assign( + {}, + order === undefined + ? {} + : Object.fromEntries(order.map((key) => [key, undefined])), + ), + value, + ); + return orderedValue; + } + + /** + * Freezes the given object and all its nested objects recursively + * + * Note: `$` is used to indicate that this function is not a pure function - it mutates given object + * Note: This function mutates the object and returns the original (but mutated-deep-freezed) object + * + * @returns The same object as the input, but deeply frozen + * @public exported from `@promptbook/utils` + */ + function $deepFreeze(objectValue) { + var e_1, _a; + if (Array.isArray(objectValue)) { + return Object.freeze(objectValue.map((item) => $deepFreeze(item))); + } + var propertyNames = Object.getOwnPropertyNames(objectValue); + try { + for ( + var propertyNames_1 = __values(propertyNames), + propertyNames_1_1 = propertyNames_1.next(); + !propertyNames_1_1.done; + propertyNames_1_1 = propertyNames_1.next() + ) { + var propertyName = propertyNames_1_1.value; + var value = objectValue[propertyName]; + if (value && typeof value === "object") { + $deepFreeze(value); + } + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if ( + propertyNames_1_1 && + !propertyNames_1_1.done && + (_a = propertyNames_1.return) + ) + _a.call(propertyNames_1); + } finally { + if (e_1) throw e_1.error; + } + } + Object.freeze(objectValue); + return objectValue; + } + /** + * TODO: [🧠] Is there a way how to meaningfully test this utility + */ + + /** + * Checks if the value is [🚉] serializable as JSON + * If not, throws an UnexpectedError with a rich error message and tracking + * + * - Almost all primitives are serializable BUT: + * - `undefined` is not serializable + * - `NaN` is not serializable + * - Objects and arrays are serializable if all their properties are serializable + * - Functions are not serializable + * - Circular references are not serializable + * - `Date` objects are not serializable + * - `Map` and `Set` objects are not serializable + * - `RegExp` objects are not serializable + * - `Error` objects are not serializable + * - `Symbol` objects are not serializable + * - And much more... + * + * @throws UnexpectedError if the value is not serializable as JSON + * @public exported from `@promptbook/utils` + */ + function checkSerializableAsJson(options) { + var e_1, _a; + var value = options.value, + name = options.name, + message = options.message; + if (value === undefined) { + throw new UnexpectedError("".concat(name, " is undefined")); + } else if (value === null) { + return; + } else if (typeof value === "boolean") { + return; + } else if (typeof value === "number" && !isNaN(value)) { + return; + } else if (typeof value === "string") { + return; + } else if (typeof value === "symbol") { + throw new UnexpectedError("".concat(name, " is symbol")); + } else if (typeof value === "function") { + throw new UnexpectedError("".concat(name, " is function")); + } else if (typeof value === "object" && Array.isArray(value)) { + for (var i = 0; i < value.length; i++) { + checkSerializableAsJson({ + name: "".concat(name, "[").concat(i, "]"), + value: value[i], + message: message, + }); + } + } else if (typeof value === "object") { + if (value instanceof Date) { + throw new UnexpectedError( + spaceTrim__default["default"]((block) => + "\n `" + .concat( + name, + "` is Date\n\n Use `string_date_iso8601` instead\n\n Additional message for `", + ) + .concat(name, "`:\n ") + .concat( + block(message || "(nothing)"), + "\n ", + ), + ), + ); + } else if (value instanceof Map) { + throw new UnexpectedError("".concat(name, " is Map")); + } else if (value instanceof Set) { + throw new UnexpectedError("".concat(name, " is Set")); + } else if (value instanceof RegExp) { + throw new UnexpectedError("".concat(name, " is RegExp")); + } else if (value instanceof Error) { + throw new UnexpectedError( + spaceTrim__default["default"]((block) => + "\n `" + .concat( + name, + "` is unserialized Error\n\n Use function `serializeError`\n\n Additional message for `", + ) + .concat(name, "`:\n ") + .concat( + block(message || "(nothing)"), + "\n\n ", + ), + ), + ); + } else { + try { + for ( + var _b = __values(Object.entries(value)), _c = _b.next(); + !_c.done; + _c = _b.next() + ) { + var _d = __read(_c.value, 2), + subName = _d[0], + subValue = _d[1]; + if (subValue === undefined) { + // Note: undefined in object is serializable - it is just omited + continue; + } + checkSerializableAsJson({ + name: "".concat(name, ".").concat(subName), + value: subValue, + message: message, + }); + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_c && !_c.done && (_a = _b.return)) _a.call(_b); + } finally { + if (e_1) throw e_1.error; + } + } + try { + JSON.stringify(value); // <- TODO: [0] + } catch (error) { + if (!(error instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error; + } + throw new UnexpectedError( + spaceTrim__default["default"]((block) => + "\n `" + .concat( + name, + "` is not serializable\n\n ", + ) + .concat( + block(error.stack || error.message), + "\n\n Additional message for `", + ) + .concat(name, "`:\n ") + .concat( + block(message || "(nothing)"), + "\n ", + ), + ), + ); + } + /* TODO: [0] Is there some more elegant way to check circular references? const seen = new Set(); const stack = [{ value }]; @@ -2148,27 +3096,36 @@ at step (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\node_modules\\node_modu } } */ - return; - } - } - else { - throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n `".concat(name, "` is unknown type\n\n Additional message for `").concat(name, "`:\n ").concat(block(message || '(nothing)'), "\n "); })); - } - } - /** - * TODO: Can be return type more type-safe? like `asserts options.value is JsonValue` - * TODO: [🧠][main] !!3 In-memory cache of same values to prevent multiple checks - * Note: [🐠] This is how `checkSerializableAsJson` + `isSerializableAsJson` together can just retun true/false or rich error message - */ - - /** - * @@@ - * - * @public exported from `@promptbook/utils` - */ - function deepClone(objectValue) { - return JSON.parse(JSON.stringify(objectValue)); - /* + return; + } + } else { + throw new UnexpectedError( + spaceTrim__default["default"]((block) => + "\n `" + .concat( + name, + "` is unknown type\n\n Additional message for `", + ) + .concat(name, "`:\n ") + .concat(block(message || "(nothing)"), "\n "), + ), + ); + } + } + /** + * TODO: Can be return type more type-safe? like `asserts options.value is JsonValue` + * TODO: [🧠][main] !!3 In-memory cache of same values to prevent multiple checks + * Note: [🐠] This is how `checkSerializableAsJson` + `isSerializableAsJson` together can just retun true/false or rich error message + */ + + /** + * @@@ + * + * @public exported from `@promptbook/utils` + */ + function deepClone(objectValue) { + return JSON.parse(JSON.stringify(objectValue)); + /* TODO: [🧠] Is there a better implementation? > const propertyNames = Object.getOwnPropertyNames(objectValue); > for (const propertyName of propertyNames) { @@ -2179,342 +3136,451 @@ at step (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\node_modules\\node_modu > } > return Object.assign({}, objectValue); */ - } - /** - * TODO: [🧠] Is there a way how to meaningfully test this utility - */ - - /** - * Utility to export a JSON object from a function - * - * 1) Checks if the value is serializable as JSON - * 2) Makes a deep clone of the object - * 2) Orders the object properties - * 2) Deeply freezes the cloned object - * - * Note: This function does not mutates the given object - * - * @returns The same type of object as the input but read-only and re-ordered - * @public exported from `@promptbook/utils` - */ - function exportJson(options) { - var name = options.name, value = options.value, order = options.order, message = options.message; - checkSerializableAsJson({ name: name, value: value, message: message }); - var orderedValue = - // TODO: Fix error "Type instantiation is excessively deep and possibly infinite." - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - order === undefined - ? deepClone(value) - : orderJson({ - value: value, - // <- Note: checkSerializableAsJson asserts that the value is serializable as JSON - order: order, - }); - $deepFreeze(orderedValue); - return orderedValue; - } - /** - * TODO: [🧠] Is there a way how to meaningfully test this utility - */ - - /** - * Order of keys in the pipeline JSON - * - * @public exported from `@promptbook/core` - */ - var ORDER_OF_PIPELINE_JSON = [ - // Note: [🍙] In this order will be pipeline serialized - 'title', - 'pipelineUrl', - 'bookVersion', - 'description', - 'formfactorName', - 'parameters', - 'tasks', - 'personas', - 'preparations', - 'knowledgeSources', - 'knowledgePieces', - 'sources', // <- TODO: [🧠] Where should the `sources` be - ]; - /** - * Nonce which is used for replacing things in strings - * - * @private within the repository - */ - var REPLACING_NONCE = 'u$k42k%!V2zo34w7Fu#@QUHYPW'; - /** - * @@@ - * - * @private within the repository - */ - var RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE; - /** - * @@@ - * - * @private within the repository - */ - var RESERVED_PARAMETER_RESTRICTED = 'RESTRICTED-' + REPLACING_NONCE; - /** - * The names of the parameters that are reserved for special purposes - * - * @public exported from `@promptbook/core` - */ - var RESERVED_PARAMETER_NAMES = exportJson({ - name: 'RESERVED_PARAMETER_NAMES', - message: "The names of the parameters that are reserved for special purposes", - value: [ - 'content', - 'context', - 'knowledge', - 'examples', - 'modelName', - 'currentDate', - // <- TODO: list here all command names - // <- TODO: Add more like 'date', 'modelName',... - // <- TODO: Add [emoji] + instructions ACRY when adding new reserved parameter - ], - }); - /** - * Note: [💞] Ignore a discrepancy between file name and entity name - */ - - /** - * Tests if given string is valid semantic version - * - * Note: There are two simmilar functions: - * - `isValidSemanticVersion` which tests any semantic version - * - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions - * - * @public exported from `@promptbook/utils` - */ - function isValidSemanticVersion(version) { - if (typeof version !== 'string') { - return false; - } - if (version.startsWith('0.0.0')) { - return false; - } - return /^\d+\.\d+\.\d+(-\d+)?$/i.test(version); - } - - /** - * Tests if given string is valid promptbook version - * It looks into list of known promptbook versions. - * - * @see https://www.npmjs.com/package/promptbook?activeTab=versions - * Note: When you are using for example promptbook 2.0.0 and there already is promptbook 3.0.0 it don`t know about it. - * Note: There are two simmilar functions: - * - `isValidSemanticVersion` which tests any semantic version - * - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions - * - * @public exported from `@promptbook/utils` - */ - function isValidPromptbookVersion(version) { - if (!isValidSemanticVersion(version)) { - return false; - } - if ( /* version === '1.0.0' || */version === '2.0.0' || version === '3.0.0') { - return false; - } - // <- TODO: [main] !!3 Check isValidPromptbookVersion against PROMPTBOOK_ENGINE_VERSIONS - return true; - } - - /** - * Checks if an URL is reserved for private networks or localhost. - * - * Note: There are two simmilar functions: - * - `isUrlOnPrivateNetwork` which tests full URL - * - `isHostnameOnPrivateNetwork` *(this one)* which tests just hostname - * - * @public exported from `@promptbook/utils` - */ - function isHostnameOnPrivateNetwork(hostname) { - if (hostname === 'example.com' || - hostname === 'localhost' || - hostname.endsWith('.localhost') || - hostname.endsWith('.local') || - hostname.endsWith('.test') || - hostname === '127.0.0.1' || - hostname === '::1') { - return true; - } - if (hostname.includes(':')) { - // IPv6 - var ipParts = hostname.split(':'); - return ipParts[0] === 'fc00' || ipParts[0] === 'fd00' || ipParts[0] === 'fe80'; - } - else { - // IPv4 - var ipParts = hostname.split('.').map(function (part) { return Number.parseInt(part, 10); }); - return (ipParts[0] === 10 || - (ipParts[0] === 172 && ipParts[1] >= 16 && ipParts[1] <= 31) || - (ipParts[0] === 192 && ipParts[1] === 168)); - } - } - - /** - * Checks if an IP address or hostname is reserved for private networks or localhost. - * - * Note: There are two simmilar functions: - * - `isUrlOnPrivateNetwork` *(this one)* which tests full URL - * - `isHostnameOnPrivateNetwork` which tests just hostname - * - * @param {string} ipAddress - The IP address to check. - * @returns {boolean} Returns true if the IP address is reserved for private networks or localhost, otherwise false. - * @public exported from `@promptbook/utils` - */ - function isUrlOnPrivateNetwork(url) { - if (typeof url === 'string') { - url = new URL(url); - } - return isHostnameOnPrivateNetwork(url.hostname); - } - - /** - * Tests if given string is valid URL. - * - * Note: Dataurl are considered perfectly valid. - * Note: There are two simmilar functions: - * - `isValidUrl` which tests any URL - * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL - * - * @public exported from `@promptbook/utils` - */ - function isValidUrl(url) { - if (typeof url !== 'string') { - return false; - } - try { - if (url.startsWith('blob:')) { - url = url.replace(/^blob:/, ''); - } - var urlObject = new URL(url /* because fail is handled */); - if (!['http:', 'https:', 'data:'].includes(urlObject.protocol)) { - return false; - } - return true; - } - catch (error) { - return false; - } - } - - /** - * Tests if given string is valid pipeline URL URL. - * - * Note: There are two simmilar functions: - * - `isValidUrl` which tests any URL - * - `isValidPipelineUrl` *(this one)* which tests just pipeline URL - * - * @public exported from `@promptbook/utils` - */ - function isValidPipelineUrl(url) { - if (!isValidUrl(url)) { - return false; - } - if (!url.startsWith('https://')) { - return false; - } - if (url.includes('#')) { - // TODO: [🐠] - return false; - } - if (isUrlOnPrivateNetwork(url)) { - return false; - } - return true; - } - /** - * TODO: [🐠] Maybe more info why the URL is invalid - */ - - /** - * Validates PipelineJson if it is logically valid - * - * It checks: - * - if it has correct parameters dependency - * - * It does NOT check: - * - if it is valid json - * - if it is meaningful - * - * @param pipeline valid or invalid PipelineJson - * @returns the same pipeline if it is logically valid - * @throws {PipelineLogicError} on logical error in the pipeline - * @public exported from `@promptbook/core` - */ - function validatePipeline(pipeline) { - if (IS_PIPELINE_LOGIC_VALIDATED) { - validatePipeline_InnerFunction(pipeline); - } - else { - try { - validatePipeline_InnerFunction(pipeline); - } - catch (error) { - if (!(error instanceof PipelineLogicError)) { - throw error; - } - console.error(spaceTrim.spaceTrim(function (block) { return "\n Pipeline is not valid but logic errors are temporarily disabled via `IS_PIPELINE_LOGIC_VALIDATED`\n\n ".concat(block(error.message), "\n "); })); - } - } - return pipeline; - } - /** - * @private internal function for `validatePipeline` - */ - function validatePipeline_InnerFunction(pipeline) { - // TODO: [🧠] Maybe test if promptbook is a promise and make specific error case for that - var e_1, _a, e_2, _b, e_3, _c; - var pipelineIdentification = (function () { - // Note: This is a 😐 implementation of [🚞] - var _ = []; - if (pipeline.sourceFile !== undefined) { - _.push("File: ".concat(pipeline.sourceFile)); - } - if (pipeline.pipelineUrl !== undefined) { - _.push("Url: ".concat(pipeline.pipelineUrl)); - } - return _.join('\n'); - })(); - if (pipeline.pipelineUrl !== undefined && !isValidPipelineUrl(pipeline.pipelineUrl)) { - // <- Note: [🚲] - throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Invalid promptbook URL \"".concat(pipeline.pipelineUrl, "\"\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - if (pipeline.bookVersion !== undefined && !isValidPromptbookVersion(pipeline.bookVersion)) { - // <- Note: [🚲] - throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Invalid Promptbook Version \"".concat(pipeline.bookVersion, "\"\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking - if (!Array.isArray(pipeline.parameters)) { - // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError - throw new ParseError(spaceTrim.spaceTrim(function (block) { return "\n Pipeline is valid JSON but with wrong structure\n\n `PipelineJson.parameters` expected to be an array, but got ".concat(typeof pipeline.parameters, "\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking - if (!Array.isArray(pipeline.tasks)) { - // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError - throw new ParseError(spaceTrim.spaceTrim(function (block) { return "\n Pipeline is valid JSON but with wrong structure\n\n `PipelineJson.tasks` expected to be an array, but got ".concat(typeof pipeline.tasks, "\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - var _loop_1 = function (parameter) { - if (parameter.isInput && parameter.isOutput) { - throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n\n Parameter `{".concat(parameter.name, "}` can not be both input and output\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - // Note: Testing that parameter is either intermediate or output BUT not created and unused - if (!parameter.isInput && - !parameter.isOutput && - !pipeline.tasks.some(function (task) { return task.dependentParameterNames.includes(parameter.name); })) { - throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Parameter `{".concat(parameter.name, "}` is created but not used\n\n You can declare {").concat(parameter.name, "} as output parameter by adding in the header:\n - OUTPUT PARAMETER `{").concat(parameter.name, "}` ").concat(parameter.description || '', "\n\n ").concat(block(pipelineIdentification), "\n\n "); })); - } - // Note: Testing that parameter is either input or result of some task - if (!parameter.isInput && !pipeline.tasks.some(function (task) { return task.resultingParameterName === parameter.name; })) { - throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Parameter `{".concat(parameter.name, "}` is declared but not defined\n\n You can do one of these:\n 1) Remove declaration of `{").concat(parameter.name, "}`\n 2) Add task that results in `-> {").concat(parameter.name, "}`\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - }; - try { - /* + } + /** + * TODO: [🧠] Is there a way how to meaningfully test this utility + */ + + /** + * Utility to export a JSON object from a function + * + * 1) Checks if the value is serializable as JSON + * 2) Makes a deep clone of the object + * 2) Orders the object properties + * 2) Deeply freezes the cloned object + * + * Note: This function does not mutates the given object + * + * @returns The same type of object as the input but read-only and re-ordered + * @public exported from `@promptbook/utils` + */ + function exportJson(options) { + var name = options.name, + value = options.value, + order = options.order, + message = options.message; + checkSerializableAsJson({ name: name, value: value, message: message }); + var orderedValue = + // TODO: Fix error "Type instantiation is excessively deep and possibly infinite." + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + order === undefined + ? deepClone(value) + : orderJson({ + value: value, + // <- Note: checkSerializableAsJson asserts that the value is serializable as JSON + order: order, + }); + $deepFreeze(orderedValue); + return orderedValue; + } + /** + * TODO: [🧠] Is there a way how to meaningfully test this utility + */ + + /** + * Order of keys in the pipeline JSON + * + * @public exported from `@promptbook/core` + */ + var ORDER_OF_PIPELINE_JSON = [ + // Note: [🍙] In this order will be pipeline serialized + "title", + "pipelineUrl", + "bookVersion", + "description", + "formfactorName", + "parameters", + "tasks", + "personas", + "preparations", + "knowledgeSources", + "knowledgePieces", + "sources", // <- TODO: [🧠] Where should the `sources` be + ]; + /** + * Nonce which is used for replacing things in strings + * + * @private within the repository + */ + var REPLACING_NONCE = "u$k42k%!V2zo34w7Fu#@QUHYPW"; + /** + * @@@ + * + * @private within the repository + */ + var RESERVED_PARAMETER_MISSING_VALUE = "MISSING-" + REPLACING_NONCE; + /** + * @@@ + * + * @private within the repository + */ + var RESERVED_PARAMETER_RESTRICTED = "RESTRICTED-" + REPLACING_NONCE; + /** + * The names of the parameters that are reserved for special purposes + * + * @public exported from `@promptbook/core` + */ + var RESERVED_PARAMETER_NAMES = exportJson({ + name: "RESERVED_PARAMETER_NAMES", + message: + "The names of the parameters that are reserved for special purposes", + value: [ + "content", + "context", + "knowledge", + "examples", + "modelName", + "currentDate", + // <- TODO: list here all command names + // <- TODO: Add more like 'date', 'modelName',... + // <- TODO: Add [emoji] + instructions ACRY when adding new reserved parameter + ], + }); + /** + * Note: [💞] Ignore a discrepancy between file name and entity name + */ + + /** + * Tests if given string is valid semantic version + * + * Note: There are two simmilar functions: + * - `isValidSemanticVersion` which tests any semantic version + * - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions + * + * @public exported from `@promptbook/utils` + */ + function isValidSemanticVersion(version) { + if (typeof version !== "string") { + return false; + } + if (version.startsWith("0.0.0")) { + return false; + } + return /^\d+\.\d+\.\d+(-\d+)?$/i.test(version); + } + + /** + * Tests if given string is valid promptbook version + * It looks into list of known promptbook versions. + * + * @see https://www.npmjs.com/package/promptbook?activeTab=versions + * Note: When you are using for example promptbook 2.0.0 and there already is promptbook 3.0.0 it don`t know about it. + * Note: There are two simmilar functions: + * - `isValidSemanticVersion` which tests any semantic version + * - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions + * + * @public exported from `@promptbook/utils` + */ + function isValidPromptbookVersion(version) { + if (!isValidSemanticVersion(version)) { + return false; + } + if ( + /* version === '1.0.0' || */ version === "2.0.0" || + version === "3.0.0" + ) { + return false; + } + // <- TODO: [main] !!3 Check isValidPromptbookVersion against PROMPTBOOK_ENGINE_VERSIONS + return true; + } + + /** + * Checks if an URL is reserved for private networks or localhost. + * + * Note: There are two simmilar functions: + * - `isUrlOnPrivateNetwork` which tests full URL + * - `isHostnameOnPrivateNetwork` *(this one)* which tests just hostname + * + * @public exported from `@promptbook/utils` + */ + function isHostnameOnPrivateNetwork(hostname) { + if ( + hostname === "example.com" || + hostname === "localhost" || + hostname.endsWith(".localhost") || + hostname.endsWith(".local") || + hostname.endsWith(".test") || + hostname === "127.0.0.1" || + hostname === "::1" + ) { + return true; + } + if (hostname.includes(":")) { + // IPv6 + var ipParts = hostname.split(":"); + return ( + ipParts[0] === "fc00" || + ipParts[0] === "fd00" || + ipParts[0] === "fe80" + ); + } else { + // IPv4 + var ipParts = hostname + .split(".") + .map((part) => Number.parseInt(part, 10)); + return ( + ipParts[0] === 10 || + (ipParts[0] === 172 && ipParts[1] >= 16 && ipParts[1] <= 31) || + (ipParts[0] === 192 && ipParts[1] === 168) + ); + } + } + + /** + * Checks if an IP address or hostname is reserved for private networks or localhost. + * + * Note: There are two simmilar functions: + * - `isUrlOnPrivateNetwork` *(this one)* which tests full URL + * - `isHostnameOnPrivateNetwork` which tests just hostname + * + * @param {string} ipAddress - The IP address to check. + * @returns {boolean} Returns true if the IP address is reserved for private networks or localhost, otherwise false. + * @public exported from `@promptbook/utils` + */ + function isUrlOnPrivateNetwork(url) { + if (typeof url === "string") { + url = new URL(url); + } + return isHostnameOnPrivateNetwork(url.hostname); + } + + /** + * Tests if given string is valid URL. + * + * Note: Dataurl are considered perfectly valid. + * Note: There are two simmilar functions: + * - `isValidUrl` which tests any URL + * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL + * + * @public exported from `@promptbook/utils` + */ + function isValidUrl(url) { + if (typeof url !== "string") { + return false; + } + try { + if (url.startsWith("blob:")) { + url = url.replace(/^blob:/, ""); + } + var urlObject = new URL(url /* because fail is handled */); + if (!["http:", "https:", "data:"].includes(urlObject.protocol)) { + return false; + } + return true; + } catch (error) { + return false; + } + } + + /** + * Tests if given string is valid pipeline URL URL. + * + * Note: There are two simmilar functions: + * - `isValidUrl` which tests any URL + * - `isValidPipelineUrl` *(this one)* which tests just pipeline URL + * + * @public exported from `@promptbook/utils` + */ + function isValidPipelineUrl(url) { + if (!isValidUrl(url)) { + return false; + } + if (!url.startsWith("https://")) { + return false; + } + if (url.includes("#")) { + // TODO: [🐠] + return false; + } + if (isUrlOnPrivateNetwork(url)) { + return false; + } + return true; + } + /** + * TODO: [🐠] Maybe more info why the URL is invalid + */ + + /** + * Validates PipelineJson if it is logically valid + * + * It checks: + * - if it has correct parameters dependency + * + * It does NOT check: + * - if it is valid json + * - if it is meaningful + * + * @param pipeline valid or invalid PipelineJson + * @returns the same pipeline if it is logically valid + * @throws {PipelineLogicError} on logical error in the pipeline + * @public exported from `@promptbook/core` + */ + function validatePipeline(pipeline) { + if (IS_PIPELINE_LOGIC_VALIDATED) { + validatePipeline_InnerFunction(pipeline); + } else { + try { + validatePipeline_InnerFunction(pipeline); + } catch (error) { + if (!(error instanceof PipelineLogicError)) { + throw error; + } + console.error( + spaceTrim.spaceTrim((block) => + "\n Pipeline is not valid but logic errors are temporarily disabled via `IS_PIPELINE_LOGIC_VALIDATED`\n\n ".concat( + block(error.message), + "\n ", + ), + ), + ); + } + } + return pipeline; + } + /** + * @private internal function for `validatePipeline` + */ + function validatePipeline_InnerFunction(pipeline) { + // TODO: [🧠] Maybe test if promptbook is a promise and make specific error case for that + var e_1, _a, e_2, _b, e_3, _c; + var pipelineIdentification = (() => { + // Note: This is a 😐 implementation of [🚞] + var _ = []; + if (pipeline.sourceFile !== undefined) { + _.push("File: ".concat(pipeline.sourceFile)); + } + if (pipeline.pipelineUrl !== undefined) { + _.push("Url: ".concat(pipeline.pipelineUrl)); + } + return _.join("\n"); + })(); + if ( + pipeline.pipelineUrl !== undefined && + !isValidPipelineUrl(pipeline.pipelineUrl) + ) { + // <- Note: [🚲] + throw new PipelineLogicError( + spaceTrim.spaceTrim((block) => + '\n Invalid promptbook URL "' + .concat(pipeline.pipelineUrl, '"\n\n ') + .concat(block(pipelineIdentification), "\n "), + ), + ); + } + if ( + pipeline.bookVersion !== undefined && + !isValidPromptbookVersion(pipeline.bookVersion) + ) { + // <- Note: [🚲] + throw new PipelineLogicError( + spaceTrim.spaceTrim((block) => + '\n Invalid Promptbook Version "' + .concat(pipeline.bookVersion, '"\n\n ') + .concat(block(pipelineIdentification), "\n "), + ), + ); + } + // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking + if (!Array.isArray(pipeline.parameters)) { + // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError + throw new ParseError( + spaceTrim.spaceTrim((block) => + "\n Pipeline is valid JSON but with wrong structure\n\n `PipelineJson.parameters` expected to be an array, but got " + .concat(typeof pipeline.parameters, "\n\n ") + .concat(block(pipelineIdentification), "\n "), + ), + ); + } + // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking + if (!Array.isArray(pipeline.tasks)) { + // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError + throw new ParseError( + spaceTrim.spaceTrim((block) => + "\n Pipeline is valid JSON but with wrong structure\n\n `PipelineJson.tasks` expected to be an array, but got " + .concat(typeof pipeline.tasks, "\n\n ") + .concat(block(pipelineIdentification), "\n "), + ), + ); + } + var _loop_1 = (parameter) => { + if (parameter.isInput && parameter.isOutput) { + throw new PipelineLogicError( + spaceTrim.spaceTrim((block) => + "\n\n Parameter `{" + .concat( + parameter.name, + "}` can not be both input and output\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + // Note: Testing that parameter is either intermediate or output BUT not created and unused + if ( + !parameter.isInput && + !parameter.isOutput && + !pipeline.tasks.some((task) => + task.dependentParameterNames.includes(parameter.name), + ) + ) { + throw new PipelineLogicError( + spaceTrim.spaceTrim((block) => + "\n Parameter `{" + .concat( + parameter.name, + "}` is created but not used\n\n You can declare {", + ) + .concat( + parameter.name, + "} as output parameter by adding in the header:\n - OUTPUT PARAMETER `{", + ) + .concat(parameter.name, "}` ") + .concat( + parameter.description || "", + "\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n\n ", + ), + ), + ); + } + // Note: Testing that parameter is either input or result of some task + if ( + !parameter.isInput && + !pipeline.tasks.some( + (task) => task.resultingParameterName === parameter.name, + ) + ) { + throw new PipelineLogicError( + spaceTrim.spaceTrim((block) => + "\n Parameter `{" + .concat( + parameter.name, + "}` is declared but not defined\n\n You can do one of these:\n 1) Remove declaration of `{", + ) + .concat( + parameter.name, + "}`\n 2) Add task that results in `-> {", + ) + .concat(parameter.name, "}`\n\n ") + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + }; + try { + /* TODO: [🧠][🅾] Should be empty pipeline valid or not // Note: Check that pipeline has some tasks if (pipeline.tasks.length === 0) { @@ -2529,669 +3595,986 @@ at step (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\node_modules\\node_modu ); } */ - // Note: Check each parameter individually - for (var _d = __values(pipeline.parameters), _e = _d.next(); !_e.done; _e = _d.next()) { - var parameter = _e.value; - _loop_1(parameter); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_e && !_e.done && (_a = _d.return)) _a.call(_d); - } - finally { if (e_1) throw e_1.error; } - } - // Note: All input parameters are defined - so that they can be used as result of some task - var definedParameters = new Set(pipeline.parameters.filter(function (_a) { - var isInput = _a.isInput; - return isInput; - }).map(function (_a) { - var name = _a.name; - return name; - })); - var _loop_2 = function (task) { - var e_4, _h, e_5, _j; - if (definedParameters.has(task.resultingParameterName)) { - throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Parameter `{".concat(task.resultingParameterName, "}` is defined multiple times\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - if (RESERVED_PARAMETER_NAMES.includes(task.resultingParameterName)) { - throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Parameter name {".concat(task.resultingParameterName, "} is reserved, please use different name\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - definedParameters.add(task.resultingParameterName); - if (task.jokerParameterNames && task.jokerParameterNames.length > 0) { - if (!task.format && - !task.expectations /* <- TODO: Require at least 1 -> min <- expectation to use jokers */) { - throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Joker parameters are used for {".concat(task.resultingParameterName, "} but no expectations are defined\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - var _loop_4 = function (joker) { - if (!task.dependentParameterNames.includes(joker)) { - throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Parameter `{".concat(joker, "}` is used for {").concat(task.resultingParameterName, "} as joker but not in `dependentParameterNames`\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - }; - try { - for (var _k = (e_4 = void 0, __values(task.jokerParameterNames)), _l = _k.next(); !_l.done; _l = _k.next()) { - var joker = _l.value; - _loop_4(joker); - } - } - catch (e_4_1) { e_4 = { error: e_4_1 }; } - finally { - try { - if (_l && !_l.done && (_h = _k.return)) _h.call(_k); - } - finally { if (e_4) throw e_4.error; } - } - } - if (task.expectations) { - var _loop_5 = function (unit, min, max) { - if (min !== undefined && max !== undefined && min > max) { - throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Min expectation (=".concat(min, ") of ").concat(unit, " is higher than max expectation (=").concat(max, ")\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - if (min !== undefined && min < 0) { - throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Min expectation of ".concat(unit, " must be zero or positive\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - if (max !== undefined && max <= 0) { - throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Max expectation of ".concat(unit, " must be positive\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - }; - try { - for (var _m = (e_5 = void 0, __values(Object.entries(task.expectations))), _o = _m.next(); !_o.done; _o = _m.next()) { - var _p = __read(_o.value, 2), unit = _p[0], _q = _p[1], min = _q.min, max = _q.max; - _loop_5(unit, min, max); - } - } - catch (e_5_1) { e_5 = { error: e_5_1 }; } - finally { - try { - if (_o && !_o.done && (_j = _m.return)) _j.call(_m); - } - finally { if (e_5) throw e_5.error; } - } - } - }; - try { - // Note: Checking each task individually - for (var _f = __values(pipeline.tasks), _g = _f.next(); !_g.done; _g = _f.next()) { - var task = _g.value; - _loop_2(task); - } - } - catch (e_2_1) { e_2 = { error: e_2_1 }; } - finally { - try { - if (_g && !_g.done && (_b = _f.return)) _b.call(_f); - } - finally { if (e_2) throw e_2.error; } - } - // Note: Detect circular dependencies - var resovedParameters = pipeline.parameters - .filter(function (_a) { - var isInput = _a.isInput; - return isInput; - }) - .map(function (_a) { - var name = _a.name; - return name; - }); - try { - // Note: All reserved parameters are resolved - for (var RESERVED_PARAMETER_NAMES_1 = __values(RESERVED_PARAMETER_NAMES), RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next(); !RESERVED_PARAMETER_NAMES_1_1.done; RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next()) { - var reservedParameterName = RESERVED_PARAMETER_NAMES_1_1.value; - resovedParameters = __spreadArray(__spreadArray([], __read(resovedParameters), false), [reservedParameterName], false); - } - } - catch (e_3_1) { e_3 = { error: e_3_1 }; } - finally { - try { - if (RESERVED_PARAMETER_NAMES_1_1 && !RESERVED_PARAMETER_NAMES_1_1.done && (_c = RESERVED_PARAMETER_NAMES_1.return)) _c.call(RESERVED_PARAMETER_NAMES_1); - } - finally { if (e_3) throw e_3.error; } - } - var unresovedTasks = __spreadArray([], __read(pipeline.tasks), false); - var loopLimit = LOOP_LIMIT; - var _loop_3 = function () { - if (loopLimit-- < 0) { - // Note: Really UnexpectedError not LimitReachedError - this should not happen and be caught below - throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Loop limit reached during detection of circular dependencies in `validatePipeline`\n\n ".concat(block(pipelineIdentification), "\n "); })); - } - var currentlyResovedTasks = unresovedTasks.filter(function (task) { - return task.dependentParameterNames.every(function (name) { return resovedParameters.includes(name); }); - }); - if (currentlyResovedTasks.length === 0) { - throw new PipelineLogicError( - // TODO: [🐎] DRY - spaceTrim.spaceTrim(function (block) { return "\n\n Can not resolve some parameters:\n Either you are using a parameter that is not defined, or there are some circular dependencies.\n\n ".concat(block(pipelineIdentification), "\n\n **Can not resolve:**\n ").concat(block(unresovedTasks - .map(function (_a) { - var resultingParameterName = _a.resultingParameterName, dependentParameterNames = _a.dependentParameterNames; - return "- Parameter `{".concat(resultingParameterName, "}` which depends on ").concat(dependentParameterNames - .map(function (dependentParameterName) { return "`{".concat(dependentParameterName, "}`"); }) - .join(' and ')); - }) - .join('\n')), "\n\n **Resolved:**\n ").concat(block(resovedParameters - .filter(function (name) { - return !RESERVED_PARAMETER_NAMES.includes(name); - }) - .map(function (name) { return "- Parameter `{".concat(name, "}`"); }) - .join('\n')), "\n\n\n **Reserved (which are available):**\n ").concat(block(resovedParameters - .filter(function (name) { - return RESERVED_PARAMETER_NAMES.includes(name); - }) - .map(function (name) { return "- Parameter `{".concat(name, "}`"); }) - .join('\n')), "\n\n\n "); })); - } - resovedParameters = __spreadArray(__spreadArray([], __read(resovedParameters), false), __read(currentlyResovedTasks.map(function (_a) { - var resultingParameterName = _a.resultingParameterName; - return resultingParameterName; - })), false); - unresovedTasks = unresovedTasks.filter(function (task) { return !currentlyResovedTasks.includes(task); }); - }; - while (unresovedTasks.length > 0) { - _loop_3(); - } - // Note: Check that formfactor is corresponding to the pipeline interface - // TODO: !!6 Implement this - // pipeline.formfactorName - } - /** - * TODO: [🧞‍♀️] Do not allow joker + foreach - * TODO: [🧠] Work with promptbookVersion - * TODO: Use here some json-schema, Zod or something similar and change it to: - * > /** - * > * Validates PipelineJson if it is logically valid. - * > * - * > * It checks: - * > * - it has a valid structure - * > * - ... - * > ex port function validatePipeline(promptbook: really_unknown): asserts promptbook is PipelineJson { - */ - /** - * TODO: [🧳][main] !!4 Validate that all examples match expectations - * TODO: [🧳][🐝][main] !!4 Validate that knowledge is valid (non-void) - * TODO: [🧳][main] !!4 Validate that persona can be used only with CHAT variant - * TODO: [🧳][main] !!4 Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES - * TODO: [🧳][main] !!4 Validate that reserved parameter is not used as joker - * TODO: [🧠] Validation not only logic itself but imports around - files and websites and rerefenced pipelines exists - * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools - */ - - /** - * Parses the task and returns the list of all parameter names - * - * @param template the string template with parameters in {curly} braces - * @returns the list of parameter names - * @public exported from `@promptbook/utils` - */ - function extractParameterNames(template) { - var e_1, _a; - var matches = template.matchAll(/{\w+}/g); - var parameterNames = new Set(); - try { - for (var matches_1 = __values(matches), matches_1_1 = matches_1.next(); !matches_1_1.done; matches_1_1 = matches_1.next()) { - var match = matches_1_1.value; - var parameterName = match[0].slice(1, -1); - parameterNames.add(parameterName); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (matches_1_1 && !matches_1_1.done && (_a = matches_1.return)) _a.call(matches_1); - } - finally { if (e_1) throw e_1.error; } - } - return parameterNames; - } - - /** - * Unprepare just strips the preparation data of the pipeline - * - * @deprecated In future version this function will be removed or deprecated - * @public exported from `@promptbook/core` - */ - function unpreparePipeline(pipeline) { - var personas = pipeline.personas, knowledgeSources = pipeline.knowledgeSources, tasks = pipeline.tasks; - personas = personas.map(function (persona) { return (__assign(__assign({}, persona), { modelRequirements: undefined, preparationIds: undefined })); }); - knowledgeSources = knowledgeSources.map(function (knowledgeSource) { return (__assign(__assign({}, knowledgeSource), { preparationIds: undefined })); }); - tasks = tasks.map(function (task) { - var dependentParameterNames = task.dependentParameterNames; - var parameterNames = extractParameterNames(task.preparedContent || ''); - dependentParameterNames = dependentParameterNames.filter(function (dependentParameterName) { return !parameterNames.has(dependentParameterName); }); - var taskUnprepared = __assign(__assign({}, task), { dependentParameterNames: dependentParameterNames }); - delete taskUnprepared.preparedContent; - return taskUnprepared; - }); - return exportJson({ - name: 'pipelineJson', - message: "Result of `unpreparePipeline`", - order: ORDER_OF_PIPELINE_JSON, - value: __assign(__assign({}, pipeline), { tasks: tasks, knowledgeSources: knowledgeSources, knowledgePieces: [], personas: personas, preparations: [] }), - }); - } - /** - * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline` - * TODO: Write tests for `preparePipeline` - * TODO: [🍙] Make some standard order of json properties - */ - - /** - * Library of pipelines that groups together pipelines for an application. - * This implementation is a very thin wrapper around the Array / Map of pipelines. - * - * @private internal function of `createCollectionFromJson`, use `createCollectionFromJson` instead - * @see https://github.com/webgptorg/pipeline#pipeline-collection - */ - var SimplePipelineCollection = /** @class */ (function () { - /** - * Constructs a pipeline collection from pipelines - * - * @param pipelines @@@ - * - * Note: During the construction logic of all pipelines are validated - * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead - */ - function SimplePipelineCollection() { - var e_1, _a; - var pipelines = []; - for (var _i = 0; _i < arguments.length; _i++) { - pipelines[_i] = arguments[_i]; - } - this.collection = new Map(); - try { - for (var pipelines_1 = __values(pipelines), pipelines_1_1 = pipelines_1.next(); !pipelines_1_1.done; pipelines_1_1 = pipelines_1.next()) { - var pipeline = pipelines_1_1.value; - // TODO: [👠] DRY - if (pipeline.pipelineUrl === undefined) { - throw new PipelineUrlError(spaceTrim.spaceTrim("\n Pipeline with name \"".concat(pipeline.title, "\" does not have defined URL\n\n File:\n ").concat(pipeline.sourceFile || 'Unknown', "\n\n Note: Pipelines without URLs are called anonymous pipelines\n They can be used as standalone pipelines, but they cannot be referenced by other pipelines\n And also they cannot be used in the pipeline collection\n\n "))); - } - // Note: [🐨] - validatePipeline(pipeline); - // TODO: [🦄] DRY - // Note: [🦄] - if ( - // TODO: [🐽] - this.collection.has(pipeline.pipelineUrl) && - pipelineJsonToString(unpreparePipeline(pipeline)) !== - pipelineJsonToString(unpreparePipeline(this.collection.get(pipeline.pipelineUrl)))) { - var existing = this.collection.get(pipeline.pipelineUrl); - throw new PipelineUrlError(spaceTrim.spaceTrim("\n Pipeline with URL ".concat(pipeline.pipelineUrl, " is already in the collection \uD83C\uDF4E\n\n Conflicting files:\n ").concat(existing.sourceFile || 'Unknown', "\n ").concat(pipeline.sourceFile || 'Unknown', "\n\n Note: You have probably forgotten to run \"ptbk make\" to update the collection\n Note: Pipelines with the same URL are not allowed\n Only exepction is when the pipelines are identical\n\n "))); - } - // Note: [🧠] Overwrite existing pipeline with the same URL - this.collection.set(pipeline.pipelineUrl, pipeline); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (pipelines_1_1 && !pipelines_1_1.done && (_a = pipelines_1.return)) _a.call(pipelines_1); - } - finally { if (e_1) throw e_1.error; } - } - } - /** - * Gets all pipelines in the collection - */ - SimplePipelineCollection.prototype.listPipelines = function () { - return Array.from(this.collection.keys()); - }; - /** - * Gets pipeline by its URL - * - * Note: This is not a direct fetching from the URL, but a lookup in the collection - */ - SimplePipelineCollection.prototype.getPipelineByUrl = function (url) { - var _this = this; - var pipeline = this.collection.get(url); - if (!pipeline) { - if (this.listPipelines().length === 0) { - throw new NotFoundError(spaceTrim.spaceTrim("\n Pipeline with url \"".concat(url, "\" not found\n\n No pipelines available\n "))); - } - throw new NotFoundError(spaceTrim.spaceTrim(function (block) { return "\n Pipeline with url \"".concat(url, "\" not found\n\n Available pipelines:\n ").concat(block(_this.listPipelines() - .map(function (pipelineUrl) { return "- ".concat(pipelineUrl); }) - .join('\n')), "\n\n "); })); - } - return pipeline; - }; - /** - * Checks whether given prompt was defined in any pipeline in the collection - */ - SimplePipelineCollection.prototype.isResponsibleForPrompt = function (prompt) { - return true; - }; - return SimplePipelineCollection; - }()); - - /** - * Creates PipelineCollection from array of PipelineJson or PipelineString - * - * Note: Functions `collectionToJson` and `createCollectionFromJson` are complementary - * Note: Syntax, parsing, and logic consistency checks are performed on all sources during build - * - * @param promptbookSources - * @returns PipelineCollection - * @public exported from `@promptbook/core` - */ - function createCollectionFromJson() { - var promptbooks = []; - for (var _i = 0; _i < arguments.length; _i++) { - promptbooks[_i] = arguments[_i]; - } - return new (SimplePipelineCollection.bind.apply(SimplePipelineCollection, __spreadArray([void 0], __read(promptbooks), false)))(); - } - - /** - * Deserializes the error object - * - * @public exported from `@promptbook/utils` - */ - function deserializeError(error) { - var name = error.name, stack = error.stack; - var message = error.message; - var ErrorClass = ALL_ERRORS[error.name]; - if (ErrorClass === undefined) { - ErrorClass = Error; - message = "".concat(name, ": ").concat(message); - } - if (stack !== undefined && stack !== '') { - message = spaceTrim__default["default"](function (block) { return "\n ".concat(block(message), "\n\n Original stack trace:\n ").concat(block(stack || ''), "\n "); }); - } - return new ErrorClass(message); - } - - /** - * Asserts that the execution of a Promptbook is successful - * - * @param executionResult - The partial result of the Promptbook execution - * @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred - * @public exported from `@promptbook/core` - */ - function assertsExecutionSuccessful(executionResult) { - var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors; - if (isSuccessful === true) { - return; - } - if (errors.length === 0) { - throw new PipelineExecutionError("Promptbook Execution failed because of unknown reason"); - } - else if (errors.length === 1) { - throw deserializeError(errors[0]); - } - else { - throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Multiple errors occurred during Promptbook execution\n\n ".concat(block(errors - .map(function (_a, index) { - var name = _a.name, stack = _a.stack, message = _a.message; - return spaceTrim.spaceTrim(function (block) { return "\n ".concat(name, " ").concat(index + 1, ":\n ").concat(block(stack || message), "\n "); }); - }) - .join('\n')), "\n "); })); - } - } - /** - * TODO: [🐚] This function should be removed OR changed OR be completely rewritten - * TODO: [🧠] Can this return type be better typed than void - */ - - /** - * Determine if the pipeline is fully prepared - * - * @see https://github.com/webgptorg/promptbook/discussions/196 - * - * @public exported from `@promptbook/core` - */ - function isPipelinePrepared(pipeline) { - // Note: Ignoring `pipeline.preparations` @@@ - // Note: Ignoring `pipeline.knowledgePieces` @@@ - if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) { - return false; - } - if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) { - return false; - } - if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) { - return false; - } - /* + // Note: Check each parameter individually + for ( + var _d = __values(pipeline.parameters), _e = _d.next(); + !_e.done; + _e = _d.next() + ) { + var parameter = _e.value; + _loop_1(parameter); + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_e && !_e.done && (_a = _d.return)) _a.call(_d); + } finally { + if (e_1) throw e_1.error; + } + } + // Note: All input parameters are defined - so that they can be used as result of some task + var definedParameters = new Set( + pipeline.parameters + .filter((_a) => { + var isInput = _a.isInput; + return isInput; + }) + .map((_a) => { + var name = _a.name; + return name; + }), + ); + var _loop_2 = (task) => { + var e_4, _h, e_5, _j; + if (definedParameters.has(task.resultingParameterName)) { + throw new PipelineLogicError( + spaceTrim.spaceTrim((block) => + "\n Parameter `{" + .concat( + task.resultingParameterName, + "}` is defined multiple times\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + if (RESERVED_PARAMETER_NAMES.includes(task.resultingParameterName)) { + throw new PipelineLogicError( + spaceTrim.spaceTrim((block) => + "\n Parameter name {" + .concat( + task.resultingParameterName, + "} is reserved, please use different name\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + definedParameters.add(task.resultingParameterName); + if (task.jokerParameterNames && task.jokerParameterNames.length > 0) { + if ( + !task.format && + !task.expectations /* <- TODO: Require at least 1 -> min <- expectation to use jokers */ + ) { + throw new PipelineLogicError( + spaceTrim.spaceTrim((block) => + "\n Joker parameters are used for {" + .concat( + task.resultingParameterName, + "} but no expectations are defined\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + var _loop_4 = (joker) => { + if (!task.dependentParameterNames.includes(joker)) { + throw new PipelineLogicError( + spaceTrim.spaceTrim((block) => + "\n Parameter `{" + .concat(joker, "}` is used for {") + .concat( + task.resultingParameterName, + "} as joker but not in `dependentParameterNames`\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + }; + try { + for ( + var _k = ((e_4 = void 0), __values(task.jokerParameterNames)), + _l = _k.next(); + !_l.done; + _l = _k.next() + ) { + var joker = _l.value; + _loop_4(joker); + } + } catch (e_4_1) { + e_4 = { error: e_4_1 }; + } finally { + try { + if (_l && !_l.done && (_h = _k.return)) _h.call(_k); + } finally { + if (e_4) throw e_4.error; + } + } + } + if (task.expectations) { + var _loop_5 = (unit, min, max) => { + if (min !== undefined && max !== undefined && min > max) { + throw new PipelineLogicError( + spaceTrim.spaceTrim((block) => + "\n Min expectation (=" + .concat(min, ") of ") + .concat(unit, " is higher than max expectation (=") + .concat(max, ")\n\n ") + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + if (min !== undefined && min < 0) { + throw new PipelineLogicError( + spaceTrim.spaceTrim((block) => + "\n Min expectation of " + .concat( + unit, + " must be zero or positive\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + if (max !== undefined && max <= 0) { + throw new PipelineLogicError( + spaceTrim.spaceTrim((block) => + "\n Max expectation of " + .concat( + unit, + " must be positive\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + }; + try { + for ( + var _m = + ((e_5 = void 0), __values(Object.entries(task.expectations))), + _o = _m.next(); + !_o.done; + _o = _m.next() + ) { + var _p = __read(_o.value, 2), + unit = _p[0], + _q = _p[1], + min = _q.min, + max = _q.max; + _loop_5(unit, min, max); + } + } catch (e_5_1) { + e_5 = { error: e_5_1 }; + } finally { + try { + if (_o && !_o.done && (_j = _m.return)) _j.call(_m); + } finally { + if (e_5) throw e_5.error; + } + } + } + }; + try { + // Note: Checking each task individually + for ( + var _f = __values(pipeline.tasks), _g = _f.next(); + !_g.done; + _g = _f.next() + ) { + var task = _g.value; + _loop_2(task); + } + } catch (e_2_1) { + e_2 = { error: e_2_1 }; + } finally { + try { + if (_g && !_g.done && (_b = _f.return)) _b.call(_f); + } finally { + if (e_2) throw e_2.error; + } + } + // Note: Detect circular dependencies + var resovedParameters = pipeline.parameters + .filter((_a) => { + var isInput = _a.isInput; + return isInput; + }) + .map((_a) => { + var name = _a.name; + return name; + }); + try { + // Note: All reserved parameters are resolved + for ( + var RESERVED_PARAMETER_NAMES_1 = __values(RESERVED_PARAMETER_NAMES), + RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next(); + !RESERVED_PARAMETER_NAMES_1_1.done; + RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next() + ) { + var reservedParameterName = RESERVED_PARAMETER_NAMES_1_1.value; + resovedParameters = __spreadArray( + __spreadArray([], __read(resovedParameters), false), + [reservedParameterName], + false, + ); + } + } catch (e_3_1) { + e_3 = { error: e_3_1 }; + } finally { + try { + if ( + RESERVED_PARAMETER_NAMES_1_1 && + !RESERVED_PARAMETER_NAMES_1_1.done && + (_c = RESERVED_PARAMETER_NAMES_1.return) + ) + _c.call(RESERVED_PARAMETER_NAMES_1); + } finally { + if (e_3) throw e_3.error; + } + } + var unresovedTasks = __spreadArray([], __read(pipeline.tasks), false); + var loopLimit = LOOP_LIMIT; + var _loop_3 = () => { + if (loopLimit-- < 0) { + // Note: Really UnexpectedError not LimitReachedError - this should not happen and be caught below + throw new UnexpectedError( + spaceTrim.spaceTrim((block) => + "\n Loop limit reached during detection of circular dependencies in `validatePipeline`\n\n ".concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + var currentlyResovedTasks = unresovedTasks.filter((task) => + task.dependentParameterNames.every((name) => + resovedParameters.includes(name), + ), + ); + if (currentlyResovedTasks.length === 0) { + throw new PipelineLogicError( + // TODO: [🐎] DRY + spaceTrim.spaceTrim((block) => + "\n\n Can not resolve some parameters:\n Either you are using a parameter that is not defined, or there are some circular dependencies.\n\n " + .concat( + block(pipelineIdentification), + "\n\n **Can not resolve:**\n ", + ) + .concat( + block( + unresovedTasks + .map((_a) => { + var resultingParameterName = _a.resultingParameterName, + dependentParameterNames = _a.dependentParameterNames; + return "- Parameter `{" + .concat( + resultingParameterName, + "}` which depends on ", + ) + .concat( + dependentParameterNames + .map((dependentParameterName) => + "`{".concat(dependentParameterName, "}`"), + ) + .join(" and "), + ); + }) + .join("\n"), + ), + "\n\n **Resolved:**\n ", + ) + .concat( + block( + resovedParameters + .filter( + (name) => !RESERVED_PARAMETER_NAMES.includes(name), + ) + .map((name) => "- Parameter `{".concat(name, "}`")) + .join("\n"), + ), + "\n\n\n **Reserved (which are available):**\n ", + ) + .concat( + block( + resovedParameters + .filter((name) => RESERVED_PARAMETER_NAMES.includes(name)) + .map((name) => "- Parameter `{".concat(name, "}`")) + .join("\n"), + ), + "\n\n\n ", + ), + ), + ); + } + resovedParameters = __spreadArray( + __spreadArray([], __read(resovedParameters), false), + __read( + currentlyResovedTasks.map((_a) => { + var resultingParameterName = _a.resultingParameterName; + return resultingParameterName; + }), + ), + false, + ); + unresovedTasks = unresovedTasks.filter( + (task) => !currentlyResovedTasks.includes(task), + ); + }; + while (unresovedTasks.length > 0) { + _loop_3(); + } + // Note: Check that formfactor is corresponding to the pipeline interface + // TODO: !!6 Implement this + // pipeline.formfactorName + } + /** + * TODO: [🧞‍♀️] Do not allow joker + foreach + * TODO: [🧠] Work with promptbookVersion + * TODO: Use here some json-schema, Zod or something similar and change it to: + * > /** + * > * Validates PipelineJson if it is logically valid. + * > * + * > * It checks: + * > * - it has a valid structure + * > * - ... + * > ex port function validatePipeline(promptbook: really_unknown): asserts promptbook is PipelineJson { + */ + /** + * TODO: [🧳][main] !!4 Validate that all examples match expectations + * TODO: [🧳][🐝][main] !!4 Validate that knowledge is valid (non-void) + * TODO: [🧳][main] !!4 Validate that persona can be used only with CHAT variant + * TODO: [🧳][main] !!4 Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES + * TODO: [🧳][main] !!4 Validate that reserved parameter is not used as joker + * TODO: [🧠] Validation not only logic itself but imports around - files and websites and rerefenced pipelines exists + * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools + */ + + /** + * Parses the task and returns the list of all parameter names + * + * @param template the string template with parameters in {curly} braces + * @returns the list of parameter names + * @public exported from `@promptbook/utils` + */ + function extractParameterNames(template) { + var e_1, _a; + var matches = template.matchAll(/{\w+}/g); + var parameterNames = new Set(); + try { + for ( + var matches_1 = __values(matches), matches_1_1 = matches_1.next(); + !matches_1_1.done; + matches_1_1 = matches_1.next() + ) { + var match = matches_1_1.value; + var parameterName = match[0].slice(1, -1); + parameterNames.add(parameterName); + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (matches_1_1 && !matches_1_1.done && (_a = matches_1.return)) + _a.call(matches_1); + } finally { + if (e_1) throw e_1.error; + } + } + return parameterNames; + } + + /** + * Unprepare just strips the preparation data of the pipeline + * + * @deprecated In future version this function will be removed or deprecated + * @public exported from `@promptbook/core` + */ + function unpreparePipeline(pipeline) { + var personas = pipeline.personas, + knowledgeSources = pipeline.knowledgeSources, + tasks = pipeline.tasks; + personas = personas.map((persona) => + __assign(__assign({}, persona), { + modelRequirements: undefined, + preparationIds: undefined, + }), + ); + knowledgeSources = knowledgeSources.map((knowledgeSource) => + __assign(__assign({}, knowledgeSource), { preparationIds: undefined }), + ); + tasks = tasks.map((task) => { + var dependentParameterNames = task.dependentParameterNames; + var parameterNames = extractParameterNames(task.preparedContent || ""); + dependentParameterNames = dependentParameterNames.filter( + (dependentParameterName) => + !parameterNames.has(dependentParameterName), + ); + var taskUnprepared = __assign(__assign({}, task), { + dependentParameterNames: dependentParameterNames, + }); + delete taskUnprepared.preparedContent; + return taskUnprepared; + }); + return exportJson({ + name: "pipelineJson", + message: "Result of `unpreparePipeline`", + order: ORDER_OF_PIPELINE_JSON, + value: __assign(__assign({}, pipeline), { + tasks: tasks, + knowledgeSources: knowledgeSources, + knowledgePieces: [], + personas: personas, + preparations: [], + }), + }); + } + /** + * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline` + * TODO: Write tests for `preparePipeline` + * TODO: [🍙] Make some standard order of json properties + */ + + /** + * Library of pipelines that groups together pipelines for an application. + * This implementation is a very thin wrapper around the Array / Map of pipelines. + * + * @private internal function of `createCollectionFromJson`, use `createCollectionFromJson` instead + * @see https://github.com/webgptorg/pipeline#pipeline-collection + */ + var SimplePipelineCollection = /** @class */ (() => { + /** + * Constructs a pipeline collection from pipelines + * + * @param pipelines @@@ + * + * Note: During the construction logic of all pipelines are validated + * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead + */ + function SimplePipelineCollection() { + var e_1, _a; + var pipelines = []; + for (var _i = 0; _i < arguments.length; _i++) { + pipelines[_i] = arguments[_i]; + } + this.collection = new Map(); + try { + for ( + var pipelines_1 = __values(pipelines), + pipelines_1_1 = pipelines_1.next(); + !pipelines_1_1.done; + pipelines_1_1 = pipelines_1.next() + ) { + var pipeline = pipelines_1_1.value; + // TODO: [👠] DRY + if (pipeline.pipelineUrl === undefined) { + throw new PipelineUrlError( + spaceTrim.spaceTrim( + '\n Pipeline with name "' + .concat( + pipeline.title, + '" does not have defined URL\n\n File:\n ', + ) + .concat( + pipeline.sourceFile || "Unknown", + "\n\n Note: Pipelines without URLs are called anonymous pipelines\n They can be used as standalone pipelines, but they cannot be referenced by other pipelines\n And also they cannot be used in the pipeline collection\n\n ", + ), + ), + ); + } + // Note: [🐨] + validatePipeline(pipeline); + // TODO: [🦄] DRY + // Note: [🦄] + if ( + // TODO: [🐽] + this.collection.has(pipeline.pipelineUrl) && + pipelineJsonToString(unpreparePipeline(pipeline)) !== + pipelineJsonToString( + unpreparePipeline(this.collection.get(pipeline.pipelineUrl)), + ) + ) { + var existing = this.collection.get(pipeline.pipelineUrl); + throw new PipelineUrlError( + spaceTrim.spaceTrim( + "\n Pipeline with URL " + .concat( + pipeline.pipelineUrl, + " is already in the collection \uD83C\uDF4E\n\n Conflicting files:\n ", + ) + .concat( + existing.sourceFile || "Unknown", + "\n ", + ) + .concat( + pipeline.sourceFile || "Unknown", + '\n\n Note: You have probably forgotten to run "ptbk make" to update the collection\n Note: Pipelines with the same URL are not allowed\n Only exepction is when the pipelines are identical\n\n ', + ), + ), + ); + } + // Note: [🧠] Overwrite existing pipeline with the same URL + this.collection.set(pipeline.pipelineUrl, pipeline); + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if ( + pipelines_1_1 && + !pipelines_1_1.done && + (_a = pipelines_1.return) + ) + _a.call(pipelines_1); + } finally { + if (e_1) throw e_1.error; + } + } + } + /** + * Gets all pipelines in the collection + */ + SimplePipelineCollection.prototype.listPipelines = function () { + return Array.from(this.collection.keys()); + }; + /** + * Gets pipeline by its URL + * + * Note: This is not a direct fetching from the URL, but a lookup in the collection + */ + SimplePipelineCollection.prototype.getPipelineByUrl = function (url) { + var pipeline = this.collection.get(url); + if (!pipeline) { + if (this.listPipelines().length === 0) { + throw new NotFoundError( + spaceTrim.spaceTrim( + '\n Pipeline with url "'.concat( + url, + '" not found\n\n No pipelines available\n ', + ), + ), + ); + } + throw new NotFoundError( + spaceTrim.spaceTrim((block) => + '\n Pipeline with url "' + .concat( + url, + '" not found\n\n Available pipelines:\n ', + ) + .concat( + block( + this.listPipelines() + .map((pipelineUrl) => "- ".concat(pipelineUrl)) + .join("\n"), + ), + "\n\n ", + ), + ), + ); + } + return pipeline; + }; + /** + * Checks whether given prompt was defined in any pipeline in the collection + */ + SimplePipelineCollection.prototype.isResponsibleForPrompt = (prompt) => + true; + return SimplePipelineCollection; + })(); + + /** + * Creates PipelineCollection from array of PipelineJson or PipelineString + * + * Note: Functions `collectionToJson` and `createCollectionFromJson` are complementary + * Note: Syntax, parsing, and logic consistency checks are performed on all sources during build + * + * @param promptbookSources + * @returns PipelineCollection + * @public exported from `@promptbook/core` + */ + function createCollectionFromJson() { + var promptbooks = []; + for (var _i = 0; _i < arguments.length; _i++) { + promptbooks[_i] = arguments[_i]; + } + return new (SimplePipelineCollection.bind.apply( + SimplePipelineCollection, + __spreadArray([void 0], __read(promptbooks), false), + ))(); + } + + /** + * Deserializes the error object + * + * @public exported from `@promptbook/utils` + */ + function deserializeError(error) { + var name = error.name, + stack = error.stack; + var message = error.message; + var ErrorClass = ALL_ERRORS[error.name]; + if (ErrorClass === undefined) { + ErrorClass = Error; + message = "".concat(name, ": ").concat(message); + } + if (stack !== undefined && stack !== "") { + message = spaceTrim__default["default"]((block) => + "\n " + .concat( + block(message), + "\n\n Original stack trace:\n ", + ) + .concat(block(stack || ""), "\n "), + ); + } + return new ErrorClass(message); + } + + /** + * Asserts that the execution of a Promptbook is successful + * + * @param executionResult - The partial result of the Promptbook execution + * @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred + * @public exported from `@promptbook/core` + */ + function assertsExecutionSuccessful(executionResult) { + var isSuccessful = executionResult.isSuccessful, + errors = executionResult.errors; + if (isSuccessful === true) { + return; + } + if (errors.length === 0) { + throw new PipelineExecutionError( + "Promptbook Execution failed because of unknown reason", + ); + } else if (errors.length === 1) { + throw deserializeError(errors[0]); + } else { + throw new PipelineExecutionError( + spaceTrim.spaceTrim((block) => + "\n Multiple errors occurred during Promptbook execution\n\n ".concat( + block( + errors + .map((_a, index) => { + var name = _a.name, + stack = _a.stack, + message = _a.message; + return spaceTrim.spaceTrim((block) => + "\n " + .concat(name, " ") + .concat( + index + 1, + ":\n ", + ) + .concat( + block(stack || message), + "\n ", + ), + ); + }) + .join("\n"), + ), + "\n ", + ), + ), + ); + } + } + /** + * TODO: [🐚] This function should be removed OR changed OR be completely rewritten + * TODO: [🧠] Can this return type be better typed than void + */ + + /** + * Determine if the pipeline is fully prepared + * + * @see https://github.com/webgptorg/promptbook/discussions/196 + * + * @public exported from `@promptbook/core` + */ + function isPipelinePrepared(pipeline) { + // Note: Ignoring `pipeline.preparations` @@@ + // Note: Ignoring `pipeline.knowledgePieces` @@@ + if ( + pipeline.title === undefined || + pipeline.title === "" || + pipeline.title === DEFAULT_BOOK_TITLE + ) { + return false; + } + if ( + !pipeline.personas.every( + (persona) => persona.modelRequirements !== undefined, + ) + ) { + return false; + } + if ( + !pipeline.knowledgeSources.every( + (knowledgeSource) => knowledgeSource.preparationIds !== undefined, + ) + ) { + return false; + } + /* TODO: [🧠][🍫] `tasks` can not be determined if they are fully prepared SO ignoring them > if (!pipeline.tasks.every(({ preparedContent }) => preparedContent === undefined)) { > return false; > } */ - return true; - } - /** - * TODO: [🔃][main] If the pipeline was prepared with different version or different set of models, prepare it once again - * TODO: [🐠] Maybe base this on `makeValidator` - * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared - * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline` - * - [🏍] ? Is context in each task - * - [♨] Are examples prepared - * - [♨] Are tasks prepared - */ - - /** - * Format either small or big number - * - * @public exported from `@promptbook/utils` - */ - function numberToString(value) { - if (value === 0) { - return '0'; - } - else if (Number.isNaN(value)) { - return VALUE_STRINGS.nan; - } - else if (value === Infinity) { - return VALUE_STRINGS.infinity; - } - else if (value === -Infinity) { - return VALUE_STRINGS.negativeInfinity; - } - for (var exponent = 0; exponent < 15; exponent++) { - var factor = Math.pow(10, exponent); - var valueRounded = Math.round(value * factor) / factor; - if (Math.abs(value - valueRounded) / value < SMALL_NUMBER) { - return valueRounded.toFixed(exponent); - } - } - return value.toString(); - } - - /** - * Function `valueToString` will convert the given value to string - * This is useful and used in the `templateParameters` function - * - * Note: This function is not just calling `toString` method - * It's more complex and can handle this conversion specifically for LLM models - * See `VALUE_STRINGS` - * - * Note: There are 2 similar functions - * - `valueToString` converts value to string for LLM models as human-readable string - * - `asSerializable` converts value to string to preserve full information to be able to convert it back - * - * @public exported from `@promptbook/utils` - */ - function valueToString(value) { - try { - if (value === '') { - return VALUE_STRINGS.empty; - } - else if (value === null) { - return VALUE_STRINGS.null; - } - else if (value === undefined) { - return VALUE_STRINGS.undefined; - } - else if (typeof value === 'string') { - return value; - } - else if (typeof value === 'number') { - return numberToString(value); - } - else if (value instanceof Date) { - return value.toISOString(); - } - else { - return JSON.stringify(value); - } - } - catch (error) { - if (!(error instanceof Error)) {console.log('!(error instanceof Error)') - throw error; - } - console.error(error); - return VALUE_STRINGS.unserializable; - } - } - - /** - * Represents the usage with no resources consumed - * - * @public exported from `@promptbook/core` - */ - var ZERO_USAGE = $deepFreeze({ - price: { value: 0 }, - input: { - tokensCount: { value: 0 }, - charactersCount: { value: 0 }, - wordsCount: { value: 0 }, - sentencesCount: { value: 0 }, - linesCount: { value: 0 }, - paragraphsCount: { value: 0 }, - pagesCount: { value: 0 }, - }, - output: { - tokensCount: { value: 0 }, - charactersCount: { value: 0 }, - wordsCount: { value: 0 }, - sentencesCount: { value: 0 }, - linesCount: { value: 0 }, - paragraphsCount: { value: 0 }, - pagesCount: { value: 0 }, - }, - }); - /** - * Represents the usage with unknown resources consumed - * - * @public exported from `@promptbook/core` - */ - $deepFreeze({ - price: { value: 0, isUncertain: true }, - input: { - tokensCount: { value: 0, isUncertain: true }, - charactersCount: { value: 0, isUncertain: true }, - wordsCount: { value: 0, isUncertain: true }, - sentencesCount: { value: 0, isUncertain: true }, - linesCount: { value: 0, isUncertain: true }, - paragraphsCount: { value: 0, isUncertain: true }, - pagesCount: { value: 0, isUncertain: true }, - }, - output: { - tokensCount: { value: 0, isUncertain: true }, - charactersCount: { value: 0, isUncertain: true }, - wordsCount: { value: 0, isUncertain: true }, - sentencesCount: { value: 0, isUncertain: true }, - linesCount: { value: 0, isUncertain: true }, - paragraphsCount: { value: 0, isUncertain: true }, - pagesCount: { value: 0, isUncertain: true }, - }, - }); - /** - * Note: [💞] Ignore a discrepancy between file name and entity name - */ - - /** - * Function `addUsage` will add multiple usages into one - * - * Note: If you provide 0 values, it returns ZERO_USAGE - * - * @public exported from `@promptbook/core` - */ - function addUsage() { - var usageItems = []; - for (var _i = 0; _i < arguments.length; _i++) { - usageItems[_i] = arguments[_i]; - } - return usageItems.reduce(function (acc, item) { - var e_1, _a, e_2, _b; - var _c; - acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0; - try { - for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) { - var key = _e.value; - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - if (item.input[key]) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - acc.input[key].value += item.input[key].value || 0; - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - if (item.input[key].isUncertain) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - acc.input[key].isUncertain = true; - } - } - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_e && !_e.done && (_a = _d.return)) _a.call(_d); - } - finally { if (e_1) throw e_1.error; } - } - try { - for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) { - var key = _g.value; - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - if (item.output[key]) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - acc.output[key].value += item.output[key].value || 0; - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - if (item.output[key].isUncertain) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore - acc.output[key].isUncertain = true; - } - } - } - } - catch (e_2_1) { e_2 = { error: e_2_1 }; } - finally { - try { - if (_g && !_g.done && (_b = _f.return)) _b.call(_f); - } - finally { if (e_2) throw e_2.error; } - } - return acc; - }, deepClone(ZERO_USAGE)); - } - - /** - * Parses the given script and returns the list of all used variables that are not defined in the script - * - * @param script from which to extract the variables - * @returns the list of variable names - * @throws {ParseError} if the script is invalid - * @public exported from `@promptbook/utils` <- Note: [👖] This is usable elsewhere than in Promptbook, so keeping in utils - */ - function extractVariablesFromScript(script) { - var variables = new Set(); - var originalScript = script; - script = "(()=>{".concat(script, "})()"); - try { - for (var i = 0; i < 100 /* <- TODO: This limit to configuration */; i++) - try { - eval(script); - } - catch (error) { - if (!(error instanceof ReferenceError)) { - throw error; - } - /* + return true; + } + /** + * TODO: [🔃][main] If the pipeline was prepared with different version or different set of models, prepare it once again + * TODO: [🐠] Maybe base this on `makeValidator` + * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared + * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline` + * - [🏍] ? Is context in each task + * - [♨] Are examples prepared + * - [♨] Are tasks prepared + */ + + /** + * Format either small or big number + * + * @public exported from `@promptbook/utils` + */ + function numberToString(value) { + if (value === 0) { + return "0"; + } else if (Number.isNaN(value)) { + return VALUE_STRINGS.nan; + } else if (value === Number.POSITIVE_INFINITY) { + return VALUE_STRINGS.infinity; + } else if (value === Number.NEGATIVE_INFINITY) { + return VALUE_STRINGS.negativeInfinity; + } + for (var exponent = 0; exponent < 15; exponent++) { + var factor = Math.pow(10, exponent); + var valueRounded = Math.round(value * factor) / factor; + if (Math.abs(value - valueRounded) / value < SMALL_NUMBER) { + return valueRounded.toFixed(exponent); + } + } + return value.toString(); + } + + /** + * Function `valueToString` will convert the given value to string + * This is useful and used in the `templateParameters` function + * + * Note: This function is not just calling `toString` method + * It's more complex and can handle this conversion specifically for LLM models + * See `VALUE_STRINGS` + * + * Note: There are 2 similar functions + * - `valueToString` converts value to string for LLM models as human-readable string + * - `asSerializable` converts value to string to preserve full information to be able to convert it back + * + * @public exported from `@promptbook/utils` + */ + function valueToString(value) { + try { + if (value === "") { + return VALUE_STRINGS.empty; + } else if (value === null) { + return VALUE_STRINGS.null; + } else if (value === undefined) { + return VALUE_STRINGS.undefined; + } else if (typeof value === "string") { + return value; + } else if (typeof value === "number") { + return numberToString(value); + } else if (value instanceof Date) { + return value.toISOString(); + } else { + return JSON.stringify(value); + } + } catch (error) { + if (!(error instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error; + } + console.error(error); + return VALUE_STRINGS.unserializable; + } + } + + /** + * Represents the usage with no resources consumed + * + * @public exported from `@promptbook/core` + */ + var ZERO_USAGE = $deepFreeze({ + price: { value: 0 }, + input: { + tokensCount: { value: 0 }, + charactersCount: { value: 0 }, + wordsCount: { value: 0 }, + sentencesCount: { value: 0 }, + linesCount: { value: 0 }, + paragraphsCount: { value: 0 }, + pagesCount: { value: 0 }, + }, + output: { + tokensCount: { value: 0 }, + charactersCount: { value: 0 }, + wordsCount: { value: 0 }, + sentencesCount: { value: 0 }, + linesCount: { value: 0 }, + paragraphsCount: { value: 0 }, + pagesCount: { value: 0 }, + }, + }); + /** + * Represents the usage with unknown resources consumed + * + * @public exported from `@promptbook/core` + */ + $deepFreeze({ + price: { value: 0, isUncertain: true }, + input: { + tokensCount: { value: 0, isUncertain: true }, + charactersCount: { value: 0, isUncertain: true }, + wordsCount: { value: 0, isUncertain: true }, + sentencesCount: { value: 0, isUncertain: true }, + linesCount: { value: 0, isUncertain: true }, + paragraphsCount: { value: 0, isUncertain: true }, + pagesCount: { value: 0, isUncertain: true }, + }, + output: { + tokensCount: { value: 0, isUncertain: true }, + charactersCount: { value: 0, isUncertain: true }, + wordsCount: { value: 0, isUncertain: true }, + sentencesCount: { value: 0, isUncertain: true }, + linesCount: { value: 0, isUncertain: true }, + paragraphsCount: { value: 0, isUncertain: true }, + pagesCount: { value: 0, isUncertain: true }, + }, + }); + /** + * Note: [💞] Ignore a discrepancy between file name and entity name + */ + + /** + * Function `addUsage` will add multiple usages into one + * + * Note: If you provide 0 values, it returns ZERO_USAGE + * + * @public exported from `@promptbook/core` + */ + function addUsage() { + var usageItems = []; + for (var _i = 0; _i < arguments.length; _i++) { + usageItems[_i] = arguments[_i]; + } + return usageItems.reduce((acc, item) => { + var e_1, _a, e_2, _b; + var _c; + acc.price.value += + ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || + 0; + try { + for ( + var _d = __values(Object.keys(acc.input)), _e = _d.next(); + !_e.done; + _e = _d.next() + ) { + var key = _e.value; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + if (item.input[key]) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + acc.input[key].value += item.input[key].value || 0; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + if (item.input[key].isUncertain) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + acc.input[key].isUncertain = true; + } + } + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_e && !_e.done && (_a = _d.return)) _a.call(_d); + } finally { + if (e_1) throw e_1.error; + } + } + try { + for ( + var _f = __values(Object.keys(acc.output)), _g = _f.next(); + !_g.done; + _g = _f.next() + ) { + var key = _g.value; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + if (item.output[key]) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + acc.output[key].value += item.output[key].value || 0; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + if (item.output[key].isUncertain) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + acc.output[key].isUncertain = true; + } + } + } + } catch (e_2_1) { + e_2 = { error: e_2_1 }; + } finally { + try { + if (_g && !_g.done && (_b = _f.return)) _b.call(_f); + } finally { + if (e_2) throw e_2.error; + } + } + return acc; + }, deepClone(ZERO_USAGE)); + } + + /** + * Parses the given script and returns the list of all used variables that are not defined in the script + * + * @param script from which to extract the variables + * @returns the list of variable names + * @throws {ParseError} if the script is invalid + * @public exported from `@promptbook/utils` <- Note: [👖] This is usable elsewhere than in Promptbook, so keeping in utils + */ + function extractVariablesFromScript(script) { + var variables = new Set(); + var originalScript = script; + script = "(()=>{".concat(script, "})()"); + try { + for (var i = 0; i < 100 /* <- TODO: This limit to configuration */; i++) + try { + eval(script); + } catch (error) { + if (!(error instanceof ReferenceError)) { + throw error; + } + /* Note: Parsing the error 🌟 Most devices: [PipelineUrlError: thing is not defined] @@ -3199,1045 +4582,1424 @@ at step (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\node_modules\\node_modu 🍏 iPhone`s Safari: [PipelineUrlError: Can't find variable: thing] */ - var variableName = undefined; - if (error.message.startsWith("Can't")) { - // 🍏 Case - variableName = error.message.split(' ').pop(); - } - else { - // 🌟 Case - variableName = error.message.split(' ').shift(); - } - if (variableName === undefined) { - throw error; - } - if (script.includes(variableName + '(')) { - script = "const ".concat(variableName, " = ()=>'';") + script; - } - else { - variables.add(variableName); - script = "const ".concat(variableName, " = '';") + script; - } - } - } - catch (error) { - if (!(error instanceof Error)) {console.log('!(error instanceof Error)') - throw error; - } - throw new ParseError(spaceTrim.spaceTrim(function (block) { return "\n Can not extract variables from the script\n ".concat(block(error.stack || error.message), "\n\n Found variables:\n ").concat(Array.from(variables) - .map(function (variableName, i) { return "".concat(i + 1, ") ").concat(variableName); }) - .join('\n'), "\n\n\n The script:\n\n ```javascript\n ").concat(block(originalScript), "\n ```\n "); })); - } - return variables; - } - /** - * TODO: [🔣] Support for multiple languages - python, java,... - */ - - /** - * Parses the task and returns the set of all used parameters - * - * @param task the task with used parameters - * @returns the set of parameter names - * @throws {ParseError} if the script is invalid - * @public exported from `@promptbook/core` <- Note: [👖] This utility is so tightly interconnected with the Promptbook that it is not exported as util but in core - */ - function extractParameterNamesFromTask(task) { - var e_1, _a, e_2, _b, e_3, _c, e_4, _d; - var title = task.title, description = task.description, taskType = task.taskType, content = task.content, preparedContent = task.preparedContent, jokerParameterNames = task.jokerParameterNames, foreach = task.foreach; - var parameterNames = new Set(); - try { - for (var _e = __values(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameterNames(title)), false), __read(extractParameterNames(description || '')), false), __read(extractParameterNames(content)), false), __read(extractParameterNames(preparedContent || '')), false)), _f = _e.next(); !_f.done; _f = _e.next()) { - var parameterName = _f.value; - parameterNames.add(parameterName); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_f && !_f.done && (_a = _e.return)) _a.call(_e); - } - finally { if (e_1) throw e_1.error; } - } - if (taskType === 'SCRIPT_TASK') { - try { - for (var _g = __values(extractVariablesFromScript(content)), _h = _g.next(); !_h.done; _h = _g.next()) { - var parameterName = _h.value; - parameterNames.add(parameterName); - } - } - catch (e_2_1) { e_2 = { error: e_2_1 }; } - finally { - try { - if (_h && !_h.done && (_b = _g.return)) _b.call(_g); - } - finally { if (e_2) throw e_2.error; } - } - } - try { - for (var _j = __values(jokerParameterNames || []), _k = _j.next(); !_k.done; _k = _j.next()) { - var jokerName = _k.value; - parameterNames.add(jokerName); - } - } - catch (e_3_1) { e_3 = { error: e_3_1 }; } - finally { - try { - if (_k && !_k.done && (_c = _j.return)) _c.call(_j); - } - finally { if (e_3) throw e_3.error; } - } - parameterNames.delete('content'); - // <- Note {websiteContent} is used in `preparedContent` - // Note: [🍭] Fixing dependent subparameterName from FOREACH command - if (foreach !== undefined) { - try { - for (var _l = __values(foreach.inputSubparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) { - var subparameterName = _m.value; - if (parameterNames.has(subparameterName)) { - parameterNames.delete(subparameterName); - parameterNames.add(foreach.parameterName); - // <- TODO: [🚎] Warn/logic error when `subparameterName` not used - } - } - } - catch (e_4_1) { e_4 = { error: e_4_1 }; } - finally { - try { - if (_m && !_m.done && (_d = _l.return)) _d.call(_l); - } - finally { if (e_4) throw e_4.error; } - } - } - return parameterNames; - } - /** - * TODO: [🔣] If script require contentLanguage - */ - - /** - * Create difference set of two sets. - * - * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set - * @public exported from `@promptbook/utils` - */ - function difference(a, b, isEqual) { - var e_1, _a; - if (isEqual === void 0) { isEqual = function (a, b) { return a === b; }; } - var diff = new Set(); - var _loop_1 = function (itemA) { - if (!Array.from(b).some(function (itemB) { return isEqual(itemA, itemB); })) { - diff.add(itemA); - } - }; - try { - for (var _b = __values(Array.from(a)), _c = _b.next(); !_c.done; _c = _b.next()) { - var itemA = _c.value; - _loop_1(itemA); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_c && !_c.done && (_a = _b.return)) _a.call(_b); - } - finally { if (e_1) throw e_1.error; } - } - return diff; - } - /** - * TODO: [🧠][💯] Maybe also implement symmetricDifference - */ - - /** - * Creates a new set with all elements that are present in either set - * - * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set - * @public exported from `@promptbook/utils` - */ - function union() { - var e_1, _a, e_2, _b; - var sets = []; - for (var _i = 0; _i < arguments.length; _i++) { - sets[_i] = arguments[_i]; - } - var union = new Set(); - try { - for (var sets_1 = __values(sets), sets_1_1 = sets_1.next(); !sets_1_1.done; sets_1_1 = sets_1.next()) { - var set = sets_1_1.value; - try { - for (var _c = (e_2 = void 0, __values(Array.from(set))), _d = _c.next(); !_d.done; _d = _c.next()) { - var item = _d.value; - union.add(item); - } - } - catch (e_2_1) { e_2 = { error: e_2_1 }; } - finally { - try { - if (_d && !_d.done && (_b = _c.return)) _b.call(_c); - } - finally { if (e_2) throw e_2.error; } - } - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (sets_1_1 && !sets_1_1.done && (_a = sets_1.return)) _a.call(sets_1); - } - finally { if (e_1) throw e_1.error; } - } - return union; - } - - /** - * @@@ - * - * @public exported from `@promptbook/core` - */ - var MANDATORY_CSV_SETTINGS = Object.freeze({ - header: true, - // encoding: 'utf-8', - }); - - /** - * Definition for CSV spreadsheet - * - * @public exported from `@promptbook/core` - * <- TODO: [🏢] Export from package `@promptbook/csv` - */ - var CsvFormatDefinition = { - formatName: 'CSV', - aliases: ['SPREADSHEET', 'TABLE'], - isValid: function (value, settings, schema) { - return true; - }, - canBeValid: function (partialValue, settings, schema) { - return true; - }, - heal: function (value, settings, schema) { - throw new Error('Not implemented'); - }, - subvalueDefinitions: [ - { - subvalueName: 'ROW', - mapValues: function (value, outputParameterName, settings, mapCallback) { - return __awaiter(this, void 0, void 0, function () { - var csv, mappedData; - var _this = this; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS)); - if (csv.errors.length !== 0) { - throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n Error(s) from CSV parsing:\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n The CSV setings:\n ").concat(block(JSON.stringify(__assign(__assign({}, settings), MANDATORY_CSV_SETTINGS), null, 2)), "\n\n The CSV data:\n ").concat(block(value), "\n "); })); - } - return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () { - var _a, _b; - var _c; - return __generator(this, function (_d) { - switch (_d.label) { - case 0: - if (row[outputParameterName]) { - throw new CsvFormatError("Can not overwrite existing column \"".concat(outputParameterName, "\" in CSV row")); - } - _a = [__assign({}, row)]; - _c = {}; - _b = outputParameterName; - return [4 /*yield*/, mapCallback(row, index)]; - case 1: return [2 /*return*/, __assign.apply(void 0, _a.concat([(_c[_b] = _d.sent(), _c)]))]; - } - }); - }); }))]; - case 1: - mappedData = _a.sent(); - return [2 /*return*/, papaparse.unparse(mappedData, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS))]; - } - }); - }); - }, - }, - { - subvalueName: 'CELL', - mapValues: function (value, outputParameterName, settings, mapCallback) { - return __awaiter(this, void 0, void 0, function () { - var csv, mappedData; - var _this = this; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS)); - if (csv.errors.length !== 0) { - throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n Error(s) from CSV parsing:\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n The CSV setings:\n ").concat(block(JSON.stringify(__assign(__assign({}, settings), MANDATORY_CSV_SETTINGS), null, 2)), "\n\n The CSV data:\n ").concat(block(value), "\n "); })); - } - return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () { - var _this = this; - return __generator(this, function (_a) { - return [2 /*return*/, /* not await */ Promise.all(Object.entries(row).map(function (_a, columnIndex) { - var _b = __read(_a, 2), key = _b[0], value = _b[1]; - return __awaiter(_this, void 0, void 0, function () { - var index; - var _c; - return __generator(this, function (_d) { - index = rowIndex * Object.keys(row).length + columnIndex; - return [2 /*return*/, /* not await */ mapCallback((_c = {}, _c[key] = value, _c), index)]; - }); - }); - }))]; - }); - }); }))]; - case 1: - mappedData = _a.sent(); - return [2 /*return*/, papaparse.unparse(mappedData, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS))]; - } - }); - }); - }, - }, - ], - }; - /** - * TODO: [🍓] In `CsvFormatDefinition` implement simple `isValid` - * TODO: [🍓] In `CsvFormatDefinition` implement partial `canBeValid` - * TODO: [🍓] In `CsvFormatDefinition` implement `heal - * TODO: [🍓] In `CsvFormatDefinition` implement `subvalueDefinitions` - * TODO: [🏢] Allow to expect something inside CSV objects and other formats - */ - - /** - * Definition for JSON format - * - * @private still in development [🏢] - */ - var JsonFormatDefinition = { - formatName: 'JSON', - mimeType: 'application/json', - isValid: function (value, settings, schema) { - return isValidJsonString(value); - }, - canBeValid: function (partialValue, settings, schema) { - return true; - }, - heal: function (value, settings, schema) { - throw new Error('Not implemented'); - }, - subvalueDefinitions: [], - }; - /** - * TODO: [🧠] Maybe propper instance of object - * TODO: [0] Make string_serialized_json - * TODO: [1] Make type for JSON Settings and Schema - * TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...? - * TODO: [🍓] In `JsonFormatDefinition` implement simple `isValid` - * TODO: [🍓] In `JsonFormatDefinition` implement partial `canBeValid` - * TODO: [🍓] In `JsonFormatDefinition` implement `heal - * TODO: [🍓] In `JsonFormatDefinition` implement `subvalueDefinitions` - * TODO: [🏢] Allow to expect something inside JSON objects and other formats - */ - - /** - * Definition for any text - this will be always valid - * - * Note: This is not useful for validation, but for splitting and mapping with `subvalueDefinitions` - * - * @public exported from `@promptbook/core` - */ - var TextFormatDefinition = { - formatName: 'TEXT', - isValid: function (value) { - return typeof value === 'string'; - }, - canBeValid: function (partialValue) { - return typeof partialValue === 'string'; - }, - heal: function () { - throw new UnexpectedError('It does not make sense to call `TextFormatDefinition.heal`'); - }, - subvalueDefinitions: [ - { - subvalueName: 'LINE', - mapValues: function (value, outputParameterName, settings, mapCallback) { - return __awaiter(this, void 0, void 0, function () { - var lines, mappedLines; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - lines = value.split('\n'); - return [4 /*yield*/, Promise.all(lines.map(function (lineContent, lineNumber) { - // TODO: [🧠] Maybe option to skip empty line - /* not await */ return mapCallback({ - lineContent: lineContent, - // TODO: [🧠] Maybe also put here `lineNumber` - }, lineNumber); - }))]; - case 1: - mappedLines = _a.sent(); - return [2 /*return*/, mappedLines.join('\n')]; - } - }); - }); - }, - }, - // <- TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues - ], - }; - /** - * TODO: [1] Make type for XML Text and Schema - * TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues - * TODO: [🍓] In `TextFormatDefinition` implement simple `isValid` - * TODO: [🍓] In `TextFormatDefinition` implement partial `canBeValid` - * TODO: [🍓] In `TextFormatDefinition` implement `heal - * TODO: [🍓] In `TextFormatDefinition` implement `subvalueDefinitions` - * TODO: [🏢] Allow to expect something inside each item of list and other formats - */ - - /** - * Definition for XML format - * - * @private still in development [🏢] - */ - var XmlFormatDefinition = { - formatName: 'XML', - mimeType: 'application/xml', - isValid: function (value, settings, schema) { - return true; - }, - canBeValid: function (partialValue, settings, schema) { - return true; - }, - heal: function (value, settings, schema) { - throw new Error('Not implemented'); - }, - subvalueDefinitions: [], - }; - /** - * TODO: [🧠] Maybe propper instance of object - * TODO: [0] Make string_serialized_xml - * TODO: [1] Make type for XML Settings and Schema - * TODO: [🧠] What to use for validating XMLs - XSD,... - * TODO: [🍓] In `XmlFormatDefinition` implement simple `isValid` - * TODO: [🍓] In `XmlFormatDefinition` implement partial `canBeValid` - * TODO: [🍓] In `XmlFormatDefinition` implement `heal - * TODO: [🍓] In `XmlFormatDefinition` implement `subvalueDefinitions` - * TODO: [🏢] Allow to expect something inside XML and other formats - */ - - /** - * Definitions for all formats supported by Promptbook - * - * @private internal index of `...` <- TODO [🏢] - */ - var FORMAT_DEFINITIONS = [ - JsonFormatDefinition, - XmlFormatDefinition, - TextFormatDefinition, - CsvFormatDefinition, - ]; - /** - * Note: [💞] Ignore a discrepancy between file name and entity name - */ - - /** - * Maps available parameters to expected parameters - * - * The strategy is: - * 1) @@@ - * 2) @@@ - * - * @throws {PipelineExecutionError} @@@ - * @private within the repository used in `createPipelineExecutor` - */ - function mapAvailableToExpectedParameters(options) { - var e_1, _a; - var expectedParameters = options.expectedParameters, availableParameters = options.availableParameters; - var availableParametersNames = new Set(Object.keys(availableParameters)); - var expectedParameterNames = new Set(Object.keys(expectedParameters)); - var mappedParameters = {}; - try { - // Phase 1️⃣: Matching mapping - for (var _b = __values(Array.from(union(availableParametersNames, expectedParameterNames))), _c = _b.next(); !_c.done; _c = _b.next()) { - var parameterName = _c.value; - // Situation: Parameter is available and expected - if (availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) { - mappedParameters[parameterName] = availableParameters[parameterName]; - // <- Note: [👩‍👩‍👧] Maybe detect parameter collision here? - availableParametersNames.delete(parameterName); - expectedParameterNames.delete(parameterName); - } - // Situation: Parameter is available but NOT expected - else if (availableParametersNames.has(parameterName) && !expectedParameterNames.has(parameterName)) { - // [🐱‍👤] Do not pass this parameter to prompt - Maybe use it non-matching mapping - } - // Situation: Parameter is NOT available BUT expected - else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) { - // Do nothing here - this will be maybe fixed in the non-matching mapping - } - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_c && !_c.done && (_a = _b.return)) _a.call(_b); - } - finally { if (e_1) throw e_1.error; } - } - if (expectedParameterNames.size === 0) { - // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@ - Object.freeze(mappedParameters); - return mappedParameters; - } - // Phase 2️⃣: Non-matching mapping - if (expectedParameterNames.size !== availableParametersNames.size) { - throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n Can not map available parameters to expected parameters\n\n Mapped parameters:\n ".concat(block(Object.keys(mappedParameters) - .map(function (parameterName) { return "- {".concat(parameterName, "}"); }) - .join('\n')), "\n\n Expected parameters which can not be mapped:\n ").concat(block(Array.from(expectedParameterNames) - .map(function (parameterName) { return "- {".concat(parameterName, "}"); }) - .join('\n')), "\n\n Remaining available parameters:\n ").concat(block(Array.from(availableParametersNames) - .map(function (parameterName) { return "- {".concat(parameterName, "}"); }) - .join('\n')), "\n\n "); })); - } - var expectedParameterNamesArray = Array.from(expectedParameterNames); - var availableParametersNamesArray = Array.from(availableParametersNames); - for (var i = 0; i < expectedParameterNames.size; i++) { - mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]]; - } - // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@ - Object.freeze(mappedParameters); - return mappedParameters; - } - - /** - * Extracts all code blocks from markdown. - * - * Note: There are multiple simmilar function: - * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing - * - `extractJsonBlock` extracts exactly one valid JSON code block - * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block - * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block - * - * @param markdown any valid markdown - * @returns code blocks with language and content - * @throws {ParseError} if block is not closed properly - * @public exported from `@promptbook/markdown-utils` - */ - function extractAllBlocksFromMarkdown(markdown) { - var e_1, _a; - var codeBlocks = []; - var lines = markdown.split('\n'); - // Note: [0] Ensure that the last block notated by gt > will be closed - lines.push(''); - var currentCodeBlock = null; - try { - for (var lines_1 = __values(lines), lines_1_1 = lines_1.next(); !lines_1_1.done; lines_1_1 = lines_1.next()) { - var line = lines_1_1.value; - if (line.startsWith('> ') || line === '>') { - if (currentCodeBlock === null) { - currentCodeBlock = { blockNotation: '>', language: null, content: '' }; - } /* not else */ - if (currentCodeBlock.blockNotation === '>') { - if (currentCodeBlock.content !== '') { - currentCodeBlock.content += '\n'; - } - currentCodeBlock.content += line.slice(2); - } - } - else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '>' /* <- Note: [0] */) { - codeBlocks.push(currentCodeBlock); - currentCodeBlock = null; - } - /* not else */ - if (line.startsWith('```')) { - var language = line.slice(3).trim() || null; - if (currentCodeBlock === null) { - currentCodeBlock = { blockNotation: '```', language: language, content: '' }; - } - else { - if (language !== null) { - throw new ParseError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed and already opening new ").concat(language, " code block")); - } - codeBlocks.push(currentCodeBlock); - currentCodeBlock = null; - } - } - else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '```') { - if (currentCodeBlock.content !== '') { - currentCodeBlock.content += '\n'; - } - currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make propper unescape */; - } - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (lines_1_1 && !lines_1_1.done && (_a = lines_1.return)) _a.call(lines_1); - } - finally { if (e_1) throw e_1.error; } - } - if (currentCodeBlock !== null) { - throw new ParseError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed at the end of the markdown")); - } - return codeBlocks; - } - /** - * TODO: Maybe name for `blockNotation` instead of '```' and '>' - */ - - /** - * Extracts extracts exactly one valid JSON code block - * - * - When given string is a valid JSON as it is, it just returns it - * - When there is no JSON code block the function throws a `ParseError` - * - When there are multiple JSON code blocks the function throws a `ParseError` - * - * Note: It is not important if marked as ```json BUT if it is VALID JSON - * Note: There are multiple simmilar function: - * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing - * - `extractJsonBlock` extracts exactly one valid JSON code block - * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block - * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block - * - * @public exported from `@promptbook/markdown-utils` - * @throws {ParseError} if there is no valid JSON block in the markdown - */ - function extractJsonBlock(markdown) { - if (isValidJsonString(markdown)) { - return markdown; - } - var codeBlocks = extractAllBlocksFromMarkdown(markdown); - var jsonBlocks = codeBlocks.filter(function (_a) { - var content = _a.content; - return isValidJsonString(content); - }); - if (jsonBlocks.length === 0) { - throw new Error('There is no valid JSON block in the markdown'); - } - if (jsonBlocks.length > 1) { - throw new Error('There are multiple JSON code blocks in the markdown'); - } - return jsonBlocks[0].content; - } - /** - * TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc. - * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString` - */ - - /** - * Takes an item or an array of items and returns an array of items - * - * 1) Any item except array and undefined returns array with that one item (also null) - * 2) Undefined returns empty array - * 3) Array returns itself - * - * @private internal utility - */ - function arrayableToArray(input) { - if (input === undefined) { - return []; - } - if (input instanceof Array) { - return input; - } - return [input]; - } - - /** - * Replaces parameters in template with values from parameters object - * - * Note: This function is not places strings into string, - * It's more complex and can handle this operation specifically for LLM models - * - * @param template the template with parameters in {curly} braces - * @param parameters the object with parameters - * @returns the template with replaced parameters - * @throws {PipelineExecutionError} if parameter is not defined, not closed, or not opened - * @public exported from `@promptbook/utils` - */ - function templateParameters(template, parameters) { - var e_1, _a; - try { - for (var _b = __values(Object.entries(parameters)), _c = _b.next(); !_c.done; _c = _b.next()) { - var _d = __read(_c.value, 2), parameterName = _d[0], parameterValue = _d[1]; - if (parameterValue === RESERVED_PARAMETER_MISSING_VALUE) { - throw new UnexpectedError("Parameter `{".concat(parameterName, "}` has missing value")); - } - else if (parameterValue === RESERVED_PARAMETER_RESTRICTED) { - // TODO: [🍵] - throw new UnexpectedError("Parameter `{".concat(parameterName, "}` is restricted to use")); - } - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_c && !_c.done && (_a = _b.return)) _a.call(_b); - } - finally { if (e_1) throw e_1.error; } - } - var replacedTemplates = template; - var match; - var loopLimit = LOOP_LIMIT; - var _loop_1 = function () { - if (loopLimit-- < 0) { - throw new LimitReachedError('Loop limit reached during parameters replacement in `templateParameters`'); - } - var precol = match.groups.precol; - var parameterName = match.groups.parameterName; - if (parameterName === '') { - return "continue"; - } - if (parameterName.indexOf('{') !== -1 || parameterName.indexOf('}') !== -1) { - throw new PipelineExecutionError('Parameter is already opened or not closed'); - } - if (parameters[parameterName] === undefined) { - throw new PipelineExecutionError("Parameter `{".concat(parameterName, "}` is not defined")); - } - var parameterValue = parameters[parameterName]; - if (parameterValue === undefined) { - throw new PipelineExecutionError("Parameter `{".concat(parameterName, "}` is not defined")); - } - parameterValue = valueToString(parameterValue); - // Escape curly braces in parameter values to prevent prompt-injection - parameterValue = parameterValue.replace(/[{}]/g, '\\$&'); - if (parameterValue.includes('\n') && /^\s*\W{0,3}\s*$/.test(precol)) { - parameterValue = parameterValue - .split('\n') - .map(function (line, index) { return (index === 0 ? line : "".concat(precol).concat(line)); }) - .join('\n'); - } - replacedTemplates = - replacedTemplates.substring(0, match.index + precol.length) + - parameterValue + - replacedTemplates.substring(match.index + precol.length + parameterName.length + 2); - }; - while ((match = /^(?.*){(?\w+)}(.*)/m /* <- Not global */ - .exec(replacedTemplates))) { - _loop_1(); - } - // [💫] Check if there are parameters that are not closed properly - if (/{\w+$/.test(replacedTemplates)) { - throw new PipelineExecutionError('Parameter is not closed'); - } - // [💫] Check if there are parameters that are not opened properly - if (/^\w+}/.test(replacedTemplates)) { - throw new PipelineExecutionError('Parameter is not opened'); - } - return replacedTemplates; - } - - /** - * Counts number of characters in the text - * - * @public exported from `@promptbook/utils` - */ - function countCharacters(text) { - // Remove null characters - text = text.replace(/\0/g, ''); - // Replace emojis (and also ZWJ sequence) with hyphens - text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1'); - text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1'); - text = text.replace(/\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, '-'); - return text.length; - } - - /** - * Number of characters per standard line with 11pt Arial font size. - * - * @public exported from `@promptbook/utils` - */ - var CHARACTERS_PER_STANDARD_LINE = 63; - /** - * Number of lines per standard A4 page with 11pt Arial font size and standard margins and spacing. - * - * @public exported from `@promptbook/utils` - */ - var LINES_PER_STANDARD_PAGE = 44; - /** - * TODO: [🧠] Should be this `constants.ts` or `config.ts`? - * Note: [💞] Ignore a discrepancy between file name and entity name - */ - - /** - * Counts number of lines in the text - * - * Note: This does not check only for the presence of newlines, but also for the length of the standard line. - * - * @public exported from `@promptbook/utils` - */ - function countLines(text) { - text = text.replace('\r\n', '\n'); - text = text.replace('\r', '\n'); - var lines = text.split('\n'); - return lines.reduce(function (count, line) { return count + Math.ceil(line.length / CHARACTERS_PER_STANDARD_LINE); }, 0); - } - - /** - * Counts number of pages in the text - * - * Note: This does not check only for the count of newlines, but also for the length of the standard line and length of the standard page. - * - * @public exported from `@promptbook/utils` - */ - function countPages(text) { - return Math.ceil(countLines(text) / LINES_PER_STANDARD_PAGE); - } - - /** - * Counts number of paragraphs in the text - * - * @public exported from `@promptbook/utils` - */ - function countParagraphs(text) { - return text.split(/\n\s*\n/).filter(function (paragraph) { return paragraph.trim() !== ''; }).length; - } - - /** - * Split text into sentences - * - * @public exported from `@promptbook/utils` - */ - function splitIntoSentences(text) { - return text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; }); - } - /** - * Counts number of sentences in the text - * - * @public exported from `@promptbook/utils` - */ - function countSentences(text) { - return splitIntoSentences(text).length; - } - - var defaultDiacriticsRemovalMap = [ - { - base: 'A', - letters: '\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F', - }, - { base: 'AA', letters: '\uA732' }, - { base: 'AE', letters: '\u00C6\u01FC\u01E2' }, - { base: 'AO', letters: '\uA734' }, - { base: 'AU', letters: '\uA736' }, - { base: 'AV', letters: '\uA738\uA73A' }, - { base: 'AY', letters: '\uA73C' }, - { - base: 'B', - letters: '\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181', - }, - { - base: 'C', - letters: '\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E', - }, - { - base: 'D', - letters: '\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779\u00D0', - }, - { base: 'DZ', letters: '\u01F1\u01C4' }, - { base: 'Dz', letters: '\u01F2\u01C5' }, - { - base: 'E', - letters: '\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E', - }, - { base: 'F', letters: '\u0046\u24BB\uFF26\u1E1E\u0191\uA77B' }, - { - base: 'G', - letters: '\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E', - }, - { - base: 'H', - letters: '\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D', - }, - { - base: 'I', - letters: '\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197', - }, - { base: 'J', letters: '\u004A\u24BF\uFF2A\u0134\u0248' }, - { - base: 'K', - letters: '\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2', - }, - { - base: 'L', - letters: '\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780', - }, - { base: 'LJ', letters: '\u01C7' }, - { base: 'Lj', letters: '\u01C8' }, - { base: 'M', letters: '\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C' }, - { - base: 'N', - letters: '\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4', - }, - { base: 'NJ', letters: '\u01CA' }, - { base: 'Nj', letters: '\u01CB' }, - { - base: 'O', - letters: '\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C', - }, - { base: 'OI', letters: '\u01A2' }, - { base: 'OO', letters: '\uA74E' }, - { base: 'OU', letters: '\u0222' }, - { base: 'OE', letters: '\u008C\u0152' }, - { base: 'oe', letters: '\u009C\u0153' }, - { - base: 'P', - letters: '\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754', - }, - { base: 'Q', letters: '\u0051\u24C6\uFF31\uA756\uA758\u024A' }, - { - base: 'R', - letters: '\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782', - }, - { - base: 'S', - letters: '\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784', - }, - { - base: 'T', - letters: '\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786', - }, - { base: 'TZ', letters: '\uA728' }, - { - base: 'U', - letters: '\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244', - }, - { base: 'V', letters: '\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245' }, - { base: 'VY', letters: '\uA760' }, - { - base: 'W', - letters: '\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72', - }, - { base: 'X', letters: '\u0058\u24CD\uFF38\u1E8A\u1E8C' }, - { - base: 'Y', - letters: '\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE', - }, - { - base: 'Z', - letters: '\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762', - }, - { - base: 'a', - letters: '\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250', - }, - { base: 'aa', letters: '\uA733' }, - { base: 'ae', letters: '\u00E6\u01FD\u01E3' }, - { base: 'ao', letters: '\uA735' }, - { base: 'au', letters: '\uA737' }, - { base: 'av', letters: '\uA739\uA73B' }, - { base: 'ay', letters: '\uA73D' }, - { - base: 'b', - letters: '\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253', - }, - { - base: 'c', - letters: '\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184', - }, - { - base: 'd', - letters: '\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A', - }, - { base: 'dz', letters: '\u01F3\u01C6' }, - { - base: 'e', - letters: '\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD', - }, - { base: 'f', letters: '\u0066\u24D5\uFF46\u1E1F\u0192\uA77C' }, - { - base: 'g', - letters: '\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F', - }, - { - base: 'h', - letters: '\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265', - }, - { base: 'hv', letters: '\u0195' }, - { - base: 'i', - letters: '\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131', - }, - { base: 'j', letters: '\u006A\u24D9\uFF4A\u0135\u01F0\u0249' }, - { - base: 'k', - letters: '\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3', - }, - { - base: 'l', - letters: '\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747', - }, - { base: 'lj', letters: '\u01C9' }, - { base: 'm', letters: '\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F' }, - { - base: 'n', - letters: '\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5', - }, - { base: 'nj', letters: '\u01CC' }, - { - base: 'o', - letters: '\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275', - }, - { base: 'oi', letters: '\u01A3' }, - { base: 'ou', letters: '\u0223' }, - { base: 'oo', letters: '\uA74F' }, - { - base: 'p', - letters: '\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755', - }, - { base: 'q', letters: '\u0071\u24E0\uFF51\u024B\uA757\uA759' }, - { - base: 'r', - letters: '\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783', - }, - { - base: 's', - letters: '\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B', - }, - { - base: 't', - letters: '\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787', - }, - { base: 'tz', letters: '\uA729' }, - { - base: 'u', - letters: '\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289', - }, - { base: 'v', letters: '\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C' }, - { base: 'vy', letters: '\uA761' }, - { - base: 'w', - letters: '\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73', - }, - { base: 'x', letters: '\u0078\u24E7\uFF58\u1E8B\u1E8D' }, - { - base: 'y', - letters: '\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF', - }, - { - base: 'z', - letters: '\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763', - }, - ]; - /** - * Map of letters from diacritic variant to diacritless variant - * Contains lowercase and uppercase separatelly - * - * > "á" => "a" - * > "ě" => "e" - * > "Ă" => "A" - * > ... - * - * @public exported from `@promptbook/utils` - */ - var DIACRITIC_VARIANTS_LETTERS = {}; - // tslint:disable-next-line: prefer-for-of - for (var i = 0; i < defaultDiacriticsRemovalMap.length; i++) { - var letters = defaultDiacriticsRemovalMap[i].letters; - // tslint:disable-next-line: prefer-for-of - for (var j = 0; j < letters.length; j++) { - DIACRITIC_VARIANTS_LETTERS[letters[j]] = defaultDiacriticsRemovalMap[i].base; - } - } - // <- TODO: [🍓] Put to maker function to save execution time if not needed - /* + var variableName = undefined; + if (error.message.startsWith("Can't")) { + // 🍏 Case + variableName = error.message.split(" ").pop(); + } else { + // 🌟 Case + variableName = error.message.split(" ").shift(); + } + if (variableName === undefined) { + throw error; + } + if (script.includes(variableName + "(")) { + script = "const ".concat(variableName, " = ()=>'';") + script; + } else { + variables.add(variableName); + script = "const ".concat(variableName, " = '';") + script; + } + } + } catch (error) { + if (!(error instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error; + } + throw new ParseError( + spaceTrim.spaceTrim((block) => + "\n Can not extract variables from the script\n " + .concat( + block(error.stack || error.message), + "\n\n Found variables:\n ", + ) + .concat( + Array.from(variables) + .map((variableName, i) => + "".concat(i + 1, ") ").concat(variableName), + ) + .join("\n"), + "\n\n\n The script:\n\n ```javascript\n ", + ) + .concat( + block(originalScript), + "\n ```\n ", + ), + ), + ); + } + return variables; + } + /** + * TODO: [🔣] Support for multiple languages - python, java,... + */ + + /** + * Parses the task and returns the set of all used parameters + * + * @param task the task with used parameters + * @returns the set of parameter names + * @throws {ParseError} if the script is invalid + * @public exported from `@promptbook/core` <- Note: [👖] This utility is so tightly interconnected with the Promptbook that it is not exported as util but in core + */ + function extractParameterNamesFromTask(task) { + var e_1, _a, e_2, _b, e_3, _c, e_4, _d; + var title = task.title, + description = task.description, + taskType = task.taskType, + content = task.content, + preparedContent = task.preparedContent, + jokerParameterNames = task.jokerParameterNames, + foreach = task.foreach; + var parameterNames = new Set(); + try { + for ( + var _e = __values( + __spreadArray( + __spreadArray( + __spreadArray( + __spreadArray( + [], + __read(extractParameterNames(title)), + false, + ), + __read(extractParameterNames(description || "")), + false, + ), + __read(extractParameterNames(content)), + false, + ), + __read(extractParameterNames(preparedContent || "")), + false, + ), + ), + _f = _e.next(); + !_f.done; + _f = _e.next() + ) { + var parameterName = _f.value; + parameterNames.add(parameterName); + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_f && !_f.done && (_a = _e.return)) _a.call(_e); + } finally { + if (e_1) throw e_1.error; + } + } + if (taskType === "SCRIPT_TASK") { + try { + for ( + var _g = __values(extractVariablesFromScript(content)), + _h = _g.next(); + !_h.done; + _h = _g.next() + ) { + var parameterName = _h.value; + parameterNames.add(parameterName); + } + } catch (e_2_1) { + e_2 = { error: e_2_1 }; + } finally { + try { + if (_h && !_h.done && (_b = _g.return)) _b.call(_g); + } finally { + if (e_2) throw e_2.error; + } + } + } + try { + for ( + var _j = __values(jokerParameterNames || []), _k = _j.next(); + !_k.done; + _k = _j.next() + ) { + var jokerName = _k.value; + parameterNames.add(jokerName); + } + } catch (e_3_1) { + e_3 = { error: e_3_1 }; + } finally { + try { + if (_k && !_k.done && (_c = _j.return)) _c.call(_j); + } finally { + if (e_3) throw e_3.error; + } + } + parameterNames.delete("content"); + // <- Note {websiteContent} is used in `preparedContent` + // Note: [🍭] Fixing dependent subparameterName from FOREACH command + if (foreach !== undefined) { + try { + for ( + var _l = __values(foreach.inputSubparameterNames), _m = _l.next(); + !_m.done; + _m = _l.next() + ) { + var subparameterName = _m.value; + if (parameterNames.has(subparameterName)) { + parameterNames.delete(subparameterName); + parameterNames.add(foreach.parameterName); + // <- TODO: [🚎] Warn/logic error when `subparameterName` not used + } + } + } catch (e_4_1) { + e_4 = { error: e_4_1 }; + } finally { + try { + if (_m && !_m.done && (_d = _l.return)) _d.call(_l); + } finally { + if (e_4) throw e_4.error; + } + } + } + return parameterNames; + } + /** + * TODO: [🔣] If script require contentLanguage + */ + + /** + * Create difference set of two sets. + * + * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set + * @public exported from `@promptbook/utils` + */ + function difference(a, b, isEqual) { + var e_1, _a; + if (isEqual === void 0) { + isEqual = (a, b) => a === b; + } + var diff = new Set(); + var _loop_1 = (itemA) => { + if (!Array.from(b).some((itemB) => isEqual(itemA, itemB))) { + diff.add(itemA); + } + }; + try { + for ( + var _b = __values(Array.from(a)), _c = _b.next(); + !_c.done; + _c = _b.next() + ) { + var itemA = _c.value; + _loop_1(itemA); + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_c && !_c.done && (_a = _b.return)) _a.call(_b); + } finally { + if (e_1) throw e_1.error; + } + } + return diff; + } + /** + * TODO: [🧠][💯] Maybe also implement symmetricDifference + */ + + /** + * Creates a new set with all elements that are present in either set + * + * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set + * @public exported from `@promptbook/utils` + */ + function union() { + var e_1, _a, e_2, _b; + var sets = []; + for (var _i = 0; _i < arguments.length; _i++) { + sets[_i] = arguments[_i]; + } + var union = new Set(); + try { + for ( + var sets_1 = __values(sets), sets_1_1 = sets_1.next(); + !sets_1_1.done; + sets_1_1 = sets_1.next() + ) { + var set = sets_1_1.value; + try { + for ( + var _c = ((e_2 = void 0), __values(Array.from(set))), + _d = _c.next(); + !_d.done; + _d = _c.next() + ) { + var item = _d.value; + union.add(item); + } + } catch (e_2_1) { + e_2 = { error: e_2_1 }; + } finally { + try { + if (_d && !_d.done && (_b = _c.return)) _b.call(_c); + } finally { + if (e_2) throw e_2.error; + } + } + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (sets_1_1 && !sets_1_1.done && (_a = sets_1.return)) + _a.call(sets_1); + } finally { + if (e_1) throw e_1.error; + } + } + return union; + } + + /** + * @@@ + * + * @public exported from `@promptbook/core` + */ + var MANDATORY_CSV_SETTINGS = Object.freeze({ + header: true, + // encoding: 'utf-8', + }); + + /** + * Definition for CSV spreadsheet + * + * @public exported from `@promptbook/core` + * <- TODO: [🏢] Export from package `@promptbook/csv` + */ + var CsvFormatDefinition = { + formatName: "CSV", + aliases: ["SPREADSHEET", "TABLE"], + isValid: (value, settings, schema) => true, + canBeValid: (partialValue, settings, schema) => true, + heal: (value, settings, schema) => { + throw new Error("Not implemented"); + }, + subvalueDefinitions: [ + { + subvalueName: "ROW", + mapValues: function ( + value, + outputParameterName, + settings, + mapCallback, + ) { + return __awaiter(this, void 0, void 0, function () { + var csv, mappedData; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + csv = papaparse.parse( + value, + __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS), + ); + if (csv.errors.length !== 0) { + throw new CsvFormatError( + spaceTrim__default["default"]((block) => + "\n CSV parsing error\n\n Error(s) from CSV parsing:\n " + .concat( + block( + csv.errors + .map((error) => error.message) + .join("\n\n"), + ), + "\n\n The CSV setings:\n ", + ) + .concat( + block( + JSON.stringify( + __assign( + __assign({}, settings), + MANDATORY_CSV_SETTINGS, + ), + null, + 2, + ), + ), + "\n\n The CSV data:\n ", + ) + .concat( + block(value), + "\n ", + ), + ), + ); + } + return [ + 4 /*yield*/, + Promise.all( + csv.data.map((row, index) => + __awaiter(this, void 0, void 0, function () { + var _a, _b; + var _c; + return __generator(this, (_d) => { + switch (_d.label) { + case 0: + if (row[outputParameterName]) { + throw new CsvFormatError( + 'Can not overwrite existing column "'.concat( + outputParameterName, + '" in CSV row', + ), + ); + } + _a = [__assign({}, row)]; + _c = {}; + _b = outputParameterName; + return [4 /*yield*/, mapCallback(row, index)]; + case 1: + return [ + 2 /*return*/, + __assign.apply( + void 0, + _a.concat([((_c[_b] = _d.sent()), _c)]), + ), + ]; + } + }); + }), + ), + ), + ]; + case 1: + mappedData = _a.sent(); + return [ + 2 /*return*/, + papaparse.unparse( + mappedData, + __assign( + __assign({}, settings), + MANDATORY_CSV_SETTINGS, + ), + ), + ]; + } + }); + }); + }, + }, + { + subvalueName: "CELL", + mapValues: function ( + value, + outputParameterName, + settings, + mapCallback, + ) { + return __awaiter(this, void 0, void 0, function () { + var csv, mappedData; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + csv = papaparse.parse( + value, + __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS), + ); + if (csv.errors.length !== 0) { + throw new CsvFormatError( + spaceTrim__default["default"]((block) => + "\n CSV parsing error\n\n Error(s) from CSV parsing:\n " + .concat( + block( + csv.errors + .map((error) => error.message) + .join("\n\n"), + ), + "\n\n The CSV setings:\n ", + ) + .concat( + block( + JSON.stringify( + __assign( + __assign({}, settings), + MANDATORY_CSV_SETTINGS, + ), + null, + 2, + ), + ), + "\n\n The CSV data:\n ", + ) + .concat( + block(value), + "\n ", + ), + ), + ); + } + return [ + 4 /*yield*/, + Promise.all( + csv.data.map((row, rowIndex) => + __awaiter(this, void 0, void 0, function () { + return __generator(this, (_a) => [ + 2 /*return*/, + /* not await */ Promise.all( + Object.entries(row).map((_a, columnIndex) => { + var _b = __read(_a, 2), + key = _b[0], + value = _b[1]; + return __awaiter( + this, + void 0, + void 0, + function () { + var index; + var _c; + return __generator(this, (_d) => { + index = + rowIndex * Object.keys(row).length + + columnIndex; + return [ + 2 /*return*/, + /* not await */ mapCallback( + ((_c = {}), (_c[key] = value), _c), + index, + ), + ]; + }); + }, + ); + }), + ), + ]); + }), + ), + ), + ]; + case 1: + mappedData = _a.sent(); + return [ + 2 /*return*/, + papaparse.unparse( + mappedData, + __assign( + __assign({}, settings), + MANDATORY_CSV_SETTINGS, + ), + ), + ]; + } + }); + }); + }, + }, + ], + }; + /** + * TODO: [🍓] In `CsvFormatDefinition` implement simple `isValid` + * TODO: [🍓] In `CsvFormatDefinition` implement partial `canBeValid` + * TODO: [🍓] In `CsvFormatDefinition` implement `heal + * TODO: [🍓] In `CsvFormatDefinition` implement `subvalueDefinitions` + * TODO: [🏢] Allow to expect something inside CSV objects and other formats + */ + + /** + * Definition for JSON format + * + * @private still in development [🏢] + */ + var JsonFormatDefinition = { + formatName: "JSON", + mimeType: "application/json", + isValid: (value, settings, schema) => isValidJsonString(value), + canBeValid: (partialValue, settings, schema) => true, + heal: (value, settings, schema) => { + throw new Error("Not implemented"); + }, + subvalueDefinitions: [], + }; + /** + * TODO: [🧠] Maybe propper instance of object + * TODO: [0] Make string_serialized_json + * TODO: [1] Make type for JSON Settings and Schema + * TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...? + * TODO: [🍓] In `JsonFormatDefinition` implement simple `isValid` + * TODO: [🍓] In `JsonFormatDefinition` implement partial `canBeValid` + * TODO: [🍓] In `JsonFormatDefinition` implement `heal + * TODO: [🍓] In `JsonFormatDefinition` implement `subvalueDefinitions` + * TODO: [🏢] Allow to expect something inside JSON objects and other formats + */ + + /** + * Definition for any text - this will be always valid + * + * Note: This is not useful for validation, but for splitting and mapping with `subvalueDefinitions` + * + * @public exported from `@promptbook/core` + */ + var TextFormatDefinition = { + formatName: "TEXT", + isValid: (value) => typeof value === "string", + canBeValid: (partialValue) => typeof partialValue === "string", + heal: () => { + throw new UnexpectedError( + "It does not make sense to call `TextFormatDefinition.heal`", + ); + }, + subvalueDefinitions: [ + { + subvalueName: "LINE", + mapValues: function ( + value, + outputParameterName, + settings, + mapCallback, + ) { + return __awaiter(this, void 0, void 0, function () { + var lines, mappedLines; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + lines = value.split("\n"); + return [ + 4 /*yield*/, + Promise.all( + lines.map((lineContent, lineNumber) => { + // TODO: [🧠] Maybe option to skip empty line + /* not await */ return mapCallback( + { + lineContent: lineContent, + // TODO: [🧠] Maybe also put here `lineNumber` + }, + lineNumber, + ); + }), + ), + ]; + case 1: + mappedLines = _a.sent(); + return [2 /*return*/, mappedLines.join("\n")]; + } + }); + }); + }, + }, + // <- TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues + ], + }; + /** + * TODO: [1] Make type for XML Text and Schema + * TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues + * TODO: [🍓] In `TextFormatDefinition` implement simple `isValid` + * TODO: [🍓] In `TextFormatDefinition` implement partial `canBeValid` + * TODO: [🍓] In `TextFormatDefinition` implement `heal + * TODO: [🍓] In `TextFormatDefinition` implement `subvalueDefinitions` + * TODO: [🏢] Allow to expect something inside each item of list and other formats + */ + + /** + * Definition for XML format + * + * @private still in development [🏢] + */ + var XmlFormatDefinition = { + formatName: "XML", + mimeType: "application/xml", + isValid: (value, settings, schema) => true, + canBeValid: (partialValue, settings, schema) => true, + heal: (value, settings, schema) => { + throw new Error("Not implemented"); + }, + subvalueDefinitions: [], + }; + /** + * TODO: [🧠] Maybe propper instance of object + * TODO: [0] Make string_serialized_xml + * TODO: [1] Make type for XML Settings and Schema + * TODO: [🧠] What to use for validating XMLs - XSD,... + * TODO: [🍓] In `XmlFormatDefinition` implement simple `isValid` + * TODO: [🍓] In `XmlFormatDefinition` implement partial `canBeValid` + * TODO: [🍓] In `XmlFormatDefinition` implement `heal + * TODO: [🍓] In `XmlFormatDefinition` implement `subvalueDefinitions` + * TODO: [🏢] Allow to expect something inside XML and other formats + */ + + /** + * Definitions for all formats supported by Promptbook + * + * @private internal index of `...` <- TODO [🏢] + */ + var FORMAT_DEFINITIONS = [ + JsonFormatDefinition, + XmlFormatDefinition, + TextFormatDefinition, + CsvFormatDefinition, + ]; + /** + * Note: [💞] Ignore a discrepancy between file name and entity name + */ + + /** + * Maps available parameters to expected parameters + * + * The strategy is: + * 1) @@@ + * 2) @@@ + * + * @throws {PipelineExecutionError} @@@ + * @private within the repository used in `createPipelineExecutor` + */ + function mapAvailableToExpectedParameters(options) { + var e_1, _a; + var expectedParameters = options.expectedParameters, + availableParameters = options.availableParameters; + var availableParametersNames = new Set(Object.keys(availableParameters)); + var expectedParameterNames = new Set(Object.keys(expectedParameters)); + var mappedParameters = {}; + try { + // Phase 1️⃣: Matching mapping + for ( + var _b = __values( + Array.from( + union(availableParametersNames, expectedParameterNames), + ), + ), + _c = _b.next(); + !_c.done; + _c = _b.next() + ) { + var parameterName = _c.value; + // Situation: Parameter is available and expected + if ( + availableParametersNames.has(parameterName) && + expectedParameterNames.has(parameterName) + ) { + mappedParameters[parameterName] = + availableParameters[parameterName]; + // <- Note: [👩‍👩‍👧] Maybe detect parameter collision here? + availableParametersNames.delete(parameterName); + expectedParameterNames.delete(parameterName); + } + // Situation: Parameter is available but NOT expected + else if ( + availableParametersNames.has(parameterName) && + !expectedParameterNames.has(parameterName) + ) { + // [🐱‍👤] Do not pass this parameter to prompt - Maybe use it non-matching mapping + } + // Situation: Parameter is NOT available BUT expected + else if ( + !availableParametersNames.has(parameterName) && + expectedParameterNames.has(parameterName) + ) { + // Do nothing here - this will be maybe fixed in the non-matching mapping + } + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_c && !_c.done && (_a = _b.return)) _a.call(_b); + } finally { + if (e_1) throw e_1.error; + } + } + if (expectedParameterNames.size === 0) { + // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@ + Object.freeze(mappedParameters); + return mappedParameters; + } + // Phase 2️⃣: Non-matching mapping + if (expectedParameterNames.size !== availableParametersNames.size) { + throw new PipelineExecutionError( + spaceTrim__default["default"]((block) => + "\n Can not map available parameters to expected parameters\n\n Mapped parameters:\n " + .concat( + block( + Object.keys(mappedParameters) + .map((parameterName) => "- {".concat(parameterName, "}")) + .join("\n"), + ), + "\n\n Expected parameters which can not be mapped:\n ", + ) + .concat( + block( + Array.from(expectedParameterNames) + .map((parameterName) => "- {".concat(parameterName, "}")) + .join("\n"), + ), + "\n\n Remaining available parameters:\n ", + ) + .concat( + block( + Array.from(availableParametersNames) + .map((parameterName) => "- {".concat(parameterName, "}")) + .join("\n"), + ), + "\n\n ", + ), + ), + ); + } + var expectedParameterNamesArray = Array.from(expectedParameterNames); + var availableParametersNamesArray = Array.from(availableParametersNames); + for (var i = 0; i < expectedParameterNames.size; i++) { + mappedParameters[expectedParameterNamesArray[i]] = + availableParameters[availableParametersNamesArray[i]]; + } + // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@ + Object.freeze(mappedParameters); + return mappedParameters; + } + + /** + * Extracts all code blocks from markdown. + * + * Note: There are multiple simmilar function: + * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing + * - `extractJsonBlock` extracts exactly one valid JSON code block + * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block + * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block + * + * @param markdown any valid markdown + * @returns code blocks with language and content + * @throws {ParseError} if block is not closed properly + * @public exported from `@promptbook/markdown-utils` + */ + function extractAllBlocksFromMarkdown(markdown) { + var e_1, _a; + var codeBlocks = []; + var lines = markdown.split("\n"); + // Note: [0] Ensure that the last block notated by gt > will be closed + lines.push(""); + var currentCodeBlock = null; + try { + for ( + var lines_1 = __values(lines), lines_1_1 = lines_1.next(); + !lines_1_1.done; + lines_1_1 = lines_1.next() + ) { + var line = lines_1_1.value; + if (line.startsWith("> ") || line === ">") { + if (currentCodeBlock === null) { + currentCodeBlock = { + blockNotation: ">", + language: null, + content: "", + }; + } /* not else */ + if (currentCodeBlock.blockNotation === ">") { + if (currentCodeBlock.content !== "") { + currentCodeBlock.content += "\n"; + } + currentCodeBlock.content += line.slice(2); + } + } else if ( + currentCodeBlock !== null && + currentCodeBlock.blockNotation === ">" /* <- Note: [0] */ + ) { + codeBlocks.push(currentCodeBlock); + currentCodeBlock = null; + } + /* not else */ + if (line.startsWith("```")) { + var language = line.slice(3).trim() || null; + if (currentCodeBlock === null) { + currentCodeBlock = { + blockNotation: "```", + language: language, + content: "", + }; + } else { + if (language !== null) { + throw new ParseError( + "" + .concat( + capitalize(currentCodeBlock.language || "the"), + " code block was not closed and already opening new ", + ) + .concat(language, " code block"), + ); + } + codeBlocks.push(currentCodeBlock); + currentCodeBlock = null; + } + } else if ( + currentCodeBlock !== null && + currentCodeBlock.blockNotation === "```" + ) { + if (currentCodeBlock.content !== "") { + currentCodeBlock.content += "\n"; + } + currentCodeBlock.content += line + .split("\\`\\`\\`") + .join("```") /* <- TODO: Maybe make propper unescape */; + } + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (lines_1_1 && !lines_1_1.done && (_a = lines_1.return)) + _a.call(lines_1); + } finally { + if (e_1) throw e_1.error; + } + } + if (currentCodeBlock !== null) { + throw new ParseError( + "".concat( + capitalize(currentCodeBlock.language || "the"), + " code block was not closed at the end of the markdown", + ), + ); + } + return codeBlocks; + } + /** + * TODO: Maybe name for `blockNotation` instead of '```' and '>' + */ + + /** + * Extracts extracts exactly one valid JSON code block + * + * - When given string is a valid JSON as it is, it just returns it + * - When there is no JSON code block the function throws a `ParseError` + * - When there are multiple JSON code blocks the function throws a `ParseError` + * + * Note: It is not important if marked as ```json BUT if it is VALID JSON + * Note: There are multiple simmilar function: + * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing + * - `extractJsonBlock` extracts exactly one valid JSON code block + * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block + * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block + * + * @public exported from `@promptbook/markdown-utils` + * @throws {ParseError} if there is no valid JSON block in the markdown + */ + function extractJsonBlock(markdown) { + if (isValidJsonString(markdown)) { + return markdown; + } + var codeBlocks = extractAllBlocksFromMarkdown(markdown); + var jsonBlocks = codeBlocks.filter((_a) => { + var content = _a.content; + return isValidJsonString(content); + }); + if (jsonBlocks.length === 0) { + throw new Error("There is no valid JSON block in the markdown"); + } + if (jsonBlocks.length > 1) { + throw new Error("There are multiple JSON code blocks in the markdown"); + } + return jsonBlocks[0].content; + } + /** + * TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc. + * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString` + */ + + /** + * Takes an item or an array of items and returns an array of items + * + * 1) Any item except array and undefined returns array with that one item (also null) + * 2) Undefined returns empty array + * 3) Array returns itself + * + * @private internal utility + */ + function arrayableToArray(input) { + if (input === undefined) { + return []; + } + if (input instanceof Array) { + return input; + } + return [input]; + } + + /** + * Replaces parameters in template with values from parameters object + * + * Note: This function is not places strings into string, + * It's more complex and can handle this operation specifically for LLM models + * + * @param template the template with parameters in {curly} braces + * @param parameters the object with parameters + * @returns the template with replaced parameters + * @throws {PipelineExecutionError} if parameter is not defined, not closed, or not opened + * @public exported from `@promptbook/utils` + */ + function templateParameters(template, parameters) { + var e_1, _a; + try { + for ( + var _b = __values(Object.entries(parameters)), _c = _b.next(); + !_c.done; + _c = _b.next() + ) { + var _d = __read(_c.value, 2), + parameterName = _d[0], + parameterValue = _d[1]; + if (parameterValue === RESERVED_PARAMETER_MISSING_VALUE) { + throw new UnexpectedError( + "Parameter `{".concat(parameterName, "}` has missing value"), + ); + } else if (parameterValue === RESERVED_PARAMETER_RESTRICTED) { + // TODO: [🍵] + throw new UnexpectedError( + "Parameter `{".concat(parameterName, "}` is restricted to use"), + ); + } + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_c && !_c.done && (_a = _b.return)) _a.call(_b); + } finally { + if (e_1) throw e_1.error; + } + } + var replacedTemplates = template; + var match; + var loopLimit = LOOP_LIMIT; + var _loop_1 = () => { + if (loopLimit-- < 0) { + throw new LimitReachedError( + "Loop limit reached during parameters replacement in `templateParameters`", + ); + } + var precol = match.groups.precol; + var parameterName = match.groups.parameterName; + if (parameterName === "") { + return "continue"; + } + if ( + parameterName.indexOf("{") !== -1 || + parameterName.indexOf("}") !== -1 + ) { + throw new PipelineExecutionError( + "Parameter is already opened or not closed", + ); + } + if (parameters[parameterName] === undefined) { + throw new PipelineExecutionError( + "Parameter `{".concat(parameterName, "}` is not defined"), + ); + } + var parameterValue = parameters[parameterName]; + if (parameterValue === undefined) { + throw new PipelineExecutionError( + "Parameter `{".concat(parameterName, "}` is not defined"), + ); + } + parameterValue = valueToString(parameterValue); + // Escape curly braces in parameter values to prevent prompt-injection + parameterValue = parameterValue.replace(/[{}]/g, "\\$&"); + if (parameterValue.includes("\n") && /^\s*\W{0,3}\s*$/.test(precol)) { + parameterValue = parameterValue + .split("\n") + .map((line, index) => + index === 0 ? line : "".concat(precol).concat(line), + ) + .join("\n"); + } + replacedTemplates = + replacedTemplates.substring(0, match.index + precol.length) + + parameterValue + + replacedTemplates.substring( + match.index + precol.length + parameterName.length + 2, + ); + }; + while ( + (match = + /^(?.*){(?\w+)}(.*)/m /* <- Not global */ + .exec(replacedTemplates)) + ) { + _loop_1(); + } + // [💫] Check if there are parameters that are not closed properly + if (/{\w+$/.test(replacedTemplates)) { + throw new PipelineExecutionError("Parameter is not closed"); + } + // [💫] Check if there are parameters that are not opened properly + if (/^\w+}/.test(replacedTemplates)) { + throw new PipelineExecutionError("Parameter is not opened"); + } + return replacedTemplates; + } + + /** + * Counts number of characters in the text + * + * @public exported from `@promptbook/utils` + */ + function countCharacters(text) { + // Remove null characters + text = text.replace(/\0/g, ""); + // Replace emojis (and also ZWJ sequence) with hyphens + text = text.replace( + /(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, + "$1", + ); + text = text.replace( + /(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, + "$1", + ); + text = text.replace( + /\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, + "-", + ); + return text.length; + } + + /** + * Number of characters per standard line with 11pt Arial font size. + * + * @public exported from `@promptbook/utils` + */ + var CHARACTERS_PER_STANDARD_LINE = 63; + /** + * Number of lines per standard A4 page with 11pt Arial font size and standard margins and spacing. + * + * @public exported from `@promptbook/utils` + */ + var LINES_PER_STANDARD_PAGE = 44; + /** + * TODO: [🧠] Should be this `constants.ts` or `config.ts`? + * Note: [💞] Ignore a discrepancy between file name and entity name + */ + + /** + * Counts number of lines in the text + * + * Note: This does not check only for the presence of newlines, but also for the length of the standard line. + * + * @public exported from `@promptbook/utils` + */ + function countLines(text) { + text = text.replace("\r\n", "\n"); + text = text.replace("\r", "\n"); + var lines = text.split("\n"); + return lines.reduce( + (count, line) => + count + Math.ceil(line.length / CHARACTERS_PER_STANDARD_LINE), + 0, + ); + } + + /** + * Counts number of pages in the text + * + * Note: This does not check only for the count of newlines, but also for the length of the standard line and length of the standard page. + * + * @public exported from `@promptbook/utils` + */ + function countPages(text) { + return Math.ceil(countLines(text) / LINES_PER_STANDARD_PAGE); + } + + /** + * Counts number of paragraphs in the text + * + * @public exported from `@promptbook/utils` + */ + function countParagraphs(text) { + return text + .split(/\n\s*\n/) + .filter((paragraph) => paragraph.trim() !== "").length; + } + + /** + * Split text into sentences + * + * @public exported from `@promptbook/utils` + */ + function splitIntoSentences(text) { + return text.split(/[.!?]+/).filter((sentence) => sentence.trim() !== ""); + } + /** + * Counts number of sentences in the text + * + * @public exported from `@promptbook/utils` + */ + function countSentences(text) { + return splitIntoSentences(text).length; + } + + var defaultDiacriticsRemovalMap = [ + { + base: "A", + letters: + "\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F", + }, + { base: "AA", letters: "\uA732" }, + { base: "AE", letters: "\u00C6\u01FC\u01E2" }, + { base: "AO", letters: "\uA734" }, + { base: "AU", letters: "\uA736" }, + { base: "AV", letters: "\uA738\uA73A" }, + { base: "AY", letters: "\uA73C" }, + { + base: "B", + letters: "\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181", + }, + { + base: "C", + letters: + "\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E", + }, + { + base: "D", + letters: + "\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779\u00D0", + }, + { base: "DZ", letters: "\u01F1\u01C4" }, + { base: "Dz", letters: "\u01F2\u01C5" }, + { + base: "E", + letters: + "\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E", + }, + { base: "F", letters: "\u0046\u24BB\uFF26\u1E1E\u0191\uA77B" }, + { + base: "G", + letters: + "\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E", + }, + { + base: "H", + letters: + "\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D", + }, + { + base: "I", + letters: + "\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197", + }, + { base: "J", letters: "\u004A\u24BF\uFF2A\u0134\u0248" }, + { + base: "K", + letters: + "\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2", + }, + { + base: "L", + letters: + "\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780", + }, + { base: "LJ", letters: "\u01C7" }, + { base: "Lj", letters: "\u01C8" }, + { + base: "M", + letters: "\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C", + }, + { + base: "N", + letters: + "\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4", + }, + { base: "NJ", letters: "\u01CA" }, + { base: "Nj", letters: "\u01CB" }, + { + base: "O", + letters: + "\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C", + }, + { base: "OI", letters: "\u01A2" }, + { base: "OO", letters: "\uA74E" }, + { base: "OU", letters: "\u0222" }, + { base: "OE", letters: "\u008C\u0152" }, + { base: "oe", letters: "\u009C\u0153" }, + { + base: "P", + letters: "\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754", + }, + { base: "Q", letters: "\u0051\u24C6\uFF31\uA756\uA758\u024A" }, + { + base: "R", + letters: + "\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782", + }, + { + base: "S", + letters: + "\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784", + }, + { + base: "T", + letters: + "\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786", + }, + { base: "TZ", letters: "\uA728" }, + { + base: "U", + letters: + "\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244", + }, + { + base: "V", + letters: "\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245", + }, + { base: "VY", letters: "\uA760" }, + { + base: "W", + letters: "\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72", + }, + { base: "X", letters: "\u0058\u24CD\uFF38\u1E8A\u1E8C" }, + { + base: "Y", + letters: + "\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE", + }, + { + base: "Z", + letters: + "\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762", + }, + { + base: "a", + letters: + "\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250", + }, + { base: "aa", letters: "\uA733" }, + { base: "ae", letters: "\u00E6\u01FD\u01E3" }, + { base: "ao", letters: "\uA735" }, + { base: "au", letters: "\uA737" }, + { base: "av", letters: "\uA739\uA73B" }, + { base: "ay", letters: "\uA73D" }, + { + base: "b", + letters: "\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253", + }, + { + base: "c", + letters: + "\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184", + }, + { + base: "d", + letters: + "\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A", + }, + { base: "dz", letters: "\u01F3\u01C6" }, + { + base: "e", + letters: + "\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD", + }, + { base: "f", letters: "\u0066\u24D5\uFF46\u1E1F\u0192\uA77C" }, + { + base: "g", + letters: + "\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F", + }, + { + base: "h", + letters: + "\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265", + }, + { base: "hv", letters: "\u0195" }, + { + base: "i", + letters: + "\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131", + }, + { base: "j", letters: "\u006A\u24D9\uFF4A\u0135\u01F0\u0249" }, + { + base: "k", + letters: + "\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3", + }, + { + base: "l", + letters: + "\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747", + }, + { base: "lj", letters: "\u01C9" }, + { + base: "m", + letters: "\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F", + }, + { + base: "n", + letters: + "\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5", + }, + { base: "nj", letters: "\u01CC" }, + { + base: "o", + letters: + "\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275", + }, + { base: "oi", letters: "\u01A3" }, + { base: "ou", letters: "\u0223" }, + { base: "oo", letters: "\uA74F" }, + { + base: "p", + letters: "\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755", + }, + { base: "q", letters: "\u0071\u24E0\uFF51\u024B\uA757\uA759" }, + { + base: "r", + letters: + "\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783", + }, + { + base: "s", + letters: + "\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B", + }, + { + base: "t", + letters: + "\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787", + }, + { base: "tz", letters: "\uA729" }, + { + base: "u", + letters: + "\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289", + }, + { + base: "v", + letters: "\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C", + }, + { base: "vy", letters: "\uA761" }, + { + base: "w", + letters: + "\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73", + }, + { base: "x", letters: "\u0078\u24E7\uFF58\u1E8B\u1E8D" }, + { + base: "y", + letters: + "\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF", + }, + { + base: "z", + letters: + "\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763", + }, + ]; + /** + * Map of letters from diacritic variant to diacritless variant + * Contains lowercase and uppercase separatelly + * + * > "á" => "a" + * > "ě" => "e" + * > "Ă" => "A" + * > ... + * + * @public exported from `@promptbook/utils` + */ + var DIACRITIC_VARIANTS_LETTERS = {}; + // tslint:disable-next-line: prefer-for-of + for (var i = 0; i < defaultDiacriticsRemovalMap.length; i++) { + var letters = defaultDiacriticsRemovalMap[i].letters; + // tslint:disable-next-line: prefer-for-of + for (var j = 0; j < letters.length; j++) { + DIACRITIC_VARIANTS_LETTERS[letters[j]] = + defaultDiacriticsRemovalMap[i].base; + } + } + // <- TODO: [🍓] Put to maker function to save execution time if not needed + /* @see https://stackoverflow.com/questions/990904/remove-accents-diacritics-in-a-string-in-javascript Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -4252,1819 +6014,3084 @@ at step (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\node_modules\\node_modu limitations under the License. */ - /** - * @@@ - * - * @param input @@@ - * @returns @@@ - * @public exported from `@promptbook/utils` - */ - function removeDiacritics(input) { - /*eslint no-control-regex: "off"*/ - return input.replace(/[^\u0000-\u007E]/g, function (a) { - return DIACRITIC_VARIANTS_LETTERS[a] || a; - }); - } - /** - * TODO: [Ж] Variant for cyrillic (and in general non-latin) letters - */ - - /** - * Counts number of words in the text - * - * @public exported from `@promptbook/utils` - */ - function countWords(text) { - text = text.replace(/[\p{Extended_Pictographic}]/gu, 'a'); - text = removeDiacritics(text); - return text.split(/[^a-zа-я0-9]+/i).filter(function (word) { return word.length > 0; }).length; - } - - /** - * Index of all counter functions - * - * @public exported from `@promptbook/utils` - */ - var CountUtils = { - CHARACTERS: countCharacters, - WORDS: countWords, - SENTENCES: countSentences, - PARAGRAPHS: countParagraphs, - LINES: countLines, - PAGES: countPages, - }; - /** - * TODO: [🧠][🤠] This should be probbably as part of `TextFormatDefinition` - * Note: [💞] Ignore a discrepancy between file name and entity name - */ - - /** - * Function checkExpectations will check if the expectations on given value are met - * - * Note: There are two simmilar functions: - * - `checkExpectations` which throws an error if the expectations are not met - * - `isPassingExpectations` which returns a boolean - * - * @throws {ExpectError} if the expectations are not met - * @returns {void} Nothing - * @private internal function of `createPipelineExecutor` - */ - function checkExpectations(expectations, value) { - var e_1, _a; - try { - for (var _b = __values(Object.entries(expectations)), _c = _b.next(); !_c.done; _c = _b.next()) { - var _d = __read(_c.value, 2), unit = _d[0], _e = _d[1], max = _e.max, min = _e.min; - var amount = CountUtils[unit.toUpperCase()](value); - if (min && amount < min) { - throw new ExpectError("Expected at least ".concat(min, " ").concat(unit, " but got ").concat(amount)); - } /* not else */ - if (max && amount > max) { - throw new ExpectError("Expected at most ".concat(max, " ").concat(unit, " but got ").concat(amount)); - } - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_c && !_c.done && (_a = _b.return)) _a.call(_b); - } - finally { if (e_1) throw e_1.error; } - } - } - /** - * TODO: [💝] Unite object for expecting amount and format - * TODO: [🧠][🤠] This should be part of `TextFormatDefinition` - * Note: [💝] and [🤠] are interconnected together - */ - - /** - * @@@ - * - * @private internal utility of `createPipelineExecutor` - */ - function executeAttempts(options) { - return __awaiter(this, void 0, void 0, function () { - var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, task, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTaskResult, _llms, llmTools, _loop_1, attempt, state_1; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, task = options.task, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts; - $ongoingTaskResult = { - $result: null, - $resultString: null, - $expectError: null, - $scriptPipelineExecutionErrors: [], - }; - _llms = arrayableToArray(tools.llm); - llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false)); - _loop_1 = function (attempt) { - var isJokerAttempt, jokerParameterName, _b, modelRequirements, _c, _d, _e, _f, _g, scriptTools, _h, error_1, e_1_1, _j, _k, _l, functionName, postprocessingError, _m, _o, scriptTools, _p, error_2, e_2_1, e_3_1, error_3; - var e_1, _q, e_3, _r, e_2, _s; - return __generator(this, function (_t) { - switch (_t.label) { - case 0: - isJokerAttempt = attempt < 0; - jokerParameterName = jokerParameterNames[jokerParameterNames.length + attempt]; - // TODO: [🧠][🍭] JOKERS, EXPECTATIONS, POSTPROCESSING and FOREACH - if (isJokerAttempt && !jokerParameterName) { - throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Joker not found in attempt ".concat(attempt, "\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - $ongoingTaskResult.$result = null; - $ongoingTaskResult.$resultString = null; - $ongoingTaskResult.$expectError = null; - if (isJokerAttempt) { - if (parameters[jokerParameterName] === undefined) { - throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Joker parameter {".concat(jokerParameterName, "} not defined\n\n ").concat(block(pipelineIdentification), "\n "); })); - // <- TODO: This is maybe `PipelineLogicError` which should be detected in `validatePipeline` and here just thrown as `UnexpectedError` - } - else { - $ongoingTaskResult.$resultString = parameters[jokerParameterName]; - } - } - _t.label = 1; - case 1: - _t.trys.push([1, 43, 44, 45]); - if (!!isJokerAttempt) return [3 /*break*/, 25]; - _b = task.taskType; - switch (_b) { - case 'SIMPLE_TASK': return [3 /*break*/, 2]; - case 'PROMPT_TASK': return [3 /*break*/, 3]; - case 'SCRIPT_TASK': return [3 /*break*/, 11]; - case 'DIALOG_TASK': return [3 /*break*/, 22]; - } - return [3 /*break*/, 24]; - case 2: - $ongoingTaskResult.$resultString = templateParameters(preparedContent, parameters); - return [3 /*break*/, 25]; - case 3: - modelRequirements = __assign(__assign({ modelVariant: 'CHAT' }, (preparedPipeline.defaultModelRequirements || {})), (task.modelRequirements || {})); - $ongoingTaskResult.$prompt = { - title: task.title, - pipelineUrl: "".concat(preparedPipeline.pipelineUrl - ? preparedPipeline.pipelineUrl - : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(task.name - // <- TODO: Here should be maybe also subformat index to distinguish between same task with different subformat values - ), - parameters: parameters, - content: preparedContent, - modelRequirements: modelRequirements, - expectations: __assign(__assign({}, (preparedPipeline.personas.find(function (_a) { - var name = _a.name; - return name === task.personaName; - }) || {})), task.expectations), - format: task.format, - postprocessingFunctionNames: task.postprocessingFunctionNames, - }; // <- TODO: Not very good type guard - _c = modelRequirements.modelVariant; - switch (_c) { - case 'CHAT': return [3 /*break*/, 4]; - case 'COMPLETION': return [3 /*break*/, 6]; - case 'EMBEDDING': return [3 /*break*/, 8]; - } - return [3 /*break*/, 9]; - case 4: - _d = $ongoingTaskResult; - return [4 /*yield*/, llmTools.callChatModel( - // <- TODO: [🧁] Check that `callChatModel` is defined - $deepFreeze($ongoingTaskResult.$prompt))]; - case 5: - _d.$chatResult = _t.sent(); - // TODO: [🍬] Destroy chatThread - $ongoingTaskResult.$result = $ongoingTaskResult.$chatResult; - $ongoingTaskResult.$resultString = $ongoingTaskResult.$chatResult.content; - return [3 /*break*/, 10]; - case 6: - _e = $ongoingTaskResult; - return [4 /*yield*/, llmTools.callCompletionModel( - // <- TODO: [🧁] Check that `callCompletionModel` is defined - $deepFreeze($ongoingTaskResult.$prompt))]; - case 7: - _e.$completionResult = _t.sent(); - $ongoingTaskResult.$result = $ongoingTaskResult.$completionResult; - $ongoingTaskResult.$resultString = $ongoingTaskResult.$completionResult.content; - return [3 /*break*/, 10]; - case 8: throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Embedding model can not be used in pipeline\n\n This should be catched during parsing\n\n ".concat(block(pipelineIdentification), "\n\n "); })); - case 9: throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Unknown model variant \"".concat(task.modelRequirements.modelVariant, "\"\n\n ").concat(block(pipelineIdentification), "\n\n "); })); - case 10: return [3 /*break*/, 25]; - case 11: - if (arrayableToArray(tools.script).length === 0) { - throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n No script execution tools are available\n\n ".concat(block(pipelineIdentification), "\n "); })); - } - if (!task.contentLanguage) { - throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Script language is not defined for SCRIPT TASK \"".concat(task.name, "\"\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - _t.label = 12; - case 12: - _t.trys.push([12, 19, 20, 21]); - _f = (e_1 = void 0, __values(arrayableToArray(tools.script))), _g = _f.next(); - _t.label = 13; - case 13: - if (!!_g.done) return [3 /*break*/, 18]; - scriptTools = _g.value; - _t.label = 14; - case 14: - _t.trys.push([14, 16, , 17]); - _h = $ongoingTaskResult; - return [4 /*yield*/, scriptTools.execute($deepFreeze({ - scriptLanguage: task.contentLanguage, - script: preparedContent, - parameters: parameters, - }))]; - case 15: - _h.$resultString = _t.sent(); - return [3 /*break*/, 18]; - case 16: - error_1 = _t.sent(); - if (!(error_1 instanceof Error)) {console.log('!(error instanceof Error)') - throw error_1; - } - if (error_1 instanceof UnexpectedError) { - throw error_1; - } - $ongoingTaskResult.$scriptPipelineExecutionErrors.push(error_1); - return [3 /*break*/, 17]; - case 17: - _g = _f.next(); - return [3 /*break*/, 13]; - case 18: return [3 /*break*/, 21]; - case 19: - e_1_1 = _t.sent(); - e_1 = { error: e_1_1 }; - return [3 /*break*/, 21]; - case 20: - try { - if (_g && !_g.done && (_q = _f.return)) _q.call(_f); - } - finally { if (e_1) throw e_1.error; } - return [7 /*endfinally*/]; - case 21: - if ($ongoingTaskResult.$resultString !== null) { - return [3 /*break*/, 25]; - } - if ($ongoingTaskResult.$scriptPipelineExecutionErrors.length === 1) { - throw $ongoingTaskResult.$scriptPipelineExecutionErrors[0]; - } - else { - throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Script execution failed ".concat($ongoingTaskResult.$scriptPipelineExecutionErrors.length, "x\n\n ").concat(block(pipelineIdentification), "\n\n ").concat(block($ongoingTaskResult.$scriptPipelineExecutionErrors - .map(function (error) { return '- ' + error.message; }) - .join('\n\n')), "\n "); })); - } - case 22: - if (tools.userInterface === undefined) { - throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n User interface tools are not available\n\n ".concat(block(pipelineIdentification), "\n "); })); - } - // TODO: [🌹] When making next attempt for `DIALOG TASK`, preserve the previous user input - _j = $ongoingTaskResult; - return [4 /*yield*/, tools.userInterface.promptDialog($deepFreeze({ - promptTitle: task.title, - promptMessage: templateParameters(task.description || '', parameters), - defaultValue: templateParameters(preparedContent, parameters), - // TODO: [🧠] Figure out how to define placeholder in `.book` file - placeholder: undefined, - priority: priority, - }))]; - case 23: - // TODO: [🌹] When making next attempt for `DIALOG TASK`, preserve the previous user input - _j.$resultString = _t.sent(); - return [3 /*break*/, 25]; - case 24: throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Unknown execution type \"".concat(task.taskType, "\"\n\n ").concat(block(pipelineIdentification), "\n "); })); - case 25: - if (!(!isJokerAttempt && task.postprocessingFunctionNames)) return [3 /*break*/, 42]; - _t.label = 26; - case 26: - _t.trys.push([26, 40, 41, 42]); - _k = (e_3 = void 0, __values(task.postprocessingFunctionNames)), _l = _k.next(); - _t.label = 27; - case 27: - if (!!_l.done) return [3 /*break*/, 39]; - functionName = _l.value; - postprocessingError = null; - _t.label = 28; - case 28: - _t.trys.push([28, 35, 36, 37]); - _m = (e_2 = void 0, __values(arrayableToArray(tools.script))), _o = _m.next(); - _t.label = 29; - case 29: - if (!!_o.done) return [3 /*break*/, 34]; - scriptTools = _o.value; - _t.label = 30; - case 30: - _t.trys.push([30, 32, , 33]); - _p = $ongoingTaskResult; - return [4 /*yield*/, scriptTools.execute({ - scriptLanguage: "javascript" /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */, - script: "".concat(functionName, "(resultString)"), - parameters: { - resultString: $ongoingTaskResult.$resultString || '', - // Note: No ...parametersForTask, because working with result only - }, - })]; - case 31: - _p.$resultString = _t.sent(); - postprocessingError = null; - return [3 /*break*/, 34]; - case 32: - error_2 = _t.sent(); - if (!(error_2 instanceof Error)) {console.log('!(error instanceof Error)') - throw error_2; - } - if (error_2 instanceof UnexpectedError) { - throw error_2; - } - postprocessingError = error_2; - $ongoingTaskResult.$scriptPipelineExecutionErrors.push(error_2); - return [3 /*break*/, 33]; - case 33: - _o = _m.next(); - return [3 /*break*/, 29]; - case 34: return [3 /*break*/, 37]; - case 35: - e_2_1 = _t.sent(); - e_2 = { error: e_2_1 }; - return [3 /*break*/, 37]; - case 36: - try { - if (_o && !_o.done && (_s = _m.return)) _s.call(_m); - } - finally { if (e_2) throw e_2.error; } - return [7 /*endfinally*/]; - case 37: - if (postprocessingError) { - throw postprocessingError; - } - _t.label = 38; - case 38: - _l = _k.next(); - return [3 /*break*/, 27]; - case 39: return [3 /*break*/, 42]; - case 40: - e_3_1 = _t.sent(); - e_3 = { error: e_3_1 }; - return [3 /*break*/, 42]; - case 41: - try { - if (_l && !_l.done && (_r = _k.return)) _r.call(_k); - } - finally { if (e_3) throw e_3.error; } - return [7 /*endfinally*/]; - case 42: - // TODO: [💝] Unite object for expecting amount and format - if (task.format) { - if (task.format === 'JSON') { - if (!isValidJsonString($ongoingTaskResult.$resultString || '')) { - // TODO: [🏢] Do more universally via `FormatDefinition` - try { - $ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || ''); - } - catch (error) { - throw new ExpectError(spaceTrim.spaceTrim(function (block) { return "\n Expected valid JSON string\n\n ".concat(block( - /*<- Note: No need for `pipelineIdentification`, it will be catched and added later */ ''), "\n "); })); - } - } - } - else { - throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Unknown format \"".concat(task.format, "\"\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - } - // TODO: [💝] Unite object for expecting amount and format - if (task.expectations) { - checkExpectations(task.expectations, $ongoingTaskResult.$resultString || ''); - } - return [2 /*return*/, "break-attempts"]; - case 43: - error_3 = _t.sent(); - if (!(error_3 instanceof ExpectError)) { - throw error_3; - } - $ongoingTaskResult.$expectError = error_3; - return [3 /*break*/, 45]; - case 44: - if (!isJokerAttempt && - task.taskType === 'PROMPT_TASK' && - $ongoingTaskResult.$prompt - // <- Note: [2] When some expected parameter is not defined, error will occur in templateParameters - // In that case we don’t want to make a report about it because it’s not a llm execution error - ) { - // TODO: [🧠] Maybe put other taskTypes into report - $executionReport.promptExecutions.push({ - prompt: __assign({}, $ongoingTaskResult.$prompt), - result: $ongoingTaskResult.$result || undefined, - error: $ongoingTaskResult.$expectError === null - ? undefined - : serializeError($ongoingTaskResult.$expectError), - }); - } - return [7 /*endfinally*/]; - case 45: - if ($ongoingTaskResult.$expectError !== null && attempt === maxAttempts - 1) { - throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { - var _a, _b, _c; - return "\n LLM execution failed ".concat(maxExecutionAttempts, "x\n\n ").concat(block(pipelineIdentification), "\n\n ---\n The Prompt:\n ").concat(block((((_a = $ongoingTaskResult.$prompt) === null || _a === void 0 ? void 0 : _a.content) || '') - .split('\n') - .map(function (line) { return "> ".concat(line); }) - .join('\n')), "\n\n Last error ").concat(((_b = $ongoingTaskResult.$expectError) === null || _b === void 0 ? void 0 : _b.name) || '', ":\n ").concat(block((((_c = $ongoingTaskResult.$expectError) === null || _c === void 0 ? void 0 : _c.message) || '') - .split('\n') - .map(function (line) { return "> ".concat(line); }) - .join('\n')), "\n\n Last result:\n ").concat(block($ongoingTaskResult.$resultString === null - ? 'null' - : $ongoingTaskResult.$resultString - .split('\n') - .map(function (line) { return "> ".concat(line); }) - .join('\n')), "\n ---\n "); - })); - } - return [2 /*return*/]; - } - }); - }; - attempt = -jokerParameterNames.length; - _a.label = 1; - case 1: - if (!(attempt < maxAttempts)) return [3 /*break*/, 4]; - return [5 /*yield**/, _loop_1(attempt)]; - case 2: - state_1 = _a.sent(); - switch (state_1) { - case "break-attempts": return [3 /*break*/, 4]; - } - _a.label = 3; - case 3: - attempt++; - return [3 /*break*/, 1]; - case 4: - if ($ongoingTaskResult.$resultString === null) { - throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Something went wrong and prompt result is null\n\n ".concat(block(pipelineIdentification), "\n "); })); - } - return [2 /*return*/, $ongoingTaskResult.$resultString]; - } - }); - }); - } - /** - * TODO: Break into smaller functions - */ - - /** - * @@@ - * - * @private internal utility of `createPipelineExecutor` - */ - function executeFormatSubvalues(options) { - return __awaiter(this, void 0, void 0, function () { - var task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString; - var _this = this; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - task = options.task, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification; - if (task.foreach === undefined) { - return [2 /*return*/, /* not await */ executeAttempts(options)]; - } - if (jokerParameterNames.length !== 0) { - throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n JOKER parameters are not supported together with FOREACH command\n\n [\uD83E\uDDDE\u200D\u2640\uFE0F] This should be prevented in `validatePipeline`\n\n ".concat(block(pipelineIdentification), "\n "); })); - } - parameterValue = parameters[task.foreach.parameterName] || ''; - formatDefinition = FORMAT_DEFINITIONS.find(function (formatDefinition) { - return __spreadArray([formatDefinition.formatName], __read((formatDefinition.aliases || [])), false).includes(task.foreach.formatName); - }); - if (formatDefinition === undefined) { - throw new UnexpectedError( - // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError) - spaceTrim__default["default"](function (block) { return "\n Unsupported format \"".concat(task.foreach.formatName, "\"\n\n Available formats:\n ").concat(block(FORMAT_DEFINITIONS.map(function (formatDefinition) { return formatDefinition.formatName; }) - .map(function (formatName) { return "- ".concat(formatName); }) - .join('\n')), "\n\n [\u26F7] This should never happen because format name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - subvalueDefinition = formatDefinition.subvalueDefinitions.find(function (subvalueDefinition) { - return __spreadArray([subvalueDefinition.subvalueName], __read((subvalueDefinition.aliases || [])), false).includes(task.foreach.subformatName); - }); - if (subvalueDefinition === undefined) { - throw new UnexpectedError( - // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError) - spaceTrim__default["default"](function (block) { return "\n Unsupported subformat name \"".concat(task.foreach.subformatName, "\" for format \"").concat(task.foreach.formatName, "\"\n\n Available subformat names for format \"").concat(formatDefinition.formatName, "\":\n ").concat(block(formatDefinition.subvalueDefinitions - .map(function (subvalueDefinition) { return subvalueDefinition.subvalueName; }) - .map(function (subvalueName) { return "- ".concat(subvalueName); }) - .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - if (formatDefinition.formatName === 'CSV') { - formatSettings = csvSettings; - // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text - } - return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () { - var mappedParameters, allSubparameters, subresultString; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions - // TODO: When done [🐚] Report progress also for each subvalue here - try { - mappedParameters = mapAvailableToExpectedParameters({ - expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map(function (subparameterName) { return [subparameterName, null]; })), - availableParameters: subparameters, - }); - } - catch (error) { - if (!(error instanceof PipelineExecutionError)) { - throw error; - } - throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n ".concat(error.message, "\n\n This is error in FOREACH command\n You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command\n\n ").concat(block(pipelineIdentification), "\n Subparameter index: ").concat(index, "\n "); })); - } - allSubparameters = __assign(__assign({}, parameters), mappedParameters); - // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed - Object.freeze(allSubparameters); - return [4 /*yield*/, executeAttempts(__assign(__assign({}, options), { priority: priority + index, parameters: allSubparameters, pipelineIdentification: spaceTrim__default["default"](function (block) { return "\n ".concat(block(pipelineIdentification), "\n Subparameter index: ").concat(index, "\n "); }) }))]; - case 1: - subresultString = _a.sent(); - return [2 /*return*/, subresultString]; - } - }); - }); })]; - case 1: - resultString = _a.sent(); - return [2 /*return*/, resultString]; - } - }); - }); - } - - /** - * @@@ - * - * @private internal utility of `createPipelineExecutor` - */ - function getContextForTask(task) { - return __awaiter(this, void 0, void 0, function () { - return __generator(this, function (_a) { - return [2 /*return*/, RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [🏍] Implement */]; - }); - }); - } - - /** - * @@@ - * - * @private internal utility of `createPipelineExecutor` - */ - function getExamplesForTask(task) { - return __awaiter(this, void 0, void 0, function () { - return __generator(this, function (_a) { - return [2 /*return*/, RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [♨] Implement */]; - }); - }); - } - - /** - * @@@ - * - * @private internal utility of `createPipelineExecutor` - */ - function getKnowledgeForTask(options) { - return __awaiter(this, void 0, void 0, function () { - var preparedPipeline; - return __generator(this, function (_a) { - preparedPipeline = options.preparedPipeline, options.task; - return [2 /*return*/, preparedPipeline.knowledgePieces.map(function (_a) { - var content = _a.content; - return "- ".concat(content); - }).join('\n')]; - }); - }); - } - - /** - * @@@ - * - * @private internal utility of `createPipelineExecutor` - */ - function getReservedParametersForTask(options) { - return __awaiter(this, void 0, void 0, function () { - var preparedPipeline, task, pipelineIdentification, context, knowledge, examples, currentDate, modelName, reservedParameters, _loop_1, RESERVED_PARAMETER_NAMES_1, RESERVED_PARAMETER_NAMES_1_1, parameterName; - var e_1, _a; - return __generator(this, function (_b) { - switch (_b.label) { - case 0: - preparedPipeline = options.preparedPipeline, task = options.task, pipelineIdentification = options.pipelineIdentification; - return [4 /*yield*/, getContextForTask()]; - case 1: - context = _b.sent(); - return [4 /*yield*/, getKnowledgeForTask({ preparedPipeline: preparedPipeline, task: task })]; - case 2: - knowledge = _b.sent(); - return [4 /*yield*/, getExamplesForTask()]; - case 3: - examples = _b.sent(); - currentDate = new Date().toISOString(); - modelName = RESERVED_PARAMETER_MISSING_VALUE; - reservedParameters = { - content: RESERVED_PARAMETER_RESTRICTED, - context: context, - knowledge: knowledge, - examples: examples, - currentDate: currentDate, - modelName: modelName, - }; - _loop_1 = function (parameterName) { - if (reservedParameters[parameterName] === undefined) { - throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Reserved parameter {".concat(parameterName, "} is not defined\n\n ").concat(block(pipelineIdentification), "\n "); })); - } - }; - try { - // Note: Doublecheck that ALL reserved parameters are defined: - for (RESERVED_PARAMETER_NAMES_1 = __values(RESERVED_PARAMETER_NAMES), RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next(); !RESERVED_PARAMETER_NAMES_1_1.done; RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next()) { - parameterName = RESERVED_PARAMETER_NAMES_1_1.value; - _loop_1(parameterName); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (RESERVED_PARAMETER_NAMES_1_1 && !RESERVED_PARAMETER_NAMES_1_1.done && (_a = RESERVED_PARAMETER_NAMES_1.return)) _a.call(RESERVED_PARAMETER_NAMES_1); - } - finally { if (e_1) throw e_1.error; } - } - return [2 /*return*/, reservedParameters]; - } - }); - }); - } - - /** - * @@@ - * - * @private internal utility of `createPipelineExecutor` - */ - function executeTask(options) { - return __awaiter(this, void 0, void 0, function () { - var currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString; - var e_1, _f, _g; - return __generator(this, function (_h) { - switch (_h.label) { - case 0: - currentTask = options.currentTask, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts, maxParallelCount = options.maxParallelCount, csvSettings = options.csvSettings, isVerbose = options.isVerbose, rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, isAutoInstalled = options.isAutoInstalled, isNotPreparedWarningSupressed = options.isNotPreparedWarningSupressed; - name = "pipeline-executor-frame-".concat(currentTask.name); - title = currentTask.title; - priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask); - return [4 /*yield*/, onProgress({ - name: name, - title: title, - isStarted: false, - isDone: false, - taskType: currentTask.taskType, - parameterName: currentTask.resultingParameterName, - parameterValue: null, - // <- [🍸] - })]; - case 1: - _h.sent(); - usedParameterNames = extractParameterNamesFromTask(currentTask); - dependentParameterNames = new Set(currentTask.dependentParameterNames); - // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters` - if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) { - throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Dependent parameters are not consistent with used parameters:\n\n Dependent parameters:\n ".concat(Array.from(dependentParameterNames) - .map(function (name) { return "{".concat(name, "}"); }) - .join(', '), "\n\n Used parameters:\n ").concat(Array.from(usedParameterNames) - .map(function (name) { return "{".concat(name, "}"); }) - .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); })); - } - _b = (_a = Object).freeze; - _c = [{}]; - return [4 /*yield*/, getReservedParametersForTask({ - preparedPipeline: preparedPipeline, - task: currentTask, - pipelineIdentification: pipelineIdentification, - })]; - case 2: - definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]); - definedParameterNames = new Set(Object.keys(definedParameters)); - parameters = {}; - _loop_1 = function (parameterName) { - // Situation: Parameter is defined and used - if (definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) { - parameters[parameterName] = definedParameters[parameterName]; - } - // Situation: Parameter is defined but NOT used - else if (definedParameterNames.has(parameterName) && !usedParameterNames.has(parameterName)) ; - // Situation: Parameter is NOT defined BUT used - else if (!definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) { - // Houston, we have a problem - // Note: Checking part is also done in `validatePipeline`, but it’s good to doublecheck - throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Parameter `{".concat(parameterName, "}` is NOT defined\n BUT used in task \"").concat(currentTask.title || currentTask.name, "\"\n\n This should be catched in `validatePipeline`\n\n ").concat(block(pipelineIdentification), "\n\n "); })); - } - }; - try { - // Note: [2] Check that all used parameters are defined and removing unused parameters for this task - // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters` - for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) { - parameterName = _e.value; - _loop_1(parameterName); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_e && !_e.done && (_f = _d.return)) _f.call(_d); - } - finally { if (e_1) throw e_1.error; } - } - // Note: [👨‍👨‍👧] Now we can freeze `parameters` because we are sure that all and only used parameters are defined and are not going to be changed - Object.freeze(parameters); - maxAttempts = currentTask.taskType === 'DIALOG_TASK' ? Infinity : maxExecutionAttempts; - jokerParameterNames = currentTask.jokerParameterNames || []; - preparedContent = (currentTask.preparedContent || '{content}').split('{content}').join(currentTask.content); - return [4 /*yield*/, executeFormatSubvalues({ - jokerParameterNames: jokerParameterNames, - priority: priority, - maxAttempts: maxAttempts, - preparedContent: preparedContent, - parameters: parameters, - task: currentTask, - preparedPipeline: preparedPipeline, - tools: tools, - $executionReport: $executionReport, - pipelineIdentification: pipelineIdentification, - maxExecutionAttempts: maxExecutionAttempts, - maxParallelCount: maxParallelCount, - csvSettings: csvSettings, - isVerbose: isVerbose, - rootDirname: rootDirname, - cacheDirname: cacheDirname, - intermediateFilesStrategy: intermediateFilesStrategy, - isAutoInstalled: isAutoInstalled, - isNotPreparedWarningSupressed: isNotPreparedWarningSupressed, - })]; - case 3: - resultString = _h.sent(); - return [4 /*yield*/, onProgress({ - name: name, - title: title, - isStarted: true, - isDone: true, - taskType: currentTask.taskType, - parameterName: currentTask.resultingParameterName, - parameterValue: resultString, - // <- [🍸] - })]; - case 4: - _h.sent(); - return [2 /*return*/, Object.freeze((_g = {}, - _g[currentTask.resultingParameterName] = - // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction - resultString, - _g))]; - } - }); - }); - } - /** - * TODO: [🤹‍♂️] - */ - /** - * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited - */ - - /** - * @@@ - * - * @private internal utility of `createPipelineExecutor` - */ - function filterJustOutputParameters(options) { - var e_1, _a; - var preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, $warnings = options.$warnings, pipelineIdentification = options.pipelineIdentification; - var outputParameters = {}; - var _loop_1 = function (parameter) { - if (parametersToPass[parameter.name] === undefined) { - // [4] - $warnings.push(new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Parameter `{".concat(parameter.name, "}` should be an output parameter, but it was not generated during pipeline execution\n\n ").concat(block(pipelineIdentification), "\n "); }))); - return "continue"; - } - outputParameters[parameter.name] = parametersToPass[parameter.name] || ''; - }; - try { - // Note: Filter ONLY output parameters - // TODO: [👩🏾‍🤝‍👩🏻] Maybe use here `mapAvailableToExpectedParameters` - for (var _b = __values(preparedPipeline.parameters.filter(function (_a) { - var isOutput = _a.isOutput; - return isOutput; - })), _c = _b.next(); !_c.done; _c = _b.next()) { - var parameter = _c.value; - _loop_1(parameter); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_c && !_c.done && (_a = _b.return)) _a.call(_b); - } - finally { if (e_1) throw e_1.error; } - } - return outputParameters; - } - - /** - * @@@ - * - * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor` - * - * @private internal utility of `createPipelineExecutor` - */ - function executePipeline(options) { - return __awaiter(this, void 0, void 0, function () { - var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _a, _b, parameter, e_1_1, _loop_1, _c, _d, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTasks_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters; - var e_1, _e, e_2, _f; - return __generator(this, function (_g) { - switch (_g.label) { - case 0: - inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, isVerbose = options.isVerbose; - preparedPipeline = options.preparedPipeline; - if (!(preparedPipeline === undefined)) return [3 /*break*/, 2]; - return [4 /*yield*/, preparePipeline(pipeline, tools, { - rootDirname: rootDirname, - isVerbose: isVerbose, - maxParallelCount: maxParallelCount, - })]; - case 1: - preparedPipeline = _g.sent(); - setPreparedPipeline(preparedPipeline); - _g.label = 2; - case 2: - errors = []; - warnings = []; - executionReport = { - pipelineUrl: preparedPipeline.pipelineUrl, - title: preparedPipeline.title, - promptbookUsedVersion: PROMPTBOOK_ENGINE_VERSION, - promptbookRequestedVersion: preparedPipeline.bookVersion, - description: preparedPipeline.description, - promptExecutions: [], - }; - isReturned = false; - _g.label = 3; - case 3: - _g.trys.push([3, 9, 10, 11]); - _a = __values(preparedPipeline.parameters.filter(function (_a) { - var isInput = _a.isInput; - return isInput; - })), _b = _a.next(); - _g.label = 4; - case 4: - if (!!_b.done) return [3 /*break*/, 8]; - parameter = _b.value; - if (!(inputParameters[parameter.name] === undefined)) return [3 /*break*/, 7]; - isReturned = true; - if (!(onProgress !== undefined)) return [3 /*break*/, 6]; - // Note: Wait a short time to prevent race conditions - return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)]; - case 5: - // Note: Wait a short time to prevent race conditions - _g.sent(); - _g.label = 6; - case 6: return [2 /*return*/, exportJson({ - name: "executionReport", - message: "Unuccessful PipelineExecutorResult (with missing parameter {".concat(parameter.name, "}) PipelineExecutorResult"), - order: [], - value: { - isSuccessful: false, - errors: __spreadArray([ - new PipelineExecutionError("Parameter `{".concat(parameter.name, "}` is required as an input parameter")) - ], __read(errors), false).map(serializeError), - warnings: [], - executionReport: executionReport, - outputParameters: {}, - usage: ZERO_USAGE, - preparedPipeline: preparedPipeline, - }, - })]; - case 7: - _b = _a.next(); - return [3 /*break*/, 4]; - case 8: return [3 /*break*/, 11]; - case 9: - e_1_1 = _g.sent(); - e_1 = { error: e_1_1 }; - return [3 /*break*/, 11]; - case 10: - try { - if (_b && !_b.done && (_e = _a.return)) _e.call(_a); - } - finally { if (e_1) throw e_1.error; } - return [7 /*endfinally*/]; - case 11: - _loop_1 = function (parameterName) { - var parameter; - return __generator(this, function (_h) { - switch (_h.label) { - case 0: - parameter = preparedPipeline.parameters.find(function (_a) { - var name = _a.name; - return name === parameterName; - }); - if (!(parameter === undefined)) return [3 /*break*/, 1]; - warnings.push(new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Extra parameter {".concat(parameterName, "} is being passed which is not part of the pipeline.\n\n ").concat(block(pipelineIdentification), "\n "); }))); - return [3 /*break*/, 4]; - case 1: - if (!(parameter.isInput === false)) return [3 /*break*/, 4]; - isReturned = true; - if (!(onProgress !== undefined)) return [3 /*break*/, 3]; - // Note: Wait a short time to prevent race conditions - return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)]; - case 2: - // Note: Wait a short time to prevent race conditions - _h.sent(); - _h.label = 3; - case 3: return [2 /*return*/, { value: exportJson({ - name: 'pipelineExecutorResult', - message: spaceTrim.spaceTrim(function (block) { return "\n Unuccessful PipelineExecutorResult (with extra parameter {".concat(parameter.name, "}) PipelineExecutorResult\n\n ").concat(block(pipelineIdentification), "\n "); }), - order: [], - value: { - isSuccessful: false, - errors: __spreadArray([ - new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Parameter `{".concat(parameter.name, "}` is passed as input parameter but it is not input\n\n ").concat(block(pipelineIdentification), "\n "); })) - ], __read(errors), false).map(serializeError), - warnings: warnings.map(serializeError), - executionReport: executionReport, - outputParameters: {}, - usage: ZERO_USAGE, - preparedPipeline: preparedPipeline, - }, - }) }]; - case 4: return [2 /*return*/]; - } - }); - }; - _g.label = 12; - case 12: - _g.trys.push([12, 17, 18, 19]); - _c = __values(Object.keys(inputParameters)), _d = _c.next(); - _g.label = 13; - case 13: - if (!!_d.done) return [3 /*break*/, 16]; - parameterName = _d.value; - return [5 /*yield**/, _loop_1(parameterName)]; - case 14: - state_1 = _g.sent(); - if (typeof state_1 === "object") - return [2 /*return*/, state_1.value]; - _g.label = 15; - case 15: - _d = _c.next(); - return [3 /*break*/, 13]; - case 16: return [3 /*break*/, 19]; - case 17: - e_2_1 = _g.sent(); - e_2 = { error: e_2_1 }; - return [3 /*break*/, 19]; - case 18: - try { - if (_d && !_d.done && (_f = _c.return)) _f.call(_c); - } - finally { if (e_2) throw e_2.error; } - return [7 /*endfinally*/]; - case 19: - parametersToPass = Object.fromEntries(Object.entries(inputParameters).map(function (_a) { - var _b = __read(_a, 2), key = _b[0], value = _b[1]; - return [key, valueToString(value)]; - })); - _g.label = 20; - case 20: - _g.trys.push([20, 25, , 28]); - resovedParameterNames_1 = preparedPipeline.parameters - .filter(function (_a) { - var isInput = _a.isInput; - return isInput; - }) - .map(function (_a) { - var name = _a.name; - return name; - }); - unresovedTasks_1 = __spreadArray([], __read(preparedPipeline.tasks), false); - resolving_1 = []; - loopLimit = LOOP_LIMIT; - _loop_2 = function () { - var currentTask, work_1; - return __generator(this, function (_j) { - switch (_j.label) { - case 0: - if (loopLimit-- < 0) { - // Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline - throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Loop limit reached during resolving parameters pipeline execution\n\n ".concat(block(pipelineIdentification), "\n "); })); - } - currentTask = unresovedTasks_1.find(function (task) { - return task.dependentParameterNames.every(function (name) { - return __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), __read(RESERVED_PARAMETER_NAMES), false).includes(name); - }); - }); - if (!(!currentTask && resolving_1.length === 0)) return [3 /*break*/, 1]; - throw new UnexpectedError( - // TODO: [🐎] DRY - spaceTrim.spaceTrim(function (block) { return "\n Can not resolve some parameters:\n\n ".concat(block(pipelineIdentification), "\n\n **Can not resolve:**\n ").concat(block(unresovedTasks_1 - .map(function (_a) { - var resultingParameterName = _a.resultingParameterName, dependentParameterNames = _a.dependentParameterNames; - return "- Parameter `{".concat(resultingParameterName, "}` which depends on ").concat(dependentParameterNames - .map(function (dependentParameterName) { return "`{".concat(dependentParameterName, "}`"); }) - .join(' and ')); - }) - .join('\n')), "\n\n **Resolved:**\n ").concat(block(resovedParameterNames_1 - .filter(function (name) { - return !RESERVED_PARAMETER_NAMES.includes(name); - }) - .map(function (name) { return "- Parameter `{".concat(name, "}`"); }) - .join('\n')), "\n\n **Reserved (which are available):**\n ").concat(block(resovedParameterNames_1 - .filter(function (name) { - return RESERVED_PARAMETER_NAMES.includes(name); - }) - .map(function (name) { return "- Parameter `{".concat(name, "}`"); }) - .join('\n')), "\n\n *Note: This should be catched in `validatePipeline`*\n "); })); - case 1: - if (!!currentTask) return [3 /*break*/, 3]; - /* [🤹‍♂️] */ return [4 /*yield*/, Promise.race(resolving_1)]; - case 2: - /* [🤹‍♂️] */ _j.sent(); - return [3 /*break*/, 4]; - case 3: - unresovedTasks_1 = unresovedTasks_1.filter(function (task) { return task !== currentTask; }); - work_1 = executeTask(__assign(__assign({}, options), { currentTask: currentTask, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) { - if (isReturned) { - throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4) - .split('\n') - .map(function (line) { return "> ".concat(line); }) - .join('\n')), "\n "); })); - } - if (onProgress) { - onProgress(progress); - } - }, $executionReport: executionReport, pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Task name: ").concat(currentTask.name, "\n Task title: ").concat(currentTask.title, "\n "); }) })) - .then(function (newParametersToPass) { - parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass); - resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTask.resultingParameterName], false); - }) - .then(function () { - resolving_1 = resolving_1.filter(function (w) { return w !== work_1; }); - }); - // <- Note: Errors are catched here [3] - // TODO: BUT if in multiple tasks are errors, only the first one is catched so maybe we should catch errors here and save them to errors array here - resolving_1.push(work_1); - _j.label = 4; - case 4: return [2 /*return*/]; - } - }); - }; - _g.label = 21; - case 21: - if (!(unresovedTasks_1.length > 0)) return [3 /*break*/, 23]; - return [5 /*yield**/, _loop_2()]; - case 22: - _g.sent(); - return [3 /*break*/, 21]; - case 23: return [4 /*yield*/, Promise.all(resolving_1)]; - case 24: - _g.sent(); - return [3 /*break*/, 28]; - case 25: - error_1 = _g.sent(); - if (!(error_1 instanceof Error)) {console.log('!(error instanceof Error)') - throw error_1; - } - usage_1 = addUsage.apply(void 0, __spreadArray([], __read(executionReport.promptExecutions.map(function (_a) { - var result = _a.result; - return (result === null || result === void 0 ? void 0 : result.usage) || ZERO_USAGE; - })), false)); - outputParameters_1 = filterJustOutputParameters({ - preparedPipeline: preparedPipeline, - parametersToPass: parametersToPass, - $warnings: warnings, - pipelineIdentification: pipelineIdentification, - }); - isReturned = true; - if (!(onProgress !== undefined)) return [3 /*break*/, 27]; - // Note: Wait a short time to prevent race conditions - return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)]; - case 26: - // Note: Wait a short time to prevent race conditions - _g.sent(); - _g.label = 27; - case 27: return [2 /*return*/, exportJson({ - name: 'pipelineExecutorResult', - message: "Unuccessful PipelineExecutorResult (with misc errors) PipelineExecutorResult", - order: [], - value: { - isSuccessful: false, - errors: __spreadArray([error_1], __read(errors), false).map(serializeError), - warnings: warnings.map(serializeError), - usage: usage_1, - executionReport: executionReport, - outputParameters: outputParameters_1, - preparedPipeline: preparedPipeline, - }, - })]; - case 28: - usage = addUsage.apply(void 0, __spreadArray([], __read(executionReport.promptExecutions.map(function (_a) { - var result = _a.result; - return (result === null || result === void 0 ? void 0 : result.usage) || ZERO_USAGE; - })), false)); - outputParameters = filterJustOutputParameters({ - preparedPipeline: preparedPipeline, - parametersToPass: parametersToPass, - $warnings: warnings, - pipelineIdentification: pipelineIdentification, - }); - isReturned = true; - if (!(onProgress !== undefined)) return [3 /*break*/, 30]; - // Note: Wait a short time to prevent race conditions - return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)]; - case 29: - // Note: Wait a short time to prevent race conditions - _g.sent(); - _g.label = 30; - case 30: return [2 /*return*/, exportJson({ - name: 'pipelineExecutorResult', - message: "Successful PipelineExecutorResult", - order: [], - value: { - isSuccessful: true, - errors: errors.map(serializeError), - warnings: warnings.map(serializeError), - usage: usage, - executionReport: executionReport, - outputParameters: outputParameters, - preparedPipeline: preparedPipeline, - }, - })]; - } - }); - }); - } - /** - * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited - */ - - /** - * Creates executor function from pipeline and execution tools. - * - * @returns The executor function - * @throws {PipelineLogicError} on logical error in the pipeline - * @public exported from `@promptbook/core` - */ - function createPipelineExecutor(options) { - var _this = this; - var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.cacheDirname, cacheDirname = _f === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _f, _g = options.intermediateFilesStrategy, intermediateFilesStrategy = _g === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _g, _h = options.isAutoInstalled, isAutoInstalled = _h === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _h, _j = options.rootDirname, rootDirname = _j === void 0 ? null : _j; - validatePipeline(pipeline); - var pipelineIdentification = (function () { - // Note: This is a 😐 implementation of [🚞] - var _ = []; - if (pipeline.sourceFile !== undefined) { - _.push("File: ".concat(pipeline.sourceFile)); - } - if (pipeline.pipelineUrl !== undefined) { - _.push("Url: ".concat(pipeline.pipelineUrl)); - } - return _.join('\n'); - })(); - var preparedPipeline; - if (isPipelinePrepared(pipeline)) { - preparedPipeline = pipeline; - } - else if (isNotPreparedWarningSupressed !== true) { - console.warn(spaceTrim.spaceTrim(function (block) { return "\n Pipeline is not prepared\n\n ".concat(block(pipelineIdentification), "\n\n It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n But it is recommended to prepare the pipeline during collection preparation\n\n @see more at https://ptbk.io/prepare-pipeline\n "); })); - } - var runCount = 0; - var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () { - return __generator(this, function (_a) { - runCount++; - return [2 /*return*/, /* not await */ executePipeline({ - pipeline: pipeline, - preparedPipeline: preparedPipeline, - setPreparedPipeline: function (newPreparedPipeline) { - preparedPipeline = newPreparedPipeline; - }, - inputParameters: inputParameters, - tools: tools, - onProgress: onProgress, - pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }), - maxExecutionAttempts: maxExecutionAttempts, - maxParallelCount: maxParallelCount, - csvSettings: csvSettings, - isVerbose: isVerbose, - isNotPreparedWarningSupressed: isNotPreparedWarningSupressed, - rootDirname: rootDirname, - cacheDirname: cacheDirname, - intermediateFilesStrategy: intermediateFilesStrategy, - isAutoInstalled: isAutoInstalled, - })]; - }); - }); }; - return pipelineExecutor; - } - /** - * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited - */ - - /** - * Async version of Array.forEach - * - * @param array - Array to iterate over - * @param options - Options for the function - * @param callbackfunction - Function to call for each item - * @public exported from `@promptbook/utils` - * @deprecated [🪂] Use queues instead - */ - function forEachAsync(array, options, callbackfunction) { - return __awaiter(this, void 0, void 0, function () { - var _a, maxParallelCount, index, runningTasks, tasks, _loop_1, _b, _c, item, e_1_1; - var e_1, _d; - return __generator(this, function (_e) { - switch (_e.label) { - case 0: - _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? Infinity : _a; - index = 0; - runningTasks = []; - tasks = []; - _loop_1 = function (item) { - var currentIndex, task; - return __generator(this, function (_f) { - switch (_f.label) { - case 0: - currentIndex = index++; - task = callbackfunction(item, currentIndex, array); - tasks.push(task); - runningTasks.push(task); - /* not await */ Promise.resolve(task).then(function () { - runningTasks = runningTasks.filter(function (t) { return t !== task; }); - }); - if (!(maxParallelCount < runningTasks.length)) return [3 /*break*/, 2]; - return [4 /*yield*/, Promise.race(runningTasks)]; - case 1: - _f.sent(); - _f.label = 2; - case 2: return [2 /*return*/]; - } - }); - }; - _e.label = 1; - case 1: - _e.trys.push([1, 6, 7, 8]); - _b = __values(array), _c = _b.next(); - _e.label = 2; - case 2: - if (!!_c.done) return [3 /*break*/, 5]; - item = _c.value; - return [5 /*yield**/, _loop_1(item)]; - case 3: - _e.sent(); - _e.label = 4; - case 4: - _c = _b.next(); - return [3 /*break*/, 2]; - case 5: return [3 /*break*/, 8]; - case 6: - e_1_1 = _e.sent(); - e_1 = { error: e_1_1 }; - return [3 /*break*/, 8]; - case 7: - try { - if (_c && !_c.done && (_d = _b.return)) _d.call(_b); - } - finally { if (e_1) throw e_1.error; } - return [7 /*endfinally*/]; - case 8: return [4 /*yield*/, Promise.all(tasks)]; - case 9: - _e.sent(); - return [2 /*return*/]; - } - }); - }); - } - - /** - * Intercepts LLM tools and counts total usage of the tools - * - * @param llmTools LLM tools to be intercepted with usage counting - * @returns LLM tools with same functionality with added total cost counting - * @public exported from `@promptbook/core` - */ - function countTotalUsage(llmTools) { - var _this = this; - var totalUsage = ZERO_USAGE; - var proxyTools = { - get title() { - // TODO: [🧠] Maybe put here some suffix - return llmTools.title; - }, - get description() { - // TODO: [🧠] Maybe put here some suffix - return llmTools.description; - }, - checkConfiguration: function () { - return __awaiter(this, void 0, void 0, function () { - return __generator(this, function (_a) { - return [2 /*return*/, /* not await */ llmTools.checkConfiguration()]; - }); - }); - }, - listModels: function () { - return /* not await */ llmTools.listModels(); - }, - getTotalUsage: function () { - // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied - return totalUsage; - }, - }; - if (llmTools.callChatModel !== undefined) { - proxyTools.callChatModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () { - var promptResult; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: return [4 /*yield*/, llmTools.callChatModel(prompt)]; - case 1: - promptResult = _a.sent(); - totalUsage = addUsage(totalUsage, promptResult.usage); - return [2 /*return*/, promptResult]; - } - }); - }); }; - } - if (llmTools.callCompletionModel !== undefined) { - proxyTools.callCompletionModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () { - var promptResult; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: return [4 /*yield*/, llmTools.callCompletionModel(prompt)]; - case 1: - promptResult = _a.sent(); - totalUsage = addUsage(totalUsage, promptResult.usage); - return [2 /*return*/, promptResult]; - } - }); - }); }; - } - if (llmTools.callEmbeddingModel !== undefined) { - proxyTools.callEmbeddingModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () { - var promptResult; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: return [4 /*yield*/, llmTools.callEmbeddingModel(prompt)]; - case 1: - promptResult = _a.sent(); - totalUsage = addUsage(totalUsage, promptResult.usage); - return [2 /*return*/, promptResult]; - } - }); - }); }; - } - // <- Note: [🤖] - return proxyTools; - } - /** - * TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?) - * TODO: [🧠] Is there some meaningfull way how to test this util - * TODO: [🧠][🌯] Maybe a way how to hide ability to `get totalUsage` - * > const [llmToolsWithUsage,getUsage] = countTotalUsage(llmTools); - * TODO: [👷‍♂️] @@@ Manual about construction of llmTools - */ - - /** - * Prepares the persona for the pipeline - * - * @see https://github.com/webgptorg/promptbook/discussions/22 - * @public exported from `@promptbook/core` - */ - function preparePersona(personaDescription, tools, options) { - return __awaiter(this, void 0, void 0, function () { - var _a, isVerbose, collection, preparePersonaExecutor, _b, _llms, llmTools, availableModels, availableModelNames, result, outputParameters, modelRequirementsRaw, modelRequirements, modelName, systemMessage, temperature; - var _c; - return __generator(this, function (_d) { - switch (_d.label) { - case 0: - _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a; - if (tools === undefined || tools.llm === undefined) { - throw new MissingToolsError('LLM tools are required for preparing persona'); - } - collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false)); - _b = createPipelineExecutor; - _c = {}; - return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.book')]; - case 1: - preparePersonaExecutor = _b.apply(void 0, [(_c.pipeline = _d.sent(), - _c.tools = tools, - _c)]); - _llms = arrayableToArray(tools.llm); - llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false)); - return [4 /*yield*/, llmTools.listModels()]; - case 2: - availableModels = _d.sent(); - availableModelNames = availableModels - .filter(function (_a) { - var modelVariant = _a.modelVariant; - return modelVariant === 'CHAT'; - }) - .map(function (_a) { - var modelName = _a.modelName; - return modelName; - }) - .join(','); - return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })]; - case 3: - result = _d.sent(); - assertsExecutionSuccessful(result); - outputParameters = result.outputParameters; - modelRequirementsRaw = outputParameters.modelRequirements; - modelRequirements = JSON.parse(modelRequirementsRaw); - if (isVerbose) { - console.info("PERSONA ".concat(personaDescription), modelRequirements); - } - modelName = modelRequirements.modelName, systemMessage = modelRequirements.systemMessage, temperature = modelRequirements.temperature; - return [2 /*return*/, { - modelVariant: 'CHAT', - modelName: modelName, - systemMessage: systemMessage, - temperature: temperature, - }]; - } - }); - }); - } - /** - * TODO: [🔃][main] If the persona was prepared with different version or different set of models, prepare it once again - * TODO: [🏢] Check validity of `modelName` in pipeline - * TODO: [🏢] Check validity of `systemMessage` in pipeline - * TODO: [🏢] Check validity of `temperature` in pipeline - */ - - /** - * @@@ - * - * Note: `$` is used to indicate that this interacts with the global scope - * @singleton Only one instance of each register is created per build, but thare can be more @@@ - * @public exported from `@promptbook/core` - */ - var $scrapersMetadataRegister = new $Register('scrapers_metadata'); - /** - * TODO: [®] DRY Register logic - */ - - /** - * Creates a message with all registered scrapers - * - * Note: This function is used to create a (error) message when there is no scraper for particular mime type - * - * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv` - */ - function $registeredScrapersMessage(availableScrapers) { - var e_1, _a, e_2, _b, e_3, _c; - /** - * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister - */ - var all = []; - var _loop_1 = function (packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser) { - if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) { - return "continue"; - } - all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser }); - }; - try { - for (var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); !_e.done; _e = _d.next()) { - var _f = _e.value, packageName = _f.packageName, className = _f.className, mimeTypes = _f.mimeTypes, documentationUrl = _f.documentationUrl, isAvilableInBrowser = _f.isAvilableInBrowser; - _loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_e && !_e.done && (_a = _d.return)) _a.call(_d); - } - finally { if (e_1) throw e_1.error; } - } - var _loop_2 = function (packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser) { - if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) { - return "continue"; - } - all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser }); - }; - try { - for (var _g = __values($scrapersRegister.list()), _h = _g.next(); !_h.done; _h = _g.next()) { - var _j = _h.value, packageName = _j.packageName, className = _j.className, mimeTypes = _j.mimeTypes, documentationUrl = _j.documentationUrl, isAvilableInBrowser = _j.isAvilableInBrowser; - _loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser); - } - } - catch (e_2_1) { e_2 = { error: e_2_1 }; } - finally { - try { - if (_h && !_h.done && (_b = _g.return)) _b.call(_g); - } - finally { if (e_2) throw e_2.error; } - } - try { - for (var availableScrapers_1 = __values(availableScrapers), availableScrapers_1_1 = availableScrapers_1.next(); !availableScrapers_1_1.done; availableScrapers_1_1 = availableScrapers_1.next()) { - var metadata_1 = availableScrapers_1_1.value.metadata; - all.push(metadata_1); - } - } - catch (e_3_1) { e_3 = { error: e_3_1 }; } - finally { - try { - if (availableScrapers_1_1 && !availableScrapers_1_1.done && (_c = availableScrapers_1.return)) _c.call(availableScrapers_1); - } - finally { if (e_3) throw e_3.error; } - } - var metadata = all.map(function (metadata) { - var isMetadataAviailable = $scrapersMetadataRegister - .list() - .find(function (_a) { - var packageName = _a.packageName, className = _a.className; - return metadata.packageName === packageName && metadata.className === className; - }); - var isInstalled = $scrapersRegister - .list() - .find(function (_a) { - var packageName = _a.packageName, className = _a.className; - return metadata.packageName === packageName && metadata.className === className; - }); - var isAvilableInTools = availableScrapers.some(function (_a) { - var _b = _a.metadata, packageName = _b.packageName, className = _b.className; - return metadata.packageName === packageName && metadata.className === className; - }); - return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isAvilableInTools: isAvilableInTools }); - }); - if (metadata.length === 0) { - return spaceTrim__default["default"]("\n **No scrapers are available**\n\n This is a unexpected behavior, you are probably using some broken version of Promptbook\n At least there should be available the metadata of the scrapers\n "); - } - return spaceTrim__default["default"](function (block) { return "\n Available scrapers are:\n ".concat(block(metadata - .map(function (_a, i) { - var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser, isAvilableInTools = _a.isAvilableInTools; - var more = []; - // TODO: [🧠] Maybe use `documentationUrl` - if (isMetadataAviailable) { - more.push("\u2B1C Metadata registered"); - } // not else - if (isInstalled) { - more.push("\uD83D\uDFE9 Installed"); - } // not else - if (isAvilableInTools) { - more.push("\uD83D\uDFE6 Available in tools"); - } // not else - if (!isMetadataAviailable && isInstalled) { - more.push("When no metadata registered but scraper is installed, it is an unexpected behavior"); - } // not else - if (!isInstalled && isAvilableInTools) { - more.push("When the scraper is not installed but available in tools, it is an unexpected compatibility behavior"); - } // not else - if (!isAvilableInBrowser) { - more.push("Not usable in browser"); - } - var moreText = more.length === 0 ? '' : " *(".concat(more.join('; '), ")*"); - return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes - .map(function (mimeType) { return "\"".concat(mimeType, "\""); }) - .join(', ')).concat(moreText); - }) - .join('\n')), "\n\n Legend:\n - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n "); }); - } - /** - * TODO: [®] DRY Register logic - */ - - /** - * @@@ - * - * @param text @@@ - * @returns @@@ - * @example 'hello-world' - * @example 'i-love-promptbook' - * @public exported from `@promptbook/utils` - */ - function normalizeToKebabCase(text) { - var e_1, _a; - text = removeDiacritics(text); - var charType; - var lastCharType = 'OTHER'; - var normalizedName = ''; - try { - for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) { - var char = text_1_1.value; - var normalizedChar = void 0; - if (/^[a-z]$/.test(char)) { - charType = 'LOWERCASE'; - normalizedChar = char; - } - else if (/^[A-Z]$/.test(char)) { - charType = 'UPPERCASE'; - normalizedChar = char.toLowerCase(); - } - else if (/^[0-9]$/.test(char)) { - charType = 'NUMBER'; - normalizedChar = char; - } - else { - charType = 'OTHER'; - normalizedChar = '-'; - } - if (charType !== lastCharType && - !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') && - !(lastCharType === 'NUMBER') && - !(charType === 'NUMBER')) { - normalizedName += '-'; - } - normalizedName += normalizedChar; - lastCharType = charType; - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1); - } - finally { if (e_1) throw e_1.error; } - } - normalizedName = normalizedName.split(/-+/g).join('-'); - normalizedName = normalizedName.split(/-?\/-?/g).join('/'); - normalizedName = normalizedName.replace(/^-/, ''); - normalizedName = normalizedName.replace(/-$/, ''); - return normalizedName; - } - /** - * Note: [💞] Ignore a discrepancy between file name and entity name - */ - - /** - * Creates unique name for the source - * - * @public exported from `@promptbook/editable` - */ - function knowledgeSourceContentToName(knowledgeSourceContent) { - var hash = cryptoJs.SHA256(hexEncoder__default["default"].parse(JSON.stringify(knowledgeSourceContent))) - // <- TODO: [🥬] Encapsulate sha256 to some private utility function - .toString( /* hex */) - .substring(0, 20); - // <- TODO: [🥬] Make some system for hashes and ids of promptbook - var semanticName = normalizeToKebabCase(knowledgeSourceContent.substring(0, 20)); - var pieces = ['source', semanticName, hash].filter(function (piece) { return piece !== ''; }); - var name = pieces.join('-').split('--').join('-'); - // <- TODO: Use MAX_FILENAME_LENGTH - return name; - } - /** - * TODO: [🐱‍🐉][🧠] Make some smart crop NOT source-i-m-pavol-a-develop-... BUT source-i-m-pavol-a-developer-... - */ - - /** - * Convert file extension to mime type - * - * @private within the repository - */ - function extensionToMimeType(value) { - return mimeTypes.lookup(value) || 'application/octet-stream'; - } - - /** - * Get the file extension from a file name - * - * @private within the repository - */ - function getFileExtension(value) { - var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i); - return match ? match[1].toLowerCase() : null; - } - - /** - * Checks if the file exists - * - * @private within the repository - */ - function isFileExisting(filename, fs) { - return __awaiter(this, void 0, void 0, function () { - var isReadAccessAllowed, isFile; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: return [4 /*yield*/, fs - .access(filename, fs.constants.R_OK) - .then(function () { return true; }) - .catch(function () { return false; })]; - case 1: - isReadAccessAllowed = _a.sent(); - if (!isReadAccessAllowed) { - return [2 /*return*/, false]; - } - return [4 /*yield*/, fs - .stat(filename) - .then(function (fileStat) { return fileStat.isFile(); }) - .catch(function () { return false; })]; - case 2: - isFile = _a.sent(); - return [2 /*return*/, isFile]; - } - }); - }); - } - /** - * Note: Not [~🟢~] because it is not directly dependent on `fs - * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages - * TODO: [🖇] What about symlinks? - */ - - /** - * Tests if given string is valid URL. - * - * Note: This does not check if the file exists only if the path is valid - * @public exported from `@promptbook/utils` - */ - function isValidFilePath(filename) { - if (typeof filename !== 'string') { - return false; - } - if (filename.split('\n').length > 1) { - return false; - } - if (filename.split(' ').length > - 5 /* <- TODO: [🧠][🈷] Make some better non-arbitrary way how to distinct filenames from informational texts */) { - return false; - } - var filenameSlashes = filename.split('\\').join('/'); - // Absolute Unix path: /hello.txt - if (/^(\/)/i.test(filenameSlashes)) { - // console.log(filename, 'Absolute Unix path: /hello.txt'); - return true; - } - // Absolute Windows path: /hello.txt - if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) { - // console.log(filename, 'Absolute Windows path: /hello.txt'); - return true; - } - // Relative path: ./hello.txt - if (/^(\.\.?\/)+/i.test(filenameSlashes)) { - // console.log(filename, 'Relative path: ./hello.txt'); - return true; - } - // Allow paths like foo/hello - if (/^[^/]+\/[^/]+/i.test(filenameSlashes)) { - // console.log(filename, 'Allow paths like foo/hello'); - return true; - } - // Allow paths like hello.book - if (/^[^/]+\.[^/]+$/i.test(filenameSlashes)) { - // console.log(filename, 'Allow paths like hello.book'); - return true; - } - return false; - } - /** - * TODO: [🍏] Implement for MacOs - */ - - /** - * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers - * - * @private as default `fetch` function used in Promptbook scrapers - */ - var scraperFetch = function (url, init) { return __awaiter(void 0, void 0, void 0, function () { - var error_1; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - _a.trys.push([0, 2, , 3]); - return [4 /*yield*/, fetch(url, init)]; - case 1: return [2 /*return*/, _a.sent()]; - case 2: - error_1 = _a.sent(); - if (!(error_1 instanceof Error)) {console.log('!(error instanceof Error)') - throw error_1; - } - throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Can not fetch \"".concat(url, "\"\n\n Fetch error:\n ").concat(block(error_1.message), "\n\n "); })); - case 3: return [2 /*return*/]; - } - }); - }); }; - /** - * TODO: [🧠] Maybe rename because it is not used only for scrapers but also in `$getCompiledBook` - */ - - /** - * @@@ - * - * @public exported from `@promptbook/core` - */ - function makeKnowledgeSourceHandler(knowledgeSource, tools, options) { - var _a; - return __awaiter(this, void 0, void 0, function () { - var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, filename_1, fileExtension, mimeType; - return __generator(this, function (_f) { - switch (_f.label) { - case 0: - _b = tools.fetch, fetch = _b === void 0 ? scraperFetch : _b; - knowledgeSourceContent = knowledgeSource.knowledgeSourceContent; - name = knowledgeSource.name; - _c = options || {}, _d = _c.rootDirname, rootDirname = _d === void 0 ? null : _d, _c.isVerbose; - if (!name) { - name = knowledgeSourceContentToName(knowledgeSourceContent); - } - if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/, 2]; - url = knowledgeSourceContent; - return [4 /*yield*/, fetch(url)]; - case 1: - response_1 = _f.sent(); - mimeType = ((_a = response_1.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html'; - return [2 /*return*/, { - source: name, - filename: null, - url: url, - mimeType: mimeType, - /* + /** + * @@@ + * + * @param input @@@ + * @returns @@@ + * @public exported from `@promptbook/utils` + */ + function removeDiacritics(input) { + /*eslint no-control-regex: "off"*/ + return input.replace( + /[^\u0000-\u007E]/g, + (a) => DIACRITIC_VARIANTS_LETTERS[a] || a, + ); + } + /** + * TODO: [Ж] Variant for cyrillic (and in general non-latin) letters + */ + + /** + * Counts number of words in the text + * + * @public exported from `@promptbook/utils` + */ + function countWords(text) { + text = text.replace(/[\p{Extended_Pictographic}]/gu, "a"); + text = removeDiacritics(text); + return text.split(/[^a-zа-я0-9]+/i).filter((word) => word.length > 0) + .length; + } + + /** + * Index of all counter functions + * + * @public exported from `@promptbook/utils` + */ + var CountUtils = { + CHARACTERS: countCharacters, + WORDS: countWords, + SENTENCES: countSentences, + PARAGRAPHS: countParagraphs, + LINES: countLines, + PAGES: countPages, + }; + /** + * TODO: [🧠][🤠] This should be probbably as part of `TextFormatDefinition` + * Note: [💞] Ignore a discrepancy between file name and entity name + */ + + /** + * Function checkExpectations will check if the expectations on given value are met + * + * Note: There are two simmilar functions: + * - `checkExpectations` which throws an error if the expectations are not met + * - `isPassingExpectations` which returns a boolean + * + * @throws {ExpectError} if the expectations are not met + * @returns {void} Nothing + * @private internal function of `createPipelineExecutor` + */ + function checkExpectations(expectations, value) { + var e_1, _a; + try { + for ( + var _b = __values(Object.entries(expectations)), _c = _b.next(); + !_c.done; + _c = _b.next() + ) { + var _d = __read(_c.value, 2), + unit = _d[0], + _e = _d[1], + max = _e.max, + min = _e.min; + var amount = CountUtils[unit.toUpperCase()](value); + if (min && amount < min) { + throw new ExpectError( + "Expected at least " + .concat(min, " ") + .concat(unit, " but got ") + .concat(amount), + ); + } /* not else */ + if (max && amount > max) { + throw new ExpectError( + "Expected at most " + .concat(max, " ") + .concat(unit, " but got ") + .concat(amount), + ); + } + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_c && !_c.done && (_a = _b.return)) _a.call(_b); + } finally { + if (e_1) throw e_1.error; + } + } + } + /** + * TODO: [💝] Unite object for expecting amount and format + * TODO: [🧠][🤠] This should be part of `TextFormatDefinition` + * Note: [💝] and [🤠] are interconnected together + */ + + /** + * @@@ + * + * @private internal utility of `createPipelineExecutor` + */ + function executeAttempts(options) { + return __awaiter(this, void 0, void 0, function () { + var jokerParameterNames, + priority, + maxAttempts, + preparedContent, + parameters, + task, + preparedPipeline, + tools, + $executionReport, + pipelineIdentification, + maxExecutionAttempts, + $ongoingTaskResult, + _llms, + llmTools, + _loop_1, + attempt, + state_1; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + (jokerParameterNames = options.jokerParameterNames), + (priority = options.priority), + (maxAttempts = options.maxAttempts), + (preparedContent = options.preparedContent), + (parameters = options.parameters), + (task = options.task), + (preparedPipeline = options.preparedPipeline), + (tools = options.tools), + ($executionReport = options.$executionReport), + (pipelineIdentification = options.pipelineIdentification), + (maxExecutionAttempts = options.maxExecutionAttempts); + $ongoingTaskResult = { + $result: null, + $resultString: null, + $expectError: null, + $scriptPipelineExecutionErrors: [], + }; + _llms = arrayableToArray(tools.llm); + llmTools = + _llms.length === 1 + ? _llms[0] + : joinLlmExecutionTools.apply( + void 0, + __spreadArray([], __read(_llms), false), + ); + _loop_1 = function (attempt) { + var isJokerAttempt, + jokerParameterName, + _b, + modelRequirements, + _c, + _d, + _e, + _f, + _g, + scriptTools, + _h, + error_1, + e_1_1, + _j, + _k, + _l, + functionName, + postprocessingError, + _m, + _o, + scriptTools, + _p, + error_2, + e_2_1, + e_3_1, + error_3; + var e_1, _q, e_3, _r, e_2, _s; + return __generator(this, (_t) => { + switch (_t.label) { + case 0: + isJokerAttempt = attempt < 0; + jokerParameterName = + jokerParameterNames[ + jokerParameterNames.length + attempt + ]; + // TODO: [🧠][🍭] JOKERS, EXPECTATIONS, POSTPROCESSING and FOREACH + if (isJokerAttempt && !jokerParameterName) { + throw new UnexpectedError( + spaceTrim.spaceTrim((block) => + "\n Joker not found in attempt " + .concat(attempt, "\n\n ") + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + $ongoingTaskResult.$result = null; + $ongoingTaskResult.$resultString = null; + $ongoingTaskResult.$expectError = null; + if (isJokerAttempt) { + if (parameters[jokerParameterName] === undefined) { + throw new PipelineExecutionError( + spaceTrim.spaceTrim((block) => + "\n Joker parameter {" + .concat( + jokerParameterName, + "} not defined\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + // <- TODO: This is maybe `PipelineLogicError` which should be detected in `validatePipeline` and here just thrown as `UnexpectedError` + } else { + $ongoingTaskResult.$resultString = + parameters[jokerParameterName]; + } + } + _t.label = 1; + case 1: + _t.trys.push([1, 43, 44, 45]); + if (!!isJokerAttempt) return [3 /*break*/, 25]; + _b = task.taskType; + switch (_b) { + case "SIMPLE_TASK": + return [3 /*break*/, 2]; + case "PROMPT_TASK": + return [3 /*break*/, 3]; + case "SCRIPT_TASK": + return [3 /*break*/, 11]; + case "DIALOG_TASK": + return [3 /*break*/, 22]; + } + return [3 /*break*/, 24]; + case 2: + $ongoingTaskResult.$resultString = templateParameters( + preparedContent, + parameters, + ); + return [3 /*break*/, 25]; + case 3: + modelRequirements = __assign( + __assign( + { modelVariant: "CHAT" }, + preparedPipeline.defaultModelRequirements || {}, + ), + task.modelRequirements || {}, + ); + $ongoingTaskResult.$prompt = { + title: task.title, + pipelineUrl: "" + .concat( + preparedPipeline.pipelineUrl + ? preparedPipeline.pipelineUrl + : "anonymous" /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, + "#", + ) + .concat( + task.name, + // <- TODO: Here should be maybe also subformat index to distinguish between same task with different subformat values + ), + parameters: parameters, + content: preparedContent, + modelRequirements: modelRequirements, + expectations: __assign( + __assign( + {}, + preparedPipeline.personas.find((_a) => { + var name = _a.name; + return name === task.personaName; + }) || {}, + ), + task.expectations, + ), + format: task.format, + postprocessingFunctionNames: + task.postprocessingFunctionNames, + }; // <- TODO: Not very good type guard + _c = modelRequirements.modelVariant; + switch (_c) { + case "CHAT": + return [3 /*break*/, 4]; + case "COMPLETION": + return [3 /*break*/, 6]; + case "EMBEDDING": + return [3 /*break*/, 8]; + } + return [3 /*break*/, 9]; + case 4: + _d = $ongoingTaskResult; + return [ + 4 /*yield*/, + llmTools.callChatModel( + // <- TODO: [🧁] Check that `callChatModel` is defined + $deepFreeze($ongoingTaskResult.$prompt), + ), + ]; + case 5: + _d.$chatResult = _t.sent(); + // TODO: [🍬] Destroy chatThread + $ongoingTaskResult.$result = + $ongoingTaskResult.$chatResult; + $ongoingTaskResult.$resultString = + $ongoingTaskResult.$chatResult.content; + return [3 /*break*/, 10]; + case 6: + _e = $ongoingTaskResult; + return [ + 4 /*yield*/, + llmTools.callCompletionModel( + // <- TODO: [🧁] Check that `callCompletionModel` is defined + $deepFreeze($ongoingTaskResult.$prompt), + ), + ]; + case 7: + _e.$completionResult = _t.sent(); + $ongoingTaskResult.$result = + $ongoingTaskResult.$completionResult; + $ongoingTaskResult.$resultString = + $ongoingTaskResult.$completionResult.content; + return [3 /*break*/, 10]; + case 8: + throw new PipelineExecutionError( + spaceTrim.spaceTrim((block) => + "\n Embedding model can not be used in pipeline\n\n This should be catched during parsing\n\n ".concat( + block(pipelineIdentification), + "\n\n ", + ), + ), + ); + case 9: + throw new PipelineExecutionError( + spaceTrim.spaceTrim((block) => + '\n Unknown model variant "' + .concat( + task.modelRequirements.modelVariant, + '"\n\n ', + ) + .concat( + block(pipelineIdentification), + "\n\n ", + ), + ), + ); + case 10: + return [3 /*break*/, 25]; + case 11: + if (arrayableToArray(tools.script).length === 0) { + throw new PipelineExecutionError( + spaceTrim.spaceTrim((block) => + "\n No script execution tools are available\n\n ".concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + if (!task.contentLanguage) { + throw new PipelineExecutionError( + spaceTrim.spaceTrim((block) => + '\n Script language is not defined for SCRIPT TASK "' + .concat( + task.name, + '"\n\n ', + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + _t.label = 12; + case 12: + _t.trys.push([12, 19, 20, 21]); + (_f = + ((e_1 = void 0), + __values(arrayableToArray(tools.script)))), + (_g = _f.next()); + _t.label = 13; + case 13: + if (!!_g.done) return [3 /*break*/, 18]; + scriptTools = _g.value; + _t.label = 14; + case 14: + _t.trys.push([14, 16, , 17]); + _h = $ongoingTaskResult; + return [ + 4 /*yield*/, + scriptTools.execute( + $deepFreeze({ + scriptLanguage: task.contentLanguage, + script: preparedContent, + parameters: parameters, + }), + ), + ]; + case 15: + _h.$resultString = _t.sent(); + return [3 /*break*/, 18]; + case 16: + error_1 = _t.sent(); + if (!(error_1 instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error_1; + } + if (error_1 instanceof UnexpectedError) { + throw error_1; + } + $ongoingTaskResult.$scriptPipelineExecutionErrors.push( + error_1, + ); + return [3 /*break*/, 17]; + case 17: + _g = _f.next(); + return [3 /*break*/, 13]; + case 18: + return [3 /*break*/, 21]; + case 19: + e_1_1 = _t.sent(); + e_1 = { error: e_1_1 }; + return [3 /*break*/, 21]; + case 20: + try { + if (_g && !_g.done && (_q = _f.return)) _q.call(_f); + } finally { + if (e_1) throw e_1.error; + } + return [7 /*endfinally*/]; + case 21: + if ($ongoingTaskResult.$resultString !== null) { + return [3 /*break*/, 25]; + } + if ( + $ongoingTaskResult.$scriptPipelineExecutionErrors + .length === 1 + ) { + throw $ongoingTaskResult + .$scriptPipelineExecutionErrors[0]; + } else { + throw new PipelineExecutionError( + spaceTrim.spaceTrim((block) => + "\n Script execution failed " + .concat( + $ongoingTaskResult + .$scriptPipelineExecutionErrors.length, + "x\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n\n ", + ) + .concat( + block( + $ongoingTaskResult.$scriptPipelineExecutionErrors + .map((error) => "- " + error.message) + .join("\n\n"), + ), + "\n ", + ), + ), + ); + } + case 22: + if (tools.userInterface === undefined) { + throw new PipelineExecutionError( + spaceTrim.spaceTrim((block) => + "\n User interface tools are not available\n\n ".concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + // TODO: [🌹] When making next attempt for `DIALOG TASK`, preserve the previous user input + _j = $ongoingTaskResult; + return [ + 4 /*yield*/, + tools.userInterface.promptDialog( + $deepFreeze({ + promptTitle: task.title, + promptMessage: templateParameters( + task.description || "", + parameters, + ), + defaultValue: templateParameters( + preparedContent, + parameters, + ), + // TODO: [🧠] Figure out how to define placeholder in `.book` file + placeholder: undefined, + priority: priority, + }), + ), + ]; + case 23: + // TODO: [🌹] When making next attempt for `DIALOG TASK`, preserve the previous user input + _j.$resultString = _t.sent(); + return [3 /*break*/, 25]; + case 24: + throw new PipelineExecutionError( + spaceTrim.spaceTrim((block) => + '\n Unknown execution type "' + .concat( + task.taskType, + '"\n\n ', + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + case 25: + if ( + !(!isJokerAttempt && task.postprocessingFunctionNames) + ) + return [3 /*break*/, 42]; + _t.label = 26; + case 26: + _t.trys.push([26, 40, 41, 42]); + (_k = + ((e_3 = void 0), + __values(task.postprocessingFunctionNames))), + (_l = _k.next()); + _t.label = 27; + case 27: + if (!!_l.done) return [3 /*break*/, 39]; + functionName = _l.value; + postprocessingError = null; + _t.label = 28; + case 28: + _t.trys.push([28, 35, 36, 37]); + (_m = + ((e_2 = void 0), + __values(arrayableToArray(tools.script)))), + (_o = _m.next()); + _t.label = 29; + case 29: + if (!!_o.done) return [3 /*break*/, 34]; + scriptTools = _o.value; + _t.label = 30; + case 30: + _t.trys.push([30, 32, , 33]); + _p = $ongoingTaskResult; + return [ + 4 /*yield*/, + scriptTools.execute({ + scriptLanguage: + "javascript" /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */, + script: "".concat(functionName, "(resultString)"), + parameters: { + resultString: + $ongoingTaskResult.$resultString || "", + // Note: No ...parametersForTask, because working with result only + }, + }), + ]; + case 31: + _p.$resultString = _t.sent(); + postprocessingError = null; + return [3 /*break*/, 34]; + case 32: + error_2 = _t.sent(); + if (!(error_2 instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error_2; + } + if (error_2 instanceof UnexpectedError) { + throw error_2; + } + postprocessingError = error_2; + $ongoingTaskResult.$scriptPipelineExecutionErrors.push( + error_2, + ); + return [3 /*break*/, 33]; + case 33: + _o = _m.next(); + return [3 /*break*/, 29]; + case 34: + return [3 /*break*/, 37]; + case 35: + e_2_1 = _t.sent(); + e_2 = { error: e_2_1 }; + return [3 /*break*/, 37]; + case 36: + try { + if (_o && !_o.done && (_s = _m.return)) _s.call(_m); + } finally { + if (e_2) throw e_2.error; + } + return [7 /*endfinally*/]; + case 37: + if (postprocessingError) { + throw postprocessingError; + } + _t.label = 38; + case 38: + _l = _k.next(); + return [3 /*break*/, 27]; + case 39: + return [3 /*break*/, 42]; + case 40: + e_3_1 = _t.sent(); + e_3 = { error: e_3_1 }; + return [3 /*break*/, 42]; + case 41: + try { + if (_l && !_l.done && (_r = _k.return)) _r.call(_k); + } finally { + if (e_3) throw e_3.error; + } + return [7 /*endfinally*/]; + case 42: + // TODO: [💝] Unite object for expecting amount and format + if (task.format) { + if (task.format === "JSON") { + if ( + !isValidJsonString( + $ongoingTaskResult.$resultString || "", + ) + ) { + // TODO: [🏢] Do more universally via `FormatDefinition` + try { + $ongoingTaskResult.$resultString = + extractJsonBlock( + $ongoingTaskResult.$resultString || "", + ); + } catch (error) { + throw new ExpectError( + spaceTrim.spaceTrim((block) => + "\n Expected valid JSON string\n\n ".concat( + block( + /*<- Note: No need for `pipelineIdentification`, it will be catched and added later */ "", + ), + "\n ", + ), + ), + ); + } + } + } else { + throw new UnexpectedError( + spaceTrim.spaceTrim((block) => + '\n Unknown format "' + .concat( + task.format, + '"\n\n ', + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + } + // TODO: [💝] Unite object for expecting amount and format + if (task.expectations) { + checkExpectations( + task.expectations, + $ongoingTaskResult.$resultString || "", + ); + } + return [2 /*return*/, "break-attempts"]; + case 43: + error_3 = _t.sent(); + if (!(error_3 instanceof ExpectError)) { + throw error_3; + } + $ongoingTaskResult.$expectError = error_3; + return [3 /*break*/, 45]; + case 44: + if ( + !isJokerAttempt && + task.taskType === "PROMPT_TASK" && + $ongoingTaskResult.$prompt + // <- Note: [2] When some expected parameter is not defined, error will occur in templateParameters + // In that case we don’t want to make a report about it because it’s not a llm execution error + ) { + // TODO: [🧠] Maybe put other taskTypes into report + $executionReport.promptExecutions.push({ + prompt: __assign({}, $ongoingTaskResult.$prompt), + result: $ongoingTaskResult.$result || undefined, + error: + $ongoingTaskResult.$expectError === null + ? undefined + : serializeError($ongoingTaskResult.$expectError), + }); + } + return [7 /*endfinally*/]; + case 45: + if ( + $ongoingTaskResult.$expectError !== null && + attempt === maxAttempts - 1 + ) { + throw new PipelineExecutionError( + spaceTrim.spaceTrim((block) => { + var _a, _b, _c; + return "\n LLM execution failed " + .concat( + maxExecutionAttempts, + "x\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n\n ---\n The Prompt:\n ", + ) + .concat( + block( + ( + ((_a = $ongoingTaskResult.$prompt) === + null || _a === void 0 + ? void 0 + : _a.content) || "" + ) + .split("\n") + .map((line) => "> ".concat(line)) + .join("\n"), + ), + "\n\n Last error ", + ) + .concat( + ((_b = $ongoingTaskResult.$expectError) === + null || _b === void 0 + ? void 0 + : _b.name) || "", + ":\n ", + ) + .concat( + block( + ( + ((_c = $ongoingTaskResult.$expectError) === + null || _c === void 0 + ? void 0 + : _c.message) || "" + ) + .split("\n") + .map((line) => "> ".concat(line)) + .join("\n"), + ), + "\n\n Last result:\n ", + ) + .concat( + block( + $ongoingTaskResult.$resultString === null + ? "null" + : $ongoingTaskResult.$resultString + .split("\n") + .map((line) => "> ".concat(line)) + .join("\n"), + ), + "\n ---\n ", + ); + }), + ); + } + return [2 /*return*/]; + } + }); + }; + attempt = -jokerParameterNames.length; + _a.label = 1; + case 1: + if (!(attempt < maxAttempts)) return [3 /*break*/, 4]; + return [5 /*yield**/, _loop_1(attempt)]; + case 2: + state_1 = _a.sent(); + switch (state_1) { + case "break-attempts": + return [3 /*break*/, 4]; + } + _a.label = 3; + case 3: + attempt++; + return [3 /*break*/, 1]; + case 4: + if ($ongoingTaskResult.$resultString === null) { + throw new UnexpectedError( + spaceTrim.spaceTrim((block) => + "\n Something went wrong and prompt result is null\n\n ".concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + return [2 /*return*/, $ongoingTaskResult.$resultString]; + } + }); + }); + } + /** + * TODO: Break into smaller functions + */ + + /** + * @@@ + * + * @private internal utility of `createPipelineExecutor` + */ + function executeFormatSubvalues(options) { + return __awaiter(this, void 0, void 0, function () { + var task, + jokerParameterNames, + parameters, + priority, + csvSettings, + pipelineIdentification, + parameterValue, + formatDefinition, + subvalueDefinition, + formatSettings, + resultString; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + (task = options.task), + (jokerParameterNames = options.jokerParameterNames), + (parameters = options.parameters), + (priority = options.priority), + (csvSettings = options.csvSettings), + (pipelineIdentification = options.pipelineIdentification); + if (task.foreach === undefined) { + return [2 /*return*/, /* not await */ executeAttempts(options)]; + } + if (jokerParameterNames.length !== 0) { + throw new UnexpectedError( + spaceTrim__default["default"]((block) => + "\n JOKER parameters are not supported together with FOREACH command\n\n [\uD83E\uDDDE\u200D\u2640\uFE0F] This should be prevented in `validatePipeline`\n\n ".concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + parameterValue = parameters[task.foreach.parameterName] || ""; + formatDefinition = FORMAT_DEFINITIONS.find((formatDefinition) => + __spreadArray( + [formatDefinition.formatName], + __read(formatDefinition.aliases || []), + false, + ).includes(task.foreach.formatName), + ); + if (formatDefinition === undefined) { + throw new UnexpectedError( + // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError) + spaceTrim__default["default"]((block) => + '\n Unsupported format "' + .concat( + task.foreach.formatName, + '"\n\n Available formats:\n ', + ) + .concat( + block( + FORMAT_DEFINITIONS.map( + (formatDefinition) => formatDefinition.formatName, + ) + .map((formatName) => "- ".concat(formatName)) + .join("\n"), + ), + "\n\n [\u26F7] This should never happen because format name should be validated during parsing\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + subvalueDefinition = formatDefinition.subvalueDefinitions.find( + (subvalueDefinition) => + __spreadArray( + [subvalueDefinition.subvalueName], + __read(subvalueDefinition.aliases || []), + false, + ).includes(task.foreach.subformatName), + ); + if (subvalueDefinition === undefined) { + throw new UnexpectedError( + // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError) + spaceTrim__default["default"]((block) => + '\n Unsupported subformat name "' + .concat(task.foreach.subformatName, '" for format "') + .concat( + task.foreach.formatName, + '"\n\n Available subformat names for format "', + ) + .concat( + formatDefinition.formatName, + '":\n ', + ) + .concat( + block( + formatDefinition.subvalueDefinitions + .map( + (subvalueDefinition) => + subvalueDefinition.subvalueName, + ) + .map((subvalueName) => "- ".concat(subvalueName)) + .join("\n"), + ), + "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + if (formatDefinition.formatName === "CSV") { + formatSettings = csvSettings; + // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text + } + return [ + 4 /*yield*/, + subvalueDefinition.mapValues( + parameterValue, + task.foreach.outputSubparameterName, + formatSettings, + (subparameters, index) => + __awaiter(this, void 0, void 0, function () { + var mappedParameters, allSubparameters, subresultString; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions + // TODO: When done [🐚] Report progress also for each subvalue here + try { + mappedParameters = + mapAvailableToExpectedParameters({ + expectedParameters: Object.fromEntries( + task.foreach.inputSubparameterNames.map( + (subparameterName) => [ + subparameterName, + null, + ], + ), + ), + availableParameters: subparameters, + }); + } catch (error) { + if (!(error instanceof PipelineExecutionError)) { + throw error; + } + throw new PipelineExecutionError( + spaceTrim__default["default"]((block) => + "\n " + .concat( + error.message, + "\n\n This is error in FOREACH command\n You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n Subparameter index: ", + ) + .concat(index, "\n "), + ), + ); + } + allSubparameters = __assign( + __assign({}, parameters), + mappedParameters, + ); + // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed + Object.freeze(allSubparameters); + return [ + 4 /*yield*/, + executeAttempts( + __assign(__assign({}, options), { + priority: priority + index, + parameters: allSubparameters, + pipelineIdentification: spaceTrim__default[ + "default" + ]((block) => + "\n " + .concat( + block(pipelineIdentification), + "\n Subparameter index: ", + ) + .concat(index, "\n "), + ), + }), + ), + ]; + case 1: + subresultString = _a.sent(); + return [2 /*return*/, subresultString]; + } + }); + }), + ), + ]; + case 1: + resultString = _a.sent(); + return [2 /*return*/, resultString]; + } + }); + }); + } + + /** + * @@@ + * + * @private internal utility of `createPipelineExecutor` + */ + function getContextForTask(task) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, (_a) => [ + 2 /*return*/, + RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [🏍] Implement */, + ]); + }); + } + + /** + * @@@ + * + * @private internal utility of `createPipelineExecutor` + */ + function getExamplesForTask(task) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, (_a) => [ + 2 /*return*/, + RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [♨] Implement */, + ]); + }); + } + + /** + * @@@ + * + * @private internal utility of `createPipelineExecutor` + */ + function getKnowledgeForTask(options) { + return __awaiter(this, void 0, void 0, function () { + var preparedPipeline; + return __generator(this, (_a) => { + (preparedPipeline = options.preparedPipeline), options.task; + return [ + 2 /*return*/, + preparedPipeline.knowledgePieces + .map((_a) => { + var content = _a.content; + return "- ".concat(content); + }) + .join("\n"), + ]; + }); + }); + } + + /** + * @@@ + * + * @private internal utility of `createPipelineExecutor` + */ + function getReservedParametersForTask(options) { + return __awaiter(this, void 0, void 0, function () { + var preparedPipeline, + task, + pipelineIdentification, + context, + knowledge, + examples, + currentDate, + modelName, + reservedParameters, + _loop_1, + RESERVED_PARAMETER_NAMES_1, + RESERVED_PARAMETER_NAMES_1_1, + parameterName; + var e_1, _a; + return __generator(this, (_b) => { + switch (_b.label) { + case 0: + (preparedPipeline = options.preparedPipeline), + (task = options.task), + (pipelineIdentification = options.pipelineIdentification); + return [4 /*yield*/, getContextForTask()]; + case 1: + context = _b.sent(); + return [ + 4 /*yield*/, + getKnowledgeForTask({ + preparedPipeline: preparedPipeline, + task: task, + }), + ]; + case 2: + knowledge = _b.sent(); + return [4 /*yield*/, getExamplesForTask()]; + case 3: + examples = _b.sent(); + currentDate = new Date().toISOString(); + modelName = RESERVED_PARAMETER_MISSING_VALUE; + reservedParameters = { + content: RESERVED_PARAMETER_RESTRICTED, + context: context, + knowledge: knowledge, + examples: examples, + currentDate: currentDate, + modelName: modelName, + }; + _loop_1 = (parameterName) => { + if (reservedParameters[parameterName] === undefined) { + throw new UnexpectedError( + spaceTrim.spaceTrim((block) => + "\n Reserved parameter {" + .concat( + parameterName, + "} is not defined\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + }; + try { + // Note: Doublecheck that ALL reserved parameters are defined: + for ( + RESERVED_PARAMETER_NAMES_1 = __values( + RESERVED_PARAMETER_NAMES, + ), + RESERVED_PARAMETER_NAMES_1_1 = + RESERVED_PARAMETER_NAMES_1.next(); + !RESERVED_PARAMETER_NAMES_1_1.done; + RESERVED_PARAMETER_NAMES_1_1 = + RESERVED_PARAMETER_NAMES_1.next() + ) { + parameterName = RESERVED_PARAMETER_NAMES_1_1.value; + _loop_1(parameterName); + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if ( + RESERVED_PARAMETER_NAMES_1_1 && + !RESERVED_PARAMETER_NAMES_1_1.done && + (_a = RESERVED_PARAMETER_NAMES_1.return) + ) + _a.call(RESERVED_PARAMETER_NAMES_1); + } finally { + if (e_1) throw e_1.error; + } + } + return [2 /*return*/, reservedParameters]; + } + }); + }); + } + + /** + * @@@ + * + * @private internal utility of `createPipelineExecutor` + */ + function executeTask(options) { + return __awaiter(this, void 0, void 0, function () { + var currentTask, + preparedPipeline, + parametersToPass, + tools, + onProgress, + $executionReport, + pipelineIdentification, + maxExecutionAttempts, + maxParallelCount, + csvSettings, + isVerbose, + rootDirname, + cacheDirname, + intermediateFilesStrategy, + isAutoInstalled, + isNotPreparedWarningSupressed, + name, + title, + priority, + usedParameterNames, + dependentParameterNames, + definedParameters, + _a, + _b, + _c, + definedParameterNames, + parameters, + _loop_1, + _d, + _e, + parameterName, + maxAttempts, + jokerParameterNames, + preparedContent, + resultString; + var e_1, _f, _g; + return __generator(this, (_h) => { + switch (_h.label) { + case 0: + (currentTask = options.currentTask), + (preparedPipeline = options.preparedPipeline), + (parametersToPass = options.parametersToPass), + (tools = options.tools), + (onProgress = options.onProgress), + ($executionReport = options.$executionReport), + (pipelineIdentification = options.pipelineIdentification), + (maxExecutionAttempts = options.maxExecutionAttempts), + (maxParallelCount = options.maxParallelCount), + (csvSettings = options.csvSettings), + (isVerbose = options.isVerbose), + (rootDirname = options.rootDirname), + (cacheDirname = options.cacheDirname), + (intermediateFilesStrategy = options.intermediateFilesStrategy), + (isAutoInstalled = options.isAutoInstalled), + (isNotPreparedWarningSupressed = + options.isNotPreparedWarningSupressed); + name = "pipeline-executor-frame-".concat(currentTask.name); + title = currentTask.title; + priority = + preparedPipeline.tasks.length - + preparedPipeline.tasks.indexOf(currentTask); + return [ + 4 /*yield*/, + onProgress({ + name: name, + title: title, + isStarted: false, + isDone: false, + taskType: currentTask.taskType, + parameterName: currentTask.resultingParameterName, + parameterValue: null, + // <- [🍸] + }), + ]; + case 1: + _h.sent(); + usedParameterNames = extractParameterNamesFromTask(currentTask); + dependentParameterNames = new Set( + currentTask.dependentParameterNames, + ); + // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters` + if ( + union( + difference(usedParameterNames, dependentParameterNames), + difference(dependentParameterNames, usedParameterNames), + ).size !== 0 + ) { + throw new UnexpectedError( + spaceTrim.spaceTrim((block) => + "\n Dependent parameters are not consistent with used parameters:\n\n Dependent parameters:\n " + .concat( + Array.from(dependentParameterNames) + .map((name) => "{".concat(name, "}")) + .join(", "), + "\n\n Used parameters:\n ", + ) + .concat( + Array.from(usedParameterNames) + .map((name) => "{".concat(name, "}")) + .join(", "), + "\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n\n ", + ), + ), + ); + } + _b = (_a = Object).freeze; + _c = [{}]; + return [ + 4 /*yield*/, + getReservedParametersForTask({ + preparedPipeline: preparedPipeline, + task: currentTask, + pipelineIdentification: pipelineIdentification, + }), + ]; + case 2: + definedParameters = _b.apply(_a, [ + __assign.apply(void 0, [ + __assign.apply(void 0, _c.concat([_h.sent()])), + parametersToPass, + ]), + ]); + definedParameterNames = new Set(Object.keys(definedParameters)); + parameters = {}; + _loop_1 = (parameterName) => { + // Situation: Parameter is defined and used + if ( + definedParameterNames.has(parameterName) && + usedParameterNames.has(parameterName) + ) { + parameters[parameterName] = definedParameters[parameterName]; + } + // Situation: Parameter is defined but NOT used + else if ( + definedParameterNames.has(parameterName) && + !usedParameterNames.has(parameterName) + ); + // Situation: Parameter is NOT defined BUT used + else if ( + !definedParameterNames.has(parameterName) && + usedParameterNames.has(parameterName) + ) { + // Houston, we have a problem + // Note: Checking part is also done in `validatePipeline`, but it’s good to doublecheck + throw new UnexpectedError( + spaceTrim.spaceTrim((block) => + "\n Parameter `{" + .concat( + parameterName, + '}` is NOT defined\n BUT used in task "', + ) + .concat( + currentTask.title || currentTask.name, + '"\n\n This should be catched in `validatePipeline`\n\n ', + ) + .concat( + block(pipelineIdentification), + "\n\n ", + ), + ), + ); + } + }; + try { + // Note: [2] Check that all used parameters are defined and removing unused parameters for this task + // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters` + for ( + _d = __values( + Array.from( + union( + definedParameterNames, + usedParameterNames, + dependentParameterNames, + ), + ), + ), + _e = _d.next(); + !_e.done; + _e = _d.next() + ) { + parameterName = _e.value; + _loop_1(parameterName); + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_e && !_e.done && (_f = _d.return)) _f.call(_d); + } finally { + if (e_1) throw e_1.error; + } + } + // Note: [👨‍👨‍👧] Now we can freeze `parameters` because we are sure that all and only used parameters are defined and are not going to be changed + Object.freeze(parameters); + maxAttempts = + currentTask.taskType === "DIALOG_TASK" + ? Number.POSITIVE_INFINITY + : maxExecutionAttempts; + jokerParameterNames = currentTask.jokerParameterNames || []; + preparedContent = (currentTask.preparedContent || "{content}") + .split("{content}") + .join(currentTask.content); + return [ + 4 /*yield*/, + executeFormatSubvalues({ + jokerParameterNames: jokerParameterNames, + priority: priority, + maxAttempts: maxAttempts, + preparedContent: preparedContent, + parameters: parameters, + task: currentTask, + preparedPipeline: preparedPipeline, + tools: tools, + $executionReport: $executionReport, + pipelineIdentification: pipelineIdentification, + maxExecutionAttempts: maxExecutionAttempts, + maxParallelCount: maxParallelCount, + csvSettings: csvSettings, + isVerbose: isVerbose, + rootDirname: rootDirname, + cacheDirname: cacheDirname, + intermediateFilesStrategy: intermediateFilesStrategy, + isAutoInstalled: isAutoInstalled, + isNotPreparedWarningSupressed: isNotPreparedWarningSupressed, + }), + ]; + case 3: + resultString = _h.sent(); + return [ + 4 /*yield*/, + onProgress({ + name: name, + title: title, + isStarted: true, + isDone: true, + taskType: currentTask.taskType, + parameterName: currentTask.resultingParameterName, + parameterValue: resultString, + // <- [🍸] + }), + ]; + case 4: + _h.sent(); + return [ + 2 /*return*/, + Object.freeze( + ((_g = {}), + (_g[currentTask.resultingParameterName] = + // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction + resultString), + _g), + ), + ]; + } + }); + }); + } + /** + * TODO: [🤹‍♂️] + */ + /** + * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited + */ + + /** + * @@@ + * + * @private internal utility of `createPipelineExecutor` + */ + function filterJustOutputParameters(options) { + var e_1, _a; + var preparedPipeline = options.preparedPipeline, + parametersToPass = options.parametersToPass, + $warnings = options.$warnings, + pipelineIdentification = options.pipelineIdentification; + var outputParameters = {}; + var _loop_1 = (parameter) => { + if (parametersToPass[parameter.name] === undefined) { + // [4] + $warnings.push( + new PipelineExecutionError( + spaceTrim.spaceTrim((block) => + "\n Parameter `{" + .concat( + parameter.name, + "}` should be an output parameter, but it was not generated during pipeline execution\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ), + ); + return "continue"; + } + outputParameters[parameter.name] = + parametersToPass[parameter.name] || ""; + }; + try { + // Note: Filter ONLY output parameters + // TODO: [👩🏾‍🤝‍👩🏻] Maybe use here `mapAvailableToExpectedParameters` + for ( + var _b = __values( + preparedPipeline.parameters.filter((_a) => { + var isOutput = _a.isOutput; + return isOutput; + }), + ), + _c = _b.next(); + !_c.done; + _c = _b.next() + ) { + var parameter = _c.value; + _loop_1(parameter); + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_c && !_c.done && (_a = _b.return)) _a.call(_b); + } finally { + if (e_1) throw e_1.error; + } + } + return outputParameters; + } + + /** + * @@@ + * + * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor` + * + * @private internal utility of `createPipelineExecutor` + */ + function executePipeline(options) { + return __awaiter(this, void 0, void 0, function () { + var inputParameters, + tools, + onProgress, + pipeline, + setPreparedPipeline, + pipelineIdentification, + maxParallelCount, + rootDirname, + isVerbose, + preparedPipeline, + errors, + warnings, + executionReport, + isReturned, + _a, + _b, + parameter, + e_1_1, + _loop_1, + _c, + _d, + parameterName, + state_1, + e_2_1, + parametersToPass, + resovedParameterNames_1, + unresovedTasks_1, + resolving_1, + loopLimit, + _loop_2, + error_1, + usage_1, + outputParameters_1, + usage, + outputParameters; + var e_1, _e, e_2, _f; + return __generator(this, (_g) => { + switch (_g.label) { + case 0: + (inputParameters = options.inputParameters), + (tools = options.tools), + (onProgress = options.onProgress), + (pipeline = options.pipeline), + (setPreparedPipeline = options.setPreparedPipeline), + (pipelineIdentification = options.pipelineIdentification), + (maxParallelCount = options.maxParallelCount), + (rootDirname = options.rootDirname), + (isVerbose = options.isVerbose); + preparedPipeline = options.preparedPipeline; + if (!(preparedPipeline === undefined)) return [3 /*break*/, 2]; + return [ + 4 /*yield*/, + preparePipeline(pipeline, tools, { + rootDirname: rootDirname, + isVerbose: isVerbose, + maxParallelCount: maxParallelCount, + }), + ]; + case 1: + preparedPipeline = _g.sent(); + setPreparedPipeline(preparedPipeline); + _g.label = 2; + case 2: + errors = []; + warnings = []; + executionReport = { + pipelineUrl: preparedPipeline.pipelineUrl, + title: preparedPipeline.title, + promptbookUsedVersion: PROMPTBOOK_ENGINE_VERSION, + promptbookRequestedVersion: preparedPipeline.bookVersion, + description: preparedPipeline.description, + promptExecutions: [], + }; + isReturned = false; + _g.label = 3; + case 3: + _g.trys.push([3, 9, 10, 11]); + (_a = __values( + preparedPipeline.parameters.filter((_a) => { + var isInput = _a.isInput; + return isInput; + }), + )), + (_b = _a.next()); + _g.label = 4; + case 4: + if (!!_b.done) return [3 /*break*/, 8]; + parameter = _b.value; + if (!(inputParameters[parameter.name] === undefined)) + return [3 /*break*/, 7]; + isReturned = true; + if (!(onProgress !== undefined)) return [3 /*break*/, 6]; + // Note: Wait a short time to prevent race conditions + return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)]; + case 5: + // Note: Wait a short time to prevent race conditions + _g.sent(); + _g.label = 6; + case 6: + return [ + 2 /*return*/, + exportJson({ + name: "executionReport", + message: + "Unuccessful PipelineExecutorResult (with missing parameter {".concat( + parameter.name, + "}) PipelineExecutorResult", + ), + order: [], + value: { + isSuccessful: false, + errors: __spreadArray( + [ + new PipelineExecutionError( + "Parameter `{".concat( + parameter.name, + "}` is required as an input parameter", + ), + ), + ], + __read(errors), + false, + ).map(serializeError), + warnings: [], + executionReport: executionReport, + outputParameters: {}, + usage: ZERO_USAGE, + preparedPipeline: preparedPipeline, + }, + }), + ]; + case 7: + _b = _a.next(); + return [3 /*break*/, 4]; + case 8: + return [3 /*break*/, 11]; + case 9: + e_1_1 = _g.sent(); + e_1 = { error: e_1_1 }; + return [3 /*break*/, 11]; + case 10: + try { + if (_b && !_b.done && (_e = _a.return)) _e.call(_a); + } finally { + if (e_1) throw e_1.error; + } + return [7 /*endfinally*/]; + case 11: + _loop_1 = function (parameterName) { + var parameter; + return __generator(this, (_h) => { + switch (_h.label) { + case 0: + parameter = preparedPipeline.parameters.find((_a) => { + var name = _a.name; + return name === parameterName; + }); + if (!(parameter === undefined)) return [3 /*break*/, 1]; + warnings.push( + new PipelineExecutionError( + spaceTrim.spaceTrim((block) => + "\n Extra parameter {" + .concat( + parameterName, + "} is being passed which is not part of the pipeline.\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ), + ); + return [3 /*break*/, 4]; + case 1: + if (!(parameter.isInput === false)) + return [3 /*break*/, 4]; + isReturned = true; + if (!(onProgress !== undefined)) return [3 /*break*/, 3]; + // Note: Wait a short time to prevent race conditions + return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)]; + case 2: + // Note: Wait a short time to prevent race conditions + _h.sent(); + _h.label = 3; + case 3: + return [ + 2 /*return*/, + { + value: exportJson({ + name: "pipelineExecutorResult", + message: spaceTrim.spaceTrim((block) => + "\n Unuccessful PipelineExecutorResult (with extra parameter {" + .concat( + parameter.name, + "}) PipelineExecutorResult\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + order: [], + value: { + isSuccessful: false, + errors: __spreadArray( + [ + new PipelineExecutionError( + spaceTrim.spaceTrim((block) => + "\n Parameter `{" + .concat( + parameter.name, + "}` is passed as input parameter but it is not input\n\n ", + ) + .concat( + block(pipelineIdentification), + "\n ", + ), + ), + ), + ], + __read(errors), + false, + ).map(serializeError), + warnings: warnings.map(serializeError), + executionReport: executionReport, + outputParameters: {}, + usage: ZERO_USAGE, + preparedPipeline: preparedPipeline, + }, + }), + }, + ]; + case 4: + return [2 /*return*/]; + } + }); + }; + _g.label = 12; + case 12: + _g.trys.push([12, 17, 18, 19]); + (_c = __values(Object.keys(inputParameters))), (_d = _c.next()); + _g.label = 13; + case 13: + if (!!_d.done) return [3 /*break*/, 16]; + parameterName = _d.value; + return [5 /*yield**/, _loop_1(parameterName)]; + case 14: + state_1 = _g.sent(); + if (typeof state_1 === "object") + return [2 /*return*/, state_1.value]; + _g.label = 15; + case 15: + _d = _c.next(); + return [3 /*break*/, 13]; + case 16: + return [3 /*break*/, 19]; + case 17: + e_2_1 = _g.sent(); + e_2 = { error: e_2_1 }; + return [3 /*break*/, 19]; + case 18: + try { + if (_d && !_d.done && (_f = _c.return)) _f.call(_c); + } finally { + if (e_2) throw e_2.error; + } + return [7 /*endfinally*/]; + case 19: + parametersToPass = Object.fromEntries( + Object.entries(inputParameters).map((_a) => { + var _b = __read(_a, 2), + key = _b[0], + value = _b[1]; + return [key, valueToString(value)]; + }), + ); + _g.label = 20; + case 20: + _g.trys.push([20, 25, , 28]); + resovedParameterNames_1 = preparedPipeline.parameters + .filter((_a) => { + var isInput = _a.isInput; + return isInput; + }) + .map((_a) => { + var name = _a.name; + return name; + }); + unresovedTasks_1 = __spreadArray( + [], + __read(preparedPipeline.tasks), + false, + ); + resolving_1 = []; + loopLimit = LOOP_LIMIT; + _loop_2 = function () { + var currentTask, work_1; + return __generator(this, (_j) => { + switch (_j.label) { + case 0: + if (loopLimit-- < 0) { + // Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline + throw new UnexpectedError( + spaceTrim.spaceTrim((block) => + "\n Loop limit reached during resolving parameters pipeline execution\n\n ".concat( + block(pipelineIdentification), + "\n ", + ), + ), + ); + } + currentTask = unresovedTasks_1.find((task) => + task.dependentParameterNames.every((name) => + __spreadArray( + __spreadArray( + [], + __read(resovedParameterNames_1), + false, + ), + __read(RESERVED_PARAMETER_NAMES), + false, + ).includes(name), + ), + ); + if (!(!currentTask && resolving_1.length === 0)) + return [3 /*break*/, 1]; + throw new UnexpectedError( + // TODO: [🐎] DRY + spaceTrim.spaceTrim((block) => + "\n Can not resolve some parameters:\n\n " + .concat( + block(pipelineIdentification), + "\n\n **Can not resolve:**\n ", + ) + .concat( + block( + unresovedTasks_1 + .map((_a) => { + var resultingParameterName = + _a.resultingParameterName, + dependentParameterNames = + _a.dependentParameterNames; + return "- Parameter `{" + .concat( + resultingParameterName, + "}` which depends on ", + ) + .concat( + dependentParameterNames + .map((dependentParameterName) => + "`{".concat( + dependentParameterName, + "}`", + ), + ) + .join(" and "), + ); + }) + .join("\n"), + ), + "\n\n **Resolved:**\n ", + ) + .concat( + block( + resovedParameterNames_1 + .filter( + (name) => + !RESERVED_PARAMETER_NAMES.includes(name), + ) + .map((name) => + "- Parameter `{".concat(name, "}`"), + ) + .join("\n"), + ), + "\n\n **Reserved (which are available):**\n ", + ) + .concat( + block( + resovedParameterNames_1 + .filter((name) => + RESERVED_PARAMETER_NAMES.includes(name), + ) + .map((name) => + "- Parameter `{".concat(name, "}`"), + ) + .join("\n"), + ), + "\n\n *Note: This should be catched in `validatePipeline`*\n ", + ), + ), + ); + case 1: + if (!!currentTask) return [3 /*break*/, 3]; + /* [🤹‍♂️] */ return [ + 4 /*yield*/, + Promise.race(resolving_1), + ]; + case 2: + /* [🤹‍♂️] */ _j.sent(); + return [3 /*break*/, 4]; + case 3: + unresovedTasks_1 = unresovedTasks_1.filter( + (task) => task !== currentTask, + ); + work_1 = executeTask( + __assign(__assign({}, options), { + currentTask: currentTask, + preparedPipeline: preparedPipeline, + parametersToPass: parametersToPass, + tools: tools, + onProgress: (progress) => { + if (isReturned) { + throw new UnexpectedError( + spaceTrim.spaceTrim((block) => + "\n Can not call `onProgress` after pipeline execution is finished\n\n " + .concat( + block(pipelineIdentification), + "\n\n ", + ) + .concat( + block( + JSON.stringify(progress, null, 4) + .split("\n") + .map((line) => "> ".concat(line)) + .join("\n"), + ), + "\n ", + ), + ), + ); + } + if (onProgress) { + onProgress(progress); + } + }, + $executionReport: executionReport, + pipelineIdentification: spaceTrim.spaceTrim((block) => + "\n " + .concat( + block(pipelineIdentification), + "\n Task name: ", + ) + .concat( + currentTask.name, + "\n Task title: ", + ) + .concat( + currentTask.title, + "\n ", + ), + ), + }), + ) + .then((newParametersToPass) => { + parametersToPass = __assign( + __assign({}, newParametersToPass), + parametersToPass, + ); + resovedParameterNames_1 = __spreadArray( + __spreadArray( + [], + __read(resovedParameterNames_1), + false, + ), + [currentTask.resultingParameterName], + false, + ); + }) + .then(() => { + resolving_1 = resolving_1.filter((w) => w !== work_1); + }); + // <- Note: Errors are catched here [3] + // TODO: BUT if in multiple tasks are errors, only the first one is catched so maybe we should catch errors here and save them to errors array here + resolving_1.push(work_1); + _j.label = 4; + case 4: + return [2 /*return*/]; + } + }); + }; + _g.label = 21; + case 21: + if (!(unresovedTasks_1.length > 0)) return [3 /*break*/, 23]; + return [5 /*yield**/, _loop_2()]; + case 22: + _g.sent(); + return [3 /*break*/, 21]; + case 23: + return [4 /*yield*/, Promise.all(resolving_1)]; + case 24: + _g.sent(); + return [3 /*break*/, 28]; + case 25: + error_1 = _g.sent(); + if (!(error_1 instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error_1; + } + usage_1 = addUsage.apply( + void 0, + __spreadArray( + [], + __read( + executionReport.promptExecutions.map((_a) => { + var result = _a.result; + return ( + (result === null || result === void 0 + ? void 0 + : result.usage) || ZERO_USAGE + ); + }), + ), + false, + ), + ); + outputParameters_1 = filterJustOutputParameters({ + preparedPipeline: preparedPipeline, + parametersToPass: parametersToPass, + $warnings: warnings, + pipelineIdentification: pipelineIdentification, + }); + isReturned = true; + if (!(onProgress !== undefined)) return [3 /*break*/, 27]; + // Note: Wait a short time to prevent race conditions + return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)]; + case 26: + // Note: Wait a short time to prevent race conditions + _g.sent(); + _g.label = 27; + case 27: + return [ + 2 /*return*/, + exportJson({ + name: "pipelineExecutorResult", + message: + "Unuccessful PipelineExecutorResult (with misc errors) PipelineExecutorResult", + order: [], + value: { + isSuccessful: false, + errors: __spreadArray([error_1], __read(errors), false).map( + serializeError, + ), + warnings: warnings.map(serializeError), + usage: usage_1, + executionReport: executionReport, + outputParameters: outputParameters_1, + preparedPipeline: preparedPipeline, + }, + }), + ]; + case 28: + usage = addUsage.apply( + void 0, + __spreadArray( + [], + __read( + executionReport.promptExecutions.map((_a) => { + var result = _a.result; + return ( + (result === null || result === void 0 + ? void 0 + : result.usage) || ZERO_USAGE + ); + }), + ), + false, + ), + ); + outputParameters = filterJustOutputParameters({ + preparedPipeline: preparedPipeline, + parametersToPass: parametersToPass, + $warnings: warnings, + pipelineIdentification: pipelineIdentification, + }); + isReturned = true; + if (!(onProgress !== undefined)) return [3 /*break*/, 30]; + // Note: Wait a short time to prevent race conditions + return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)]; + case 29: + // Note: Wait a short time to prevent race conditions + _g.sent(); + _g.label = 30; + case 30: + return [ + 2 /*return*/, + exportJson({ + name: "pipelineExecutorResult", + message: "Successful PipelineExecutorResult", + order: [], + value: { + isSuccessful: true, + errors: errors.map(serializeError), + warnings: warnings.map(serializeError), + usage: usage, + executionReport: executionReport, + outputParameters: outputParameters, + preparedPipeline: preparedPipeline, + }, + }), + ]; + } + }); + }); + } + /** + * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited + */ + + /** + * Creates executor function from pipeline and execution tools. + * + * @returns The executor function + * @throws {PipelineLogicError} on logical error in the pipeline + * @public exported from `@promptbook/core` + */ + function createPipelineExecutor(options) { + var pipeline = options.pipeline, + tools = options.tools, + _a = options.maxExecutionAttempts, + maxExecutionAttempts = + _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, + _b = options.maxParallelCount, + maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, + _c = options.csvSettings, + csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, + _d = options.isVerbose, + isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, + _e = options.isNotPreparedWarningSupressed, + isNotPreparedWarningSupressed = _e === void 0 ? false : _e, + _f = options.cacheDirname, + cacheDirname = _f === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _f, + _g = options.intermediateFilesStrategy, + intermediateFilesStrategy = + _g === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _g, + _h = options.isAutoInstalled, + isAutoInstalled = _h === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _h, + _j = options.rootDirname, + rootDirname = _j === void 0 ? null : _j; + validatePipeline(pipeline); + var pipelineIdentification = (() => { + // Note: This is a 😐 implementation of [🚞] + var _ = []; + if (pipeline.sourceFile !== undefined) { + _.push("File: ".concat(pipeline.sourceFile)); + } + if (pipeline.pipelineUrl !== undefined) { + _.push("Url: ".concat(pipeline.pipelineUrl)); + } + return _.join("\n"); + })(); + var preparedPipeline; + if (isPipelinePrepared(pipeline)) { + preparedPipeline = pipeline; + } else if (isNotPreparedWarningSupressed !== true) { + console.warn( + spaceTrim.spaceTrim((block) => + "\n Pipeline is not prepared\n\n ".concat( + block(pipelineIdentification), + "\n\n It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n But it is recommended to prepare the pipeline during collection preparation\n\n @see more at https://ptbk.io/prepare-pipeline\n ", + ), + ), + ); + } + var runCount = 0; + var pipelineExecutor = (inputParameters, onProgress) => + __awaiter(this, void 0, void 0, function () { + return __generator(this, (_a) => { + runCount++; + return [ + 2 /*return*/, + /* not await */ executePipeline({ + pipeline: pipeline, + preparedPipeline: preparedPipeline, + setPreparedPipeline: (newPreparedPipeline) => { + preparedPipeline = newPreparedPipeline; + }, + inputParameters: inputParameters, + tools: tools, + onProgress: onProgress, + pipelineIdentification: spaceTrim.spaceTrim((block) => + "\n " + .concat( + block(pipelineIdentification), + "\n ", + ) + .concat( + runCount === 1 ? "" : "Run #".concat(runCount), + "\n ", + ), + ), + maxExecutionAttempts: maxExecutionAttempts, + maxParallelCount: maxParallelCount, + csvSettings: csvSettings, + isVerbose: isVerbose, + isNotPreparedWarningSupressed: isNotPreparedWarningSupressed, + rootDirname: rootDirname, + cacheDirname: cacheDirname, + intermediateFilesStrategy: intermediateFilesStrategy, + isAutoInstalled: isAutoInstalled, + }), + ]; + }); + }); + return pipelineExecutor; + } + /** + * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited + */ + + /** + * Async version of Array.forEach + * + * @param array - Array to iterate over + * @param options - Options for the function + * @param callbackfunction - Function to call for each item + * @public exported from `@promptbook/utils` + * @deprecated [🪂] Use queues instead + */ + function forEachAsync(array, options, callbackfunction) { + return __awaiter(this, void 0, void 0, function () { + var _a, + maxParallelCount, + index, + runningTasks, + tasks, + _loop_1, + _b, + _c, + item, + e_1_1; + var e_1, _d; + return __generator(this, (_e) => { + switch (_e.label) { + case 0: + (_a = options.maxParallelCount), + (maxParallelCount = + _a === void 0 ? Number.POSITIVE_INFINITY : _a); + index = 0; + runningTasks = []; + tasks = []; + _loop_1 = function (item) { + var currentIndex, task; + return __generator(this, (_f) => { + switch (_f.label) { + case 0: + currentIndex = index++; + task = callbackfunction(item, currentIndex, array); + tasks.push(task); + runningTasks.push(task); + /* not await */ Promise.resolve(task).then(() => { + runningTasks = runningTasks.filter((t) => t !== task); + }); + if (!(maxParallelCount < runningTasks.length)) + return [3 /*break*/, 2]; + return [4 /*yield*/, Promise.race(runningTasks)]; + case 1: + _f.sent(); + _f.label = 2; + case 2: + return [2 /*return*/]; + } + }); + }; + _e.label = 1; + case 1: + _e.trys.push([1, 6, 7, 8]); + (_b = __values(array)), (_c = _b.next()); + _e.label = 2; + case 2: + if (!!_c.done) return [3 /*break*/, 5]; + item = _c.value; + return [5 /*yield**/, _loop_1(item)]; + case 3: + _e.sent(); + _e.label = 4; + case 4: + _c = _b.next(); + return [3 /*break*/, 2]; + case 5: + return [3 /*break*/, 8]; + case 6: + e_1_1 = _e.sent(); + e_1 = { error: e_1_1 }; + return [3 /*break*/, 8]; + case 7: + try { + if (_c && !_c.done && (_d = _b.return)) _d.call(_b); + } finally { + if (e_1) throw e_1.error; + } + return [7 /*endfinally*/]; + case 8: + return [4 /*yield*/, Promise.all(tasks)]; + case 9: + _e.sent(); + return [2 /*return*/]; + } + }); + }); + } + + /** + * Intercepts LLM tools and counts total usage of the tools + * + * @param llmTools LLM tools to be intercepted with usage counting + * @returns LLM tools with same functionality with added total cost counting + * @public exported from `@promptbook/core` + */ + function countTotalUsage(llmTools) { + var totalUsage = ZERO_USAGE; + var proxyTools = { + get title() { + // TODO: [🧠] Maybe put here some suffix + return llmTools.title; + }, + get description() { + // TODO: [🧠] Maybe put here some suffix + return llmTools.description; + }, + checkConfiguration: function () { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, (_a) => [ + 2 /*return*/, + /* not await */ llmTools.checkConfiguration(), + ]); + }); + }, + listModels: () => llmTools.listModels(), + getTotalUsage: () => { + // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied + return totalUsage; + }, + }; + if (llmTools.callChatModel !== undefined) { + proxyTools.callChatModel = (prompt) => + __awaiter(this, void 0, void 0, function () { + var promptResult; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + return [4 /*yield*/, llmTools.callChatModel(prompt)]; + case 1: + promptResult = _a.sent(); + totalUsage = addUsage(totalUsage, promptResult.usage); + return [2 /*return*/, promptResult]; + } + }); + }); + } + if (llmTools.callCompletionModel !== undefined) { + proxyTools.callCompletionModel = (prompt) => + __awaiter(this, void 0, void 0, function () { + var promptResult; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + return [4 /*yield*/, llmTools.callCompletionModel(prompt)]; + case 1: + promptResult = _a.sent(); + totalUsage = addUsage(totalUsage, promptResult.usage); + return [2 /*return*/, promptResult]; + } + }); + }); + } + if (llmTools.callEmbeddingModel !== undefined) { + proxyTools.callEmbeddingModel = (prompt) => + __awaiter(this, void 0, void 0, function () { + var promptResult; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + return [4 /*yield*/, llmTools.callEmbeddingModel(prompt)]; + case 1: + promptResult = _a.sent(); + totalUsage = addUsage(totalUsage, promptResult.usage); + return [2 /*return*/, promptResult]; + } + }); + }); + } + // <- Note: [🤖] + return proxyTools; + } + /** + * TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?) + * TODO: [🧠] Is there some meaningfull way how to test this util + * TODO: [🧠][🌯] Maybe a way how to hide ability to `get totalUsage` + * > const [llmToolsWithUsage,getUsage] = countTotalUsage(llmTools); + * TODO: [👷‍♂️] @@@ Manual about construction of llmTools + */ + + /** + * Prepares the persona for the pipeline + * + * @see https://github.com/webgptorg/promptbook/discussions/22 + * @public exported from `@promptbook/core` + */ + function preparePersona(personaDescription, tools, options) { + return __awaiter(this, void 0, void 0, function () { + var _a, + isVerbose, + collection, + preparePersonaExecutor, + _b, + _llms, + llmTools, + availableModels, + availableModelNames, + result, + outputParameters, + modelRequirementsRaw, + modelRequirements, + modelName, + systemMessage, + temperature; + var _c; + return __generator(this, (_d) => { + switch (_d.label) { + case 0: + (_a = options.isVerbose), + (isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a); + if (tools === undefined || tools.llm === undefined) { + throw new MissingToolsError( + "LLM tools are required for preparing persona", + ); + } + collection = createCollectionFromJson.apply( + void 0, + __spreadArray([], __read(PipelineCollection), false), + ); + _b = createPipelineExecutor; + _c = {}; + return [ + 4 /*yield*/, + collection.getPipelineByUrl( + "https://promptbook.studio/promptbook/prepare-persona.book", + ), + ]; + case 1: + preparePersonaExecutor = _b.apply(void 0, [ + ((_c.pipeline = _d.sent()), (_c.tools = tools), _c), + ]); + _llms = arrayableToArray(tools.llm); + llmTools = + _llms.length === 1 + ? _llms[0] + : joinLlmExecutionTools.apply( + void 0, + __spreadArray([], __read(_llms), false), + ); + return [4 /*yield*/, llmTools.listModels()]; + case 2: + availableModels = _d.sent(); + availableModelNames = availableModels + .filter((_a) => { + var modelVariant = _a.modelVariant; + return modelVariant === "CHAT"; + }) + .map((_a) => { + var modelName = _a.modelName; + return modelName; + }) + .join(","); + return [ + 4 /*yield*/, + preparePersonaExecutor({ + availableModelNames: availableModelNames, + personaDescription: personaDescription, + }), + ]; + case 3: + result = _d.sent(); + assertsExecutionSuccessful(result); + outputParameters = result.outputParameters; + modelRequirementsRaw = outputParameters.modelRequirements; + modelRequirements = JSON.parse(modelRequirementsRaw); + if (isVerbose) { + console.info( + "PERSONA ".concat(personaDescription), + modelRequirements, + ); + } + (modelName = modelRequirements.modelName), + (systemMessage = modelRequirements.systemMessage), + (temperature = modelRequirements.temperature); + return [ + 2 /*return*/, + { + modelVariant: "CHAT", + modelName: modelName, + systemMessage: systemMessage, + temperature: temperature, + }, + ]; + } + }); + }); + } + /** + * TODO: [🔃][main] If the persona was prepared with different version or different set of models, prepare it once again + * TODO: [🏢] Check validity of `modelName` in pipeline + * TODO: [🏢] Check validity of `systemMessage` in pipeline + * TODO: [🏢] Check validity of `temperature` in pipeline + */ + + /** + * @@@ + * + * Note: `$` is used to indicate that this interacts with the global scope + * @singleton Only one instance of each register is created per build, but thare can be more @@@ + * @public exported from `@promptbook/core` + */ + var $scrapersMetadataRegister = new $Register("scrapers_metadata"); + /** + * TODO: [®] DRY Register logic + */ + + /** + * Creates a message with all registered scrapers + * + * Note: This function is used to create a (error) message when there is no scraper for particular mime type + * + * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv` + */ + function $registeredScrapersMessage(availableScrapers) { + var e_1, _a, e_2, _b, e_3, _c; + /** + * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister + */ + var all = []; + var _loop_1 = ( + packageName, + className, + mimeTypes, + documentationUrl, + isAvilableInBrowser, + ) => { + if ( + all.some( + (item) => + item.packageName === packageName && item.className === className, + ) + ) { + return "continue"; + } + all.push({ + packageName: packageName, + className: className, + mimeTypes: mimeTypes, + documentationUrl: documentationUrl, + isAvilableInBrowser: isAvilableInBrowser, + }); + }; + try { + for ( + var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); + !_e.done; + _e = _d.next() + ) { + var _f = _e.value, + packageName = _f.packageName, + className = _f.className, + mimeTypes = _f.mimeTypes, + documentationUrl = _f.documentationUrl, + isAvilableInBrowser = _f.isAvilableInBrowser; + _loop_1( + packageName, + className, + mimeTypes, + documentationUrl, + isAvilableInBrowser, + ); + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (_e && !_e.done && (_a = _d.return)) _a.call(_d); + } finally { + if (e_1) throw e_1.error; + } + } + var _loop_2 = ( + packageName, + className, + mimeTypes, + documentationUrl, + isAvilableInBrowser, + ) => { + if ( + all.some( + (item) => + item.packageName === packageName && item.className === className, + ) + ) { + return "continue"; + } + all.push({ + packageName: packageName, + className: className, + mimeTypes: mimeTypes, + documentationUrl: documentationUrl, + isAvilableInBrowser: isAvilableInBrowser, + }); + }; + try { + for ( + var _g = __values($scrapersRegister.list()), _h = _g.next(); + !_h.done; + _h = _g.next() + ) { + var _j = _h.value, + packageName = _j.packageName, + className = _j.className, + mimeTypes = _j.mimeTypes, + documentationUrl = _j.documentationUrl, + isAvilableInBrowser = _j.isAvilableInBrowser; + _loop_2( + packageName, + className, + mimeTypes, + documentationUrl, + isAvilableInBrowser, + ); + } + } catch (e_2_1) { + e_2 = { error: e_2_1 }; + } finally { + try { + if (_h && !_h.done && (_b = _g.return)) _b.call(_g); + } finally { + if (e_2) throw e_2.error; + } + } + try { + for ( + var availableScrapers_1 = __values(availableScrapers), + availableScrapers_1_1 = availableScrapers_1.next(); + !availableScrapers_1_1.done; + availableScrapers_1_1 = availableScrapers_1.next() + ) { + var metadata_1 = availableScrapers_1_1.value.metadata; + all.push(metadata_1); + } + } catch (e_3_1) { + e_3 = { error: e_3_1 }; + } finally { + try { + if ( + availableScrapers_1_1 && + !availableScrapers_1_1.done && + (_c = availableScrapers_1.return) + ) + _c.call(availableScrapers_1); + } finally { + if (e_3) throw e_3.error; + } + } + var metadata = all.map((metadata) => { + var isMetadataAviailable = $scrapersMetadataRegister + .list() + .find((_a) => { + var packageName = _a.packageName, + className = _a.className; + return ( + metadata.packageName === packageName && + metadata.className === className + ); + }); + var isInstalled = $scrapersRegister.list().find((_a) => { + var packageName = _a.packageName, + className = _a.className; + return ( + metadata.packageName === packageName && + metadata.className === className + ); + }); + var isAvilableInTools = availableScrapers.some((_a) => { + var _b = _a.metadata, + packageName = _b.packageName, + className = _b.className; + return ( + metadata.packageName === packageName && + metadata.className === className + ); + }); + return __assign(__assign({}, metadata), { + isMetadataAviailable: isMetadataAviailable, + isInstalled: isInstalled, + isAvilableInTools: isAvilableInTools, + }); + }); + if (metadata.length === 0) { + return spaceTrim__default["default"]( + "\n **No scrapers are available**\n\n This is a unexpected behavior, you are probably using some broken version of Promptbook\n At least there should be available the metadata of the scrapers\n ", + ); + } + return spaceTrim__default["default"]((block) => + "\n Available scrapers are:\n ".concat( + block( + metadata + .map((_a, i) => { + var packageName = _a.packageName, + className = _a.className, + isMetadataAviailable = _a.isMetadataAviailable, + isInstalled = _a.isInstalled, + mimeTypes = _a.mimeTypes, + isAvilableInBrowser = _a.isAvilableInBrowser, + isAvilableInTools = _a.isAvilableInTools; + var more = []; + // TODO: [🧠] Maybe use `documentationUrl` + if (isMetadataAviailable) { + more.push("\u2B1C Metadata registered"); + } // not else + if (isInstalled) { + more.push("\uD83D\uDFE9 Installed"); + } // not else + if (isAvilableInTools) { + more.push("\uD83D\uDFE6 Available in tools"); + } // not else + if (!isMetadataAviailable && isInstalled) { + more.push( + "When no metadata registered but scraper is installed, it is an unexpected behavior", + ); + } // not else + if (!isInstalled && isAvilableInTools) { + more.push( + "When the scraper is not installed but available in tools, it is an unexpected compatibility behavior", + ); + } // not else + if (!isAvilableInBrowser) { + more.push("Not usable in browser"); + } + var moreText = + more.length === 0 ? "" : " *(".concat(more.join("; "), ")*"); + return "" + .concat(i + 1, ") `") + .concat(className, "` from `") + .concat(packageName, "` compatible to scrape ") + .concat( + mimeTypes + .map((mimeType) => '"'.concat(mimeType, '"')) + .join(", "), + ) + .concat(moreText); + }) + .join("\n"), + ), + "\n\n Legend:\n - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n ", + ), + ); + } + /** + * TODO: [®] DRY Register logic + */ + + /** + * @@@ + * + * @param text @@@ + * @returns @@@ + * @example 'hello-world' + * @example 'i-love-promptbook' + * @public exported from `@promptbook/utils` + */ + function normalizeToKebabCase(text) { + var e_1, _a; + text = removeDiacritics(text); + var charType; + var lastCharType = "OTHER"; + var normalizedName = ""; + try { + for ( + var text_1 = __values(text), text_1_1 = text_1.next(); + !text_1_1.done; + text_1_1 = text_1.next() + ) { + var char = text_1_1.value; + var normalizedChar = void 0; + if (/^[a-z]$/.test(char)) { + charType = "LOWERCASE"; + normalizedChar = char; + } else if (/^[A-Z]$/.test(char)) { + charType = "UPPERCASE"; + normalizedChar = char.toLowerCase(); + } else if (/^[0-9]$/.test(char)) { + charType = "NUMBER"; + normalizedChar = char; + } else { + charType = "OTHER"; + normalizedChar = "-"; + } + if ( + charType !== lastCharType && + !(lastCharType === "UPPERCASE" && charType === "LOWERCASE") && + !(lastCharType === "NUMBER") && + !(charType === "NUMBER") + ) { + normalizedName += "-"; + } + normalizedName += normalizedChar; + lastCharType = charType; + } + } catch (e_1_1) { + e_1 = { error: e_1_1 }; + } finally { + try { + if (text_1_1 && !text_1_1.done && (_a = text_1.return)) + _a.call(text_1); + } finally { + if (e_1) throw e_1.error; + } + } + normalizedName = normalizedName.split(/-+/g).join("-"); + normalizedName = normalizedName.split(/-?\/-?/g).join("/"); + normalizedName = normalizedName.replace(/^-/, ""); + normalizedName = normalizedName.replace(/-$/, ""); + return normalizedName; + } + /** + * Note: [💞] Ignore a discrepancy between file name and entity name + */ + + /** + * Creates unique name for the source + * + * @public exported from `@promptbook/editable` + */ + function knowledgeSourceContentToName(knowledgeSourceContent) { + var hash = cryptoJs + .SHA256( + hexEncoder__default["default"].parse( + JSON.stringify(knowledgeSourceContent), + ), + ) + // <- TODO: [🥬] Encapsulate sha256 to some private utility function + .toString(/* hex */) + .substring(0, 20); + // <- TODO: [🥬] Make some system for hashes and ids of promptbook + var semanticName = normalizeToKebabCase( + knowledgeSourceContent.substring(0, 20), + ); + var pieces = ["source", semanticName, hash].filter( + (piece) => piece !== "", + ); + var name = pieces.join("-").split("--").join("-"); + // <- TODO: Use MAX_FILENAME_LENGTH + return name; + } + /** + * TODO: [🐱‍🐉][🧠] Make some smart crop NOT source-i-m-pavol-a-develop-... BUT source-i-m-pavol-a-developer-... + */ + + /** + * Convert file extension to mime type + * + * @private within the repository + */ + function extensionToMimeType(value) { + return mimeTypes.lookup(value) || "application/octet-stream"; + } + + /** + * Get the file extension from a file name + * + * @private within the repository + */ + function getFileExtension(value) { + var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i); + return match ? match[1].toLowerCase() : null; + } + + /** + * Checks if the file exists + * + * @private within the repository + */ + function isFileExisting(filename, fs) { + return __awaiter(this, void 0, void 0, function () { + var isReadAccessAllowed, isFile; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + return [ + 4 /*yield*/, + fs + .access(filename, fs.constants.R_OK) + .then(() => true) + .catch(() => false), + ]; + case 1: + isReadAccessAllowed = _a.sent(); + if (!isReadAccessAllowed) { + return [2 /*return*/, false]; + } + return [ + 4 /*yield*/, + fs + .stat(filename) + .then((fileStat) => fileStat.isFile()) + .catch(() => false), + ]; + case 2: + isFile = _a.sent(); + return [2 /*return*/, isFile]; + } + }); + }); + } + /** + * Note: Not [~🟢~] because it is not directly dependent on `fs + * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages + * TODO: [🖇] What about symlinks? + */ + + /** + * Tests if given string is valid URL. + * + * Note: This does not check if the file exists only if the path is valid + * @public exported from `@promptbook/utils` + */ + function isValidFilePath(filename) { + if (typeof filename !== "string") { + return false; + } + if (filename.split("\n").length > 1) { + return false; + } + if ( + filename.split(" ").length > + 5 /* <- TODO: [🧠][🈷] Make some better non-arbitrary way how to distinct filenames from informational texts */ + ) { + return false; + } + var filenameSlashes = filename.split("\\").join("/"); + // Absolute Unix path: /hello.txt + if (/^(\/)/i.test(filenameSlashes)) { + // console.log(filename, 'Absolute Unix path: /hello.txt'); + return true; + } + // Absolute Windows path: /hello.txt + if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) { + // console.log(filename, 'Absolute Windows path: /hello.txt'); + return true; + } + // Relative path: ./hello.txt + if (/^(\.\.?\/)+/i.test(filenameSlashes)) { + // console.log(filename, 'Relative path: ./hello.txt'); + return true; + } + // Allow paths like foo/hello + if (/^[^/]+\/[^/]+/i.test(filenameSlashes)) { + // console.log(filename, 'Allow paths like foo/hello'); + return true; + } + // Allow paths like hello.book + if (/^[^/]+\.[^/]+$/i.test(filenameSlashes)) { + // console.log(filename, 'Allow paths like hello.book'); + return true; + } + return false; + } + /** + * TODO: [🍏] Implement for MacOs + */ + + /** + * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers + * + * @private as default `fetch` function used in Promptbook scrapers + */ + var scraperFetch = (url, init) => + __awaiter(void 0, void 0, void 0, function () { + var error_1; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + _a.trys.push([0, 2, , 3]); + return [4 /*yield*/, fetch(url, init)]; + case 1: + return [2 /*return*/, _a.sent()]; + case 2: + error_1 = _a.sent(); + if (!(error_1 instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error_1; + } + throw new KnowledgeScrapeError( + spaceTrim__default["default"]((block) => + '\n Can not fetch "' + .concat( + url, + '"\n\n Fetch error:\n ', + ) + .concat(block(error_1.message), "\n\n "), + ), + ); + case 3: + return [2 /*return*/]; + } + }); + }); + /** + * TODO: [🧠] Maybe rename because it is not used only for scrapers but also in `$getCompiledBook` + */ + + /** + * @@@ + * + * @public exported from `@promptbook/core` + */ + function makeKnowledgeSourceHandler(knowledgeSource, tools, options) { + var _a; + return __awaiter(this, void 0, void 0, function () { + var _b, + fetch, + knowledgeSourceContent, + name, + _c, + _d, + rootDirname, + url, + response_1, + mimeType, + filename_1, + fileExtension, + mimeType; + return __generator(this, (_f) => { + switch (_f.label) { + case 0: + (_b = tools.fetch), (fetch = _b === void 0 ? scraperFetch : _b); + knowledgeSourceContent = knowledgeSource.knowledgeSourceContent; + name = knowledgeSource.name; + (_c = options || {}), + (_d = _c.rootDirname), + (rootDirname = _d === void 0 ? null : _d), + _c.isVerbose; + if (!name) { + name = knowledgeSourceContentToName(knowledgeSourceContent); + } + if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/, 2]; + url = knowledgeSourceContent; + return [4 /*yield*/, fetch(url)]; + case 1: + response_1 = _f.sent(); + mimeType = + ((_a = response_1.headers.get("content-type")) === null || + _a === void 0 + ? void 0 + : _a.split(";")[0]) || "text/html"; + return [ + 2 /*return*/, + { + source: name, + filename: null, + url: url, + mimeType: mimeType, + /* TODO: [🥽] > async asBlob() { > // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed @@ -6072,58 +9099,80 @@ at step (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\node_modules\\node_modu > return content; > }, */ - asJson: function () { - return __awaiter(this, void 0, void 0, function () { - var content; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: return [4 /*yield*/, response_1.json()]; - case 1: - content = _a.sent(); - return [2 /*return*/, content]; - } - }); - }); - }, - asText: function () { - return __awaiter(this, void 0, void 0, function () { - var content; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: return [4 /*yield*/, response_1.text()]; - case 1: - content = _a.sent(); - return [2 /*return*/, content]; - } - }); - }); - }, - }]; - case 2: - if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 4]; - if (tools.fs === undefined) { - throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools'); - // <- TODO: [🧠] What is the best error type here` - } - if (rootDirname === null) { - throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline'); - // <- TODO: [🧠] What is the best error type here` - } - filename_1 = path.join(rootDirname, knowledgeSourceContent).split('\\').join('/'); - fileExtension = getFileExtension(filename_1); - mimeType = extensionToMimeType(fileExtension || ''); - return [4 /*yield*/, isFileExisting(filename_1, tools.fs)]; - case 3: - if (!(_f.sent())) { - throw new NotFoundError(spaceTrim__default["default"](function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(knowledgeSourceContent), "\n\n Full file path:\n ").concat(block(filename_1), "\n "); })); - } - // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`) - return [2 /*return*/, { - source: name, - filename: filename_1, - url: null, - mimeType: mimeType, - /* + asJson: function () { + return __awaiter(this, void 0, void 0, function () { + var content; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + return [4 /*yield*/, response_1.json()]; + case 1: + content = _a.sent(); + return [2 /*return*/, content]; + } + }); + }); + }, + asText: function () { + return __awaiter(this, void 0, void 0, function () { + var content; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + return [4 /*yield*/, response_1.text()]; + case 1: + content = _a.sent(); + return [2 /*return*/, content]; + } + }); + }); + }, + }, + ]; + case 2: + if (!isValidFilePath(knowledgeSourceContent)) + return [3 /*break*/, 4]; + if (tools.fs === undefined) { + throw new EnvironmentMismatchError( + "Can not import file knowledge without filesystem tools", + ); + // <- TODO: [🧠] What is the best error type here` + } + if (rootDirname === null) { + throw new EnvironmentMismatchError( + "Can not import file knowledge in non-file pipeline", + ); + // <- TODO: [🧠] What is the best error type here` + } + filename_1 = path + .join(rootDirname, knowledgeSourceContent) + .split("\\") + .join("/"); + fileExtension = getFileExtension(filename_1); + mimeType = extensionToMimeType(fileExtension || ""); + return [4 /*yield*/, isFileExisting(filename_1, tools.fs)]; + case 3: + if (!_f.sent()) { + throw new NotFoundError( + spaceTrim__default["default"]((block) => + "\n Can not make source handler for file which does not exist:\n\n File:\n " + .concat( + block(knowledgeSourceContent), + "\n\n Full file path:\n ", + ) + .concat(block(filename_1), "\n "), + ), + ); + } + // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`) + return [ + 2 /*return*/, + { + source: name, + filename: filename_1, + url: null, + mimeType: mimeType, + /* TODO: [🥽] > async asBlob() { > const content = await tools.fs!.readFile(filename); @@ -6136,42 +9185,55 @@ at step (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\node_modules\\node_modu > ); > }, */ - asJson: function () { - return __awaiter(this, void 0, void 0, function () { - var _a, _b; - return __generator(this, function (_c) { - switch (_c.label) { - case 0: - _b = (_a = JSON).parse; - return [4 /*yield*/, tools.fs.readFile(filename_1, 'utf-8')]; - case 1: return [2 /*return*/, _b.apply(_a, [_c.sent()])]; - } - }); - }); - }, - asText: function () { - return __awaiter(this, void 0, void 0, function () { - return __generator(this, function (_a) { - switch (_a.label) { - case 0: return [4 /*yield*/, tools.fs.readFile(filename_1, 'utf-8')]; - case 1: return [2 /*return*/, _a.sent()]; - } - }); - }); - }, - }]; - case 4: return [2 /*return*/, { - source: name, - filename: null, - url: null, - mimeType: 'text/markdown', - asText: function () { - return knowledgeSource.knowledgeSourceContent; - }, - asJson: function () { - throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`'); - }, - /* + asJson: function () { + return __awaiter(this, void 0, void 0, function () { + var _a, _b; + return __generator(this, (_c) => { + switch (_c.label) { + case 0: + _b = (_a = JSON).parse; + return [ + 4 /*yield*/, + tools.fs.readFile(filename_1, "utf-8"), + ]; + case 1: + return [2 /*return*/, _b.apply(_a, [_c.sent()])]; + } + }); + }); + }, + asText: function () { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + return [ + 4 /*yield*/, + tools.fs.readFile(filename_1, "utf-8"), + ]; + case 1: + return [2 /*return*/, _a.sent()]; + } + }); + }); + }, + }, + ]; + case 4: + return [ + 2 /*return*/, + { + source: name, + filename: null, + url: null, + mimeType: "text/markdown", + asText: () => knowledgeSource.knowledgeSourceContent, + asJson: () => { + throw new UnexpectedError( + "Did not expect that `markdownScraper` would need to get the content `asJson`", + ); + }, + /* TODO: [🥽] > asBlob() { > throw new UnexpectedError( @@ -6179,119 +9241,226 @@ at step (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\node_modules\\node_modu > ); > }, */ - }]; - } - }); - }); - } - - /** - * Prepares the knowle - * - * @see https://github.com/webgptorg/promptbook/discussions/41 - * @public exported from `@promptbook/core` - */ - function prepareKnowledgePieces(knowledgeSources, tools, options) { - return __awaiter(this, void 0, void 0, function () { - var _a, maxParallelCount, rootDirname, _b, isVerbose, knowledgePreparedUnflatten, knowledgePrepared; - var _this = this; - return __generator(this, function (_c) { - switch (_c.label) { - case 0: - _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b; - knowledgePreparedUnflatten = new Array(knowledgeSources.length); - return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () { - var partialPieces, sourceHandler, scrapers, _loop_1, scrapers_1, scrapers_1_1, scraper, state_1, e_1_1, pieces; - var e_1, _a; - return __generator(this, function (_b) { - switch (_b.label) { - case 0: - partialPieces = null; - return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })]; - case 1: - sourceHandler = _b.sent(); - scrapers = arrayableToArray(tools.scrapers); - _loop_1 = function (scraper) { - var partialPiecesUnchecked; - return __generator(this, function (_c) { - switch (_c.label) { - case 0: - if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType) - // <- TODO: [🦔] Implement mime-type wildcards - ) { - return [2 /*return*/, "continue"]; - } - return [4 /*yield*/, scraper.scrape(sourceHandler)]; - case 1: - partialPiecesUnchecked = _c.sent(); - if (partialPiecesUnchecked !== null) { - partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false); - return [2 /*return*/, "break"]; - } - console.warn(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n\n The source:\n ").concat(block(knowledgeSource.knowledgeSourceContent - .split('\n') - .map(function (line) { return "> ".concat(line); }) - .join('\n')), "\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); })); - return [2 /*return*/]; - } - }); - }; - _b.label = 2; - case 2: - _b.trys.push([2, 7, 8, 9]); - scrapers_1 = __values(scrapers), scrapers_1_1 = scrapers_1.next(); - _b.label = 3; - case 3: - if (!!scrapers_1_1.done) return [3 /*break*/, 6]; - scraper = scrapers_1_1.value; - return [5 /*yield**/, _loop_1(scraper)]; - case 4: - state_1 = _b.sent(); - if (state_1 === "break") - return [3 /*break*/, 6]; - _b.label = 5; - case 5: - scrapers_1_1 = scrapers_1.next(); - return [3 /*break*/, 3]; - case 6: return [3 /*break*/, 9]; - case 7: - e_1_1 = _b.sent(); - e_1 = { error: e_1_1 }; - return [3 /*break*/, 9]; - case 8: - try { - if (scrapers_1_1 && !scrapers_1_1.done && (_a = scrapers_1.return)) _a.call(scrapers_1); - } - finally { if (e_1) throw e_1.error; } - return [7 /*endfinally*/]; - case 9: - if (partialPieces === null) { - throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge\n\n The source:\n > ".concat(block(knowledgeSource.knowledgeSourceContent - .split('\n') - .map(function (line) { return "> ".concat(line); }) - .join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); })); - } - pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [ - { - name: knowledgeSource.name, - // line, column <- TODO: [☀] - // <- TODO: [❎] - }, - ] })); }); - knowledgePreparedUnflatten[index] = pieces; - return [2 /*return*/]; - } - }); - }); })]; - case 1: - _c.sent(); - knowledgePrepared = knowledgePreparedUnflatten.flat(); - return [2 /*return*/, knowledgePrepared]; - } - }); - }); - } - /* + }, + ]; + } + }); + }); + } + + /** + * Prepares the knowle + * + * @see https://github.com/webgptorg/promptbook/discussions/41 + * @public exported from `@promptbook/core` + */ + function prepareKnowledgePieces(knowledgeSources, tools, options) { + return __awaiter(this, void 0, void 0, function () { + var _a, + maxParallelCount, + rootDirname, + _b, + isVerbose, + knowledgePreparedUnflatten, + knowledgePrepared; + return __generator(this, (_c) => { + switch (_c.label) { + case 0: + (_a = options.maxParallelCount), + (maxParallelCount = + _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a), + (rootDirname = options.rootDirname), + (_b = options.isVerbose), + (isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b); + knowledgePreparedUnflatten = new Array(knowledgeSources.length); + return [ + 4 /*yield*/, + forEachAsync( + knowledgeSources, + { maxParallelCount: maxParallelCount }, + (knowledgeSource, index) => + __awaiter(this, void 0, void 0, function () { + var partialPieces, + sourceHandler, + scrapers, + _loop_1, + scrapers_1, + scrapers_1_1, + scraper, + state_1, + e_1_1, + pieces; + var e_1, _a; + return __generator(this, (_b) => { + switch (_b.label) { + case 0: + partialPieces = null; + return [ + 4 /*yield*/, + makeKnowledgeSourceHandler( + knowledgeSource, + tools, + { + rootDirname: rootDirname, + isVerbose: isVerbose, + }, + ), + ]; + case 1: + sourceHandler = _b.sent(); + scrapers = arrayableToArray(tools.scrapers); + _loop_1 = function (scraper) { + var partialPiecesUnchecked; + return __generator(this, (_c) => { + switch (_c.label) { + case 0: + if ( + !scraper.metadata.mimeTypes.includes( + sourceHandler.mimeType, + ) + // <- TODO: [🦔] Implement mime-type wildcards + ) { + return [2 /*return*/, "continue"]; + } + return [ + 4 /*yield*/, + scraper.scrape(sourceHandler), + ]; + case 1: + partialPiecesUnchecked = _c.sent(); + if (partialPiecesUnchecked !== null) { + partialPieces = __spreadArray( + [], + __read(partialPiecesUnchecked), + false, + ); + return [2 /*return*/, "break"]; + } + console.warn( + spaceTrim__default["default"]((block) => + "\n Cannot scrape knowledge from source despite the scraper `" + .concat( + scraper.metadata.className, + '` supports the mime type "', + ) + .concat( + sourceHandler.mimeType, + '".\n\n The source:\n ', + ) + .concat( + block( + knowledgeSource.knowledgeSourceContent + .split("\n") + .map((line) => + "> ".concat(line), + ) + .join("\n"), + ), + "\n\n ", + ) + .concat( + block( + $registeredScrapersMessage( + scrapers, + ), + ), + "\n\n\n ", + ), + ), + ); + return [2 /*return*/]; + } + }); + }; + _b.label = 2; + case 2: + _b.trys.push([2, 7, 8, 9]); + (scrapers_1 = __values(scrapers)), + (scrapers_1_1 = scrapers_1.next()); + _b.label = 3; + case 3: + if (!!scrapers_1_1.done) return [3 /*break*/, 6]; + scraper = scrapers_1_1.value; + return [5 /*yield**/, _loop_1(scraper)]; + case 4: + state_1 = _b.sent(); + if (state_1 === "break") return [3 /*break*/, 6]; + _b.label = 5; + case 5: + scrapers_1_1 = scrapers_1.next(); + return [3 /*break*/, 3]; + case 6: + return [3 /*break*/, 9]; + case 7: + e_1_1 = _b.sent(); + e_1 = { error: e_1_1 }; + return [3 /*break*/, 9]; + case 8: + try { + if ( + scrapers_1_1 && + !scrapers_1_1.done && + (_a = scrapers_1.return) + ) + _a.call(scrapers_1); + } finally { + if (e_1) throw e_1.error; + } + return [7 /*endfinally*/]; + case 9: + if (partialPieces === null) { + throw new KnowledgeScrapeError( + spaceTrim__default["default"]((block) => + "\n Cannot scrape knowledge\n\n The source:\n > " + .concat( + block( + knowledgeSource.knowledgeSourceContent + .split("\n") + .map((line) => "> ".concat(line)) + .join("\n"), + ), + '\n\n No scraper found for the mime type "', + ) + .concat( + sourceHandler.mimeType, + '"\n\n ', + ) + .concat( + block( + $registeredScrapersMessage(scrapers), + ), + "\n\n\n ", + ), + ), + ); + } + pieces = partialPieces.map((partialPiece) => + __assign(__assign({}, partialPiece), { + sources: [ + { + name: knowledgeSource.name, + // line, column <- TODO: [☀] + // <- TODO: [❎] + }, + ], + }), + ); + knowledgePreparedUnflatten[index] = pieces; + return [2 /*return*/]; + } + }); + }), + ), + ]; + case 1: + _c.sent(); + knowledgePrepared = knowledgePreparedUnflatten.flat(); + return [2 /*return*/, knowledgePrepared]; + } + }); + }); + } + /* TODO: [🧊] This is how it can look in future > type PrepareKnowledgeKnowledge = { > /** @@ -6305,564 +9474,902 @@ at step (C:\\Users\\me\\work\\webgpt\\promptbook-studio\\node_modules\\node_modu > options: PrepareAndScrapeOptions, > ): */ - /** - * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time - * Put `knowledgePieces` into `PrepareKnowledgeOptions` - * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]] - * TODO: [🧠][❎] Do here propper M:N mapping - * [x] One source can make multiple pieces - * [ ] One piece can have multiple sources - */ - - /** - * @@@ - * - * @public exported from `@promptbook/core` - */ - function prepareTasks(pipeline, tools, options) { - return __awaiter(this, void 0, void 0, function () { - var _a, maxParallelCount, tasks, knowledgePiecesCount, tasksPrepared; - var _this = this; - return __generator(this, function (_b) { - switch (_b.label) { - case 0: - _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a; - tasks = pipeline.tasks, pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount; - tasksPrepared = new Array(tasks.length); - return [4 /*yield*/, forEachAsync(tasks, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (task, index) { return __awaiter(_this, void 0, void 0, function () { - var dependentParameterNames, preparedContent, preparedTask; - return __generator(this, function (_a) { - dependentParameterNames = task.dependentParameterNames; - preparedContent = undefined; - if (knowledgePiecesCount > 0 && !dependentParameterNames.includes('knowledge')) { - preparedContent = spaceTrim.spaceTrim("\n {content}\n\n ## Knowledge\n\n {knowledge}\n "); - // <- TODO: [🧠][🧻] Cutomize shape/language/formatting of the addition to the prompt - dependentParameterNames = __spreadArray(__spreadArray([], __read(dependentParameterNames), false), [ - 'knowledge', - ], false); - } - preparedTask = __assign(__assign({}, task), { dependentParameterNames: dependentParameterNames, preparedContent: preparedContent }); - tasksPrepared[index] = preparedTask; - return [2 /*return*/]; - }); - }); })]; - case 1: - _b.sent(); - return [2 /*return*/, { tasksPrepared: tasksPrepared }]; - } - }); - }); - } - /** - * TODO: [😂] Adding knowledge should be convert to async high-level abstractions, simmilar thing with expectations to sync high-level abstractions - * TODO: [🧠] Add context to each task (if missing) - * TODO: [🧠] What is better name `prepareTask` or `prepareTaskAndParameters` - * TODO: [♨][main] !!3 Prepare index the examples and maybe tasks - * TODO: Write tests for `preparePipeline` - * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch - * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time - * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools - */ - - /** - * Prepare pipeline locally - * - * @see https://github.com/webgptorg/promptbook/discussions/196 - * - * Note: This function does not validate logic of the pipeline - * Note: This function acts as part of compilation process - * Note: When the pipeline is already prepared, it returns the same pipeline - * @public exported from `@promptbook/core` - */ - function preparePipeline(pipeline, tools, options) { - return __awaiter(this, void 0, void 0, function () { - - // throw new Error('Bzz'); - - - var rootDirname, _a, maxParallelCount, _b, isVerbose, parameters, tasks, - /* + /** + * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time + * Put `knowledgePieces` into `PrepareKnowledgeOptions` + * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]] + * TODO: [🧠][❎] Do here propper M:N mapping + * [x] One source can make multiple pieces + * [ ] One piece can have multiple sources + */ + + /** + * @@@ + * + * @public exported from `@promptbook/core` + */ + function prepareTasks(pipeline, tools, options) { + return __awaiter(this, void 0, void 0, function () { + var _a, maxParallelCount, tasks, knowledgePiecesCount, tasksPrepared; + return __generator(this, (_b) => { + switch (_b.label) { + case 0: + (_a = options.maxParallelCount), + (maxParallelCount = + _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a); + (tasks = pipeline.tasks), + pipeline.parameters, + (knowledgePiecesCount = pipeline.knowledgePiecesCount); + tasksPrepared = new Array(tasks.length); + return [ + 4 /*yield*/, + forEachAsync( + tasks, + { + maxParallelCount: + maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */, + }, + (task, index) => + __awaiter(this, void 0, void 0, function () { + var dependentParameterNames, + preparedContent, + preparedTask; + return __generator(this, (_a) => { + dependentParameterNames = task.dependentParameterNames; + preparedContent = undefined; + if ( + knowledgePiecesCount > 0 && + !dependentParameterNames.includes("knowledge") + ) { + preparedContent = spaceTrim.spaceTrim( + "\n {content}\n\n ## Knowledge\n\n {knowledge}\n ", + ); + // <- TODO: [🧠][🧻] Cutomize shape/language/formatting of the addition to the prompt + dependentParameterNames = __spreadArray( + __spreadArray( + [], + __read(dependentParameterNames), + false, + ), + ["knowledge"], + false, + ); + } + preparedTask = __assign(__assign({}, task), { + dependentParameterNames: dependentParameterNames, + preparedContent: preparedContent, + }); + tasksPrepared[index] = preparedTask; + return [2 /*return*/]; + }); + }), + ), + ]; + case 1: + _b.sent(); + return [2 /*return*/, { tasksPrepared: tasksPrepared }]; + } + }); + }); + } + /** + * TODO: [😂] Adding knowledge should be convert to async high-level abstractions, simmilar thing with expectations to sync high-level abstractions + * TODO: [🧠] Add context to each task (if missing) + * TODO: [🧠] What is better name `prepareTask` or `prepareTaskAndParameters` + * TODO: [♨][main] !!3 Prepare index the examples and maybe tasks + * TODO: Write tests for `preparePipeline` + * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch + * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time + * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools + */ + + /** + * Prepare pipeline locally + * + * @see https://github.com/webgptorg/promptbook/discussions/196 + * + * Note: This function does not validate logic of the pipeline + * Note: This function acts as part of compilation process + * Note: When the pipeline is already prepared, it returns the same pipeline + * @public exported from `@promptbook/core` + */ + function preparePipeline(pipeline, tools, options) { + return __awaiter(this, void 0, void 0, function () { + // throw new Error('Bzz'); + + var rootDirname, + _a, + maxParallelCount, + _b, + isVerbose, + parameters, + tasks, + /* <- TODO: [🧠][🪑] `promptbookVersion` */ - knowledgeSources /* - <- TODO: [🧊] `knowledgePieces` */, personas /* - <- TODO: [🧊] `preparations` */, sources, _llms, llmTools, llmToolsWithUsage, currentPreparation, preparations, title, collection, prepareTitleExecutor, _c, result, outputParameters, titleRaw, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared, tasksPrepared /* TODO: parameters: parametersPrepared*/; - var _d; - var _this = this; - return __generator(this, function (_e) { - - console.log('preparePipeline __generator', _e.label); - - - // throw new Error('Bzz'); - - - switch (_e.label) { - case 0: - if (isPipelinePrepared(pipeline)) { - return [2 /*return*/, pipeline]; - } - rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b; - parameters = pipeline.parameters, tasks = pipeline.tasks, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas, sources = pipeline.sources; - if (tools === undefined || tools.llm === undefined) { - throw new MissingToolsError('LLM tools are required for preparing the pipeline'); - } - _llms = arrayableToArray(tools.llm); - llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false)); - llmToolsWithUsage = countTotalUsage(llmTools); - currentPreparation = { - id: 1, - // TODO: [🍥]> date: $getCurrentDate(), - promptbookVersion: PROMPTBOOK_ENGINE_VERSION, - usage: ZERO_USAGE, - }; - preparations = [ - // ...preparations - // <- TODO: [🧊] - currentPreparation, - ]; - title = pipeline.title; - if (!(title === undefined || title === '' || title === DEFAULT_BOOK_TITLE)) return [3 /*break*/, 3]; - collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false)); - _c = createPipelineExecutor; - _d = {}; - return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-title.book')]; - case 1: - prepareTitleExecutor = _c.apply(void 0, [(_d.pipeline = _e.sent(), - _d.tools = tools, - _d)]); - return [4 /*yield*/, prepareTitleExecutor({ - book: sources.map(function (_a) { - var content = _a.content; - return content; - }).join('\n\n'), - })]; - case 2: - result = _e.sent(); - assertsExecutionSuccessful(result); - outputParameters = result.outputParameters; - titleRaw = outputParameters.title; - if (isVerbose) { - console.info("The title is \"".concat(titleRaw, "\"")); - } - title = titleRaw || DEFAULT_BOOK_TITLE; - _e.label = 3; - case 3: - preparedPersonas = new Array(personas.length); - return [4 /*yield*/, forEachAsync(personas, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (persona, index) { return __awaiter(_this, void 0, void 0, function () { - var modelRequirements, preparedPersona; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: return [4 /*yield*/, preparePersona(persona.description, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), { - rootDirname: rootDirname, - maxParallelCount: maxParallelCount /* <- TODO: [🪂] */, - isVerbose: isVerbose, - })]; - case 1: - modelRequirements = _a.sent(); - preparedPersona = __assign(__assign({}, persona), { modelRequirements: modelRequirements, preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] }); - preparedPersonas[index] = preparedPersona; - return [2 /*return*/]; - } - }); - }); })]; - case 4: - _e.sent(); - knowledgeSourcesPrepared = knowledgeSources.map(function (source) { return (__assign(__assign({}, source), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); }); - return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), __assign(__assign({}, options), { rootDirname: rootDirname, maxParallelCount: maxParallelCount /* <- TODO: [🪂] */, isVerbose: isVerbose }))]; - case 5: - partialknowledgePiecesPrepared = _e.sent(); - knowledgePiecesPrepared = partialknowledgePiecesPrepared.map(function (piece) { return (__assign(__assign({}, piece), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); }); - return [4 /*yield*/, prepareTasks({ - parameters: parameters, - tasks: tasks, - knowledgePiecesCount: knowledgePiecesPrepared.length, - }, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), { - rootDirname: rootDirname, - maxParallelCount: maxParallelCount /* <- TODO: [🪂] */, - isVerbose: isVerbose, - })]; - case 6: - tasksPrepared = (_e.sent()).tasksPrepared; - // ----- /Tasks preparation ----- - // TODO: [😂] Use here all `AsyncHighLevelAbstraction` - // Note: Count total usage - currentPreparation.usage = llmToolsWithUsage.getTotalUsage(); - return [2 /*return*/, exportJson({ - name: 'pipelineJson', - message: "Result of `preparePipeline`", - order: ORDER_OF_PIPELINE_JSON, - value: __assign(__assign({}, pipeline), { - // <- TODO: Probbably deeply clone the pipeline because `$exportJson` freezes the subobjects - title: title, knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, tasks: __spreadArray([], __read(tasksPrepared), false), - // <- TODO: [🪓] Here should be no need for spreading new array, just ` tasks: tasksPrepared` - personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }), - })]; - } - }); - }); - } - /** - * TODO: Write tests for `preparePipeline` and `preparePipelineOnRemoteServer` - * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch - * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time - * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools - * TODO: [🧠][♏] Maybe if expecting JSON (In Anthropic Claude and other models without non-json) and its not specified in prompt content, append the instructions - * @see https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/increase-consistency#specify-the-desired-output-format - */ - - /** - * Remote server is a proxy server that uses its execution tools internally and exposes the executor interface externally. - * - * You can simply use `RemoteExecutionTools` on client-side javascript and connect to your remote server. - * This is useful to make all logic on browser side but not expose your API keys or no need to use customer's GPU. - * - * @see https://github.com/webgptorg/promptbook#remote-server - * @public exported from `@promptbook/remote-server` - */ - function startRemoteServer(options) { - var _this = this; - var _a = __assign({ isAnonymousModeAllowed: false, isApplicationModeAllowed: false, collection: null, createLlmExecutionTools: null }, options), port = _a.port, path = _a.path, collection = _a.collection, createLlmExecutionTools = _a.createLlmExecutionTools, isAnonymousModeAllowed = _a.isAnonymousModeAllowed, isApplicationModeAllowed = _a.isApplicationModeAllowed, _b = _a.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b; - // <- TODO: [🦪] Some helper type to be able to use discriminant union types with destructuring - var httpServer = http__default["default"].createServer({}, function (request, response) { return __awaiter(_this, void 0, void 0, function () { - var _a, _b; - var _this = this; - var _c; - return __generator(this, function (_d) { - switch (_d.label) { - case 0: - if ((_c = request.url) === null || _c === void 0 ? void 0 : _c.includes('socket.io')) { - return [2 /*return*/]; - } - _b = (_a = response).write; - return [4 /*yield*/, spaceTrim.spaceTrim(function (block) { return __awaiter(_this, void 0, void 0, function () { - var _a, _b, _c, _d, _e; - return __generator(this, function (_f) { - switch (_f.label) { - case 0: - _b = (_a = "\n Server for processing promptbook remote requests is running.\n\n Version: ".concat(PROMPTBOOK_ENGINE_VERSION, "\n Socket.io path: ").concat(path, "/socket.io\n Anonymouse mode: ").concat(isAnonymousModeAllowed ? 'enabled' : 'disabled', "\n Application mode: ").concat(isApplicationModeAllowed ? 'enabled' : 'disabled', "\n ")).concat; - _c = block; - if (!!isApplicationModeAllowed) return [3 /*break*/, 1]; - _d = ''; - return [3 /*break*/, 3]; - case 1: - _e = 'Pipelines in collection:\n'; - return [4 /*yield*/, collection.listPipelines()]; - case 2: - _d = _e + - (_f.sent()) - .map(function (pipelineUrl) { return "- ".concat(pipelineUrl); }) - .join('\n'); - _f.label = 3; - case 3: return [2 /*return*/, _b.apply(_a, [_c.apply(void 0, [_d]), "\n\n For more information look at:\n https://github.com/webgptorg/promptbook\n "])]; - } - }); - }); })]; - case 1: - _b.apply(_a, [_d.sent()]); - response.end(); - return [2 /*return*/]; - } - }); - }); }); - var server = new socket_io.Server(httpServer, { - path: path, - transports: [/*'websocket', <- TODO: [🌬] Make websocket transport work */ 'polling'], - cors: { - origin: '*', - methods: ['GET', 'POST'], - }, - }); - server.on('connection', function (socket) { - if (isVerbose) { - console.info(colors__default["default"].gray("Client connected"), socket.id); - } - var getExecutionToolsFromIdentification = function (identification) { return __awaiter(_this, void 0, void 0, function () { - var isAnonymous, llm, llmToolsConfiguration, appId, userId, customOptions, fs, executables, tools; - var _a; - return __generator(this, function (_b) { - switch (_b.label) { - case 0: - isAnonymous = identification.isAnonymous; - if (isAnonymous === true && !isAnonymousModeAllowed) { - throw new PipelineExecutionError("Anonymous mode is not allowed"); // <- TODO: [main] !!3 Test - } - if (isAnonymous === false && !isApplicationModeAllowed) { - throw new PipelineExecutionError("Application mode is not allowed"); // <- TODO: [main] !!3 Test - } - if (!(isAnonymous === true)) return [3 /*break*/, 1]; - llmToolsConfiguration = identification.llmToolsConfiguration; - llm = createLlmToolsFromConfiguration(llmToolsConfiguration, { isVerbose: isVerbose }); - return [3 /*break*/, 4]; - case 1: - if (!(isAnonymous === false && createLlmExecutionTools !== null)) return [3 /*break*/, 3]; - appId = identification.appId, userId = identification.userId, customOptions = identification.customOptions; - return [4 /*yield*/, createLlmExecutionTools({ - appId: appId, - userId: userId, - customOptions: customOptions, - })]; - case 2: - llm = _b.sent(); - return [3 /*break*/, 4]; - case 3: throw new PipelineExecutionError("You must provide either llmToolsConfiguration or non-anonymous mode must be propperly configured"); - case 4: - fs = $provideFilesystemForNode(); - return [4 /*yield*/, $provideExecutablesForNode()]; - case 5: - executables = _b.sent(); - _a = { - llm: llm, - fs: fs - }; - return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables })]; - case 6: - tools = (_a.scrapers = _b.sent(), - _a); - return [2 /*return*/, tools]; - } - }); - }); }; - // ----------- - socket.on('prompt-request', function (request) { return __awaiter(_this, void 0, void 0, function () { - var identification, prompt, executionTools, llm, _a, promptResult, _b, error_1; - return __generator(this, function (_c) { - switch (_c.label) { - case 0: - identification = request.identification, prompt = request.prompt; - if (isVerbose) { - console.info(colors__default["default"].bgWhite("Prompt:"), colors__default["default"].gray(JSON.stringify(request, null, 4))); - } - _c.label = 1; - case 1: - _c.trys.push([1, 13, 14, 15]); - return [4 /*yield*/, getExecutionToolsFromIdentification(identification)]; - case 2: - executionTools = _c.sent(); - llm = executionTools.llm; - _a = identification.isAnonymous === false && - collection !== null; - if (!_a) return [3 /*break*/, 4]; - return [4 /*yield*/, collection.isResponsibleForPrompt(prompt)]; - case 3: - _a = !(_c.sent()); - _c.label = 4; - case 4: - if (_a) { - throw new PipelineExecutionError("Pipeline is not in the collection of this server"); - } - promptResult = void 0; - _b = prompt.modelRequirements.modelVariant; - switch (_b) { - case 'CHAT': return [3 /*break*/, 5]; - case 'COMPLETION': return [3 /*break*/, 7]; - case 'EMBEDDING': return [3 /*break*/, 9]; - } - return [3 /*break*/, 11]; - case 5: - if (llm.callChatModel === undefined) { - // Note: [0] This check should not be a thing - throw new PipelineExecutionError("Chat model is not available"); - } - return [4 /*yield*/, llm.callChatModel(prompt)]; - case 6: - promptResult = _c.sent(); - return [3 /*break*/, 12]; - case 7: - if (llm.callCompletionModel === undefined) { - // Note: [0] This check should not be a thing - throw new PipelineExecutionError("Completion model is not available"); - } - return [4 /*yield*/, llm.callCompletionModel(prompt)]; - case 8: - promptResult = _c.sent(); - return [3 /*break*/, 12]; - case 9: - if (llm.callEmbeddingModel === undefined) { - // Note: [0] This check should not be a thing - throw new PipelineExecutionError("Embedding model is not available"); - } - return [4 /*yield*/, llm.callEmbeddingModel(prompt)]; - case 10: - promptResult = _c.sent(); - return [3 /*break*/, 12]; - case 11: throw new PipelineExecutionError("Unknown model variant \"".concat(prompt.modelRequirements.modelVariant, "\"")); - case 12: - if (isVerbose) { - console.info(colors__default["default"].bgGreen("PromptResult:"), colors__default["default"].green(JSON.stringify(promptResult, null, 4))); - } - socket.emit('prompt-response', { promptResult: promptResult } /* <- Note: [🤛] */); - return [3 /*break*/, 15]; - case 13: - error_1 = _c.sent(); - if (!(error_1 instanceof Error)) {console.log('!(error instanceof Error)') - throw error_1; - } - socket.emit('error', serializeError(error_1) /* <- Note: [🤛] */); - return [3 /*break*/, 15]; - case 14: - socket.disconnect(); - return [7 /*endfinally*/]; - case 15: return [2 /*return*/]; - } - }); - }); }); - // ----------- - // TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io - socket.on('listModels-request', function (request) { return __awaiter(_this, void 0, void 0, function () { - var identification, executionTools, llm, models, error_2; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - identification = request.identification; - if (isVerbose) { - console.info(colors__default["default"].bgWhite("Listing models")); - } - _a.label = 1; - case 1: - _a.trys.push([1, 4, 5, 6]); - return [4 /*yield*/, getExecutionToolsFromIdentification(identification)]; - case 2: - executionTools = _a.sent(); - llm = executionTools.llm; - return [4 /*yield*/, llm.listModels()]; - case 3: - models = _a.sent(); - socket.emit('listModels-response', { models: models } /* <- Note: [🤛] */); - return [3 /*break*/, 6]; - case 4: - error_2 = _a.sent(); - if (!(error_2 instanceof Error)) {console.log('!(error instanceof Error)') - throw error_2; - } - socket.emit('error', serializeError(error_2)); - return [3 /*break*/, 6]; - case 5: - socket.disconnect(); - return [7 /*endfinally*/]; - case 6: return [2 /*return*/]; - } - }); - }); }); - // ----------- - // TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io - socket.on('preparePipeline-request', function (request) { return __awaiter(_this, void 0, void 0, function () { - - console.log('preparePipeline-request begin'); - - - var identification, pipeline, executionTools, preparedPipeline, error_3; - - try{ - return __generator(this, function (_a) { - - console.log('preparePipeline-request __generator', _a.label); - - - switch (_a.label) { - case 0: - console.log(0); - identification = request.identification, pipeline = request.pipeline; - if (isVerbose) { - console.info(colors__default["default"].bgWhite("Prepare pipeline")); - } - _a.label = 1; - case 1: - console.log(1); - _a.trys.push([1, 4, 5, 6]); - return [4 /*yield*/, getExecutionToolsFromIdentification(identification)]; - case 2: - console.log(2); - executionTools = _a.sent(); - // throw new Error('Bzz'); - - // throw new NotYetImplementedError('Bzz') - return [4 /*yield*/, preparePipeline(pipeline, executionTools, options)]; - case 3: - console.log(3); - preparedPipeline = _a.sent(); - socket.emit('preparePipeline-response', { preparedPipeline: preparedPipeline } /* <- Note: [🤛] */); - return [3 /*break*/, 6]; - case 4: - console.log(4); - error_3 = _a.sent(); - console.log( {error_3}); - if (!(error_3 instanceof Error)) {console.log('!(error instanceof Error)') - - throw error_3; - } - console.log(4,'b'); - console.log('error_3 instanceof NotYetImplementedError',error_3 instanceof NotYetImplementedError); - - - // console.log('serializeError(error_3)',serializeError(error_3)) - // socket.emit('error',{name: 'Brr', message: 'Brr', stack:''}); - socket.emit('error', serializeError(error_3)); - - console.log(4,'c'); - return [3 /*break*/, 6]; - case 5: - console.log(5); - // socket.disconnect(); - return [7 /*endfinally*/]; - case 6: return [2 /*return*/]; - } - }); - - }catch(error){ - console.log('preparePipeline-request catch',error); - } - - console.log('preparePipeline-request end'); - - - }); }); - // ----------- - socket.on('disconnect', function () { - // TODO: Destroy here executionToolsForClient - if (isVerbose) { - console.info(colors__default["default"].gray("Client disconnected"), socket.id); - } - }); - }); - httpServer.listen(port); - // Note: We want to log this also in non-verbose mode - console.info(colors__default["default"].bgGreen("PROMPTBOOK server listening on port ".concat(port))); - if (isVerbose) { - console.info(colors__default["default"].gray("Verbose mode is enabled")); - } - var isDestroyed = false; - return { - get isDestroyed() { - return isDestroyed; - }, - destroy: function () { - if (isDestroyed) { - return; - } - isDestroyed = true; - httpServer.close(); - server.close(); - }, - }; - } - /** - * TODO: Split this file into multiple functions - handler for each request - * TODO: Maybe use `$exportJson` - * TODO: [🧠][🛍] Maybe not `isAnonymous: boolean` BUT `mode: 'ANONYMOUS'|'COLLECTION'` - * TODO: [⚖] Expose the collection to be able to connect to same collection via createCollectionFromUrl - * TODO: Handle progress - support streaming - * TODO: [🗯] Do not hang up immediately but wait until client closes OR timeout - * TODO: [🗯] Timeout on chat to free up resources - * TODO: [🃏] Pass here some security token to prevent malitious usage and/or DDoS - * TODO: [0] Set unavailable models as undefined in `RemoteLlmExecutionTools` NOT throw error here - * TODO: Allow to constrain anonymous mode for specific models / providers - */ - - exports.BOOK_LANGUAGE_VERSION = BOOK_LANGUAGE_VERSION; - exports.PROMPTBOOK_ENGINE_VERSION = PROMPTBOOK_ENGINE_VERSION; - exports.startRemoteServer = startRemoteServer; - - Object.defineProperty(exports, '__esModule', { value: true }); - -})); + knowledgeSources /* + <- TODO: [🧊] `knowledgePieces` */, + personas /* + <- TODO: [🧊] `preparations` */, + sources, + _llms, + llmTools, + llmToolsWithUsage, + currentPreparation, + preparations, + title, + collection, + prepareTitleExecutor, + _c, + result, + outputParameters, + titleRaw, + preparedPersonas, + knowledgeSourcesPrepared, + partialknowledgePiecesPrepared, + knowledgePiecesPrepared, + tasksPrepared /* TODO: parameters: parametersPrepared*/; + var _d; + return __generator(this, (_e) => { + console.log("preparePipeline __generator", _e.label); + + // throw new Error('Bzz'); + + switch (_e.label) { + case 0: + if (isPipelinePrepared(pipeline)) { + return [2 /*return*/, pipeline]; + } + (rootDirname = options.rootDirname), + (_a = options.maxParallelCount), + (maxParallelCount = + _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a), + (_b = options.isVerbose), + (isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b); + (parameters = pipeline.parameters), + (tasks = pipeline.tasks), + (knowledgeSources = pipeline.knowledgeSources), + (personas = pipeline.personas), + (sources = pipeline.sources); + if (tools === undefined || tools.llm === undefined) { + throw new MissingToolsError( + "LLM tools are required for preparing the pipeline", + ); + } + _llms = arrayableToArray(tools.llm); + llmTools = + _llms.length === 1 + ? _llms[0] + : joinLlmExecutionTools.apply( + void 0, + __spreadArray([], __read(_llms), false), + ); + llmToolsWithUsage = countTotalUsage(llmTools); + currentPreparation = { + id: 1, + // TODO: [🍥]> date: $getCurrentDate(), + promptbookVersion: PROMPTBOOK_ENGINE_VERSION, + usage: ZERO_USAGE, + }; + preparations = [ + // ...preparations + // <- TODO: [🧊] + currentPreparation, + ]; + title = pipeline.title; + if ( + !( + title === undefined || + title === "" || + title === DEFAULT_BOOK_TITLE + ) + ) + return [3 /*break*/, 3]; + collection = createCollectionFromJson.apply( + void 0, + __spreadArray([], __read(PipelineCollection), false), + ); + _c = createPipelineExecutor; + _d = {}; + return [ + 4 /*yield*/, + collection.getPipelineByUrl( + "https://promptbook.studio/promptbook/prepare-title.book", + ), + ]; + case 1: + prepareTitleExecutor = _c.apply(void 0, [ + ((_d.pipeline = _e.sent()), (_d.tools = tools), _d), + ]); + return [ + 4 /*yield*/, + prepareTitleExecutor({ + book: sources + .map((_a) => { + var content = _a.content; + return content; + }) + .join("\n\n"), + }), + ]; + case 2: + result = _e.sent(); + assertsExecutionSuccessful(result); + outputParameters = result.outputParameters; + titleRaw = outputParameters.title; + if (isVerbose) { + console.info('The title is "'.concat(titleRaw, '"')); + } + title = titleRaw || DEFAULT_BOOK_TITLE; + _e.label = 3; + case 3: + preparedPersonas = new Array(personas.length); + return [ + 4 /*yield*/, + forEachAsync( + personas, + { + maxParallelCount: + maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */, + }, + (persona, index) => + __awaiter(this, void 0, void 0, function () { + var modelRequirements, preparedPersona; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + return [ + 4 /*yield*/, + preparePersona( + persona.description, + __assign(__assign({}, tools), { + llm: llmToolsWithUsage, + }), + { + rootDirname: rootDirname, + maxParallelCount: + maxParallelCount /* <- TODO: [🪂] */, + isVerbose: isVerbose, + }, + ), + ]; + case 1: + modelRequirements = _a.sent(); + preparedPersona = __assign(__assign({}, persona), { + modelRequirements: modelRequirements, + preparationIds: [ + /* TODO: [🧊] -> */ currentPreparation.id, + ], + }); + preparedPersonas[index] = preparedPersona; + return [2 /*return*/]; + } + }); + }), + ), + ]; + case 4: + _e.sent(); + knowledgeSourcesPrepared = knowledgeSources.map((source) => + __assign(__assign({}, source), { + preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id], + }), + ); + return [ + 4 /*yield*/, + prepareKnowledgePieces( + knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, + __assign(__assign({}, tools), { llm: llmToolsWithUsage }), + __assign(__assign({}, options), { + rootDirname: rootDirname, + maxParallelCount: maxParallelCount /* <- TODO: [🪂] */, + isVerbose: isVerbose, + }), + ), + ]; + case 5: + partialknowledgePiecesPrepared = _e.sent(); + knowledgePiecesPrepared = partialknowledgePiecesPrepared.map( + (piece) => + __assign(__assign({}, piece), { + preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id], + }), + ); + return [ + 4 /*yield*/, + prepareTasks( + { + parameters: parameters, + tasks: tasks, + knowledgePiecesCount: knowledgePiecesPrepared.length, + }, + __assign(__assign({}, tools), { llm: llmToolsWithUsage }), + { + rootDirname: rootDirname, + maxParallelCount: maxParallelCount /* <- TODO: [🪂] */, + isVerbose: isVerbose, + }, + ), + ]; + case 6: + tasksPrepared = _e.sent().tasksPrepared; + // ----- /Tasks preparation ----- + // TODO: [😂] Use here all `AsyncHighLevelAbstraction` + // Note: Count total usage + currentPreparation.usage = llmToolsWithUsage.getTotalUsage(); + return [ + 2 /*return*/, + exportJson({ + name: "pipelineJson", + message: "Result of `preparePipeline`", + order: ORDER_OF_PIPELINE_JSON, + value: __assign(__assign({}, pipeline), { + // <- TODO: Probbably deeply clone the pipeline because `$exportJson` freezes the subobjects + title: title, + knowledgeSources: knowledgeSourcesPrepared, + knowledgePieces: knowledgePiecesPrepared, + tasks: __spreadArray([], __read(tasksPrepared), false), + // <- TODO: [🪓] Here should be no need for spreading new array, just ` tasks: tasksPrepared` + personas: preparedPersonas, + preparations: __spreadArray( + [], + __read(preparations), + false, + ), + }), + }), + ]; + } + }); + }); + } + /** + * TODO: Write tests for `preparePipeline` and `preparePipelineOnRemoteServer` + * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch + * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time + * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools + * TODO: [🧠][♏] Maybe if expecting JSON (In Anthropic Claude and other models without non-json) and its not specified in prompt content, append the instructions + * @see https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/increase-consistency#specify-the-desired-output-format + */ + + /** + * Remote server is a proxy server that uses its execution tools internally and exposes the executor interface externally. + * + * You can simply use `RemoteExecutionTools` on client-side javascript and connect to your remote server. + * This is useful to make all logic on browser side but not expose your API keys or no need to use customer's GPU. + * + * @see https://github.com/webgptorg/promptbook#remote-server + * @public exported from `@promptbook/remote-server` + */ + function startRemoteServer(options) { + var _a = __assign( + { + isAnonymousModeAllowed: false, + isApplicationModeAllowed: false, + collection: null, + createLlmExecutionTools: null, + }, + options, + ), + port = _a.port, + path = _a.path, + collection = _a.collection, + createLlmExecutionTools = _a.createLlmExecutionTools, + isAnonymousModeAllowed = _a.isAnonymousModeAllowed, + isApplicationModeAllowed = _a.isApplicationModeAllowed, + _b = _a.isVerbose, + isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b; + // <- TODO: [🦪] Some helper type to be able to use discriminant union types with destructuring + var httpServer = http__default["default"].createServer( + {}, + (request, response) => + __awaiter(this, void 0, void 0, function () { + var _a, _b; + var _c; + return __generator(this, (_d) => { + switch (_d.label) { + case 0: + if ( + (_c = request.url) === null || _c === void 0 + ? void 0 + : _c.includes("socket.io") + ) { + return [2 /*return*/]; + } + _b = (_a = response).write; + return [ + 4 /*yield*/, + spaceTrim.spaceTrim((block) => + __awaiter(this, void 0, void 0, function () { + var _a, _b, _c, _d, _e; + return __generator(this, (_f) => { + switch (_f.label) { + case 0: + _b = (_a = + "\n Server for processing promptbook remote requests is running.\n\n Version: " + .concat( + PROMPTBOOK_ENGINE_VERSION, + "\n Socket.io path: ", + ) + .concat( + path, + "/socket.io\n Anonymouse mode: ", + ) + .concat( + isAnonymousModeAllowed + ? "enabled" + : "disabled", + "\n Application mode: ", + ) + .concat( + isApplicationModeAllowed + ? "enabled" + : "disabled", + "\n ", + )).concat; + _c = block; + if (!!isApplicationModeAllowed) + return [3 /*break*/, 1]; + _d = ""; + return [3 /*break*/, 3]; + case 1: + _e = "Pipelines in collection:\n"; + return [4 /*yield*/, collection.listPipelines()]; + case 2: + _d = + _e + + _f + .sent() + .map((pipelineUrl) => + "- ".concat(pipelineUrl), + ) + .join("\n"); + _f.label = 3; + case 3: + return [ + 2 /*return*/, + _b.apply(_a, [ + _c.apply(void 0, [_d]), + "\n\n For more information look at:\n https://github.com/webgptorg/promptbook\n ", + ]), + ]; + } + }); + }), + ), + ]; + case 1: + _b.apply(_a, [_d.sent()]); + response.end(); + return [2 /*return*/]; + } + }); + }), + ); + var server = new socket_io.Server(httpServer, { + path: path, + transports: [ + /*'websocket', <- TODO: [🌬] Make websocket transport work */ "polling", + ], + cors: { + origin: "*", + methods: ["GET", "POST"], + }, + }); + server.on("connection", (socket) => { + if (isVerbose) { + console.info( + colors__default["default"].gray("Client connected"), + socket.id, + ); + } + var getExecutionToolsFromIdentification = (identification) => + __awaiter(this, void 0, void 0, function () { + var isAnonymous, + llm, + llmToolsConfiguration, + appId, + userId, + customOptions, + fs, + executables, + tools; + var _a; + return __generator(this, (_b) => { + switch (_b.label) { + case 0: + isAnonymous = identification.isAnonymous; + if (isAnonymous === true && !isAnonymousModeAllowed) { + throw new PipelineExecutionError( + "Anonymous mode is not allowed", + ); // <- TODO: [main] !!3 Test + } + if (isAnonymous === false && !isApplicationModeAllowed) { + throw new PipelineExecutionError( + "Application mode is not allowed", + ); // <- TODO: [main] !!3 Test + } + if (!(isAnonymous === true)) return [3 /*break*/, 1]; + llmToolsConfiguration = identification.llmToolsConfiguration; + llm = createLlmToolsFromConfiguration(llmToolsConfiguration, { + isVerbose: isVerbose, + }); + return [3 /*break*/, 4]; + case 1: + if ( + !(isAnonymous === false && createLlmExecutionTools !== null) + ) + return [3 /*break*/, 3]; + (appId = identification.appId), + (userId = identification.userId), + (customOptions = identification.customOptions); + return [ + 4 /*yield*/, + createLlmExecutionTools({ + appId: appId, + userId: userId, + customOptions: customOptions, + }), + ]; + case 2: + llm = _b.sent(); + return [3 /*break*/, 4]; + case 3: + throw new PipelineExecutionError( + "You must provide either llmToolsConfiguration or non-anonymous mode must be propperly configured", + ); + case 4: + fs = $provideFilesystemForNode(); + return [4 /*yield*/, $provideExecutablesForNode()]; + case 5: + executables = _b.sent(); + _a = { + llm: llm, + fs: fs, + }; + return [ + 4 /*yield*/, + $provideScrapersForNode({ + fs: fs, + llm: llm, + executables: executables, + }), + ]; + case 6: + tools = ((_a.scrapers = _b.sent()), _a); + return [2 /*return*/, tools]; + } + }); + }); + // ----------- + socket.on("prompt-request", (request) => + __awaiter(this, void 0, void 0, function () { + var identification, + prompt, + executionTools, + llm, + _a, + promptResult, + _b, + error_1; + return __generator(this, (_c) => { + switch (_c.label) { + case 0: + (identification = request.identification), + (prompt = request.prompt); + if (isVerbose) { + console.info( + colors__default["default"].bgWhite("Prompt:"), + colors__default["default"].gray( + JSON.stringify(request, null, 4), + ), + ); + } + _c.label = 1; + case 1: + _c.trys.push([1, 13, 14, 15]); + return [ + 4 /*yield*/, + getExecutionToolsFromIdentification(identification), + ]; + case 2: + executionTools = _c.sent(); + llm = executionTools.llm; + _a = + identification.isAnonymous === false && collection !== null; + if (!_a) return [3 /*break*/, 4]; + return [ + 4 /*yield*/, + collection.isResponsibleForPrompt(prompt), + ]; + case 3: + _a = !_c.sent(); + _c.label = 4; + case 4: + if (_a) { + throw new PipelineExecutionError( + "Pipeline is not in the collection of this server", + ); + } + promptResult = void 0; + _b = prompt.modelRequirements.modelVariant; + switch (_b) { + case "CHAT": + return [3 /*break*/, 5]; + case "COMPLETION": + return [3 /*break*/, 7]; + case "EMBEDDING": + return [3 /*break*/, 9]; + } + return [3 /*break*/, 11]; + case 5: + if (llm.callChatModel === undefined) { + // Note: [0] This check should not be a thing + throw new PipelineExecutionError( + "Chat model is not available", + ); + } + return [4 /*yield*/, llm.callChatModel(prompt)]; + case 6: + promptResult = _c.sent(); + return [3 /*break*/, 12]; + case 7: + if (llm.callCompletionModel === undefined) { + // Note: [0] This check should not be a thing + throw new PipelineExecutionError( + "Completion model is not available", + ); + } + return [4 /*yield*/, llm.callCompletionModel(prompt)]; + case 8: + promptResult = _c.sent(); + return [3 /*break*/, 12]; + case 9: + if (llm.callEmbeddingModel === undefined) { + // Note: [0] This check should not be a thing + throw new PipelineExecutionError( + "Embedding model is not available", + ); + } + return [4 /*yield*/, llm.callEmbeddingModel(prompt)]; + case 10: + promptResult = _c.sent(); + return [3 /*break*/, 12]; + case 11: + throw new PipelineExecutionError( + 'Unknown model variant "'.concat( + prompt.modelRequirements.modelVariant, + '"', + ), + ); + case 12: + if (isVerbose) { + console.info( + colors__default["default"].bgGreen("PromptResult:"), + colors__default["default"].green( + JSON.stringify(promptResult, null, 4), + ), + ); + } + socket.emit( + "prompt-response", + { promptResult: promptResult } /* <- Note: [🤛] */, + ); + return [3 /*break*/, 15]; + case 13: + error_1 = _c.sent(); + if (!(error_1 instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error_1; + } + socket.emit( + "error", + serializeError(error_1) /* <- Note: [🤛] */, + ); + return [3 /*break*/, 15]; + case 14: + socket.disconnect(); + return [7 /*endfinally*/]; + case 15: + return [2 /*return*/]; + } + }); + }), + ); + // ----------- + // TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io + socket.on("listModels-request", (request) => + __awaiter(this, void 0, void 0, function () { + var identification, executionTools, llm, models, error_2; + return __generator(this, (_a) => { + switch (_a.label) { + case 0: + identification = request.identification; + if (isVerbose) { + console.info( + colors__default["default"].bgWhite("Listing models"), + ); + } + _a.label = 1; + case 1: + _a.trys.push([1, 4, 5, 6]); + return [ + 4 /*yield*/, + getExecutionToolsFromIdentification(identification), + ]; + case 2: + executionTools = _a.sent(); + llm = executionTools.llm; + return [4 /*yield*/, llm.listModels()]; + case 3: + models = _a.sent(); + socket.emit( + "listModels-response", + { models: models } /* <- Note: [🤛] */, + ); + return [3 /*break*/, 6]; + case 4: + error_2 = _a.sent(); + if (!(error_2 instanceof Error)) { + console.log("!(error instanceof Error)"); + throw error_2; + } + socket.emit("error", serializeError(error_2)); + return [3 /*break*/, 6]; + case 5: + socket.disconnect(); + return [7 /*endfinally*/]; + case 6: + return [2 /*return*/]; + } + }); + }), + ); + // ----------- + // TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io + socket.on("preparePipeline-request", (request) => + __awaiter(this, void 0, void 0, function () { + console.log("preparePipeline-request begin"); + + var identification, + pipeline, + executionTools, + preparedPipeline, + error_3; + + try { + return __generator(this, (_a) => { + console.log("preparePipeline-request __generator", _a.label); + + switch (_a.label) { + case 0: + console.log(0); + (identification = request.identification), + (pipeline = request.pipeline); + if (isVerbose) { + console.info( + colors__default["default"].bgWhite("Prepare pipeline"), + ); + } + _a.label = 1; + case 1: + console.log(1); + _a.trys.push([1, 4, 5, 6]); + return [ + 4 /*yield*/, + getExecutionToolsFromIdentification(identification), + ]; + case 2: + console.log(2); + executionTools = _a.sent(); + // throw new Error('Bzz'); + + // throw new NotYetImplementedError('Bzz') + return [ + 4 /*yield*/, + preparePipeline(pipeline, executionTools, options), + ]; + case 3: + console.log(3); + preparedPipeline = _a.sent(); + socket.emit( + "preparePipeline-response", + { + preparedPipeline: preparedPipeline, + } /* <- Note: [🤛] */, + ); + return [3 /*break*/, 6]; + case 4: + console.log(4); + error_3 = _a.sent(); + console.log({ error_3 }); + if (!(error_3 instanceof Error)) { + console.log("!(error instanceof Error)"); + + throw error_3; + } + console.log(4, "b"); + console.log( + "error_3 instanceof NotYetImplementedError", + error_3 instanceof NotYetImplementedError, + ); + + // console.log('serializeError(error_3)',serializeError(error_3)) + // socket.emit('error',{name: 'Brr', message: 'Brr', stack:''}); + socket.emit("error", serializeError(error_3)); + + console.log(4, "c"); + return [3 /*break*/, 6]; + case 5: + console.log(5); + // socket.disconnect(); + return [7 /*endfinally*/]; + case 6: + return [2 /*return*/]; + } + }); + } catch (error) { + console.log("preparePipeline-request catch", error); + } + + console.log("preparePipeline-request end"); + }), + ); + // ----------- + socket.on("disconnect", () => { + // TODO: Destroy here executionToolsForClient + if (isVerbose) { + console.info( + colors__default["default"].gray("Client disconnected"), + socket.id, + ); + } + }); + }); + httpServer.listen(port); + // Note: We want to log this also in non-verbose mode + console.info( + colors__default["default"].bgGreen( + "PROMPTBOOK server listening on port ".concat(port), + ), + ); + if (isVerbose) { + console.info( + colors__default["default"].gray("Verbose mode is enabled"), + ); + } + var isDestroyed = false; + return { + get isDestroyed() { + return isDestroyed; + }, + destroy: () => { + if (isDestroyed) { + return; + } + isDestroyed = true; + httpServer.close(); + server.close(); + }, + }; + } + /** + * TODO: Split this file into multiple functions - handler for each request + * TODO: Maybe use `$exportJson` + * TODO: [🧠][🛍] Maybe not `isAnonymous: boolean` BUT `mode: 'ANONYMOUS'|'COLLECTION'` + * TODO: [⚖] Expose the collection to be able to connect to same collection via createCollectionFromUrl + * TODO: Handle progress - support streaming + * TODO: [🗯] Do not hang up immediately but wait until client closes OR timeout + * TODO: [🗯] Timeout on chat to free up resources + * TODO: [🃏] Pass here some security token to prevent malitious usage and/or DDoS + * TODO: [0] Set unavailable models as undefined in `RemoteLlmExecutionTools` NOT throw error here + * TODO: Allow to constrain anonymous mode for specific models / providers + */ + + exports.BOOK_LANGUAGE_VERSION = BOOK_LANGUAGE_VERSION; + exports.PROMPTBOOK_ENGINE_VERSION = PROMPTBOOK_ENGINE_VERSION; + exports.startRemoteServer = startRemoteServer; + + Object.defineProperty(exports, "__esModule", { value: true }); + }, +); //# sourceMappingURL=index.umd.js.map diff --git a/src/remote-server/socket-types/_common/PromptbookServer_Error.ts b/src/remote-server/socket-types/_common/PromptbookServer_Error.ts index e99e84fa9b..d03958c457 100644 --- a/src/remote-server/socket-types/_common/PromptbookServer_Error.ts +++ b/src/remote-server/socket-types/_common/PromptbookServer_Error.ts @@ -1,4 +1,4 @@ -import type { ErrorJson } from '../../../errors/utils/ErrorJson'; +import type { ErrorJson } from "../../../errors/utils/ErrorJson"; /** * This is sent from server to client when error occurs and stops the process diff --git a/src/remote-server/socket-types/_subtypes/PromptbookServer_Identification.ts b/src/remote-server/socket-types/_subtypes/PromptbookServer_Identification.ts index 3d9c7e1704..22a8ae7c0b 100644 --- a/src/remote-server/socket-types/_subtypes/PromptbookServer_Identification.ts +++ b/src/remote-server/socket-types/_subtypes/PromptbookServer_Identification.ts @@ -1,6 +1,6 @@ -import type { LlmToolsConfiguration } from '../../../llm-providers/_common/register/LlmToolsConfiguration'; -import type { string_user_id } from '../../../types/typeAliases'; -import type { ApplicationRemoteServerClientOptions } from '../../types/RemoteServerOptions'; +import type { LlmToolsConfiguration } from "../../../llm-providers/_common/register/LlmToolsConfiguration"; +import type { string_user_id } from "../../../types/typeAliases"; +import type { ApplicationRemoteServerClientOptions } from "../../types/RemoteServerOptions"; /** * Identification of client for Socket.io remote server @@ -9,8 +9,8 @@ import type { ApplicationRemoteServerClientOptions } from '../../types/RemoteSer * @public exported from `@promptbook/remote-client` */ export type PromptbookServer_Identification = - | PromptbookServer_ApplicationIdentification - | PromptbookServer_AnonymousIdentification; + | PromptbookServer_ApplicationIdentification + | PromptbookServer_AnonymousIdentification; /** * Application mode is situation when you run known and well-defined books with your own api keys @@ -19,12 +19,12 @@ export type PromptbookServer_Identification = * @public exported from `@promptbook/remote-client` */ export type PromptbookServer_ApplicationIdentification = - ApplicationRemoteServerClientOptions & { - /** - * Application mode - */ - readonly isAnonymous: false; - }; + ApplicationRemoteServerClientOptions & { + /** + * Application mode + */ + readonly isAnonymous: false; + }; /** * Anonymous mode is when you run arbitrary user books without api keys from user @@ -36,23 +36,23 @@ export type PromptbookServer_ApplicationIdentification = * @public exported from `@promptbook/remote-client` */ export type PromptbookServer_AnonymousIdentification = { - /** - * Anonymous mode - */ - readonly isAnonymous: true; + /** + * Anonymous mode + */ + readonly isAnonymous: true; - /** - * Identifier of the end user - * - * Note: this is passed to the certain model providers to identify misuse - * Note: In anonymous mode, there is no need to identify yourself, nor does it change the actual configuration of LLM Tools (unlike in application mode) - */ - readonly userId?: string_user_id; + /** + * Identifier of the end user + * + * Note: this is passed to the certain model providers to identify misuse + * Note: In anonymous mode, there is no need to identify yourself, nor does it change the actual configuration of LLM Tools (unlike in application mode) + */ + readonly userId?: string_user_id; - /** - * Configuration for the LLM tools - */ - readonly llmToolsConfiguration: LlmToolsConfiguration; + /** + * Configuration for the LLM tools + */ + readonly llmToolsConfiguration: LlmToolsConfiguration; }; /** diff --git a/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Request.ts b/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Request.ts index 8a42b75675..3cc9737afc 100644 --- a/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Request.ts +++ b/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Request.ts @@ -1,4 +1,4 @@ -import type { PromptbookServer_Identification } from '../_subtypes/PromptbookServer_Identification'; +import type { PromptbookServer_Identification } from "../_subtypes/PromptbookServer_Identification"; /** * List models available in the server @@ -8,10 +8,10 @@ import type { PromptbookServer_Identification } from '../_subtypes/PromptbookSer * @private internal type of remote server */ export type PromptbookServer_ListModels_Request = { - /** - * Identifier of the end user or application - */ - readonly identification: PromptbookServer_Identification; + /** + * Identifier of the end user or application + */ + readonly identification: PromptbookServer_Identification; }; /** diff --git a/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Response.ts b/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Response.ts index e70641e67f..7db845052c 100644 --- a/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Response.ts +++ b/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Response.ts @@ -1,4 +1,4 @@ -import type { AvailableModel } from '../../../execution/AvailableModel'; +import type { AvailableModel } from "../../../execution/AvailableModel"; /** * Models available for use in the server @@ -8,10 +8,10 @@ import type { AvailableModel } from '../../../execution/AvailableModel'; * @private internal type of remote server */ export type PromptbookServer_ListModels_Response = { - /** - * Available models that can be used - */ - readonly models: ReadonlyArray; + /** + * Available models that can be used + */ + readonly models: ReadonlyArray; }; /** diff --git a/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Request.ts b/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Request.ts index 2553407748..9e45e0b5fb 100644 --- a/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Request.ts +++ b/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Request.ts @@ -1,5 +1,5 @@ -import type { PipelineJson } from '../../../pipeline/PipelineJson/PipelineJson'; -import type { PromptbookServer_Identification } from '../_subtypes/PromptbookServer_Identification'; +import type { PipelineJson } from "../../../pipeline/PipelineJson/PipelineJson"; +import type { PromptbookServer_Identification } from "../_subtypes/PromptbookServer_Identification"; /** * This is a request from client to server to prepare a pipeline @@ -7,13 +7,13 @@ import type { PromptbookServer_Identification } from '../_subtypes/PromptbookSer * @private internal type of remote server */ export type PromptbookServer_PreparePipeline_Request = { - /** - * Identifier of the end user or application - */ - readonly identification: PromptbookServer_Identification; + /** + * Identifier of the end user or application + */ + readonly identification: PromptbookServer_Identification; - /** - * The Pipeline to prepare - */ - readonly pipeline: PipelineJson; + /** + * The Pipeline to prepare + */ + readonly pipeline: PipelineJson; }; diff --git a/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Response.ts b/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Response.ts index 5732a1c27a..e807279394 100644 --- a/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Response.ts +++ b/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Response.ts @@ -1,4 +1,4 @@ -import type { PipelineJson } from '../../../pipeline/PipelineJson/PipelineJson'; +import type { PipelineJson } from "../../../pipeline/PipelineJson/PipelineJson"; /** * This is sent from server to client when the pipeline is prepared @@ -6,8 +6,8 @@ import type { PipelineJson } from '../../../pipeline/PipelineJson/PipelineJson'; * @private internal type of remote server */ export type PromptbookServer_PreparePipeline_Response = { - /** - * Prepared pipeline - */ - readonly preparedPipeline: PipelineJson; + /** + * Prepared pipeline + */ + readonly preparedPipeline: PipelineJson; }; diff --git a/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Request.ts b/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Request.ts index 20ae32e409..f98c025820 100644 --- a/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Request.ts +++ b/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Request.ts @@ -1,5 +1,5 @@ -import type { Prompt } from '../../../types/Prompt'; -import type { PromptbookServer_Identification } from '../_subtypes/PromptbookServer_Identification'; +import type { Prompt } from "../../../types/Prompt"; +import type { PromptbookServer_Identification } from "../_subtypes/PromptbookServer_Identification"; /** * This is a request from client to server to execute a prompt @@ -7,13 +7,13 @@ import type { PromptbookServer_Identification } from '../_subtypes/PromptbookSer * @private internal type of remote server */ export type PromptbookServer_Prompt_Request = { - /** - * Identifier of the end user or application - */ - readonly identification: PromptbookServer_Identification; + /** + * Identifier of the end user or application + */ + readonly identification: PromptbookServer_Identification; - /** - * The Prompt to execute - */ - readonly prompt: Prompt; + /** + * The Prompt to execute + */ + readonly prompt: Prompt; }; diff --git a/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Response.ts b/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Response.ts index 1be6005f53..a07269e57e 100644 --- a/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Response.ts +++ b/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Response.ts @@ -1,4 +1,4 @@ -import type { PromptResult } from '../../../execution/PromptResult'; +import type { PromptResult } from "../../../execution/PromptResult"; /** * This is sent from server to client when the generated text is completed @@ -6,8 +6,8 @@ import type { PromptResult } from '../../../execution/PromptResult'; * @private internal type of remote server */ export type PromptbookServer_Prompt_Response = { - /** - * The result of the prompt - */ - readonly promptResult: PromptResult; + /** + * The result of the prompt + */ + readonly promptResult: PromptResult; }; diff --git a/src/remote-server/startRemoteServer.ts b/src/remote-server/startRemoteServer.ts index 4c7379324c..9382877e67 100644 --- a/src/remote-server/startRemoteServer.ts +++ b/src/remote-server/startRemoteServer.ts @@ -1,39 +1,39 @@ -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import type { IDestroyable } from 'destroyable'; -import express from 'express'; -import http from 'http'; -import { Server, Socket } from 'socket.io'; -import { spaceTrim } from 'spacetrim'; -import { forTime } from 'waitasecond'; -import { CLAIM } from '../config'; -import { DEFAULT_IS_VERBOSE } from '../config'; -import { PipelineExecutionError } from '../errors/PipelineExecutionError'; -import { serializeError } from '../errors/utils/serializeError'; -import { $provideExecutablesForNode } from '../executables/$provideExecutablesForNode'; -import { createPipelineExecutor } from '../execution/createPipelineExecutor/00-createPipelineExecutor'; -import type { ExecutionTask } from '../execution/ExecutionTask'; -import type { ExecutionTools } from '../execution/ExecutionTools'; -import type { LlmExecutionTools } from '../execution/LlmExecutionTools'; -import type { PromptResult } from '../execution/PromptResult'; -import { createLlmToolsFromConfiguration } from '../llm-providers/_common/register/createLlmToolsFromConfiguration'; -import { preparePipeline } from '../prepare/preparePipeline'; -import { $provideFilesystemForNode } from '../scrapers/_common/register/$provideFilesystemForNode'; -import { $provideScrapersForNode } from '../scrapers/_common/register/$provideScrapersForNode'; -import type { InputParameters } from '../types/typeAliases'; -import type { string_pipeline_url } from '../types/typeAliases'; -import { keepTypeImported } from '../utils/organization/keepTypeImported'; -import type { really_any } from '../utils/organization/really_any'; -import { BOOK_LANGUAGE_VERSION } from '../version'; -import { PROMPTBOOK_ENGINE_VERSION } from '../version'; -import type { PromptbookServer_Error } from './socket-types/_common/PromptbookServer_Error'; -import type { PromptbookServer_Identification } from './socket-types/_subtypes/PromptbookServer_Identification'; -import type { PromptbookServer_ListModels_Request } from './socket-types/listModels/PromptbookServer_ListModels_Request'; -import type { PromptbookServer_ListModels_Response } from './socket-types/listModels/PromptbookServer_ListModels_Response'; -import type { PromptbookServer_PreparePipeline_Request } from './socket-types/prepare/PromptbookServer_PreparePipeline_Request'; -import type { PromptbookServer_PreparePipeline_Response } from './socket-types/prepare/PromptbookServer_PreparePipeline_Response'; -import type { PromptbookServer_Prompt_Request } from './socket-types/prompt/PromptbookServer_Prompt_Request'; -import type { PromptbookServer_Prompt_Response } from './socket-types/prompt/PromptbookServer_Prompt_Response'; -import type { RemoteServerOptions } from './types/RemoteServerOptions'; +import http from "http"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import type { IDestroyable } from "destroyable"; +import express from "express"; +import { Server, type Socket } from "socket.io"; +import { spaceTrim } from "spacetrim"; +import { forTime } from "waitasecond"; +import { CLAIM } from "../config"; +import { DEFAULT_IS_VERBOSE } from "../config"; +import { PipelineExecutionError } from "../errors/PipelineExecutionError"; +import { serializeError } from "../errors/utils/serializeError"; +import { $provideExecutablesForNode } from "../executables/$provideExecutablesForNode"; +import type { ExecutionTask } from "../execution/ExecutionTask"; +import type { ExecutionTools } from "../execution/ExecutionTools"; +import type { LlmExecutionTools } from "../execution/LlmExecutionTools"; +import type { PromptResult } from "../execution/PromptResult"; +import { createPipelineExecutor } from "../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { createLlmToolsFromConfiguration } from "../llm-providers/_common/register/createLlmToolsFromConfiguration"; +import { preparePipeline } from "../prepare/preparePipeline"; +import { $provideFilesystemForNode } from "../scrapers/_common/register/$provideFilesystemForNode"; +import { $provideScrapersForNode } from "../scrapers/_common/register/$provideScrapersForNode"; +import type { InputParameters } from "../types/typeAliases"; +import type { string_pipeline_url } from "../types/typeAliases"; +import { keepTypeImported } from "../utils/organization/keepTypeImported"; +import type { really_any } from "../utils/organization/really_any"; +import { BOOK_LANGUAGE_VERSION } from "../version"; +import { PROMPTBOOK_ENGINE_VERSION } from "../version"; +import type { PromptbookServer_Error } from "./socket-types/_common/PromptbookServer_Error"; +import type { PromptbookServer_Identification } from "./socket-types/_subtypes/PromptbookServer_Identification"; +import type { PromptbookServer_ListModels_Request } from "./socket-types/listModels/PromptbookServer_ListModels_Request"; +import type { PromptbookServer_ListModels_Response } from "./socket-types/listModels/PromptbookServer_ListModels_Response"; +import type { PromptbookServer_PreparePipeline_Request } from "./socket-types/prepare/PromptbookServer_PreparePipeline_Request"; +import type { PromptbookServer_PreparePipeline_Response } from "./socket-types/prepare/PromptbookServer_PreparePipeline_Response"; +import type { PromptbookServer_Prompt_Request } from "./socket-types/prompt/PromptbookServer_Prompt_Request"; +import type { PromptbookServer_Prompt_Response } from "./socket-types/prompt/PromptbookServer_Prompt_Response"; +import type { RemoteServerOptions } from "./types/RemoteServerOptions"; keepTypeImported(); // <- Note: [🤛] keepTypeImported(); // <- Note: [🤛] @@ -49,117 +49,119 @@ keepTypeImported(); // <- Note: [🤛] * @public exported from `@promptbook/remote-server` */ export function startRemoteServer( - options: RemoteServerOptions, + options: RemoteServerOptions, ): IDestroyable { - const { - port, - - collection, - createLlmExecutionTools, - isAnonymousModeAllowed, - isApplicationModeAllowed, - isVerbose = DEFAULT_IS_VERBOSE, - } = { - isAnonymousModeAllowed: false, - isApplicationModeAllowed: false, - collection: null, - createLlmExecutionTools: null, - ...options, - }; - // <- TODO: [🦪] Some helper type to be able to use discriminant union types with destructuring - let { rootPath = '/' } = options; - - if (!rootPath.startsWith('/')) { - rootPath = `/${rootPath}`; - } /* not else */ - if (rootPath.endsWith('/')) { - rootPath = rootPath.slice(0, -1); - } /* not else */ - if (rootPath === '/') { - rootPath = ''; - } - - const socketioPath = - '/' + - `${rootPath}/socket.io` - .split('/') - .filter((part) => part !== '') - .join('/'); - - const startupDate = new Date(); - - async function getExecutionToolsFromIdentification( - identification: PromptbookServer_Identification, - ): Promise { - if (identification === null || identification === undefined) { - throw new Error(`Identification is not provided`); - } - - const { isAnonymous } = identification; - - if (isAnonymous === true && !isAnonymousModeAllowed) { - throw new PipelineExecutionError(`Anonymous mode is not allowed`); // <- TODO: [main] !!3 Test - } - - if (isAnonymous === false && !isApplicationModeAllowed) { - throw new PipelineExecutionError(`Application mode is not allowed`); // <- TODO: [main] !!3 Test - } - - // TODO: [main] !!4 Validate here userId (pass validator as dependency) - - let llm: LlmExecutionTools; - - if (isAnonymous === true) { - // Note: Anonymouse mode - // TODO: Maybe check that configuration is not empty - const { llmToolsConfiguration } = identification; - llm = createLlmToolsFromConfiguration(llmToolsConfiguration, { isVerbose }); - } else if (isAnonymous === false && createLlmExecutionTools !== null) { - // Note: Application mode - const { appId, userId, customOptions } = identification; - llm = await createLlmExecutionTools!({ - appId, - userId, - customOptions, - }); - } else { - throw new PipelineExecutionError( - `You must provide either llmToolsConfiguration or non-anonymous mode must be propperly configured`, - ); - } - - const fs = $provideFilesystemForNode(); - const executables = await $provideExecutablesForNode(); - const tools = { - llm, - fs, - scrapers: await $provideScrapersForNode({ fs, llm, executables }), - // TODO: Allow when `JavascriptExecutionTools` more secure *(without eval)*> script: [new JavascriptExecutionTools()], - }; - - return tools; - } - - const app = express(); - - app.use(express.json()); - app.use(function (request, response, next) { - response.setHeader('X-Powered-By', 'Promptbook engine'); - next(); - }); - - const runningExecutionTasks: Array = []; - - // TODO: [🧠] Do here some garbage collection of finished tasks - - app.get(['/', rootPath], async (request, response) => { - if (request.url?.includes('socket.io')) { - return; - } - - response.type('text/markdown').send( - await spaceTrim( - async (block) => ` + const { + port, + + collection, + createLlmExecutionTools, + isAnonymousModeAllowed, + isApplicationModeAllowed, + isVerbose = DEFAULT_IS_VERBOSE, + } = { + isAnonymousModeAllowed: false, + isApplicationModeAllowed: false, + collection: null, + createLlmExecutionTools: null, + ...options, + }; + // <- TODO: [🦪] Some helper type to be able to use discriminant union types with destructuring + let { rootPath = "/" } = options; + + if (!rootPath.startsWith("/")) { + rootPath = `/${rootPath}`; + } /* not else */ + if (rootPath.endsWith("/")) { + rootPath = rootPath.slice(0, -1); + } /* not else */ + if (rootPath === "/") { + rootPath = ""; + } + + const socketioPath = + "/" + + `${rootPath}/socket.io` + .split("/") + .filter((part) => part !== "") + .join("/"); + + const startupDate = new Date(); + + async function getExecutionToolsFromIdentification( + identification: PromptbookServer_Identification, + ): Promise { + if (identification === null || identification === undefined) { + throw new Error(`Identification is not provided`); + } + + const { isAnonymous } = identification; + + if (isAnonymous === true && !isAnonymousModeAllowed) { + throw new PipelineExecutionError(`Anonymous mode is not allowed`); // <- TODO: [main] !!3 Test + } + + if (isAnonymous === false && !isApplicationModeAllowed) { + throw new PipelineExecutionError(`Application mode is not allowed`); // <- TODO: [main] !!3 Test + } + + // TODO: [main] !!4 Validate here userId (pass validator as dependency) + + let llm: LlmExecutionTools; + + if (isAnonymous === true) { + // Note: Anonymouse mode + // TODO: Maybe check that configuration is not empty + const { llmToolsConfiguration } = identification; + llm = createLlmToolsFromConfiguration(llmToolsConfiguration, { + isVerbose, + }); + } else if (isAnonymous === false && createLlmExecutionTools !== null) { + // Note: Application mode + const { appId, userId, customOptions } = identification; + llm = await createLlmExecutionTools!({ + appId, + userId, + customOptions, + }); + } else { + throw new PipelineExecutionError( + `You must provide either llmToolsConfiguration or non-anonymous mode must be propperly configured`, + ); + } + + const fs = $provideFilesystemForNode(); + const executables = await $provideExecutablesForNode(); + const tools = { + llm, + fs, + scrapers: await $provideScrapersForNode({ fs, llm, executables }), + // TODO: Allow when `JavascriptExecutionTools` more secure *(without eval)*> script: [new JavascriptExecutionTools()], + }; + + return tools; + } + + const app = express(); + + app.use(express.json()); + app.use((request, response, next) => { + response.setHeader("X-Powered-By", "Promptbook engine"); + next(); + }); + + const runningExecutionTasks: Array = []; + + // TODO: [🧠] Do here some garbage collection of finished tasks + + app.get(["/", rootPath], async (request, response) => { + if (request.url?.includes("socket.io")) { + return; + } + + response.type("text/markdown").send( + await spaceTrim( + async (block) => ` # Promptbook > ${block(CLAIM)} @@ -176,16 +178,16 @@ export function startRemoteServer( **Server root path:** ${rootPath} **Socket.io path:** ${socketioPath} **Startup date:** ${startupDate.toISOString()} - **Anonymouse mode:** ${isAnonymousModeAllowed ? 'enabled' : 'disabled'} - **Application mode:** ${isApplicationModeAllowed ? 'enabled' : 'disabled'} + **Anonymouse mode:** ${isAnonymousModeAllowed ? "enabled" : "disabled"} + **Application mode:** ${isApplicationModeAllowed ? "enabled" : "disabled"} ${block( - !isApplicationModeAllowed || collection === null - ? '' - : '**Pipelines in collection:**\n' + - (await collection.listPipelines()) - .map((pipelineUrl) => `- ${pipelineUrl}`) - .join('\n'), - )} + !isApplicationModeAllowed || collection === null + ? "" + : "**Pipelines in collection:**\n" + + (await collection.listPipelines()) + .map((pipelineUrl) => `- ${pipelineUrl}`) + .join("\n"), + )} **Running executions:** ${runningExecutionTasks.length} --- @@ -193,12 +195,12 @@ export function startRemoteServer( ## Paths ${block( - app._router.stack - .map(({ route }: really_any) => route?.path || null) - .filter((path: string) => path !== null) - .map((path: string) => `- ${path}`) - .join('\n'), - )} + app._router.stack + .map(({ route }: really_any) => route?.path || null) + .filter((path: string) => path !== null) + .map((path: string) => `- ${path}`) + .join("\n"), + )} --- @@ -213,123 +215,133 @@ export function startRemoteServer( For more information look at: https://github.com/webgptorg/promptbook `, - ), - // <- TODO: [🗽] Unite branding and make single place for it - ); - }); - - app.get(`${rootPath}/books`, async (request, response) => { - if (collection === null) { - response.status(500).send('No collection available'); - return; - } - - const pipelines = await collection.listPipelines(); - // <- TODO: [🧠][👩🏾‍🤝‍🧑🏿] List `inputParameters` required for the execution - - response.send(pipelines); - }); - - // TODO: [🧠] Is it secure / good idea to expose source codes of hosted books - app.get(`${rootPath}/books/*`, async (request, response) => { - try { - if (collection === null) { - response.status(500).send('No collection nor books available'); - return; - } - - const pipelines = await collection.listPipelines(); - - const fullUrl = request.protocol + '://' + request.get('host') + request.originalUrl; - const pipelineUrl = pipelines.find((pipelineUrl) => pipelineUrl.endsWith(request.originalUrl)) || fullUrl; - - const pipeline = await collection.getPipelineByUrl(pipelineUrl); - - const source = pipeline.sources[0]; - - if (source === undefined || source.type !== 'BOOK') { - throw new Error('Pipeline source is not a book'); - } - - response - .type( - 'text/markdown', - // <- TODO: [🧠] Make custom mime-type for books - ) - .send(source.content); - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - response - .status( - 404, - // <- TODO: [👨🏼‍🤝‍👨🏻] Implement and use `errorToHttpStatus` - ) - .send({ error: serializeError(error) }); - } - }); - - app.get(`${rootPath}/executions`, async (request, response) => { - response.send( - runningExecutionTasks, - // <- TODO: [🧠][👩🏼‍🤝‍🧑🏼] Secure this through some token - // <- TODO: [🧠] Better and more information - ); - }); - - app.get(`${rootPath}/executions/:taskId`, async (request, response) => { - const { taskId } = request.params; - - const execution = runningExecutionTasks.find((executionTask) => executionTask.taskId === taskId); - - if (execution === undefined) { - response - .status( - 404, - // <- TODO: [👨🏼‍🤝‍👨🏻] Implement and use `errorToHttpStatus` - ) - .send(`Execution "${taskId}" not found`); - return; - } - - response.send(execution.currentValue); - }); - - app.post<{ - pipelineUrl: string_pipeline_url /* TODO: callbackUrl: string_url */; - inputParameters: InputParameters; - identification: PromptbookServer_Identification; - }>(`${rootPath}/executions/new`, async (request, response) => { - try { - const { inputParameters, identification } = request.body; - const pipelineUrl = request.body.pipelineUrl || request.body.book; - - // TODO: [🧠] Check `pipelineUrl` and `inputParameters` here or it should be responsibility of `collection.getPipelineByUrl` and `pipelineExecutor` - - const pipeline = await collection?.getPipelineByUrl(pipelineUrl); - - if (pipeline === undefined) { - response.status(404).send(`Pipeline "${pipelineUrl}" not found`); - return; - } - - const tools = await getExecutionToolsFromIdentification(identification); - - const pipelineExecutor = createPipelineExecutor({ pipeline, tools, ...options }); - - const executionTask = pipelineExecutor(inputParameters); - - runningExecutionTasks.push(executionTask); - - await forTime(10); - // <- Note: Wait for a while to wait for quick responses or sudden but asynchronous errors - // <- TODO: Put this into configuration - - response.send(executionTask); - - /*/ + ), + // <- TODO: [🗽] Unite branding and make single place for it + ); + }); + + app.get(`${rootPath}/books`, async (request, response) => { + if (collection === null) { + response.status(500).send("No collection available"); + return; + } + + const pipelines = await collection.listPipelines(); + // <- TODO: [🧠][👩🏾‍🤝‍🧑🏿] List `inputParameters` required for the execution + + response.send(pipelines); + }); + + // TODO: [🧠] Is it secure / good idea to expose source codes of hosted books + app.get(`${rootPath}/books/*`, async (request, response) => { + try { + if (collection === null) { + response.status(500).send("No collection nor books available"); + return; + } + + const pipelines = await collection.listPipelines(); + + const fullUrl = + request.protocol + "://" + request.get("host") + request.originalUrl; + const pipelineUrl = + pipelines.find((pipelineUrl) => + pipelineUrl.endsWith(request.originalUrl), + ) || fullUrl; + + const pipeline = await collection.getPipelineByUrl(pipelineUrl); + + const source = pipeline.sources[0]; + + if (source === undefined || source.type !== "BOOK") { + throw new Error("Pipeline source is not a book"); + } + + response + .type( + "text/markdown", + // <- TODO: [🧠] Make custom mime-type for books + ) + .send(source.content); + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + response + .status( + 404, + // <- TODO: [👨🏼‍🤝‍👨🏻] Implement and use `errorToHttpStatus` + ) + .send({ error: serializeError(error) }); + } + }); + + app.get(`${rootPath}/executions`, async (request, response) => { + response.send( + runningExecutionTasks, + // <- TODO: [🧠][👩🏼‍🤝‍🧑🏼] Secure this through some token + // <- TODO: [🧠] Better and more information + ); + }); + + app.get(`${rootPath}/executions/:taskId`, async (request, response) => { + const { taskId } = request.params; + + const execution = runningExecutionTasks.find( + (executionTask) => executionTask.taskId === taskId, + ); + + if (execution === undefined) { + response + .status( + 404, + // <- TODO: [👨🏼‍🤝‍👨🏻] Implement and use `errorToHttpStatus` + ) + .send(`Execution "${taskId}" not found`); + return; + } + + response.send(execution.currentValue); + }); + + app.post<{ + pipelineUrl: string_pipeline_url /* TODO: callbackUrl: string_url */; + inputParameters: InputParameters; + identification: PromptbookServer_Identification; + }>(`${rootPath}/executions/new`, async (request, response) => { + try { + const { inputParameters, identification } = request.body; + const pipelineUrl = request.body.pipelineUrl || request.body.book; + + // TODO: [🧠] Check `pipelineUrl` and `inputParameters` here or it should be responsibility of `collection.getPipelineByUrl` and `pipelineExecutor` + + const pipeline = await collection?.getPipelineByUrl(pipelineUrl); + + if (pipeline === undefined) { + response.status(404).send(`Pipeline "${pipelineUrl}" not found`); + return; + } + + const tools = await getExecutionToolsFromIdentification(identification); + + const pipelineExecutor = createPipelineExecutor({ + pipeline, + tools, + ...options, + }); + + const executionTask = pipelineExecutor(inputParameters); + + runningExecutionTasks.push(executionTask); + + await forTime(10); + // <- Note: Wait for a while to wait for quick responses or sudden but asynchronous errors + // <- TODO: Put this into configuration + + response.send(executionTask); + + /*/ executionTask.asObservable().subscribe({ next(partialResult) { console.info(executionTask.taskId, 'next', partialResult); @@ -343,210 +355,256 @@ export function startRemoteServer( }); /**/ - /* + /* await fetch(request.body.callbackUrl); // <- TODO: [🧠] Should be here transferred data as POST / PUT */ - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - response.status(400).send({ error: serializeError(error) }); - } - }); - - const httpServer = http.createServer(app); - - const server: Server = new Server(httpServer, { - path: socketioPath, - transports: [/*'websocket', <- TODO: [🌬] Make websocket transport work */ 'polling'], - cors: { - origin: '*', - methods: ['GET', 'POST'], - }, - }); - - server.on('connection', (socket: Socket) => { - if (isVerbose) { - console.info(colors.gray(`Client connected`), socket.id); - } - - // ----------- - - socket.on('prompt-request', async (request: PromptbookServer_Prompt_Request) => { - const { identification, prompt } = request; - - if (isVerbose) { - console.info(colors.bgWhite(`Prompt:`), colors.gray(JSON.stringify(request, null, 4))); - } - - try { - const tools = await getExecutionToolsFromIdentification(identification); - const { llm } = tools; - - if ( - identification.isAnonymous === false && - collection !== null && - !(await collection.isResponsibleForPrompt(prompt)) - ) { - throw new PipelineExecutionError(`Pipeline is not in the collection of this server`); - } - - let promptResult: PromptResult; - switch (prompt.modelRequirements.modelVariant) { - case 'CHAT': - if (llm.callChatModel === undefined) { - // Note: [0] This check should not be a thing - throw new PipelineExecutionError(`Chat model is not available`); - } - promptResult = await llm.callChatModel(prompt); - break; - - case 'COMPLETION': - if (llm.callCompletionModel === undefined) { - // Note: [0] This check should not be a thing - throw new PipelineExecutionError(`Completion model is not available`); - } - promptResult = await llm.callCompletionModel(prompt); - break; - - case 'EMBEDDING': - if (llm.callEmbeddingModel === undefined) { - // Note: [0] This check should not be a thing - throw new PipelineExecutionError(`Embedding model is not available`); - } - promptResult = await llm.callEmbeddingModel(prompt); - break; - - // <- case [🤖]: - - default: - throw new PipelineExecutionError( - `Unknown model variant "${(prompt as really_any).modelRequirements.modelVariant}"`, - ); - } - - if (isVerbose) { - console.info(colors.bgGreen(`PromptResult:`), colors.green(JSON.stringify(promptResult, null, 4))); - } - - socket.emit( - 'prompt-response', - { promptResult } satisfies PromptbookServer_Prompt_Response /* <- Note: [🤛] */, - ); - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - socket.emit('error', serializeError(error) satisfies PromptbookServer_Error /* <- Note: [🤛] */); - } finally { - socket.disconnect(); - // TODO: [🍚]> executionTools.destroy(); - } - }); - - // ----------- - - // TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io - socket.on('listModels-request', async (request: PromptbookServer_ListModels_Request) => { - const { identification } = request; - - if (isVerbose) { - console.info(colors.bgWhite(`Listing models`)); - } - - try { - const tools = await getExecutionToolsFromIdentification(identification); - const { llm } = tools; - - const models = await llm.listModels(); - - socket.emit( - 'listModels-response', - { models } satisfies PromptbookServer_ListModels_Response /* <- Note: [🤛] */, - ); - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - socket.emit('error', serializeError(error) satisfies PromptbookServer_Error); - } finally { - socket.disconnect(); - // TODO: [🍚]> executionTools.destroy(); - } - }); - - // ----------- - - // TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io - socket.on( - 'preparePipeline-request', - async (request: PromptbookServer_PreparePipeline_Request) => { - const { identification, pipeline } = request; - - if (isVerbose) { - console.info(colors.bgWhite(`Prepare pipeline`)); - } - - try { - const tools = await getExecutionToolsFromIdentification(identification); - - const preparedPipeline = await preparePipeline(pipeline, tools, options); - - socket.emit( - 'preparePipeline-response', - { preparedPipeline } satisfies PromptbookServer_PreparePipeline_Response /* <- Note: [🤛] */, - ); - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - socket.emit('error', serializeError(error) satisfies PromptbookServer_Error); - // <- TODO: [🚋] There is a problem with the remote server handling errors and sending them back to the client - } finally { - socket.disconnect(); - // TODO: [🍚]> executionTools.destroy(); - } - }, - ); - - // ----------- - - socket.on('disconnect', () => { - // TODO: Destroy here executionToolsForClient - if (isVerbose) { - console.info(colors.gray(`Client disconnected`), socket.id); - } - }); - }); - - httpServer.listen(port); - - // Note: We want to log this also in non-verbose mode - console.info(colors.bgGreen(`PROMPTBOOK server listening on port ${port}`)); - if (isVerbose) { - console.info(colors.gray(`Verbose mode is enabled`)); - } - - let isDestroyed = false; - - return { - get isDestroyed() { - return isDestroyed; - }, - destroy() { - if (isDestroyed) { - return; - } - isDestroyed = true; - httpServer.close(); - server.close(); - }, - }; + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + response.status(400).send({ error: serializeError(error) }); + } + }); + + const httpServer = http.createServer(app); + + const server: Server = new Server(httpServer, { + path: socketioPath, + transports: [ + /*'websocket', <- TODO: [🌬] Make websocket transport work */ "polling", + ], + cors: { + origin: "*", + methods: ["GET", "POST"], + }, + }); + + server.on("connection", (socket: Socket) => { + if (isVerbose) { + console.info(colors.gray(`Client connected`), socket.id); + } + + // ----------- + + socket.on( + "prompt-request", + async (request: PromptbookServer_Prompt_Request) => { + const { identification, prompt } = request; + + if (isVerbose) { + console.info( + colors.bgWhite(`Prompt:`), + colors.gray(JSON.stringify(request, null, 4)), + ); + } + + try { + const tools = + await getExecutionToolsFromIdentification(identification); + const { llm } = tools; + + if ( + identification.isAnonymous === false && + collection !== null && + !(await collection.isResponsibleForPrompt(prompt)) + ) { + throw new PipelineExecutionError( + `Pipeline is not in the collection of this server`, + ); + } + + let promptResult: PromptResult; + switch (prompt.modelRequirements.modelVariant) { + case "CHAT": + if (llm.callChatModel === undefined) { + // Note: [0] This check should not be a thing + throw new PipelineExecutionError(`Chat model is not available`); + } + promptResult = await llm.callChatModel(prompt); + break; + + case "COMPLETION": + if (llm.callCompletionModel === undefined) { + // Note: [0] This check should not be a thing + throw new PipelineExecutionError( + `Completion model is not available`, + ); + } + promptResult = await llm.callCompletionModel(prompt); + break; + + case "EMBEDDING": + if (llm.callEmbeddingModel === undefined) { + // Note: [0] This check should not be a thing + throw new PipelineExecutionError( + `Embedding model is not available`, + ); + } + promptResult = await llm.callEmbeddingModel(prompt); + break; + + // <- case [🤖]: + + default: + throw new PipelineExecutionError( + `Unknown model variant "${(prompt as really_any).modelRequirements.modelVariant}"`, + ); + } + + if (isVerbose) { + console.info( + colors.bgGreen(`PromptResult:`), + colors.green(JSON.stringify(promptResult, null, 4)), + ); + } + + socket.emit( + "prompt-response", + { + promptResult, + } satisfies PromptbookServer_Prompt_Response /* <- Note: [🤛] */, + ); + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + socket.emit( + "error", + serializeError( + error, + ) satisfies PromptbookServer_Error /* <- Note: [🤛] */, + ); + } finally { + socket.disconnect(); + // TODO: [🍚]> executionTools.destroy(); + } + }, + ); + + // ----------- + + // TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io + socket.on( + "listModels-request", + async (request: PromptbookServer_ListModels_Request) => { + const { identification } = request; + + if (isVerbose) { + console.info(colors.bgWhite(`Listing models`)); + } + + try { + const tools = + await getExecutionToolsFromIdentification(identification); + const { llm } = tools; + + const models = await llm.listModels(); + + socket.emit( + "listModels-response", + { + models, + } satisfies PromptbookServer_ListModels_Response /* <- Note: [🤛] */, + ); + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + socket.emit( + "error", + serializeError(error) satisfies PromptbookServer_Error, + ); + } finally { + socket.disconnect(); + // TODO: [🍚]> executionTools.destroy(); + } + }, + ); + + // ----------- + + // TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io + socket.on( + "preparePipeline-request", + async ( + request: PromptbookServer_PreparePipeline_Request, + ) => { + const { identification, pipeline } = request; + + if (isVerbose) { + console.info(colors.bgWhite(`Prepare pipeline`)); + } + + try { + const tools = + await getExecutionToolsFromIdentification(identification); + + const preparedPipeline = await preparePipeline( + pipeline, + tools, + options, + ); + + socket.emit( + "preparePipeline-response", + { + preparedPipeline, + } satisfies PromptbookServer_PreparePipeline_Response /* <- Note: [🤛] */, + ); + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + socket.emit( + "error", + serializeError(error) satisfies PromptbookServer_Error, + ); + // <- TODO: [🚋] There is a problem with the remote server handling errors and sending them back to the client + } finally { + socket.disconnect(); + // TODO: [🍚]> executionTools.destroy(); + } + }, + ); + + // ----------- + + socket.on("disconnect", () => { + // TODO: Destroy here executionToolsForClient + if (isVerbose) { + console.info(colors.gray(`Client disconnected`), socket.id); + } + }); + }); + + httpServer.listen(port); + + // Note: We want to log this also in non-verbose mode + console.info(colors.bgGreen(`PROMPTBOOK server listening on port ${port}`)); + if (isVerbose) { + console.info(colors.gray(`Verbose mode is enabled`)); + } + + let isDestroyed = false; + + return { + get isDestroyed() { + return isDestroyed; + }, + destroy() { + if (isDestroyed) { + return; + } + isDestroyed = true; + httpServer.close(); + server.close(); + }, + }; } /** diff --git a/src/remote-server/types/RemoteClientOptions.ts b/src/remote-server/types/RemoteClientOptions.ts index 5817b86364..6dbf734a78 100644 --- a/src/remote-server/types/RemoteClientOptions.ts +++ b/src/remote-server/types/RemoteClientOptions.ts @@ -1,7 +1,7 @@ -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; -import type { string_base_url } from '../../types/typeAliases'; -import type { string_uri } from '../../types/typeAliases'; -import type { PromptbookServer_Identification } from '../socket-types/_subtypes/PromptbookServer_Identification'; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; +import type { string_base_url } from "../../types/typeAliases"; +import type { string_uri } from "../../types/typeAliases"; +import type { PromptbookServer_Identification } from "../socket-types/_subtypes/PromptbookServer_Identification"; /** * Options for `RemoteLlmExecutionTools` @@ -9,24 +9,24 @@ import type { PromptbookServer_Identification } from '../socket-types/_subtypes/ * @public exported from `@promptbook/remote-client` */ export type RemoteClientOptions = CommonToolsOptions & { - /** - * URL of the remote server - * On this server will be connected to the socket.io server - */ - readonly remoteUrl: string_base_url; + /** + * URL of the remote server + * On this server will be connected to the socket.io server + */ + readonly remoteUrl: string_base_url; - /** - * Path for the Socket.io server to listen - * - * @default '/socket.io' - * @example '/promptbook/socket.io' - */ - readonly path: string_uri; + /** + * Path for the Socket.io server to listen + * + * @default '/socket.io' + * @example '/promptbook/socket.io' + */ + readonly path: string_uri; - /** - * Identification of client for Socket.io remote server - */ - readonly identification: PromptbookServer_Identification; + /** + * Identification of client for Socket.io remote server + */ + readonly identification: PromptbookServer_Identification; }; /** diff --git a/src/remote-server/types/RemoteServerOptions.ts b/src/remote-server/types/RemoteServerOptions.ts index 05dde99f93..1391c1346b 100644 --- a/src/remote-server/types/RemoteServerOptions.ts +++ b/src/remote-server/types/RemoteServerOptions.ts @@ -1,10 +1,10 @@ -import type { Promisable } from 'type-fest'; -import type { PipelineCollection } from '../../collection/PipelineCollection'; -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; -import type { LlmExecutionTools } from '../../execution/LlmExecutionTools'; -import type { string_app_id } from '../../types/typeAliases'; -import type { string_uri } from '../../types/typeAliases'; -import type { string_user_id } from '../../types/typeAliases'; +import type { Promisable } from "type-fest"; +import type { PipelineCollection } from "../../collection/PipelineCollection"; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; +import type { LlmExecutionTools } from "../../execution/LlmExecutionTools"; +import type { string_app_id } from "../../types/typeAliases"; +import type { string_uri } from "../../types/typeAliases"; +import type { string_user_id } from "../../types/typeAliases"; /** * @@@ @@ -21,25 +21,26 @@ import type { string_user_id } from '../../types/typeAliases'; * @public exported from `@promptbook/remote-server` */ export type RemoteServerOptions = CommonToolsOptions & { - /** - * Port on which the server will listen - */ - readonly port: number; + /** + * Port on which the server will listen + */ + readonly port: number; - /** - * Root path of the server - * - * Note: This is useful when you reverse proxy the server without changing the path - * - * @default '/' - * @example '/api/promptbook/' - */ - readonly rootPath: string_uri; + /** + * Root path of the server + * + * Note: This is useful when you reverse proxy the server without changing the path + * + * @default '/' + * @example '/api/promptbook/' + */ + readonly rootPath: string_uri; } & ( - | AnonymousRemoteServerOptions - | ApplicationRemoteServerOptions - | (AnonymousRemoteServerOptions & ApplicationRemoteServerOptions) - ); + | AnonymousRemoteServerOptions + | ApplicationRemoteServerOptions + | (AnonymousRemoteServerOptions & + ApplicationRemoteServerOptions) + ); // <- TODO: [🐛] Typescript bug in this discriminated union // This should throw typescript error but it doesn't // @@ -51,48 +52,48 @@ export type RemoteServerOptions = CommonToolsOptions & { // > }); export type AnonymousRemoteServerOptions = { - /** - * Enable anonymous mode - */ - readonly isAnonymousModeAllowed: true; + /** + * Enable anonymous mode + */ + readonly isAnonymousModeAllowed: true; }; export type ApplicationRemoteServerOptions = { - /** - * Enable application mode - */ - readonly isApplicationModeAllowed: true; + /** + * Enable application mode + */ + readonly isApplicationModeAllowed: true; - /** - * Promptbook collection to use - * - * This is used to check validity of the prompt to prevent misuse - */ - readonly collection: PipelineCollection; + /** + * Promptbook collection to use + * + * This is used to check validity of the prompt to prevent misuse + */ + readonly collection: PipelineCollection; - /** - * Creates llm execution tools for each client - */ - createLlmExecutionTools( - options: ApplicationRemoteServerClientOptions, - ): Promisable /* <- TODO: [🍚] &({}|IDestroyable) */; + /** + * Creates llm execution tools for each client + */ + createLlmExecutionTools( + options: ApplicationRemoteServerClientOptions, + ): Promisable /* <- TODO: [🍚] &({}|IDestroyable) */; }; export type ApplicationRemoteServerClientOptions = { - /** - * @@@ - */ - readonly appId: string_app_id | null; + /** + * @@@ + */ + readonly appId: string_app_id | null; - /** - * @@@ - */ - readonly userId?: string_user_id; + /** + * @@@ + */ + readonly userId?: string_user_id; - /** - * @@@ - */ - readonly customOptions?: TCustomOptions; + /** + * @@@ + */ + readonly customOptions?: TCustomOptions; }; /** diff --git a/src/scrapers/_boilerplate/BoilerplateScraper.ts b/src/scrapers/_boilerplate/BoilerplateScraper.ts index a2b778da5e..aa623d6782 100644 --- a/src/scrapers/_boilerplate/BoilerplateScraper.ts +++ b/src/scrapers/_boilerplate/BoilerplateScraper.ts @@ -1,27 +1,27 @@ -import { readFile } from 'fs/promises'; -import spaceTrim from 'spacetrim'; -import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from '../../config'; -import { DEFAULT_IS_VERBOSE } from '../../config'; -import { DEFAULT_SCRAPE_CACHE_DIRNAME } from '../../config'; -import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError'; -import { KnowledgeScrapeError } from '../../errors/KnowledgeScrapeError'; -import { MissingToolsError } from '../../errors/MissingToolsError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { KnowledgePiecePreparedJson } from '../../pipeline/PipelineJson/KnowledgePieceJson'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { $isRunningInNode } from '../../utils/environment/$isRunningInNode'; -import { $execCommand } from '../../utils/execCommand/$execCommand'; -import { getFileExtension } from '../../utils/files/getFileExtension'; -import { isFileExisting } from '../../utils/files/isFileExisting'; -import type { Converter } from '../_common/Converter'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; -import type { Scraper } from '../_common/Scraper'; -import type { ScraperSourceHandler } from '../_common/Scraper'; -import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource'; -import { getScraperIntermediateSource } from '../_common/utils/getScraperIntermediateSource'; -import { MarkdownScraper } from '../markdown/MarkdownScraper'; -import { boilerplateScraperMetadata } from './register-metadata'; +import { readFile } from "fs/promises"; +import spaceTrim from "spacetrim"; +import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from "../../config"; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import { DEFAULT_SCRAPE_CACHE_DIRNAME } from "../../config"; +import { EnvironmentMismatchError } from "../../errors/EnvironmentMismatchError"; +import { KnowledgeScrapeError } from "../../errors/KnowledgeScrapeError"; +import { MissingToolsError } from "../../errors/MissingToolsError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { KnowledgePiecePreparedJson } from "../../pipeline/PipelineJson/KnowledgePieceJson"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { $isRunningInNode } from "../../utils/environment/$isRunningInNode"; +import { $execCommand } from "../../utils/execCommand/$execCommand"; +import { getFileExtension } from "../../utils/files/getFileExtension"; +import { isFileExisting } from "../../utils/files/isFileExisting"; +import type { Converter } from "../_common/Converter"; +import type { Scraper } from "../_common/Scraper"; +import type { ScraperSourceHandler } from "../_common/Scraper"; +import type { ScraperIntermediateSource } from "../_common/ScraperIntermediateSource"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; +import { getScraperIntermediateSource } from "../_common/utils/getScraperIntermediateSource"; +import { MarkdownScraper } from "../markdown/MarkdownScraper"; +import { boilerplateScraperMetadata } from "./register-metadata"; /** * Scraper of @@@ files @@ -30,82 +30,92 @@ import { boilerplateScraperMetadata } from './register-metadata'; * @public exported from `@promptbook/boilerplate` */ export class BoilerplateScraper implements Converter, Scraper { - /** - * Metadata of the scraper which includes title, mime types, etc. - */ - public get metadata(): ScraperAndConverterMetadata { - return boilerplateScraperMetadata; - } - - /** - * Markdown scraper is used internally - */ - private readonly markdownScraper: MarkdownScraper; - - public constructor( - private readonly tools: Pick, - private readonly options: PrepareAndScrapeOptions, - ) { - this.markdownScraper = new MarkdownScraper(tools, options); - } - - /** - * Convert the `.@@@` to `.md` file and returns intermediate source - * - * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object - */ - public async $convert(source: ScraperSourceHandler): Promise { - const { - rootDirname = process.cwd(), - cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, - intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, - isVerbose = DEFAULT_IS_VERBOSE, - } = this.options; - - // TODO: @@@ Preserve or delete - if (!$isRunningInNode()) { - throw new KnowledgeScrapeError('BoilerplateScraper is only supported in Node environment'); - } - - // TODO: @@@ Preserve or delete - if (this.tools.fs === undefined) { - throw new EnvironmentMismatchError('Can not scrape boilerplates without filesystem tools'); - // <- TODO: [🧠] What is the best error type here` - } - - // TODO: @@@ Preserve, delete or modify - if (this.tools.executables?.pandocPath === undefined) { - throw new MissingToolsError('Pandoc is required for scraping .docx files'); - } - - // TODO: @@@ Preserve, delete or modify - if (source.filename === null) { - // TODO: [🧠] Maybe save file as temporary - throw new KnowledgeScrapeError('When parsing .@@@ file, it must be real file in the file system'); - } - - const extension = getFileExtension(source.filename); - - const cacheFilehandler = await getScraperIntermediateSource(source, { - rootDirname, - cacheDirname, - intermediateFilesStrategy, - extension: 'md', - isVerbose, - }); - - // TODO: @@@ Preserve, delete or modify - // Note: Running Pandoc ONLY if the file in the cache does not exist - if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { - const command = `"${this.tools.executables.pandocPath}" -f ${extension} -t markdown "${source.filename}" -o "${cacheFilehandler.filename}"`; - - await $execCommand(command); - - // Note: [0] - if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + /** + * Metadata of the scraper which includes title, mime types, etc. + */ + public get metadata(): ScraperAndConverterMetadata { + return boilerplateScraperMetadata; + } + + /** + * Markdown scraper is used internally + */ + private readonly markdownScraper: MarkdownScraper; + + public constructor( + private readonly tools: Pick, + private readonly options: PrepareAndScrapeOptions, + ) { + this.markdownScraper = new MarkdownScraper(tools, options); + } + + /** + * Convert the `.@@@` to `.md` file and returns intermediate source + * + * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object + */ + public async $convert( + source: ScraperSourceHandler, + ): Promise { + const { + rootDirname = process.cwd(), + cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, + intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, + isVerbose = DEFAULT_IS_VERBOSE, + } = this.options; + + // TODO: @@@ Preserve or delete + if (!$isRunningInNode()) { + throw new KnowledgeScrapeError( + "BoilerplateScraper is only supported in Node environment", + ); + } + + // TODO: @@@ Preserve or delete + if (this.tools.fs === undefined) { + throw new EnvironmentMismatchError( + "Can not scrape boilerplates without filesystem tools", + ); + // <- TODO: [🧠] What is the best error type here` + } + + // TODO: @@@ Preserve, delete or modify + if (this.tools.executables?.pandocPath === undefined) { + throw new MissingToolsError( + "Pandoc is required for scraping .docx files", + ); + } + + // TODO: @@@ Preserve, delete or modify + if (source.filename === null) { + // TODO: [🧠] Maybe save file as temporary + throw new KnowledgeScrapeError( + "When parsing .@@@ file, it must be real file in the file system", + ); + } + + const extension = getFileExtension(source.filename); + + const cacheFilehandler = await getScraperIntermediateSource(source, { + rootDirname, + cacheDirname, + intermediateFilesStrategy, + extension: "md", + isVerbose, + }); + + // TODO: @@@ Preserve, delete or modify + // Note: Running Pandoc ONLY if the file in the cache does not exist + if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { + const command = `"${this.tools.executables.pandocPath}" -f ${extension} -t markdown "${source.filename}" -o "${cacheFilehandler.filename}"`; + + await $execCommand(command); + + // Note: [0] + if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { + throw new UnexpectedError( + spaceTrim( + (block) => ` File that was supposed to be created by Pandoc does not exist for unknown reason Expected file: @@ -115,38 +125,40 @@ export class BoilerplateScraper implements Converter, Scraper { > ${block(command)} `, - ), - ); - } - } - - return cacheFilehandler; - } - - /** - * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it - */ - public async scrape( - source: ScraperSourceHandler, - ): Promise> | null> { - const cacheFilehandler = await this.$convert(source); - - // TODO: @@@ Preserve, delete or modify - const markdownSource = { - source: source.source, - filename: cacheFilehandler.filename, - url: null, - mimeType: 'text/markdown', - async asText() { - // Note: [0] In $convert we check that the file exists - return await readFile(cacheFilehandler.filename, 'utf-8'); - }, - asJson() { - throw new UnexpectedError( - 'Did not expect that `markdownScraper` would need to get the content `asJson`', - ); - }, - /* + ), + ); + } + } + + return cacheFilehandler; + } + + /** + * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it + */ + public async scrape( + source: ScraperSourceHandler, + ): Promise + > | null> { + const cacheFilehandler = await this.$convert(source); + + // TODO: @@@ Preserve, delete or modify + const markdownSource = { + source: source.source, + filename: cacheFilehandler.filename, + url: null, + mimeType: "text/markdown", + async asText() { + // Note: [0] In $convert we check that the file exists + return await readFile(cacheFilehandler.filename, "utf-8"); + }, + asJson() { + throw new UnexpectedError( + "Did not expect that `markdownScraper` would need to get the content `asJson`", + ); + }, + /* TODO: [🥽] > asBlob() { > throw new UnexpectedError( @@ -154,14 +166,14 @@ export class BoilerplateScraper implements Converter, Scraper { > ); > }, */ - } satisfies ScraperSourceHandler; + } satisfies ScraperSourceHandler; - const knowledge = this.markdownScraper.scrape(markdownSource); + const knowledge = this.markdownScraper.scrape(markdownSource); - await cacheFilehandler.destroy(); + await cacheFilehandler.destroy(); - return knowledge; - } + return knowledge; + } } /** diff --git a/src/scrapers/_boilerplate/createBoilerplateScraper.ts b/src/scrapers/_boilerplate/createBoilerplateScraper.ts index de4c89b340..cbe41fba22 100644 --- a/src/scrapers/_boilerplate/createBoilerplateScraper.ts +++ b/src/scrapers/_boilerplate/createBoilerplateScraper.ts @@ -1,9 +1,9 @@ -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import type { ScraperConstructor } from '../_common/register/ScraperConstructor'; -import { BoilerplateScraper } from './BoilerplateScraper'; -import { boilerplateScraperMetadata } from './register-metadata'; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import type { ScraperConstructor } from "../_common/register/ScraperConstructor"; +import { BoilerplateScraper } from "./BoilerplateScraper"; +import { boilerplateScraperMetadata } from "./register-metadata"; keepTypeImported(); @@ -13,10 +13,13 @@ keepTypeImported(); * @public exported from `@promptbook/boilerplate` */ export const createBoilerplateScraper = Object.assign( - (tools: Pick, options: PrepareAndScrapeOptions): BoilerplateScraper => { - return new BoilerplateScraper(tools, options); - }, - boilerplateScraperMetadata, + ( + tools: Pick, + options: PrepareAndScrapeOptions, + ): BoilerplateScraper => { + return new BoilerplateScraper(tools, options); + }, + boilerplateScraperMetadata, ) satisfies ScraperConstructor; /* <- Note: [🤛] */ /** diff --git a/src/scrapers/_boilerplate/playground/boilerplate-scraper-playground.ts b/src/scrapers/_boilerplate/playground/boilerplate-scraper-playground.ts index e1946ce8b9..f40c59031b 100644 --- a/src/scrapers/_boilerplate/playground/boilerplate-scraper-playground.ts +++ b/src/scrapers/_boilerplate/playground/boilerplate-scraper-playground.ts @@ -1,76 +1,78 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { writeFile } from 'fs/promises'; -import { join } from 'path'; -import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode'; -import { usageToHuman } from '../../../execution/utils/usageToHuman'; -import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground'; -import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson'; -import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode'; -import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler'; -import { BoilerplateScraper } from '../BoilerplateScraper'; +import { join } from "path"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { writeFile } from "fs/promises"; +import { $provideExecutablesForNode } from "../../../executables/$provideExecutablesForNode"; +import { usageToHuman } from "../../../execution/utils/usageToHuman"; +import { $provideLlmToolsForTestingAndScriptsAndPlayground } from "../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground"; +import { stringifyPipelineJson } from "../../../utils/editable/utils/stringifyPipelineJson"; +import { $provideFilesystemForNode } from "../../_common/register/$provideFilesystemForNode"; +import { makeKnowledgeSourceHandler } from "../../_common/utils/makeKnowledgeSourceHandler"; +import { BoilerplateScraper } from "../BoilerplateScraper"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 Scrape knowledge from @@@ (playground)`); + console.info(`🧸 Scrape knowledge from @@@ (playground)`); - // Do here stuff you want to test - //========================================> + // Do here stuff you want to test + //========================================> - const example = '10-simple.@@@'; - // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them + const example = "10-simple.@@@"; + // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them - const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true }); - const rootDirname = join(__dirname, '..', 'examples'); + const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ + isCacheReloaded: true, + }); + const rootDirname = join(__dirname, "..", "examples"); - const boilerplateScraper = new BoilerplateScraper( - { - fs: $provideFilesystemForNode(), - llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), - executables: await $provideExecutablesForNode(), - }, - { - rootDirname, - }, - ); + const boilerplateScraper = new BoilerplateScraper( + { + fs: $provideFilesystemForNode(), + llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), + executables: await $provideExecutablesForNode(), + }, + { + rootDirname, + }, + ); - const knowledge = await boilerplateScraper.scrape( - await makeKnowledgeSourceHandler( - { knowledgeSourceContent: example }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ); + const knowledge = await boilerplateScraper.scrape( + await makeKnowledgeSourceHandler( + { knowledgeSourceContent: example }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ); - console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); - console.info(colors.bgGreen(' Knowledge: ')); - console.info(knowledge); + console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); + console.info(colors.bgGreen(" Knowledge: ")); + console.info(knowledge); - await writeFile( - join( - __dirname, - `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, - ), - stringifyPipelineJson(knowledge), - 'utf-8', - ); - /**/ + await writeFile( + join( + __dirname, + `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, + ), + stringifyPipelineJson(knowledge), + "utf-8", + ); + /**/ - //========================================/ + //========================================/ } /** diff --git a/src/scrapers/_boilerplate/playground/tsconfig.json b/src/scrapers/_boilerplate/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/scrapers/_boilerplate/playground/tsconfig.json +++ b/src/scrapers/_boilerplate/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/scrapers/_boilerplate/register-constructor.ts b/src/scrapers/_boilerplate/register-constructor.ts index afc65fdfd2..8af07391f8 100644 --- a/src/scrapers/_boilerplate/register-constructor.ts +++ b/src/scrapers/_boilerplate/register-constructor.ts @@ -1,6 +1,6 @@ -import type { Registration } from '../../utils/$Register'; -import { $scrapersRegister } from '../_common/register/$scrapersRegister'; -import { createBoilerplateScraper } from './createBoilerplateScraper'; +import type { Registration } from "../../utils/$Register"; +import { $scrapersRegister } from "../_common/register/$scrapersRegister"; +import { createBoilerplateScraper } from "./createBoilerplateScraper"; /** * Registration of known scraper @@ -11,7 +11,8 @@ import { createBoilerplateScraper } from './createBoilerplateScraper'; * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _BoilerplateScraperRegistration: Registration = $scrapersRegister.register(createBoilerplateScraper); +export const _BoilerplateScraperRegistration: Registration = + $scrapersRegister.register(createBoilerplateScraper); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/scrapers/_boilerplate/register-metadata.ts b/src/scrapers/_boilerplate/register-metadata.ts index 3498777e57..000520ebb7 100644 --- a/src/scrapers/_boilerplate/register-metadata.ts +++ b/src/scrapers/_boilerplate/register-metadata.ts @@ -1,8 +1,8 @@ -import type { Registration } from '../../utils/$Register'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import { $deepFreeze } from '../../utils/serialization/$deepFreeze'; -import { $scrapersMetadataRegister } from '../_common/register/$scrapersMetadataRegister'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; +import type { Registration } from "../../utils/$Register"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import { $deepFreeze } from "../../utils/serialization/$deepFreeze"; +import { $scrapersMetadataRegister } from "../_common/register/$scrapersMetadataRegister"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; keepTypeImported(); @@ -12,19 +12,19 @@ keepTypeImported(); * @private within the scraper directory */ export const boilerplateScraperMetadata = $deepFreeze({ - title: 'Boilerplate scraper', - packageName: '@promptbook/boilerplate', - className: 'BoilerplateScraper', - mimeTypes: [ - '@@@/@@@', - // <- TODO: @@@ Add compatible mime types with Boilerplate scraper - ], - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@@', - isAvilableInBrowser: false, // <- TODO: @@@ Is it avilable in browser? - // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server - requiredExecutables: [ - /* @@@ 'Pandoc' */ - ], + title: "Boilerplate scraper", + packageName: "@promptbook/boilerplate", + className: "BoilerplateScraper", + mimeTypes: [ + "@@@/@@@", + // <- TODO: @@@ Add compatible mime types with Boilerplate scraper + ], + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@@", + isAvilableInBrowser: false, // <- TODO: @@@ Is it avilable in browser? + // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server + requiredExecutables: [ + /* @@@ 'Pandoc' */ + ], }) satisfies ScraperAndConverterMetadata; /* <- Note: [🤛] */ /** @@ -37,7 +37,7 @@ export const boilerplateScraperMetadata = $deepFreeze({ * @public exported from `@promptbook/cli` */ export const _BoilerplateScraperMetadataRegistration: Registration = - $scrapersMetadataRegister.register(boilerplateScraperMetadata); + $scrapersMetadataRegister.register(boilerplateScraperMetadata); /** * Note: [💞] Ignore a discrepancy between file name and entity name diff --git a/src/scrapers/_common/Converter.ts b/src/scrapers/_common/Converter.ts index dcebd9180e..17f8919770 100644 --- a/src/scrapers/_common/Converter.ts +++ b/src/scrapers/_common/Converter.ts @@ -1,6 +1,6 @@ -import type { ScraperAndConverterMetadata } from './register/ScraperAndConverterMetadata'; -import type { ScraperSourceHandler } from './Scraper'; -import type { ScraperIntermediateSource } from './ScraperIntermediateSource'; +import type { ScraperSourceHandler } from "./Scraper"; +import type { ScraperIntermediateSource } from "./ScraperIntermediateSource"; +import type { ScraperAndConverterMetadata } from "./register/ScraperAndConverterMetadata"; /** * @@@ @@ -8,18 +8,18 @@ import type { ScraperIntermediateSource } from './ScraperIntermediateSource'; * Note: [🌏] Converters are not usable in browser because they produce a files */ export type Converter = { - /** - * Metadata of the converter which includes title, mime types, etc. - */ - readonly metadata: ScraperAndConverterMetadata; + /** + * Metadata of the converter which includes title, mime types, etc. + */ + readonly metadata: ScraperAndConverterMetadata; - /** - * Convert the the file and returns intermediate source or `null` if it can't convert it - * - * For example, convert a `.docx` to `.doc` file - * Or convert a `.pdf` to `.md` file - * - * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object - */ - $convert(source: ScraperSourceHandler): Promise; + /** + * Convert the the file and returns intermediate source or `null` if it can't convert it + * + * For example, convert a `.docx` to `.doc` file + * Or convert a `.pdf` to `.md` file + * + * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object + */ + $convert(source: ScraperSourceHandler): Promise; }; diff --git a/src/scrapers/_common/Scraper.ts b/src/scrapers/_common/Scraper.ts index 3ec6fb81b7..ba5ae4e6b6 100644 --- a/src/scrapers/_common/Scraper.ts +++ b/src/scrapers/_common/Scraper.ts @@ -1,65 +1,67 @@ -import type { Promisable } from 'type-fest'; -import type { KnowledgePiecePreparedJson } from '../../pipeline/PipelineJson/KnowledgePieceJson'; -import type { string_filename } from '../../types/typeAliases'; -import type { string_knowledge_source_link } from '../../types/typeAliases'; -import type { string_mime_type } from '../../types/typeAliases'; -import type { string_url } from '../../types/typeAliases'; -import type { ScraperAndConverterMetadata } from './register/ScraperAndConverterMetadata'; +import type { Promisable } from "type-fest"; +import type { KnowledgePiecePreparedJson } from "../../pipeline/PipelineJson/KnowledgePieceJson"; +import type { string_filename } from "../../types/typeAliases"; +import type { string_knowledge_source_link } from "../../types/typeAliases"; +import type { string_mime_type } from "../../types/typeAliases"; +import type { string_url } from "../../types/typeAliases"; +import type { ScraperAndConverterMetadata } from "./register/ScraperAndConverterMetadata"; /** * @@@ */ export type Scraper = { - /** - * Metadata of the scraper which includes title, mime types, etc. - */ - readonly metadata: ScraperAndConverterMetadata; + /** + * Metadata of the scraper which includes title, mime types, etc. + */ + readonly metadata: ScraperAndConverterMetadata; - /** - * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it - */ - scrape( - source: ScraperSourceHandler, - ): Promisable> | null>; + /** + * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it + */ + scrape( + source: ScraperSourceHandler, + ): Promisable + > | null>; }; /** * @@@ */ export type ScraperSourceHandler = { - /** - * The source of the knowledge - */ - readonly source: string_knowledge_source_link; + /** + * The source of the knowledge + */ + readonly source: string_knowledge_source_link; - /** - * The path to the file, if it is a file - * - * Note: Typically one of the `filename` or `url` is set and the other is `null` - */ - readonly filename: string_filename | null; + /** + * The path to the file, if it is a file + * + * Note: Typically one of the `filename` or `url` is set and the other is `null` + */ + readonly filename: string_filename | null; - /** - * The URL, if it is online - * - * Note: Typically one of the `filename` or `url` is set and the other is `null` - */ - readonly url: string_url | null; + /** + * The URL, if it is online + * + * Note: Typically one of the `filename` or `url` is set and the other is `null` + */ + readonly url: string_url | null; - /** - * Mime type of the source - */ - readonly mimeType: string_mime_type; + /** + * Mime type of the source + */ + readonly mimeType: string_mime_type; - /** - * Get the content as parsed JSON - */ - asJson(): Promisable; + /** + * Get the content as parsed JSON + */ + asJson(): Promisable; - /** - * Get the content as a utf-8 string - */ - asText(): Promisable; + /** + * Get the content as a utf-8 string + */ + asText(): Promisable; }; /** diff --git a/src/scrapers/_common/ScraperIntermediateSource.ts b/src/scrapers/_common/ScraperIntermediateSource.ts index 4c97bc4c31..58d7830169 100644 --- a/src/scrapers/_common/ScraperIntermediateSource.ts +++ b/src/scrapers/_common/ScraperIntermediateSource.ts @@ -1,14 +1,14 @@ -import type { IDestroyable } from 'destroyable'; -import type { string_absolute_filename } from '../../types/typeAliases'; +import type { IDestroyable } from "destroyable"; +import type { string_absolute_filename } from "../../types/typeAliases"; /** * @@@ */ export type ScraperIntermediateSource = IDestroyable & { - /** - * @@@ - */ - readonly filename: string_absolute_filename; + /** + * @@@ + */ + readonly filename: string_absolute_filename; }; /** diff --git a/src/scrapers/_common/prepareKnowledgePieces.test.ts b/src/scrapers/_common/prepareKnowledgePieces.test.ts index fdd65641ec..e106efa7c4 100644 --- a/src/scrapers/_common/prepareKnowledgePieces.test.ts +++ b/src/scrapers/_common/prepareKnowledgePieces.test.ts @@ -1,7 +1,7 @@ -import { describe, expect, it } from '@jest/globals'; -import { join } from 'path'; -import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground'; -import { prepareKnowledgePieces } from './prepareKnowledgePieces'; +import { join } from "path"; +import { describe, expect, it } from "@jest/globals"; +import { $provideLlmToolsForTestingAndScriptsAndPlayground } from "../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground"; +import { prepareKnowledgePieces } from "./prepareKnowledgePieces"; /* // TODO: [🐝] Test markdown, file and website here, rest in their own preparers @@ -19,17 +19,17 @@ describe('all the scrapers', () => { */ -describe('how prepareKnowledge works', () => { - it('should work with empty knowledge', async () => { - const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground(); - await expect( - prepareKnowledgePieces( - [], - { llm: llmTools }, - { - rootDirname: join(__dirname, 'examples'), - }, - ), - ).resolves.toEqual([]); - }); +describe("how prepareKnowledge works", () => { + it("should work with empty knowledge", async () => { + const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground(); + await expect( + prepareKnowledgePieces( + [], + { llm: llmTools }, + { + rootDirname: join(__dirname, "examples"), + }, + ), + ).resolves.toEqual([]); + }); }); diff --git a/src/scrapers/_common/prepareKnowledgePieces.ts b/src/scrapers/_common/prepareKnowledgePieces.ts index 1d20123f0e..afa51388f5 100644 --- a/src/scrapers/_common/prepareKnowledgePieces.ts +++ b/src/scrapers/_common/prepareKnowledgePieces.ts @@ -1,16 +1,16 @@ -import spaceTrim from 'spacetrim'; -import { DEFAULT_IS_VERBOSE } from '../../config'; -import { DEFAULT_MAX_PARALLEL_COUNT } from '../../config'; -import { KnowledgeScrapeError } from '../../errors/KnowledgeScrapeError'; -import { forEachAsync } from '../../execution/utils/forEachAsync'; -import type { KnowledgePiecePreparedJson } from '../../pipeline/PipelineJson/KnowledgePieceJson'; -import type { KnowledgeSourceJson } from '../../pipeline/PipelineJson/KnowledgeSourceJson'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; - -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import { arrayableToArray } from '../../utils/arrayableToArray'; -import { $registeredScrapersMessage } from './register/$registeredScrapersMessage'; -import { makeKnowledgeSourceHandler } from './utils/makeKnowledgeSourceHandler'; +import spaceTrim from "spacetrim"; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import { DEFAULT_MAX_PARALLEL_COUNT } from "../../config"; +import { KnowledgeScrapeError } from "../../errors/KnowledgeScrapeError"; +import { forEachAsync } from "../../execution/utils/forEachAsync"; +import type { KnowledgePiecePreparedJson } from "../../pipeline/PipelineJson/KnowledgePieceJson"; +import type { KnowledgeSourceJson } from "../../pipeline/PipelineJson/KnowledgeSourceJson"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; + +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import { arrayableToArray } from "../../utils/arrayableToArray"; +import { $registeredScrapersMessage } from "./register/$registeredScrapersMessage"; +import { makeKnowledgeSourceHandler } from "./utils/makeKnowledgeSourceHandler"; /** * Prepares the knowle @@ -19,74 +19,87 @@ import { makeKnowledgeSourceHandler } from './utils/makeKnowledgeSourceHandler'; * @public exported from `@promptbook/core` */ export async function prepareKnowledgePieces( - knowledgeSources: ReadonlyArray, - tools: Pick, - options: PrepareAndScrapeOptions, -): Promise>> { - const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, rootDirname, isVerbose = DEFAULT_IS_VERBOSE } = options; - - const knowledgePreparedUnflatten: Array>> = new Array( - knowledgeSources.length, - ); - - await forEachAsync(knowledgeSources, { maxParallelCount }, async (knowledgeSource, index) => { - let partialPieces: Omit[] | null = null; - const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose }); - const scrapers = arrayableToArray(tools.scrapers); - - for (const scraper of scrapers) { - if ( - !scraper.metadata.mimeTypes.includes(sourceHandler.mimeType) - // <- TODO: [🦔] Implement mime-type wildcards - ) { - continue; - } - - const partialPiecesUnchecked = await scraper.scrape(sourceHandler); - - if (partialPiecesUnchecked !== null) { - partialPieces = [...partialPiecesUnchecked]; - // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked` - - break; - } - - console.warn( - spaceTrim( - (block) => ` + knowledgeSources: ReadonlyArray, + tools: Pick, + options: PrepareAndScrapeOptions, +): Promise>> { + const { + maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, + rootDirname, + isVerbose = DEFAULT_IS_VERBOSE, + } = options; + + const knowledgePreparedUnflatten: Array< + Array> + > = new Array(knowledgeSources.length); + + await forEachAsync( + knowledgeSources, + { maxParallelCount }, + async (knowledgeSource, index) => { + let partialPieces: + | Omit[] + | null = null; + const sourceHandler = await makeKnowledgeSourceHandler( + knowledgeSource, + tools, + { rootDirname, isVerbose }, + ); + const scrapers = arrayableToArray(tools.scrapers); + + for (const scraper of scrapers) { + if ( + !scraper.metadata.mimeTypes.includes(sourceHandler.mimeType) + // <- TODO: [🦔] Implement mime-type wildcards + ) { + continue; + } + + const partialPiecesUnchecked = await scraper.scrape(sourceHandler); + + if (partialPiecesUnchecked !== null) { + partialPieces = [...partialPiecesUnchecked]; + // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked` + + break; + } + + console.warn( + spaceTrim( + (block) => ` Cannot scrape knowledge from source despite the scraper \`${ - scraper.metadata.className - }\` supports the mime type "${sourceHandler.mimeType}". + scraper.metadata.className + }\` supports the mime type "${sourceHandler.mimeType}". The source: ${block( - knowledgeSource.knowledgeSourceContent - .split('\n') - .map((line) => `> ${line}`) - .join('\n'), - )} + knowledgeSource.knowledgeSourceContent + .split("\n") + .map((line) => `> ${line}`) + .join("\n"), + )} ${block($registeredScrapersMessage(scrapers))} `, - ), - ); - } - - if (partialPieces === null) { - throw new KnowledgeScrapeError( - spaceTrim( - (block) => ` + ), + ); + } + + if (partialPieces === null) { + throw new KnowledgeScrapeError( + spaceTrim( + (block) => ` Cannot scrape knowledge The source: > ${block( - knowledgeSource.knowledgeSourceContent - .split('\n') - .map((line) => `> ${line}`) - .join('\n'), - )} + knowledgeSource.knowledgeSourceContent + .split("\n") + .map((line) => `> ${line}`) + .join("\n"), + )} No scraper found for the mime type "${sourceHandler.mimeType}" @@ -94,28 +107,30 @@ export async function prepareKnowledgePieces( `, - ), - ); - } - - const pieces = partialPieces.map((partialPiece) => ({ - ...partialPiece, - sources: [ - { - name: knowledgeSource.name, - // line, column <- TODO: [☀] - // <- TODO: [❎] - }, - ], - })); - - knowledgePreparedUnflatten[index] = pieces; - }); - - const knowledgePrepared: ReadonlyArray> = - knowledgePreparedUnflatten.flat(); - - return knowledgePrepared; + ), + ); + } + + const pieces = partialPieces.map((partialPiece) => ({ + ...partialPiece, + sources: [ + { + name: knowledgeSource.name, + // line, column <- TODO: [☀] + // <- TODO: [❎] + }, + ], + })); + + knowledgePreparedUnflatten[index] = pieces; + }, + ); + + const knowledgePrepared: ReadonlyArray< + Omit + > = knowledgePreparedUnflatten.flat(); + + return knowledgePrepared; } /* diff --git a/src/scrapers/_common/register/$provideFilesystemForNode.ts b/src/scrapers/_common/register/$provideFilesystemForNode.ts index 398049d2c8..853425a739 100644 --- a/src/scrapers/_common/register/$provideFilesystemForNode.ts +++ b/src/scrapers/_common/register/$provideFilesystemForNode.ts @@ -1,34 +1,46 @@ -import { access, constants, mkdir, readdir, readFile, stat, writeFile } from 'fs/promises'; -import { DEFAULT_IS_VERBOSE } from '../../../config'; -import { EnvironmentMismatchError } from '../../../errors/EnvironmentMismatchError'; -import type { FilesystemTools } from '../../../execution/FilesystemTools'; -import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions'; -import { $isRunningInNode } from '../../../utils/environment/$isRunningInNode'; -import { TODO_USE } from '../../../utils/organization/TODO_USE'; +import { + constants, + access, + mkdir, + readFile, + readdir, + stat, + writeFile, +} from "fs/promises"; +import { DEFAULT_IS_VERBOSE } from "../../../config"; +import { EnvironmentMismatchError } from "../../../errors/EnvironmentMismatchError"; +import type { FilesystemTools } from "../../../execution/FilesystemTools"; +import type { PrepareAndScrapeOptions } from "../../../prepare/PrepareAndScrapeOptions"; +import { $isRunningInNode } from "../../../utils/environment/$isRunningInNode"; +import { TODO_USE } from "../../../utils/organization/TODO_USE"; /** * @@@ * * @public exported from `@promptbook/node` */ -export function $provideFilesystemForNode(options?: Pick): FilesystemTools { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment'); - } +export function $provideFilesystemForNode( + options?: Pick, +): FilesystemTools { + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Function `$provideFilesystemForNode` works only in Node.js environment", + ); + } - const { isVerbose = DEFAULT_IS_VERBOSE } = options || {}; + const { isVerbose = DEFAULT_IS_VERBOSE } = options || {}; - TODO_USE(isVerbose); + TODO_USE(isVerbose); - return { - stat, - access, - constants, - readFile, - writeFile, - readdir, - mkdir, - }; + return { + stat, + access, + constants, + readFile, + writeFile, + readdir, + mkdir, + }; } /** diff --git a/src/scrapers/_common/register/$provideScrapersForBrowser.ts b/src/scrapers/_common/register/$provideScrapersForBrowser.ts index ccdf193f46..a30efeb9a1 100644 --- a/src/scrapers/_common/register/$provideScrapersForBrowser.ts +++ b/src/scrapers/_common/register/$provideScrapersForBrowser.ts @@ -1,10 +1,10 @@ -import { EnvironmentMismatchError } from '../../../errors/EnvironmentMismatchError'; -import type { ExecutionTools } from '../../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions'; -import { $isRunningInBrowser } from '../../../utils/environment/$isRunningInBrowser'; -import { $isRunningInWebWorker } from '../../../utils/environment/$isRunningInWebWorker'; -import type { Scraper } from '../Scraper'; -import { $scrapersRegister } from './$scrapersRegister'; +import { EnvironmentMismatchError } from "../../../errors/EnvironmentMismatchError"; +import type { ExecutionTools } from "../../../execution/ExecutionTools"; +import type { PrepareAndScrapeOptions } from "../../../prepare/PrepareAndScrapeOptions"; +import { $isRunningInBrowser } from "../../../utils/environment/$isRunningInBrowser"; +import { $isRunningInWebWorker } from "../../../utils/environment/$isRunningInWebWorker"; +import type { Scraper } from "../Scraper"; +import { $scrapersRegister } from "./$scrapersRegister"; /** * @@@ @@ -15,27 +15,33 @@ import { $scrapersRegister } from './$scrapersRegister'; * @public exported from `@promptbook/browser` */ export async function $provideScrapersForBrowser( - tools: Pick, - options?: PrepareAndScrapeOptions, + tools: Pick, + options?: PrepareAndScrapeOptions, ): Promise> { - if (!$isRunningInBrowser() || $isRunningInWebWorker()) { - throw new EnvironmentMismatchError('Function `$provideScrapersForBrowser` works only in browser environment'); - } + if (!$isRunningInBrowser() || $isRunningInWebWorker()) { + throw new EnvironmentMismatchError( + "Function `$provideScrapersForBrowser` works only in browser environment", + ); + } - const { isAutoInstalled /* Note: [0] Intentionally not assigning a default value = IS_AUTO_INSTALLED */ } = - options || {}; + const { + isAutoInstalled /* Note: [0] Intentionally not assigning a default value = IS_AUTO_INSTALLED */, + } = options || {}; - if ( - isAutoInstalled === true /* <- Note: [0] Ignoring undefined, just checking EXPLICIT requirement for install */ - ) { - throw new EnvironmentMismatchError('Auto-installing is not supported in browser environment'); - } + if ( + isAutoInstalled === + true /* <- Note: [0] Ignoring undefined, just checking EXPLICIT requirement for install */ + ) { + throw new EnvironmentMismatchError( + "Auto-installing is not supported in browser environment", + ); + } - const scrapers: Array = []; - for (const scraperFactory of $scrapersRegister.list()) { - const scraper = await scraperFactory(tools, options || {}); - scrapers.push(scraper); - } + const scrapers: Array = []; + for (const scraperFactory of $scrapersRegister.list()) { + const scraper = await scraperFactory(tools, options || {}); + scrapers.push(scraper); + } - return scrapers; + return scrapers; } diff --git a/src/scrapers/_common/register/$provideScrapersForNode.ts b/src/scrapers/_common/register/$provideScrapersForNode.ts index a40a5e62dc..a5a2ad93a6 100644 --- a/src/scrapers/_common/register/$provideScrapersForNode.ts +++ b/src/scrapers/_common/register/$provideScrapersForNode.ts @@ -1,12 +1,12 @@ -import { DEFAULT_IS_AUTO_INSTALLED } from '../../../config'; -import { DEFAULT_IS_VERBOSE } from '../../../config'; -import { EnvironmentMismatchError } from '../../../errors/EnvironmentMismatchError'; -import type { ExecutionTools } from '../../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions'; -import { $isRunningInNode } from '../../../utils/environment/$isRunningInNode'; -import { TODO_USE } from '../../../utils/organization/TODO_USE'; -import type { Scraper } from '../Scraper'; -import { $scrapersRegister } from './$scrapersRegister'; +import { DEFAULT_IS_AUTO_INSTALLED } from "../../../config"; +import { DEFAULT_IS_VERBOSE } from "../../../config"; +import { EnvironmentMismatchError } from "../../../errors/EnvironmentMismatchError"; +import type { ExecutionTools } from "../../../execution/ExecutionTools"; +import type { PrepareAndScrapeOptions } from "../../../prepare/PrepareAndScrapeOptions"; +import { $isRunningInNode } from "../../../utils/environment/$isRunningInNode"; +import { TODO_USE } from "../../../utils/organization/TODO_USE"; +import type { Scraper } from "../Scraper"; +import { $scrapersRegister } from "./$scrapersRegister"; /** * @@@ @@ -17,36 +17,43 @@ import { $scrapersRegister } from './$scrapersRegister'; * @public exported from `@promptbook/node` */ export async function $provideScrapersForNode( - tools: Pick, - options?: PrepareAndScrapeOptions, + tools: Pick, + options?: PrepareAndScrapeOptions, ): Promise> { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment'); - } - - const { isAutoInstalled = DEFAULT_IS_AUTO_INSTALLED, isVerbose = DEFAULT_IS_VERBOSE } = options || {}; - - TODO_USE(isAutoInstalled); - TODO_USE(isVerbose); - - // TODO: [🔱] Do here auto-installation + auto-include of missing scrapers - use all from $scrapersMetadataRegister.list() - // TODO: [🔱][🧠] What is the best strategy for auto-install - install them all? - - const scrapers: Array = []; - for (const scraperFactory of $scrapersRegister.list()) { - const scraper = await scraperFactory(tools, options || {}); - - if ( - scraper.metadata.packageName === '@promptbook/boilerplate' || - scraper.metadata.mimeTypes.some((mimeType) => mimeType.includes('DISABLED')) - ) { - continue; - } - - scrapers.push(scraper); - } - - return scrapers; + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Function `$getScrapersForNode` works only in Node.js environment", + ); + } + + const { + isAutoInstalled = DEFAULT_IS_AUTO_INSTALLED, + isVerbose = DEFAULT_IS_VERBOSE, + } = options || {}; + + TODO_USE(isAutoInstalled); + TODO_USE(isVerbose); + + // TODO: [🔱] Do here auto-installation + auto-include of missing scrapers - use all from $scrapersMetadataRegister.list() + // TODO: [🔱][🧠] What is the best strategy for auto-install - install them all? + + const scrapers: Array = []; + for (const scraperFactory of $scrapersRegister.list()) { + const scraper = await scraperFactory(tools, options || {}); + + if ( + scraper.metadata.packageName === "@promptbook/boilerplate" || + scraper.metadata.mimeTypes.some((mimeType) => + mimeType.includes("DISABLED"), + ) + ) { + continue; + } + + scrapers.push(scraper); + } + + return scrapers; } /** diff --git a/src/scrapers/_common/register/$registeredScrapersMessage.ts b/src/scrapers/_common/register/$registeredScrapersMessage.ts index 962a3e81fd..b005aec798 100644 --- a/src/scrapers/_common/register/$registeredScrapersMessage.ts +++ b/src/scrapers/_common/register/$registeredScrapersMessage.ts @@ -1,10 +1,10 @@ -import spaceTrim from 'spacetrim'; -import type { string_markdown } from '../../../types/typeAliases'; -import type { string_markdown_text } from '../../../types/typeAliases'; -import type { Scraper } from '../Scraper'; -import { $scrapersMetadataRegister } from './$scrapersMetadataRegister'; -import { $scrapersRegister } from './$scrapersRegister'; -import type { ScraperAndConverterMetadata } from './ScraperAndConverterMetadata'; +import spaceTrim from "spacetrim"; +import type { string_markdown } from "../../../types/typeAliases"; +import type { string_markdown_text } from "../../../types/typeAliases"; +import type { Scraper } from "../Scraper"; +import { $scrapersMetadataRegister } from "./$scrapersMetadataRegister"; +import { $scrapersRegister } from "./$scrapersRegister"; +import type { ScraperAndConverterMetadata } from "./ScraperAndConverterMetadata"; /** * Creates a message with all registered scrapers @@ -13,139 +13,176 @@ import type { ScraperAndConverterMetadata } from './ScraperAndConverterMetadata' * * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv` */ -export function $registeredScrapersMessage(availableScrapers: ReadonlyArray): string_markdown { - /** - * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister - */ - const all: Array< - Pick< - ScraperAndConverterMetadata, - 'packageName' | 'className' | 'mimeTypes' | 'documentationUrl' | 'isAvilableInBrowser' - > - > = []; - - for (const { - packageName, - className, - mimeTypes, - documentationUrl, - isAvilableInBrowser, - } of $scrapersMetadataRegister.list()) { - if (all.some((item) => item.packageName === packageName && item.className === className)) { - continue; - } - all.push({ packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser }); - } - - for (const { - packageName, - className, - mimeTypes, - documentationUrl, - isAvilableInBrowser, - } of $scrapersRegister.list()) { - if (all.some((item) => item.packageName === packageName && item.className === className)) { - continue; - } - all.push({ packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser }); - } - - for (const { metadata } of availableScrapers) { - all.push(metadata); - } - - const metadata = all.map((metadata) => { - const isMetadataAviailable = $scrapersMetadataRegister - .list() - .find( - ({ packageName, className }) => - metadata.packageName === packageName && metadata.className === className, - ); - - const isInstalled = $scrapersRegister - .list() - .find( - ({ packageName, className }) => - metadata.packageName === packageName && metadata.className === className, - ); - - const isAvilableInTools = availableScrapers.some( - ({ metadata: { packageName, className } }) => - metadata.packageName === packageName && metadata.className === className, - ); - - return { ...metadata, isMetadataAviailable, isInstalled, isAvilableInTools }; - }); - - if (metadata.length === 0) { - return spaceTrim(` +export function $registeredScrapersMessage( + availableScrapers: ReadonlyArray, +): string_markdown { + /** + * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister + */ + const all: Array< + Pick< + ScraperAndConverterMetadata, + | "packageName" + | "className" + | "mimeTypes" + | "documentationUrl" + | "isAvilableInBrowser" + > + > = []; + + for (const { + packageName, + className, + mimeTypes, + documentationUrl, + isAvilableInBrowser, + } of $scrapersMetadataRegister.list()) { + if ( + all.some( + (item) => + item.packageName === packageName && item.className === className, + ) + ) { + continue; + } + all.push({ + packageName, + className, + mimeTypes, + documentationUrl, + isAvilableInBrowser, + }); + } + + for (const { + packageName, + className, + mimeTypes, + documentationUrl, + isAvilableInBrowser, + } of $scrapersRegister.list()) { + if ( + all.some( + (item) => + item.packageName === packageName && item.className === className, + ) + ) { + continue; + } + all.push({ + packageName, + className, + mimeTypes, + documentationUrl, + isAvilableInBrowser, + }); + } + + for (const { metadata } of availableScrapers) { + all.push(metadata); + } + + const metadata = all.map((metadata) => { + const isMetadataAviailable = $scrapersMetadataRegister + .list() + .find( + ({ packageName, className }) => + metadata.packageName === packageName && + metadata.className === className, + ); + + const isInstalled = $scrapersRegister + .list() + .find( + ({ packageName, className }) => + metadata.packageName === packageName && + metadata.className === className, + ); + + const isAvilableInTools = availableScrapers.some( + ({ metadata: { packageName, className } }) => + metadata.packageName === packageName && + metadata.className === className, + ); + + return { + ...metadata, + isMetadataAviailable, + isInstalled, + isAvilableInTools, + }; + }); + + if (metadata.length === 0) { + return spaceTrim(` **No scrapers are available** This is a unexpected behavior, you are probably using some broken version of Promptbook At least there should be available the metadata of the scrapers `); - } + } - return spaceTrim( - (block) => ` + return spaceTrim( + (block) => ` Available scrapers are: ${block( - metadata - .map( - ( - { - packageName, - className, - isMetadataAviailable, - isInstalled, - mimeTypes, - isAvilableInBrowser, - isAvilableInTools, - }, - i, - ) => { - const more: Array = []; - - // TODO: [🧠] Maybe use `documentationUrl` - - if (isMetadataAviailable) { - more.push(`⬜ Metadata registered`); - } // not else - - if (isInstalled) { - more.push(`🟩 Installed`); - } // not else - if (isAvilableInTools) { - more.push(`🟦 Available in tools`); - } // not else - - if (!isMetadataAviailable && isInstalled) { - more.push( - `When no metadata registered but scraper is installed, it is an unexpected behavior`, - ); - } // not else - - if (!isInstalled && isAvilableInTools) { - more.push( - `When the scraper is not installed but available in tools, it is an unexpected compatibility behavior`, - ); - } // not else - - if (!isAvilableInBrowser) { - more.push(`Not usable in browser`); - } - - const moreText = more.length === 0 ? '' : ` *(${more.join('; ')})*`; - - return `${i + 1}) \`${className}\` from \`${packageName}\` compatible to scrape ${mimeTypes - .map((mimeType) => `"${mimeType}"`) - .join( - ', ', // <- TODO: Some smart join A, B, C and D - )}${moreText}`; - }, - ) - .join('\n'), - )} + metadata + .map( + ( + { + packageName, + className, + isMetadataAviailable, + isInstalled, + mimeTypes, + isAvilableInBrowser, + isAvilableInTools, + }, + i, + ) => { + const more: Array = []; + + // TODO: [🧠] Maybe use `documentationUrl` + + if (isMetadataAviailable) { + more.push(`⬜ Metadata registered`); + } // not else + + if (isInstalled) { + more.push(`🟩 Installed`); + } // not else + if (isAvilableInTools) { + more.push(`🟦 Available in tools`); + } // not else + + if (!isMetadataAviailable && isInstalled) { + more.push( + `When no metadata registered but scraper is installed, it is an unexpected behavior`, + ); + } // not else + + if (!isInstalled && isAvilableInTools) { + more.push( + `When the scraper is not installed but available in tools, it is an unexpected compatibility behavior`, + ); + } // not else + + if (!isAvilableInBrowser) { + more.push(`Not usable in browser`); + } + + const moreText = + more.length === 0 ? "" : ` *(${more.join("; ")})*`; + + return `${i + 1}) \`${className}\` from \`${packageName}\` compatible to scrape ${mimeTypes + .map((mimeType) => `"${mimeType}"`) + .join( + ", ", // <- TODO: Some smart join A, B, C and D + )}${moreText}`; + }, + ) + .join("\n"), + )} Legend: - ⬜ **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry @@ -153,7 +190,7 @@ export function $registeredScrapersMessage(availableScrapers: ReadonlyArray('scrapers_metadata'); +export const $scrapersMetadataRegister = + new $Register("scrapers_metadata"); /** * TODO: [®] DRY Register logic diff --git a/src/scrapers/_common/register/$scrapersRegister.ts b/src/scrapers/_common/register/$scrapersRegister.ts index e9224a4a00..b02e5e86ac 100644 --- a/src/scrapers/_common/register/$scrapersRegister.ts +++ b/src/scrapers/_common/register/$scrapersRegister.ts @@ -1,5 +1,5 @@ -import { $Register } from '../../../utils/$Register'; -import type { ScraperConstructor } from './ScraperConstructor'; +import { $Register } from "../../../utils/$Register"; +import type { ScraperConstructor } from "./ScraperConstructor"; /** * @@@ @@ -8,7 +8,9 @@ import type { ScraperConstructor } from './ScraperConstructor'; * @singleton Only one instance of each register is created per build, but thare can be more @@@ * @public exported from `@promptbook/core` */ -export const $scrapersRegister = new $Register('scraper_constructors'); +export const $scrapersRegister = new $Register( + "scraper_constructors", +); /** * TODO: [®] DRY Register logic diff --git a/src/scrapers/_common/register/ScraperAndConverterMetadata.ts b/src/scrapers/_common/register/ScraperAndConverterMetadata.ts index 7e0df743f1..ccd8f47339 100644 --- a/src/scrapers/_common/register/ScraperAndConverterMetadata.ts +++ b/src/scrapers/_common/register/ScraperAndConverterMetadata.ts @@ -1,8 +1,8 @@ -import type { Executables } from '../../../execution/Executables'; -import type { string_mime_type } from '../../../types/typeAliases'; -import type { string_promptbook_documentation_url } from '../../../types/typeAliases'; -import type { string_title } from '../../../types/typeAliases'; -import type { Registered } from '../../../utils/$Register'; +import type { Executables } from "../../../execution/Executables"; +import type { string_mime_type } from "../../../types/typeAliases"; +import type { string_promptbook_documentation_url } from "../../../types/typeAliases"; +import type { string_title } from "../../../types/typeAliases"; +import type { Registered } from "../../../utils/$Register"; /** * @@@ @@ -19,30 +19,32 @@ import type { Registered } from '../../../utils/$Register'; * x) `ScraperOptions` */ export type ScraperAndConverterMetadata = Registered & { - /** - * @@@ - */ - readonly title: string_title; + /** + * @@@ + */ + readonly title: string_title; - /** - * Mime types that this scraper can handle - */ - readonly mimeTypes: ReadonlyArray; + /** + * Mime types that this scraper can handle + */ + readonly mimeTypes: ReadonlyArray; - /** - * @@@ - * - * Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server - */ - readonly isAvilableInBrowser: boolean; + /** + * @@@ + * + * Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server + */ + readonly isAvilableInBrowser: boolean; - /** - * @@@ - */ - readonly requiredExecutables: ReadonlyArray>; + /** + * @@@ + */ + readonly requiredExecutables: ReadonlyArray< + Capitalize + >; - /** - * Link to documentation - */ - readonly documentationUrl: string_promptbook_documentation_url; + /** + * Link to documentation + */ + readonly documentationUrl: string_promptbook_documentation_url; }; diff --git a/src/scrapers/_common/register/ScraperConstructor.ts b/src/scrapers/_common/register/ScraperConstructor.ts index 3cf644a385..2e468ca9a3 100644 --- a/src/scrapers/_common/register/ScraperConstructor.ts +++ b/src/scrapers/_common/register/ScraperConstructor.ts @@ -1,15 +1,18 @@ -import type { ExecutionTools } from '../../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions'; -import type { Registered } from '../../../utils/$Register'; -import type { Scraper } from '../Scraper'; -import type { ScraperAndConverterMetadata } from './ScraperAndConverterMetadata'; +import type { ExecutionTools } from "../../../execution/ExecutionTools"; +import type { PrepareAndScrapeOptions } from "../../../prepare/PrepareAndScrapeOptions"; +import type { Registered } from "../../../utils/$Register"; +import type { Scraper } from "../Scraper"; +import type { ScraperAndConverterMetadata } from "./ScraperAndConverterMetadata"; /** * @@@ */ export type ScraperConstructor = Registered & - ScraperAndConverterMetadata & - ((tools: Pick, options: PrepareAndScrapeOptions) => Scraper); + ScraperAndConverterMetadata & + (( + tools: Pick, + options: PrepareAndScrapeOptions, + ) => Scraper); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/scrapers/_common/utils/getScraperIntermediateSource.test.ts b/src/scrapers/_common/utils/getScraperIntermediateSource.test.ts index 4a77984987..6d80483a78 100644 --- a/src/scrapers/_common/utils/getScraperIntermediateSource.test.ts +++ b/src/scrapers/_common/utils/getScraperIntermediateSource.test.ts @@ -1,49 +1,52 @@ -import { describe, expect, it } from '@jest/globals'; -import { join } from 'path'; -import { getScraperIntermediateSource } from './getScraperIntermediateSource'; +import { join } from "path"; +import { describe, expect, it } from "@jest/globals"; +import { getScraperIntermediateSource } from "./getScraperIntermediateSource"; -describe('how `getScraperIntermediateSource` works', () => { - it('should create cache filename for file source', () => - expect( - getScraperIntermediateSource( - { - filename: 'prague.pdf', - url: null, - }, - { - rootDirname: '/home/user/coolproject/books/', - cacheDirname: '/home/user/coolproject/.promptbook/', - intermediateFilesStrategy: 'HIDE_AND_KEEP', - isVerbose: false, - extension: 'md', - }, - ).then(({ filename }) => filename), - ).resolves.toBe( - join( - process.cwd(), - '/home/user/coolproject/.promptbook/8/0/intermediate-prague-pdf-80efa46cc0147c9b65fd.md', - ) - .split('\\') - .join('/'), - )); +describe("how `getScraperIntermediateSource` works", () => { + it("should create cache filename for file source", () => + expect( + getScraperIntermediateSource( + { + filename: "prague.pdf", + url: null, + }, + { + rootDirname: "/home/user/coolproject/books/", + cacheDirname: "/home/user/coolproject/.promptbook/", + intermediateFilesStrategy: "HIDE_AND_KEEP", + isVerbose: false, + extension: "md", + }, + ).then(({ filename }) => filename), + ).resolves.toBe( + join( + process.cwd(), + "/home/user/coolproject/.promptbook/8/0/intermediate-prague-pdf-80efa46cc0147c9b65fd.md", + ) + .split("\\") + .join("/"), + )); - it('should create filename for url source', () => - expect( - getScraperIntermediateSource( - { filename: null, url: 'https://praha.eu/' }, - { - rootDirname: '/home/user/coolproject/books/', - cacheDirname: '/home/user/coolproject/.promptbook/', - intermediateFilesStrategy: 'HIDE_AND_KEEP', - isVerbose: false, - extension: 'md', - }, - ).then(({ filename }) => filename), - ).resolves.toBe( - join(process.cwd(), '/home/user/coolproject/.promptbook/2/0/intermediate-praha-eu-203eb08e746a0a13aad7.md') - .split('\\') - .join('/'), - )); + it("should create filename for url source", () => + expect( + getScraperIntermediateSource( + { filename: null, url: "https://praha.eu/" }, + { + rootDirname: "/home/user/coolproject/books/", + cacheDirname: "/home/user/coolproject/.promptbook/", + intermediateFilesStrategy: "HIDE_AND_KEEP", + isVerbose: false, + extension: "md", + }, + ).then(({ filename }) => filename), + ).resolves.toBe( + join( + process.cwd(), + "/home/user/coolproject/.promptbook/2/0/intermediate-praha-eu-203eb08e746a0a13aad7.md", + ) + .split("\\") + .join("/"), + )); }); /** diff --git a/src/scrapers/_common/utils/getScraperIntermediateSource.ts b/src/scrapers/_common/utils/getScraperIntermediateSource.ts index ab64b25bea..4952091b60 100644 --- a/src/scrapers/_common/utils/getScraperIntermediateSource.ts +++ b/src/scrapers/_common/utils/getScraperIntermediateSource.ts @@ -1,22 +1,25 @@ -import { SHA256 as sha256 } from 'crypto-js'; -import hexEncoder from 'crypto-js/enc-hex'; -import { mkdir, rm } from 'fs/promises'; -import { dirname, join } from 'path'; -import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions'; -import { nameToSubfolderPath } from '../../../storage/file-cache-storage/utils/nameToSubfolderPath'; -import type { string_file_extension } from '../../../types/typeAliases'; -import { normalizeToKebabCase } from '../../../utils/normalization/normalize-to-kebab-case'; -import { titleToName } from '../../../utils/normalization/titleToName'; -import { TODO_USE } from '../../../utils/organization/TODO_USE'; -import type { ScraperSourceHandler } from '../Scraper'; -import type { ScraperIntermediateSource } from '../ScraperIntermediateSource'; +import { dirname, join } from "path"; +import { SHA256 as sha256 } from "crypto-js"; +import hexEncoder from "crypto-js/enc-hex"; +import { mkdir, rm } from "fs/promises"; +import type { PrepareAndScrapeOptions } from "../../../prepare/PrepareAndScrapeOptions"; +import { nameToSubfolderPath } from "../../../storage/file-cache-storage/utils/nameToSubfolderPath"; +import type { string_file_extension } from "../../../types/typeAliases"; +import { normalizeToKebabCase } from "../../../utils/normalization/normalize-to-kebab-case"; +import { titleToName } from "../../../utils/normalization/titleToName"; +import { TODO_USE } from "../../../utils/organization/TODO_USE"; +import type { ScraperSourceHandler } from "../Scraper"; +import type { ScraperIntermediateSource } from "../ScraperIntermediateSource"; /** * @@@ * * @private internal utility of `getScraperIntermediateSource` */ -type GetScraperIntermediateSourceSource = Pick; +type GetScraperIntermediateSourceSource = Pick< + ScraperSourceHandler, + "filename" | "url" +>; /** * @@@ @@ -24,9 +27,12 @@ type GetScraperIntermediateSourceSource = Pick + Pick< + PrepareAndScrapeOptions, + "rootDirname" | "cacheDirname" | "intermediateFilesStrategy" | "isVerbose" + > > & { - readonly extension: string_file_extension; + readonly extension: string_file_extension; }; /** @@ -37,79 +43,89 @@ type GetScraperIntermediateSourceHandler = Required< * @private as internal utility for scrapers */ export async function getScraperIntermediateSource( - source: GetScraperIntermediateSourceSource, - options: GetScraperIntermediateSourceHandler, + source: GetScraperIntermediateSourceSource, + options: GetScraperIntermediateSourceHandler, ): Promise { - const { filename: sourceFilename, url } = source; - const { rootDirname, cacheDirname, intermediateFilesStrategy, extension, isVerbose } = options; + const { filename: sourceFilename, url } = source; + const { + rootDirname, + cacheDirname, + intermediateFilesStrategy, + extension, + isVerbose, + } = options; - // TODO: [👬] DRY - const hash = sha256( - // <- TODO: [🥬] Encapsulate sha256 to some private utility function - hexEncoder.parse( - sourceFilename || url || 'untitled', - // <- TODO: [🧠] Probbably hash file content instead of filename - now hash does not provide any extra value - ), - ) - .toString(/* hex */) - .substring( - 0, - 20, - // <- TODO: Use MAX_FILENAME_LENGTH - ); - // <- TODO: [🥬] Make some system for hashes and ids of promptbook + // TODO: [👬] DRY + const hash = sha256( + // <- TODO: [🥬] Encapsulate sha256 to some private utility function + hexEncoder.parse( + sourceFilename || url || "untitled", + // <- TODO: [🧠] Probbably hash file content instead of filename - now hash does not provide any extra value + ), + ) + .toString(/* hex */) + .substring( + 0, + 20, + // <- TODO: Use MAX_FILENAME_LENGTH + ); + // <- TODO: [🥬] Make some system for hashes and ids of promptbook - const semanticName = normalizeToKebabCase( - titleToName((sourceFilename || url || '').split('intermediate').join('')), - ).substring( - 0, - 20, - // <- TODO: Use MAX_FILENAME_LENGTH - // <- TODO: [🐱‍🐉] - ); - // <- TODO: [🐱‍🐉] + const semanticName = normalizeToKebabCase( + titleToName((sourceFilename || url || "").split("intermediate").join("")), + ).substring( + 0, + 20, + // <- TODO: Use MAX_FILENAME_LENGTH + // <- TODO: [🐱‍🐉] + ); + // <- TODO: [🐱‍🐉] - const pieces = ['intermediate', semanticName, hash].filter((piece) => piece !== ''); + const pieces = ["intermediate", semanticName, hash].filter( + (piece) => piece !== "", + ); - const name = pieces.join('-').split('--').join('-'); - // <- TODO: Use MAX_FILENAME_LENGTH + const name = pieces.join("-").split("--").join("-"); + // <- TODO: Use MAX_FILENAME_LENGTH - TODO_USE(rootDirname); // <- TODO: [😡] + TODO_USE(rootDirname); // <- TODO: [😡] - const cacheFilename = - join( - process.cwd(), - cacheDirname, - ...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), - name, - ) - .split('\\') - .join('/') + - '.' + - extension; + const cacheFilename = + join( + process.cwd(), + cacheDirname, + ...nameToSubfolderPath( + hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */, + ), + name, + ) + .split("\\") + .join("/") + + "." + + extension; - await mkdir(dirname(cacheFilename), { recursive: true }); + await mkdir(dirname(cacheFilename), { recursive: true }); - let isDestroyed = true; - const fileHandler = { - filename: cacheFilename, - get isDestroyed() { - return isDestroyed; - }, - async destroy() { - if (intermediateFilesStrategy === 'HIDE_AND_CLEAN') { - if (isVerbose) { - console.info('legacyDocumentScraper: Clening cache'); - } - await rm(cacheFilename); - // TODO: [🐿][🧠] Maybe remove empty folders - } + let isDestroyed = true; + const fileHandler = { + filename: cacheFilename, + get isDestroyed() { + return isDestroyed; + }, + async destroy() { + if (intermediateFilesStrategy === "HIDE_AND_CLEAN") { + if (isVerbose) { + console.info("legacyDocumentScraper: Clening cache"); + } + await rm(cacheFilename); + // TODO: [🐿][🧠] Maybe remove empty folders + } - isDestroyed = true; - }, - } satisfies ScraperIntermediateSource; + isDestroyed = true; + }, + } satisfies ScraperIntermediateSource; - return fileHandler; + return fileHandler; } /** diff --git a/src/scrapers/_common/utils/makeKnowledgeSourceHandler.ts b/src/scrapers/_common/utils/makeKnowledgeSourceHandler.ts index 02078bb39d..187b8a75ad 100644 --- a/src/scrapers/_common/utils/makeKnowledgeSourceHandler.ts +++ b/src/scrapers/_common/utils/makeKnowledgeSourceHandler.ts @@ -1,31 +1,31 @@ -import hexEncoder from 'crypto-js/enc-hex'; -import sha256 from 'crypto-js/sha256'; -import { dirname, join } from 'path'; -import spaceTrim from 'spacetrim'; -import type { SetOptional } from 'type-fest'; -import { knowledgeSourceContentToName } from '../../../commands/KNOWLEDGE/utils/knowledgeSourceContentToName'; -import { DEFAULT_DOWNLOAD_CACHE_DIRNAME } from '../../../config'; -import { DEFAULT_IS_VERBOSE } from '../../../config'; -import { DEFAULT_MAX_FILE_SIZE } from '../../../config'; -import { MAX_FILENAME_LENGTH } from '../../../config'; -import { EnvironmentMismatchError } from '../../../errors/EnvironmentMismatchError'; -import { LimitReachedError } from '../../../errors/LimitReachedError'; -import { NotFoundError } from '../../../errors/NotFoundError'; -import { UnexpectedError } from '../../../errors/UnexpectedError'; -import type { ExecutionTools } from '../../../execution/ExecutionTools'; -import type { KnowledgeSourceJson } from '../../../pipeline/PipelineJson/KnowledgeSourceJson'; -import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions'; -import { nameToSubfolderPath } from '../../../storage/file-cache-storage/utils/nameToSubfolderPath'; -import { extensionToMimeType } from '../../../utils/files/extensionToMimeType'; -import { getFileExtension } from '../../../utils/files/getFileExtension'; -import { isFileExisting } from '../../../utils/files/isFileExisting'; -import { mimeTypeToExtension } from '../../../utils/files/mimeTypeToExtension'; -import { titleToName } from '../../../utils/normalization/titleToName'; -import { TODO_USE } from '../../../utils/organization/TODO_USE'; -import { isValidFilePath } from '../../../utils/validators/filePath/isValidFilePath'; -import { isValidUrl } from '../../../utils/validators/url/isValidUrl'; -import type { ScraperSourceHandler } from '../Scraper'; -import { scraperFetch } from './scraperFetch'; +import { dirname, join } from "path"; +import hexEncoder from "crypto-js/enc-hex"; +import sha256 from "crypto-js/sha256"; +import spaceTrim from "spacetrim"; +import type { SetOptional } from "type-fest"; +import { knowledgeSourceContentToName } from "../../../commands/KNOWLEDGE/utils/knowledgeSourceContentToName"; +import { DEFAULT_DOWNLOAD_CACHE_DIRNAME } from "../../../config"; +import { DEFAULT_IS_VERBOSE } from "../../../config"; +import { DEFAULT_MAX_FILE_SIZE } from "../../../config"; +import { MAX_FILENAME_LENGTH } from "../../../config"; +import { EnvironmentMismatchError } from "../../../errors/EnvironmentMismatchError"; +import { LimitReachedError } from "../../../errors/LimitReachedError"; +import { NotFoundError } from "../../../errors/NotFoundError"; +import { UnexpectedError } from "../../../errors/UnexpectedError"; +import type { ExecutionTools } from "../../../execution/ExecutionTools"; +import type { KnowledgeSourceJson } from "../../../pipeline/PipelineJson/KnowledgeSourceJson"; +import type { PrepareAndScrapeOptions } from "../../../prepare/PrepareAndScrapeOptions"; +import { nameToSubfolderPath } from "../../../storage/file-cache-storage/utils/nameToSubfolderPath"; +import { extensionToMimeType } from "../../../utils/files/extensionToMimeType"; +import { getFileExtension } from "../../../utils/files/getFileExtension"; +import { isFileExisting } from "../../../utils/files/isFileExisting"; +import { mimeTypeToExtension } from "../../../utils/files/mimeTypeToExtension"; +import { titleToName } from "../../../utils/normalization/titleToName"; +import { TODO_USE } from "../../../utils/organization/TODO_USE"; +import { isValidFilePath } from "../../../utils/validators/filePath/isValidFilePath"; +import { isValidUrl } from "../../../utils/validators/url/isValidUrl"; +import type { ScraperSourceHandler } from "../Scraper"; +import { scraperFetch } from "./scraperFetch"; /** * @@@ @@ -33,39 +33,40 @@ import { scraperFetch } from './scraperFetch'; * @public exported from `@promptbook/core` */ export async function makeKnowledgeSourceHandler( - knowledgeSource: SetOptional, - tools: Pick, - options?: Pick, + knowledgeSource: SetOptional, + tools: Pick, + options?: Pick, ): Promise { - // console.log('!! makeKnowledgeSourceHandler', knowledgeSource); - - const { fetch = scraperFetch } = tools; - const { knowledgeSourceContent } = knowledgeSource; - let { name } = knowledgeSource; - const { - rootDirname = null, - // <- TODO: process.cwd() if running in Node.js - isVerbose = DEFAULT_IS_VERBOSE, - } = options || {}; - - TODO_USE(isVerbose); - - if (!name) { - name = knowledgeSourceContentToName(knowledgeSourceContent); - } - - if (isValidUrl(knowledgeSourceContent)) { - const url = knowledgeSourceContent; - const response = await fetch(url); // <- TODO: [🧠] Scraping and fetch proxy - const mimeType = response.headers.get('content-type')?.split(';')[0] || 'text/html'; - - if (tools.fs === undefined || !url.endsWith('.pdf' /* <- TODO: [💵] */)) { - return { - source: name, - filename: null, - url, - mimeType, - /* + // console.log('!! makeKnowledgeSourceHandler', knowledgeSource); + + const { fetch = scraperFetch } = tools; + const { knowledgeSourceContent } = knowledgeSource; + let { name } = knowledgeSource; + const { + rootDirname = null, + // <- TODO: process.cwd() if running in Node.js + isVerbose = DEFAULT_IS_VERBOSE, + } = options || {}; + + TODO_USE(isVerbose); + + if (!name) { + name = knowledgeSourceContentToName(knowledgeSourceContent); + } + + if (isValidUrl(knowledgeSourceContent)) { + const url = knowledgeSourceContent; + const response = await fetch(url); // <- TODO: [🧠] Scraping and fetch proxy + const mimeType = + response.headers.get("content-type")?.split(";")[0] || "text/html"; + + if (tools.fs === undefined || !url.endsWith(".pdf" /* <- TODO: [💵] */)) { + return { + source: name, + filename: null, + url, + mimeType, + /* TODO: [🥽] > async asBlob() { > // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed @@ -73,73 +74,90 @@ export async function makeKnowledgeSourceHandler( > return content; > }, */ - async asJson() { - // TODO: [👨🏻‍🤝‍👨🏻] - const content = await response.json(); - return content; - }, - async asText() { - // TODO: [👨🏻‍🤝‍👨🏻] - const content = await response.text(); - return content; - }, - }; - } - - const basename = url.split('/').pop() || titleToName(url); - const hash = sha256(hexEncoder.parse(url)).toString(/* hex */); - // <- TODO: [🥬] Encapsulate sha256 to some private utility function - - const rootDirname = join( - process.cwd(), - DEFAULT_DOWNLOAD_CACHE_DIRNAME, - // <- TODO: [🦒] Allow to override (pass different value into the function) - ); - - const filepath = join( - ...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), - `${basename.substring(0, MAX_FILENAME_LENGTH)}.${mimeTypeToExtension(mimeType)}`, - ); - - await tools.fs!.mkdir(dirname(join(rootDirname, filepath)), { recursive: true }); - const fileContent = Buffer.from(await response.arrayBuffer()); - - if (fileContent.length > DEFAULT_MAX_FILE_SIZE /* <- TODO: Allow to pass different value to remote server */) { - throw new LimitReachedError( - `File is too large (${Math.round( - fileContent.length / 1024 / 1024, - )}MB). Maximum allowed size is ${Math.round(DEFAULT_MAX_FILE_SIZE / 1024 / 1024)}MB.`, - ); - } - - await tools.fs!.writeFile(join(rootDirname, filepath), fileContent); - - // TODO: [💵] Check the file security - // TODO: [🧹][🧠] Delete the file after the scraping is done - - return makeKnowledgeSourceHandler({ name, knowledgeSourceContent: filepath }, tools, { - ...options, - rootDirname, - }); - } else if (isValidFilePath(knowledgeSourceContent)) { - if (tools.fs === undefined) { - throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools'); - // <- TODO: [🧠] What is the best error type here` - } - - if (rootDirname === null) { - throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline'); - // <- TODO: [🧠] What is the best error type here` - } - - const filename = join(rootDirname, knowledgeSourceContent).split('\\').join('/'); - const fileExtension = getFileExtension(filename); - const mimeType = extensionToMimeType(fileExtension || ''); - - if (!(await isFileExisting(filename, tools.fs))) { - throw new NotFoundError( - spaceTrim( - (block) => ` + async asJson() { + // TODO: [👨🏻‍🤝‍👨🏻] + const content = await response.json(); + return content; + }, + async asText() { + // TODO: [👨🏻‍🤝‍👨🏻] + const content = await response.text(); + return content; + }, + }; + } + + const basename = url.split("/").pop() || titleToName(url); + const hash = sha256(hexEncoder.parse(url)).toString(/* hex */); + // <- TODO: [🥬] Encapsulate sha256 to some private utility function + + const rootDirname = join( + process.cwd(), + DEFAULT_DOWNLOAD_CACHE_DIRNAME, + // <- TODO: [🦒] Allow to override (pass different value into the function) + ); + + const filepath = join( + ...nameToSubfolderPath( + hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */, + ), + `${basename.substring(0, MAX_FILENAME_LENGTH)}.${mimeTypeToExtension(mimeType)}`, + ); + + await tools.fs!.mkdir(dirname(join(rootDirname, filepath)), { + recursive: true, + }); + const fileContent = Buffer.from(await response.arrayBuffer()); + + if ( + fileContent.length > + DEFAULT_MAX_FILE_SIZE /* <- TODO: Allow to pass different value to remote server */ + ) { + throw new LimitReachedError( + `File is too large (${Math.round( + fileContent.length / 1024 / 1024, + )}MB). Maximum allowed size is ${Math.round(DEFAULT_MAX_FILE_SIZE / 1024 / 1024)}MB.`, + ); + } + + await tools.fs!.writeFile(join(rootDirname, filepath), fileContent); + + // TODO: [💵] Check the file security + // TODO: [🧹][🧠] Delete the file after the scraping is done + + return makeKnowledgeSourceHandler( + { name, knowledgeSourceContent: filepath }, + tools, + { + ...options, + rootDirname, + }, + ); + } else if (isValidFilePath(knowledgeSourceContent)) { + if (tools.fs === undefined) { + throw new EnvironmentMismatchError( + "Can not import file knowledge without filesystem tools", + ); + // <- TODO: [🧠] What is the best error type here` + } + + if (rootDirname === null) { + throw new EnvironmentMismatchError( + "Can not import file knowledge in non-file pipeline", + ); + // <- TODO: [🧠] What is the best error type here` + } + + const filename = join(rootDirname, knowledgeSourceContent) + .split("\\") + .join("/"); + const fileExtension = getFileExtension(filename); + const mimeType = extensionToMimeType(fileExtension || ""); + + if (!(await isFileExisting(filename, tools.fs))) { + throw new NotFoundError( + spaceTrim( + (block) => ` Can not make source handler for file which does not exist: File: @@ -148,18 +166,18 @@ export async function makeKnowledgeSourceHandler( Full file path: ${block(filename)} `, - ), - ); - } - - // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`) - - return { - source: name, - filename, - url: null, - mimeType, - /* + ), + ); + } + + // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`) + + return { + source: name, + filename, + url: null, + mimeType, + /* TODO: [🥽] > async asBlob() { > const content = await tools.fs!.readFile(filename); @@ -172,28 +190,28 @@ export async function makeKnowledgeSourceHandler( > ); > }, */ - async asJson() { - return JSON.parse(await tools.fs!.readFile(filename, 'utf-8')); - }, - async asText() { - return await tools.fs!.readFile(filename, 'utf-8'); - }, - }; - } else { - return { - source: name, - filename: null, - url: null, - mimeType: 'text/markdown', - asText() { - return knowledgeSource.knowledgeSourceContent; - }, - asJson() { - throw new UnexpectedError( - 'Did not expect that `markdownScraper` would need to get the content `asJson`', - ); - }, - /* + async asJson() { + return JSON.parse(await tools.fs!.readFile(filename, "utf-8")); + }, + async asText() { + return await tools.fs!.readFile(filename, "utf-8"); + }, + }; + } else { + return { + source: name, + filename: null, + url: null, + mimeType: "text/markdown", + asText() { + return knowledgeSource.knowledgeSourceContent; + }, + asJson() { + throw new UnexpectedError( + "Did not expect that `markdownScraper` would need to get the content `asJson`", + ); + }, + /* TODO: [🥽] > asBlob() { > throw new UnexpectedError( @@ -201,6 +219,6 @@ export async function makeKnowledgeSourceHandler( > ); > }, */ - }; - } + }; + } } diff --git a/src/scrapers/_common/utils/scraperFetch.ts b/src/scrapers/_common/utils/scraperFetch.ts index dc92593b84..24dc8b1901 100644 --- a/src/scrapers/_common/utils/scraperFetch.ts +++ b/src/scrapers/_common/utils/scraperFetch.ts @@ -1,33 +1,36 @@ -import spaceTrim from 'spacetrim'; -import { KnowledgeScrapeError } from '../../../errors/KnowledgeScrapeError'; -import type { PromptbookFetch } from '../../../execution/PromptbookFetch'; -import type { string_url } from '../../../types/typeAliases'; +import spaceTrim from "spacetrim"; +import { KnowledgeScrapeError } from "../../../errors/KnowledgeScrapeError"; +import type { PromptbookFetch } from "../../../execution/PromptbookFetch"; +import type { string_url } from "../../../types/typeAliases"; /** * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers * * @private as default `fetch` function used in Promptbook scrapers */ -export const scraperFetch: PromptbookFetch = async (url: string_url, init?: RequestInit): Promise => { - try { - return await fetch(url, init); - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } +export const scraperFetch: PromptbookFetch = async ( + url: string_url, + init?: RequestInit, +): Promise => { + try { + return await fetch(url, init); + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } - throw new KnowledgeScrapeError( - spaceTrim( - (block) => ` + throw new KnowledgeScrapeError( + spaceTrim( + (block) => ` Can not fetch "${url}" Fetch error: ${block((error as Error).message)} `, - ), - ); - } + ), + ); + } }; /** diff --git a/src/scrapers/document-legacy/LegacyDocumentScraper.test.ts b/src/scrapers/document-legacy/LegacyDocumentScraper.test.ts index a55a61a37e..37a3976efa 100644 --- a/src/scrapers/document-legacy/LegacyDocumentScraper.test.ts +++ b/src/scrapers/document-legacy/LegacyDocumentScraper.test.ts @@ -1,87 +1,93 @@ -import { describe, expect, it } from '@jest/globals'; -import { join } from 'path'; -import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode'; -import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground'; -import { $provideFilesystemForNode } from '../_common/register/$provideFilesystemForNode'; -import { makeKnowledgeSourceHandler } from '../_common/utils/makeKnowledgeSourceHandler'; -import { LegacyDocumentScraper } from './LegacyDocumentScraper'; +import { join } from "path"; +import { describe, expect, it } from "@jest/globals"; +import { $provideExecutablesForNode } from "../../executables/$provideExecutablesForNode"; +import { $provideLlmToolsForTestingAndScriptsAndPlayground } from "../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground"; +import { $provideFilesystemForNode } from "../_common/register/$provideFilesystemForNode"; +import { makeKnowledgeSourceHandler } from "../_common/utils/makeKnowledgeSourceHandler"; +import { LegacyDocumentScraper } from "./LegacyDocumentScraper"; -describe('how creating knowledge from docx works', () => { - const rootDirname = join(__dirname, 'examples'); - const legacyDocumentScraperPromise = (async () => - new LegacyDocumentScraper( - { - fs: $provideFilesystemForNode(), - llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), - executables: await $provideExecutablesForNode(), - }, - { - rootDirname, - }, - ))(); +describe("how creating knowledge from docx works", () => { + const rootDirname = join(__dirname, "examples"); + const legacyDocumentScraperPromise = (async () => + new LegacyDocumentScraper( + { + fs: $provideFilesystemForNode(), + llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), + executables: await $provideExecutablesForNode(), + }, + { + rootDirname, + }, + ))(); - it('should scrape simple information from a (legacy) .doc file', () => - expect( - Promise.all([ - legacyDocumentScraperPromise, - makeKnowledgeSourceHandler( - { - knowledgeSourceContent: '10-simple.doc', - }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ]) - .then(([legacyDocumentScraper, sourceHandler]) => legacyDocumentScraper.scrape(sourceHandler)) - .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) - .then((knowledge) => knowledge?.slice(0, 1)), - ).resolves.toMatchObject([ - { - content: expect.stringMatching(/Springfield (is )?.*/i), - }, - ])); + it("should scrape simple information from a (legacy) .doc file", () => + expect( + Promise.all([ + legacyDocumentScraperPromise, + makeKnowledgeSourceHandler( + { + knowledgeSourceContent: "10-simple.doc", + }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ]) + .then(([legacyDocumentScraper, sourceHandler]) => + legacyDocumentScraper.scrape(sourceHandler), + ) + .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) + .then((knowledge) => knowledge?.slice(0, 1)), + ).resolves.toMatchObject([ + { + content: expect.stringMatching(/Springfield (is )?.*/i), + }, + ])); - it('should scrape simple information from a .rtf file', () => - expect( - Promise.all([ - legacyDocumentScraperPromise, - makeKnowledgeSourceHandler( - { - knowledgeSourceContent: '10-simple.rtf', - }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ]) - .then(([legacyDocumentScraper, sourceHandler]) => legacyDocumentScraper.scrape(sourceHandler)) - .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) - .then((knowledge) => knowledge?.slice(0, 1)), - ).resolves.toMatchObject([ - { - content: expect.stringMatching(/Springfield (is )?.*/i), - }, - ])); + it("should scrape simple information from a .rtf file", () => + expect( + Promise.all([ + legacyDocumentScraperPromise, + makeKnowledgeSourceHandler( + { + knowledgeSourceContent: "10-simple.rtf", + }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ]) + .then(([legacyDocumentScraper, sourceHandler]) => + legacyDocumentScraper.scrape(sourceHandler), + ) + .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) + .then((knowledge) => knowledge?.slice(0, 1)), + ).resolves.toMatchObject([ + { + content: expect.stringMatching(/Springfield (is )?.*/i), + }, + ])); - it('should NOT scrape irrelevant information', () => - expect( - Promise.all([ - legacyDocumentScraperPromise, - makeKnowledgeSourceHandler( - { - knowledgeSourceContent: '10-simple.doc', - }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ]) - .then(([legacyDocumentScraper, sourceHandler]) => legacyDocumentScraper.scrape(sourceHandler)) - .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) - .then((knowledge) => knowledge?.slice(0, 1)), - ).resolves.toMatchObject([ - { - content: expect.not.stringMatching(/London (is )?.*/i), - }, - ])); + it("should NOT scrape irrelevant information", () => + expect( + Promise.all([ + legacyDocumentScraperPromise, + makeKnowledgeSourceHandler( + { + knowledgeSourceContent: "10-simple.doc", + }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ]) + .then(([legacyDocumentScraper, sourceHandler]) => + legacyDocumentScraper.scrape(sourceHandler), + ) + .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) + .then((knowledge) => knowledge?.slice(0, 1)), + ).resolves.toMatchObject([ + { + content: expect.not.stringMatching(/London (is )?.*/i), + }, + ])); }); /** diff --git a/src/scrapers/document-legacy/LegacyDocumentScraper.ts b/src/scrapers/document-legacy/LegacyDocumentScraper.ts index b92e032fac..cb1e5ce57a 100644 --- a/src/scrapers/document-legacy/LegacyDocumentScraper.ts +++ b/src/scrapers/document-legacy/LegacyDocumentScraper.ts @@ -1,30 +1,30 @@ -import type { KnowledgePiecePreparedJson } from '../../pipeline/PipelineJson/KnowledgePieceJson'; +import { dirname, join } from "path"; // TODO: [🏳‍🌈] Finally take pick of .json vs .ts // import PipelineCollection from '../../../books/books'; -import { readdir, rename, rmdir } from 'fs/promises'; -import { dirname, join } from 'path'; -import spaceTrim from 'spacetrim'; -import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from '../../config'; -import { DEFAULT_IS_VERBOSE } from '../../config'; -import { DEFAULT_SCRAPE_CACHE_DIRNAME } from '../../config'; -import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError'; -import { KnowledgeScrapeError } from '../../errors/KnowledgeScrapeError'; -import { MissingToolsError } from '../../errors/MissingToolsError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { $isRunningInNode } from '../../utils/environment/$isRunningInNode'; -import { $execCommand } from '../../utils/execCommand/$execCommand'; -import { getFileExtension } from '../../utils/files/getFileExtension'; -import { isFileExisting } from '../../utils/files/isFileExisting'; -import type { Converter } from '../_common/Converter'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; -import type { Scraper } from '../_common/Scraper'; -import type { ScraperSourceHandler } from '../_common/Scraper'; -import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource'; -import { getScraperIntermediateSource } from '../_common/utils/getScraperIntermediateSource'; -import { DocumentScraper } from '../document/DocumentScraper'; -import { legacyDocumentScraperMetadata } from './register-metadata'; +import { readdir, rename, rmdir } from "fs/promises"; +import spaceTrim from "spacetrim"; +import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from "../../config"; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import { DEFAULT_SCRAPE_CACHE_DIRNAME } from "../../config"; +import { EnvironmentMismatchError } from "../../errors/EnvironmentMismatchError"; +import { KnowledgeScrapeError } from "../../errors/KnowledgeScrapeError"; +import { MissingToolsError } from "../../errors/MissingToolsError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { KnowledgePiecePreparedJson } from "../../pipeline/PipelineJson/KnowledgePieceJson"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { $isRunningInNode } from "../../utils/environment/$isRunningInNode"; +import { $execCommand } from "../../utils/execCommand/$execCommand"; +import { getFileExtension } from "../../utils/files/getFileExtension"; +import { isFileExisting } from "../../utils/files/isFileExisting"; +import type { Converter } from "../_common/Converter"; +import type { Scraper } from "../_common/Scraper"; +import type { ScraperSourceHandler } from "../_common/Scraper"; +import type { ScraperIntermediateSource } from "../_common/ScraperIntermediateSource"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; +import { getScraperIntermediateSource } from "../_common/utils/getScraperIntermediateSource"; +import { DocumentScraper } from "../document/DocumentScraper"; +import { legacyDocumentScraperMetadata } from "./register-metadata"; /** * Scraper for old document files (like .doc and .rtf) @@ -33,91 +33,104 @@ import { legacyDocumentScraperMetadata } from './register-metadata'; * @public exported from `@promptbook/legacy-documents` */ export class LegacyDocumentScraper implements Converter, Scraper { - /** - * Metadata of the scraper which includes title, mime types, etc. - */ - public get metadata(): ScraperAndConverterMetadata { - return legacyDocumentScraperMetadata; - } - - /** - * Document scraper is used internally - */ - private readonly documentScraper: DocumentScraper; - - public constructor( - private readonly tools: Pick, - private readonly options: PrepareAndScrapeOptions, - ) { - this.documentScraper = new DocumentScraper(tools, options); - } - - /** - * Convert the `.doc` or `.rtf` to `.doc` file and returns intermediate source - * - * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object - */ - public async $convert(source: ScraperSourceHandler): Promise { - const { - rootDirname = process.cwd(), - cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, - intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, - isVerbose = DEFAULT_IS_VERBOSE, - } = this.options; - - if (!$isRunningInNode()) { - throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment'); - } - - if (this.tools.fs === undefined) { - throw new EnvironmentMismatchError('Can not scrape (legacy) documents without filesystem tools'); - // <- TODO: [🧠] What is the best error type here` - } - - if (this.tools.executables?.libreOfficePath === undefined) { - throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files'); - } - - if (source.filename === null) { - // TODO: [🧠] Maybe save file as temporary - throw new KnowledgeScrapeError('When parsing .doc or .rtf file, it must be real file in the file system'); - } - - const extension = getFileExtension(source.filename); - - const cacheFilehandler = await getScraperIntermediateSource(source, { - rootDirname, - cacheDirname, - intermediateFilesStrategy, - extension: 'docx', - isVerbose, - }); - - if (isVerbose) { - console.info(`documentScraper: Converting .${extension} -> .docx`); - } - - // Note: Running Libreoffice ONLY if the file in the cache does not exist - if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { - /** - * Note: Unfortunately, LibreOffice does not support to specify the output file path, - * so we create a subfolder `libreoffice` in the same folder as the source file - * and then move the file to the desired location - */ - const documentSourceOutdirPathForLibreOffice = join(dirname(cacheFilehandler.filename), 'libreoffice') - .split('\\') - .join('/'); - - const command = `"${this.tools.executables.libreOfficePath}" --headless --convert-to docx "${source.filename}" --outdir "${documentSourceOutdirPathForLibreOffice}"`; - - await $execCommand(command); - - const files = await readdir(documentSourceOutdirPathForLibreOffice); - - if (files.length !== 1) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + /** + * Metadata of the scraper which includes title, mime types, etc. + */ + public get metadata(): ScraperAndConverterMetadata { + return legacyDocumentScraperMetadata; + } + + /** + * Document scraper is used internally + */ + private readonly documentScraper: DocumentScraper; + + public constructor( + private readonly tools: Pick, + private readonly options: PrepareAndScrapeOptions, + ) { + this.documentScraper = new DocumentScraper(tools, options); + } + + /** + * Convert the `.doc` or `.rtf` to `.doc` file and returns intermediate source + * + * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object + */ + public async $convert( + source: ScraperSourceHandler, + ): Promise { + const { + rootDirname = process.cwd(), + cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, + intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, + isVerbose = DEFAULT_IS_VERBOSE, + } = this.options; + + if (!$isRunningInNode()) { + throw new KnowledgeScrapeError( + "Scraping .doc files is only supported in Node environment", + ); + } + + if (this.tools.fs === undefined) { + throw new EnvironmentMismatchError( + "Can not scrape (legacy) documents without filesystem tools", + ); + // <- TODO: [🧠] What is the best error type here` + } + + if (this.tools.executables?.libreOfficePath === undefined) { + throw new MissingToolsError( + "LibreOffice is required for scraping .doc and .rtf files", + ); + } + + if (source.filename === null) { + // TODO: [🧠] Maybe save file as temporary + throw new KnowledgeScrapeError( + "When parsing .doc or .rtf file, it must be real file in the file system", + ); + } + + const extension = getFileExtension(source.filename); + + const cacheFilehandler = await getScraperIntermediateSource(source, { + rootDirname, + cacheDirname, + intermediateFilesStrategy, + extension: "docx", + isVerbose, + }); + + if (isVerbose) { + console.info(`documentScraper: Converting .${extension} -> .docx`); + } + + // Note: Running Libreoffice ONLY if the file in the cache does not exist + if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { + /** + * Note: Unfortunately, LibreOffice does not support to specify the output file path, + * so we create a subfolder `libreoffice` in the same folder as the source file + * and then move the file to the desired location + */ + const documentSourceOutdirPathForLibreOffice = join( + dirname(cacheFilehandler.filename), + "libreoffice", + ) + .split("\\") + .join("/"); + + const command = `"${this.tools.executables.libreOfficePath}" --headless --convert-to docx "${source.filename}" --outdir "${documentSourceOutdirPathForLibreOffice}"`; + + await $execCommand(command); + + const files = await readdir(documentSourceOutdirPathForLibreOffice); + + if (files.length !== 1) { + throw new UnexpectedError( + spaceTrim( + (block) => ` Expected exactly 1 file in the LibreOffice output directory, got ${files.length} The temporary folder: @@ -126,19 +139,22 @@ export class LegacyDocumentScraper implements Converter, Scraper { Command: > ${block(command)} `, - ), - ); - } - - const file = files[0]!; - - await rename(join(documentSourceOutdirPathForLibreOffice, file), cacheFilehandler.filename); - await rmdir(documentSourceOutdirPathForLibreOffice); - - if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + ), + ); + } + + const file = files[0]!; + + await rename( + join(documentSourceOutdirPathForLibreOffice, file), + cacheFilehandler.filename, + ); + await rmdir(documentSourceOutdirPathForLibreOffice); + + if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { + throw new UnexpectedError( + spaceTrim( + (block) => ` File that was supposed to be created by LibreOffice does not exist for unknown reason Expected file: @@ -151,38 +167,40 @@ export class LegacyDocumentScraper implements Converter, Scraper { > ${block(command)} `, - ), - ); - } - } - - return cacheFilehandler; - } - - /** - * Scrapes the `.doc` or `.rtf` file and returns the knowledge pieces or `null` if it can't scrape it - */ - public async scrape( - source: ScraperSourceHandler, - ): Promise> | null> { - const cacheFilehandler = await this.$convert(source); - - const markdownSource = { - source: source.source, - filename: cacheFilehandler.filename, - url: null, - mimeType: 'text/markdown', - asText() { - throw new UnexpectedError( - 'Did not expect that `documentScraper` would need to get the content `asText`', - ); - }, - asJson() { - throw new UnexpectedError( - 'Did not expect that `documentScraper` would need to get the content `asJson`', - ); - }, - /* + ), + ); + } + } + + return cacheFilehandler; + } + + /** + * Scrapes the `.doc` or `.rtf` file and returns the knowledge pieces or `null` if it can't scrape it + */ + public async scrape( + source: ScraperSourceHandler, + ): Promise + > | null> { + const cacheFilehandler = await this.$convert(source); + + const markdownSource = { + source: source.source, + filename: cacheFilehandler.filename, + url: null, + mimeType: "text/markdown", + asText() { + throw new UnexpectedError( + "Did not expect that `documentScraper` would need to get the content `asText`", + ); + }, + asJson() { + throw new UnexpectedError( + "Did not expect that `documentScraper` would need to get the content `asJson`", + ); + }, + /* TODO: [🥽] > asBlob() { > throw new UnexpectedError( @@ -190,14 +208,14 @@ export class LegacyDocumentScraper implements Converter, Scraper { > ); > }, */ - } satisfies ScraperSourceHandler; + } satisfies ScraperSourceHandler; - const knowledge = this.documentScraper.scrape(markdownSource); + const knowledge = this.documentScraper.scrape(markdownSource); - await cacheFilehandler.destroy(); + await cacheFilehandler.destroy(); - return knowledge; - } + return knowledge; + } } /** diff --git a/src/scrapers/document-legacy/createLegacyDocumentScraper.ts b/src/scrapers/document-legacy/createLegacyDocumentScraper.ts index 48878ae263..245fc871c3 100644 --- a/src/scrapers/document-legacy/createLegacyDocumentScraper.ts +++ b/src/scrapers/document-legacy/createLegacyDocumentScraper.ts @@ -1,9 +1,9 @@ -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import type { ScraperConstructor } from '../_common/register/ScraperConstructor'; -import { LegacyDocumentScraper } from './LegacyDocumentScraper'; -import { legacyDocumentScraperMetadata } from './register-metadata'; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import type { ScraperConstructor } from "../_common/register/ScraperConstructor"; +import { LegacyDocumentScraper } from "./LegacyDocumentScraper"; +import { legacyDocumentScraperMetadata } from "./register-metadata"; keepTypeImported(); @@ -13,10 +13,13 @@ keepTypeImported(); * @public exported from `@promptbook/legacy-documents` */ export const createLegacyDocumentScraper = Object.assign( - (tools: Pick, options: PrepareAndScrapeOptions): LegacyDocumentScraper => { - return new LegacyDocumentScraper(tools, options); - }, - legacyDocumentScraperMetadata, + ( + tools: Pick, + options: PrepareAndScrapeOptions, + ): LegacyDocumentScraper => { + return new LegacyDocumentScraper(tools, options); + }, + legacyDocumentScraperMetadata, ) satisfies ScraperConstructor; /* <- Note: [🤛] */ /** diff --git a/src/scrapers/document-legacy/examples/10-simple.doc.knowledge.json b/src/scrapers/document-legacy/examples/10-simple.doc.knowledge.json index a07c05472e..6166054edf 100644 --- a/src/scrapers/document-legacy/examples/10-simple.doc.knowledge.json +++ b/src/scrapers/document-legacy/examples/10-simple.doc.knowledge.json @@ -1,44 +1,1287 @@ [ - { - "name": "springfield-illinois-sixth-largest-city", - "title": "Springfield: Illinois' Sixth Largest City", - "content": "---\n\nSpringfield is the sixth most populous city in Illinois, with a population of 10,566 as of 2019.", - "keywords": [ - "Springfield", - "Illinois", - "population", - "sixth most populous", - "10", - "566", - "2019" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.019289264, 0.03618999, -0.0003673661, -0.017636588, 0.008607185, 0.0005126915, 0.0048856484, 0.0038120123, -0.043162595, 0.025212359, -0.020821307, 0.013740133, 0.000021711638, 0.016478509, 0.0023855234, 0.018058805, -0.020857498, 0.03423573, 0.0003524754, -0.006858003, 0.0070148264, -0.0144156795, -0.045816526, 0.035779838, 0.025598386, 0.010627793, -0.027214872, 0.008353855, -0.025622513, -0.008191001, -0.02275144, -0.0048886645, 0.05771097, -0.006640863, 0.0073887897, -0.0066830846, 0.0035405874, -0.020121634, 0.018818794, 0.035152543, 0.014295046, -0.004632319, 0.015139479, 0.0104709705, 0.038843922, -0.039422963, 0.00031892429, -0.02738376, 0.02311334, 0.008745914, 0.024355862, -0.018239755, -0.024898713, 0.042945456, -0.0009213368, -0.027238999, 0.007684341, 0.024223166, -0.0077989427, -0.03980899, -0.001979894, -0.007883386, 0.012871573, -0.005344055, -0.0017506907, 0.0069967313, 0.008564964, -0.0053711976, 0.06441818, 0.012618243, 0.010772553, 0.04658858, 0.011580797, 0.03322241, 0.058048744, 0.0027217888, -0.017660715, -0.0023885393, -0.03271575, 0.03286051, 0.015658202, -0.0064176917, -0.003401859, 0.0003402236, 0.009463682, 0.029868804, 0.009771297, -0.06779592, -0.002287509, 0.010911282, 0.025694892, -0.0035918565, -0.0012040711, -0.03611761, -0.043693382, -0.023065086, 0.032957017, -0.0059080156, 0.009620505, 0.020616231, 0.017359132, -0.0029796425, -0.010917313, 0.025212359, 0.00038244526, 0.0096506635, -0.018456895, 0.0061522983, -0.027432011, 0.0050545353, 0.010380495, 0.002035687, -0.012286501, 0.01206333, -0.010404622, -0.0057572243, -0.02535712, -0.0075154547, 0.04198039, 0.02130384, 0.00055189733, 0.021943197, -0.025622513, 0.018493084, -0.0070691113, -0.010338274, 0.02101432, -0.013245536, 0.003293289, -0.0450686, 0.03433224, 0.005509926, -0.020495597, 0.010615731, 0.012153805, -0.039181694, -0.011840158, 0.016804218, 0.027769785, 0.020917814, -0.059399836, -0.006731338, 0.019542594, 0.0049972343, -0.00008434906, -0.014886149, -0.0125338, 0.008812263, 0.009656696, -0.029289765, -0.014524249, -0.005923095, -0.02372857, 0.0044604163, 0.00053191744, 0.031509418, -0.0018607687, -0.013848702, 0.047167618, -0.05259612, -0.01293189, -0.021955261, 0.027456138, -0.03799949, -0.022630807, -0.010766522, -0.0076964046, -0.00031609694, 0.0028755963, 0.002898215, 0.022582553, -0.01369188, -0.063839145, -0.036310624, -0.007358631, -0.010585572, -0.033560183, -0.016032165, -0.010911282, 0.03039959, -0.0028484538, -0.00058997225, 0.030737365, -0.0038843923, -0.019253075, -0.018300071, 0.0064900713, 0.007991956, 0.0268771, -0.014331236, -0.003362653, 0.044586066, 0.0050485036, -0.0054827835, 0.04017089, -0.00031628544, -0.009964311, -0.027528519, -0.0047559678, 0.024042217, -0.044899713, 0.0054888153, -0.037854727, -0.024946965, 0.0059954748, 0.018806731, -0.0014491075, -0.0018683082, 0.021279713, 0.020314647, -0.006369438, -0.0076541826, -0.025308866, 0.026515199, -0.011828095, 0.022015577, -0.009976374, -0.00009099332, -0.019047998, 0.019940684, 0.0142467925, 0.038023617, -0.000017482404, -0.028903738, -0.033367172, -0.020724801, -0.045599386, 0.034549378, -0.022003513, -0.037637588, 0.026611706, 0.027335506, -0.0130887125, 0.054477997, -0.011387783, 0.04743301, 0.013522993, 0.004249308, -0.014898213, 0.034766518, 0.025164107, 0.0026569483, 0.0033686848, -0.015223922, 0.0012817289, -0.019699417, 0.025260612, -0.014331236, 0.016225178, 0.017274689, 0.0114842905, -0.008293539, 0.05462276, -0.005989443, -0.008715756, 0.0145001225, -0.013209346, 0.021219397, 0.0002141241, -0.0066891164, -0.022293033, -0.0021683834, -0.03387383, 0.03474239, 0.0074189478, -0.012141742, 0.028807232, 0.00072870054, 0.005485799, -0.030761492, -0.000269917, -0.012811257, 0.005344055, 0.0061432505, -0.009692886, 0.030423718, 0.045454625, 0.0031877349, -0.029434524, -0.032233216, -0.028445331, 0.04608192, 0.0029208336, -0.013872829, 0.010392559, 0.0016647396, -0.018432768, -0.002213621, 0.011357625, -0.030278958, -0.01039859, 0.00026897455, 0.044586066, 0.025912033, -0.005160089, -0.014560439, 0.022437794, 0.0006725306, -0.042221654, 0.03184719, 0.034404617, 0.036793157, -0.03112339, -0.028228192, 0.007159586, -0.038868047, 0.016490571, -0.017528018, -0.03618999, -0.06480421, -0.016804218, -0.050762493, -0.013607436, -0.00066762994, 0.006206583, 0.0012447848, 0.025912033, 0.008812263, 0.029482778, -0.015067099, -0.0033415423, 0.0071897446, 0.020049253, 0.02557426, 0.0031364658, 0.054429743, -0.00158482, -0.0020387028, 0.01119477, 0.0038482023, -0.011206834, 0.022763504, -0.0028198033, -0.033198282, 0.017865792, 0.002376476, 0.030568479, 0.011725557, -0.0066167363, 0.02300477, 0.001078914, 0.008564964, 0.009439556, 0.03626237, 0.020097507, -0.013764259, -0.00072191487, -0.0070811748, -0.018891174, 0.039422963, 0.0006216385, 0.016719775, 0.05829001, -0.0071897446, 0.012473483, -0.009801456, -0.02267906, -0.0029238495, 0.024753952, -0.050231706, -0.023535557, 0.0064900713, -0.018468957, 0.004979139, -0.032257345, 0.021798437, -0.010525255, 0.0060889656, 0.030013565, 0.003223925, 0.0071173646, 0.016755965, -0.02535712, 0.014801705, -0.0014166873, 0.012919826, -0.018239755, -0.0017521987, -0.021098765, -0.005721034, 0.02416285, 0.020785118, -0.010416686, -0.02249811, -0.006212615, -0.007310378, -0.007057048, -0.013414423, -0.036962043, 0.012594116, 0.021496853, -0.028927865, 0.011604924, 0.015766772, 0.032402106, 0.029675791, -0.019494342, -0.01115858, 0.017962297, -0.00044596623, 0.016888661, -0.022401603, -0.00055566715, 0.02557426, -0.019035935, -0.0030007532, -0.01018145, -0.05814525, 0.008275445, -0.011466195, -0.006827845, 0.026273932, 0.013872829, -0.010428749, -0.020037191, -0.046685085, -0.010344305, 0.039471216, 0.015549632, 0.015501379, -0.007020858, -0.024247292, 0.002877104, 0.012943953, -0.015308365, 0.015959786, -0.012545863, 0.027721532, -0.022763504, -0.030327212, -0.023885394, -0.0036491572, -0.011864285, -0.015694393, -0.009125909, 0.051631052, -0.04781904, 0.0022226686, 0.05027996, 0.030882124, -0.03061673, 0.009162099, 0.043066088, -0.018999744, -0.004427242, 0.026563453, 0.017733095, 0.024319673, 0.019868305, 0.048373953, -0.013100776, 0.018613718, 0.009916057, 0.016526762, 0.028927865, -0.0028077401, -0.029820552, 0.01032621, -0.007509423, 0.011731588, 0.037951235, -0.023825077, 0.014632819, -0.031147517, -0.037661716, 0.03206433, 0.057662718, 0.018348325, -0.0004195777, -0.0038270913, -0.0074189478, -0.025936158, -0.049194258, 0.03647951, -0.03618999, 0.014439805, 0.0014543852, 0.019663228, 0.012871573, -0.028228192, -0.030327212, 0.018324198, -0.044972092, 0.004243276, -0.003223925, 0.056745905, -0.018372452, -0.008770041, 0.037806477, -0.0052837385, 0.053464677, 0.0018607687, -0.009916057, -0.0144156795, 0.0142467925, -0.0037999488, -0.031244025, -0.05129328, 0.04058104, 0.038240757, -0.0063754697, 0.01195476, 0.029241512, 0.020857498, -0.02318572, -0.0114842905, -0.018493084, -0.029555159, 0.0034621756, 0.010531287, 0.067120366, 0.013740133, -0.026346313, -0.020519724, 0.014560439, -0.01278713, 0.0053319917, -0.02658758, 0.026949478, 0.0071414914, -0.013607436, 0.02260668, 0.009554157, 0.004231213, -0.027673278, 0.03604523, -0.002034179, -0.0034290014, 0.03913344, 0.0059653167, -0.04521336, -0.0031093233, 0.00072191487, 0.025719019, 0.049146004, -0.02629806, -0.021038447, -0.028469458, -0.0016587079, 0.035731584, -0.011502385, -0.051872317, 0.0031937666, 0.036576018, 0.025453625, 0.013957272, -0.011562701, -0.026249805, 0.011502385, -0.012835383, 0.047915544, -0.0061824564, 0.010850965, -0.0060407124, -0.008770041, -0.004164865, -0.010483034, -0.012015076, -0.035948724, -0.030592604, 0.012039203, 0.011610955, 0.021508917, -0.0038120123, 0.00025295294, 0.017745158, 0.007937671, 0.008938927, 0.007985924, 0.009560189, 0.025815526, 0.024030153, 0.017129928, -0.006646895, 0.0032028141, 0.016369939, 0.00284393, -0.015706455, -0.0035043973, -0.0010140736, -0.006550388, 0.017540082, 0.030037692, 0.027962798, -0.0047137463, 0.0144156795, -0.011683335, -0.04487559, -0.016104545, 0.0515828, 0.050810747, 0.025188232, 0.007865291, 0.009505904, -0.00084066327, 0.014222666, -0.007883386, 0.005057551, -0.02286001, -0.052789133, 0.019023871, 0.0018140232, 0.010501129, -0.026225679, -0.020338774, -0.003944709, 0.008106558, 0.0043850206, 0.02213621, -0.011285245, -0.01934958, -0.00070268894, 0.01155667, -0.016780091, -0.035707455, 0.006948478, 0.020640356, 0.027721532, 0.0014174413, -0.014463932, 0.0031002758, -0.030496098, -0.027769785, -0.008203064, 0.014451869, -0.01191857, 0.013197283, -0.010042722, -0.0003726438, -0.008426236, -0.01130334, 0.026056793, -0.006924351, -0.0018200548, -0.007032921, -0.004647398, 0.005096757, 0.0052596116, -0.03129228, 0.003525508, 0.019325454, 0.061957262, 0.011496353, 0.02286001, -0.009662727, -0.001421965, -0.026177425, 0.009035434, -0.0019874335, 0.017672779, 0.023776824, -0.020604167, -0.0011671272, 0.0077144993, -0.011683335, 0.02231716, 0.028276445, -0.031051012, 0.019735608, 0.0054254825, -0.034501124, 0.004930886, 0.002700678, 0.01076049, 0.007563708, 0.029096752, -0.044537812, 0.0232581, 0.037420448, 0.008776072, 0.008341793, 0.0218105, -0.007509423, 0.014210602, -0.015139479, 0.017974362, -0.007075143, -0.0021563203, -0.01329379, -0.009113845, -0.008414173, -0.010205577, 0.0007237998, -0.010483034, -0.0014264887, -0.026515199, 0.045261614, 0.025550133, 0.008824326, 0.0043850206, 0.008764009, 0.03618999, -0.005814525, 0.0034621756, 0.03645538, -0.0014325205, -0.012413166, 0.034983657, 0.011466195, -0.007684341, -0.022075893, 0.022329224, -0.007093238, -0.041859753, 0.00619452, 0.0062367413, -0.0025468706, -0.026901225, -0.05380245, -0.0012131186, -0.0046715243, 0.0128957, 0.014331236, -0.033704944, 0.053995464, -0.03768584, 0.019385772, -0.015609949, -0.0012583561, 0.036576018, 0.030520225, 0.017093739, -0.045454625, -0.0039326455, -0.031026885, 0.0043277196, -0.035273176, -0.038023617, 0.017443575, 0.009988437, -0.031364657, 0.013486803, 0.037854727, -0.005775319, 0.0028167875, 0.0028559933, 0.0003585071, -0.028927865, 0.025188232, -0.002194018, -0.012702687, -0.02810756, 0.038409643, 0.006930383, 0.010078912, 0.010893187, 0.0043820045, -0.0049821553, -0.058676038, -0.0018502132, -0.031388786, -0.019759735, 0.05129328, 0.030882124, 0.00004311698, -0.0139090195, -0.0014822816, -0.0035948723, 0.010386527, 0.049411397, 0.029989438, 0.008703693, 0.007889418, 0.00858306, 0.015585822, -0.005021361, 0.010084944, -0.0009688362, -0.0046896194, 0.0038421706, 0.04354862, 0.0003566222, -0.016864534, 0.0018155312, 0.03322241, 0.002643377, 0.009729075, -0.021364158, 0.021545107, -0.040556915, 0.025622513, -0.010151292, 0.012075393, 0.011170643, 0.00023466947, -0.02115908, -0.0043126405, -0.009101782, 0.008191001, -0.013317917, -0.015790898, -0.0196391, -0.044706702, 0.0038089964, 0.009958278, -0.014174413, 0.0034470966, -0.026225679, 0.05249961, 0.0059321425, 0.015971849, 0.031509418, 0.00087760726, 0.01772103, -0.048205066, 0.008257349, 0.002718773, -0.0016602158, -0.0058838893, 0.0030686096, 0.032257345, -0.0326675, 0.027818039, -0.016261369, -0.031750683, -0.025887907, 0.0150912255, -0.0243438, -0.0041920072, -0.029506905, 0.025550133, -0.02383714, 0.027118366, -0.018312134, 0.023089213, 0.010519223, 0.0016571999, -0.016647395, 0.007581803, 0.02485046, -0.022039704, 0.00496406, -0.020604167, -0.010495097, -0.009813519, 0.023451114, 0.00056094484, 0.01202714, 0.018782604, 0.005232469, 0.007509423, -0.012799193, 0.02441618, -0.016249305, 0.027118366, 0.03973661, -0.0060015065, -0.03980899, 0.024440307, -0.017757222, -0.03143704, 0.015947722, -0.022739377, 0.017624525, -0.019204821, 0.028155811, 0.021460664, 0.009119878, -0.017383259, 0.015706455, 0.0019361645, 0.003778838, 0.029048499, -0.01354712, 0.0011942696, 0.0070691113, 0.0044905744, 0.0073164096, -0.021098765, -0.01732294, 0.0034832866, 0.002414174, -0.00070193503, -0.018782604, -0.0010766522, 0.009620505, -0.001764262, -0.0013601404, -0.024090469, 0.026008539, -0.036624268, -0.008154811, -0.015139479, -0.022124147, 0.016538825, 0.020133698, -0.014126159, -0.028758978, 0.0059562693, -0.005926111, 0.0030610699, 0.009903993, -0.0053531025, -0.016249305, 0.006206583, 0.016321685, -0.029531032, -0.02887961, -0.0011120882, 0.005090725, 0.017250562, -0.008414173, 0.014427743, -0.025719019, -0.0011331991, 0.016997231, 0.026780592, 0.008420204, -0.009517968, 0.011616986, -0.016466444, 0.022763504, 0.0049519967, -0.020833371, -0.004315656, 0.020845434, -0.011852222, -0.011623018, -0.0032088459, 0.018456895, 0.024223166, 0.007690373, -0.03901281, 0.007256093, -0.0051721525, 0.0051962794, 0.01282332, -0.041353095, -0.009409397, -0.0036310623, 0.01985624, 0.016599141, 0.0008798691, -0.004502638, -0.00316964, -0.035948724, -0.036865536, 0.012449357, -0.008136716, 0.028011052, -0.01014526, 0.008727819, -0.030737365, 0.0029826583, -0.030496098, 0.037396323, -0.027456138, 0.02101432, 0.030278958, -0.0009861772, -0.0020387028, 0.014065842, -0.009602411, 0.002286001, 0.036383003, 0.022546364, -0.015983911, 0.027407885, -0.016188988, 0.02608092, -0.0029434524, 0.019313391, -0.040774055, -0.0030218642, 0.008341793, -0.015187732, -0.028831359, -0.05481577, -0.012256343, 0.012449357, 0.014681072, -0.0038843923, 0.007153555, -0.019916557, -0.0005183462, 0.030278958, -0.0081306845, -0.0005311635, 0.049749173, 0.035200797, 0.010416686, 0.009481777, 0.0014099017, 0.058048744, 0.016032165, -0.01264237, -0.0130887125, -0.008691629, -0.011562701, -0.004243276, 0.02658758, -0.005793414, -0.010549382, -0.000053106924, 0.057324942, -0.009150036, -0.01137572, 0.008178937, -0.016164862, 0.023173656, 0.03126815, 0.010820807, -0.028059306, -0.002429253, -0.009306859, 0.03242623, -0.04152198, -0.013945209, -0.009729075, 0.0018155312, 0.020423217, -0.021352094, 0.03351193, -0.037323944, -0.0022467952, -0.031726558, -0.016008038, -0.017986424, 0.005368182, -0.008197033, 0.009343049, -0.003223925, -0.0019225932, -0.023378734, -0.0026524246, -0.0013631562, 0.014379489, -0.01094144, 0.015272176, 0.040774055, 0.021798437, -0.0221724, -0.0058537307, -0.0027926608, -0.01318522, -0.0064176917, -0.008872579, 0.049990438, 0.021219397, 0.015103289, 0.0038572496, -0.03481477, 0.031075139, 0.0027489313, 0.00072455377, -0.0053953244, -0.007666246, 0.003618999, 0.032957017, 0.03112339, -0.009337017, 0.0056486544, -0.0072500613, 0.0012651418, -0.036793157, -0.004647398, 0.027407885, 0.016345812, -0.002684091, -0.018710224, -0.003869313, -0.018251818, -0.004336767, 0.0062186467, 0.026032666, -0.012847446, -0.012943953, 0.010350337, -0.013450612, -0.003887408, 0.0036491572, -0.006948478, 0.034356363, -0.013764259, 0.028734852, 0.0048826328, -0.009361144, -0.024609193, -0.0062729316, -0.029338019, 0.013740133, 0.008070367, -0.021448601, 0.017624525, -0.0021110827, -0.026925351, 0.017757222, 0.0006352097, -0.008209096, 0.0028695646, -0.016394066, -0.01177381, 0.024753952, 0.010187482, 0.04557526, -0.018312134, 0.028276445, 0.04933902, -0.005814525, -0.00020960036, -0.009005276, 0.0007090976, 0.009451619, -0.04304196, 0.007286251, 0.013836639, -0.041256588, -0.019590847, -0.055153545, -0.009053529, -0.018324198, 0.021605425, -0.03242623, -0.0021291778, 0.022908263, 0.019204821, -0.020929877, -0.007358631, -0.007002763, -0.0150912255, -0.024705699, -0.012883636, -0.024874587, -0.0029042466, 0.03310178, -0.012594116, -0.010332242, -0.0013088713, 0.012714749, -0.0030490067, -0.020990195, 0.017805476, -0.004014073, -0.042993706, 0.018722288, 0.011502385, 0.008172906, -0.0065986416, 0.019421961, -0.007907513, 0.00576024, -0.0006514198, 0.0022603665, -0.010169387, -0.0043910523, -0.0060738865, 0.017009296, -0.000396205, 0.0055431, -0.016599141, -0.029000245, 0.0005424729, 0.016647395, -0.006369438, -0.0014649406, -0.021822564, 0.0022407635, -0.010109071, -0.022884136, 0.017901981, -0.015356619, -0.0037999488, -0.008046241, 0.015356619, 0.012141742, 0.008709724, 0.007165618, 0.0026599641, 0.0034139224, 0.027287252, 0.00044860507, 0.007183713, -0.023607936, -0.025212359, 0.0122744385, -0.007358631, 0.013378233, -0.01705755, 0.006031665, 0.0077989427, -0.0026298058, -0.0049369177, -0.008070367, 0.009952247, -0.017443575, -0.014849959, -0.017238498, -0.022051767, -0.03575571, -0.0015591853, 0.0064780083, -0.02267906, 0.026346313, 0.02314953, -0.0026298058, -0.018143248, -0.0055159577, 0.004125659, 0.017708968, 0.027021859, -0.030496098, 0.027359633, -0.016068356, -0.011785873, 0.009463682, -0.006260868, 0.0011271674, 0.00087609934, -0.0037728064, -0.01974767, 0.013414423, 0.006441818, 0.019759735, -0.012811257, 0.017407386, -0.014475996, -0.017528018, 0.018915301, 0.041232463, 0.010501129, -0.008227191, 0.0012327215, -0.026466945, 0.00847449, 0.014439805, 0.008613218, -0.037155055, 0.004973108, -0.029096752, -0.0013329979, -0.010663983, 0.020664483, 0.00009353793, 0.009873835, 0.012968079, -0.027238999, -0.024367927, 0.03264337, 0.010483034, 0.0155978855, 0.00021487806, -0.017901981, 0.020374965, -0.004409147, -0.008070367, 0.024319673, -0.035056036, -0.03343955, 0.011037947, -0.037806477, 0.020990195, 0.011236992, 0.0059683323, -0.027021859, -0.043283228, -0.0011392307, -0.02130384, -0.0018818794, -0.02430761, -0.012147773, 0.014946465, -0.0025091725, -0.009349081, 0.020543851, 0.006719275, 0.017009296, -0.007925607, -0.0009522491, 0.023535557, 0.006912288, 0.0034802707, 0.019820051, -0.024705699, -0.010567477, 0.033487804, 0.024319673, -0.005163105, -0.009885899, 0.020857498, 0.010223672, 0.014910276, -0.0045267646, 0.0035315398, 0.018649908, -0.0057029394, -0.0025227438, 0.028179938, -0.015296302, 0.020326711, 0.004590097, 0.02680472, 0.012226185, 0.011333498, 0.0008368935, -0.031533543, -0.0013646642, 0.014319172, -0.038216628, -0.004210102, -0.02278763, 0.026201552, -0.0261533, 0.008932896, -0.012165868, -0.012847446, -0.013752196, -0.022992706, 0.012835383, 0.002519728, 0.003148529, 0.011369688, 0.019832114, -0.035393808, -0.013426486, 0.00702689, -0.005989443, 0.019590847, -0.00076187466, -0.002862025, 0.0044061313, -0.039664228, -0.009916057, -0.02383714, -0.0017069612, 0.008999244, -0.0006423723, 0.01003669, 0.0067434013, -0.010066849, 0.008926864, -0.0010057801, 0.007020858, -0.017045485, -0.0041135955, -0.009391302, -0.016056292, 0.02155717, -0.017202308, 0.05056948, 0.0013156569, 0.018915301, 0.0098497085, -0.00010762752, -0.00568786, 0.0039658197, 0.0017265641, -0.0016964058, -0.024404116, 0.0015124399, -0.02058004, -0.004158833, 0.01166524, -0.019735608, 0.020302584, -0.03329479, 0.00055905996, -0.0070691113, 0.0030655938, 0.025260612, 0.009385271, 0.021315904, 0.031919573, 0.023607936, -0.01665946, 0.005681828, -0.006731338, -0.010736364, -0.010718268, 0.004900728, -0.02343905, 0.0228962, 0.02018195, 0.014102032, 0.017371194, 0.008341793, 0.0065564197, -0.0072621247, -0.03341542, -0.001242523, 0.0016240258, -0.0062186467, -0.0005202311, 0.003778838, -0.0022814772, 0.02220859, 0.016635332, 0.035635076, 0.0019572752, -0.004752952, -0.0050002504, 0.018589592, 0.009584315, 0.018541338, 0.0021910022, -0.018143248, 0.008293539, 0.010603667, -0.015947722, 0.0055431, -0.005202311, -0.00989193, -0.017226435, 0.0007434027, -0.0073465677, 0.048132688, -0.0025468706, 0.013305853, -0.017986424, -0.007919576, 0.019807987, 0.0045961286, -0.022558426, 0.013354106, -0.02513998, -0.01181, -0.008528775, -0.010971598, 0.019759735, 0.015042973, -0.035948724, 0.01665946, -0.014234729, -0.009005276, 0.016828345, 0.002216637, -0.034259856, -0.021641614, -0.016357874, 0.003347574, -0.002430761, 0.008064336, 0.013957272, -0.005199295, 0.0044302577, 0.008004019, 0.0080764, 0.002860517, -0.0077325944, -0.024186976, 0.0028002004, -0.0031213865, 0.008812263, 0.0039024872, 0.01307665, -0.016213115, 0.013921083, -0.024295546, -0.024898713, -0.0142467925, -0.017733095, -0.021038447, -0.0019889416, 0.009958278, -0.009355112, 0.027745659, 0.010000501, 0.002773058, 0.013981399, 0.009463682, -0.0015064083, 0.018493084, -0.009493841, 0.0035948723, 0.011550638, -0.020748928, 0.006924351, 0.000737748, 0.01593566, -0.008993212, 0.014463932, -0.006725306, -0.004357878, 0.02228097, 0.029458651, 0.003796933, 0.029434524, 0.036407128, 0.0022241764, -0.026491072, -0.00953003, -0.016285496, 0.019602912, 0.013535056, 0.0069183195, 0.0065805465, 0.030471971, -0.009439556, 0.003365669, 0.013703942, 0.031895444, -0.029724045, 0.013703942, 0.03271575, -0.018589592, -0.015802963, -0.00044709718, 0.026225679, 0.011249055, 0.0069605415, 0.026828846, -0.009638601, 0.010446844, -0.030206578, -0.00953003, -0.0203629, 0.00702689, 0.011442068, -0.034911275, -0.0124976095, -0.013957272, 0.020917814, 0.02170193, -0.0047046985, 0.025936158, -0.04159436, 0.011743652, -0.0025302835, 0.0014159334, 0.022872074, -0.011031915, 0.014825832, 0.06740989, 0.033608437, -0.010066849, 0.017491829, -0.006284995, -0.004412163, 0.021894943, -0.0028635329, -0.020302584, -0.014319172, 0.036358874, -0.018082932, -0.003546619, -0.0033355106, 0.0014905751, 0.0029811503, 0.007581803, -0.0089751175, -0.008070367, -0.014198539, 0.015995976, 0.014898213, -0.0010397083, -0.0069967313, 0.020809244, 0.0027851213, -0.014524249, -0.028011052, -0.013245536, -0.022727313, -0.0023734602, 0.014005526, 0.0021080668, 0.003419954, 0.01916863, -0.0015388285, 0.027890418, -0.006119124, 0.010869061, -0.0027866291, 0.015549632, -0.0016783108, 0.019880367, 0.00033965815, -0.0074973595, 0.00032495594, -0.017081674, 0.010887155, -0.012003013, 0.013390296, 0.0058235726, 0.014898213, -0.0059803957, -0.022582553, -0.0058356356, -0.008377982, -0.010844934, -0.0047619995, 0.008884642, -0.009861772, -0.0018668002, -0.022112083, -0.022124147, 0.020628294, 0.014210602, 0.0067916545, -0.010875092, 0.010452875, 0.00045237486, -0.009542094, 0.004572002, 0.0032902732, 0.014849959, -0.02300477, -0.015887406, 0.02076099, -0.002950992, -0.0082151275, -0.01039859, 0.003489318, 0.024102533, -0.04342799, -0.005959285, -0.000121575744, 0.00020903489, 0.008540837, -0.008685597, -0.0075154547, -0.015923595, 0.0013578786, -0.015272176, 0.0035194764, 0.004463432, -0.012051267, 0.0002665242, 0.0017069612, -0.015441062, -0.0023372702, 0.0000022795452, -0.003818044, -0.012992206, -0.008450363, 0.035152543, -0.02608092, 0.01985624, 0.0013058555, 0.0024910776, 0.007485296, 0.014729326, 0.009596379, 0.011056041, 0.008733851, 0.006242773, -0.0067795916, -0.008106558, 0.020833371, -0.0070811748, 0.011478258, -0.008498616, 0.0457924, -0.008570996, -0.03083387, 0.029989438, 0.009041466, -0.009596379, 0.0074973595, -0.0064900713, 0.00040374458, -0.01895149, -0.00011582681, -0.016430255, 0.010621762, 0.022835884, 0.0014966069, -0.024319673, 0.0024563957, -0.015863279, -0.0013774815, -0.004198039, 0.0128957, 0.0070691113, 0.0027263125, -0.010368432, 0.005422467, 0.021738121, 0.022329224, -0.010151292, -0.02333048, -0.0196391, 0.010850965, 0.0016436287, 0.008981149, -0.0008391554, -0.004140738, 0.009952247, -0.01593566, 0.001188238, 0.010458907, -0.00169339, 0.00316964, -0.005310881, 0.026249805, 0.0072259344, 0.008607185, 0.008667503, -0.008643376, -0.0026509168, -0.0066649895, -0.024078406, 0.002287509, -0.0030701174, -0.0038632813, -0.0056637335, -0.009125909, 0.01202714, 0.00028480767, -0.02079718, -0.008920833, -0.02709424, -0.009023371, -0.0053078653, 0.008824326, -0.0150912255, -0.010700174, -0.016273431, -0.01633375, 0.021569233, 0.014451869, 0.005199295, -0.0032510674, -0.014512186, 0.00089947204, 0.0012930381, 0.016164862, 0.021605425, 0.013486803, 0.017383259, -0.0004267403, 0.009234479, 0.012232216, -0.012943953, -0.009861772, 0.027142493, 0.015634075, -0.0037034422, -0.016804218, 0.0026659959, -0.0055038943, -0.037396323, -0.02224478, -0.0039326455, 0.00010743903, -0.021364158, -0.0147534525, -0.011580797, 0.006978636, -0.017431512, -0.004731841, 0.010537319, -0.0036793156, 0.0075456128, 0.024753952, -0.007997988, -0.022112083, 0.023921583, 0.0051570735, -0.011140485, 0.0056094485, -0.00039695896, 0.0016466446, 0.0011746667, 0.008269412, 0.011616986, -0.003920582, -0.033680815, -0.031581797, 0.022437794, 0.01014526, 0.026732339, -0.004014073, 0.008782105, -0.0020085445, 0.0071233963, 0.014403616, 0.011966824, 0.023270164, 0.002105051, 0.025526006, 0.009554157, 0.014644883, 0.03503191, -0.009101782, 0.016538825, 0.009946216, 0.0031274182, 0.007744658, 0.011562701, -0.031002758, 0.021134954, -0.037854727, 0.0032390042, -0.012811257, -0.009005276, -0.026008539, 0.003706458, -0.015477252, -0.012232216, -0.00266298, 0.016514698, -0.0021125907, 0.007539581, -0.0006284241, -0.026346313, 0.012334755, -0.006713243, -0.005350087, 0.01873435, 0.017817538, 0.033704944, -0.017648652, -0.0018577528, -0.00026690116, -0.017335005, -0.026442818, 0.0029856742, -0.002089972, 0.009246542, -0.021605425, 0.025815526, -0.00038941938, -0.007020858, -0.007274188, -0.016828345, 0.01665946, 0.0015169637, 0.008667503, -0.015561695, -0.019434024, 0.006441818, -0.002284493, -0.0031274182, -0.040556915, -0.001964815, 0.019409897, -0.016840408, -0.0036069357, 0.0028891675, -0.0013789894, -0.031026885, 0.019602912, -0.00001858978, -0.007835133, 0.010657952, -0.018915301, -0.012654433, 0.007690373, -0.010024627, 0.011224928, -0.012919826, 0.0148378955, -0.00019357874, 0.0046685087, 0.009288764, 0.019108314, 0.002789645, 0.012521736, 0.010965567, -0.017757222, 0.027673278, 0.0092767, 0.008607185, 0.0025604418, -0.0073887897, 0.0007679063, -0.0033174157, -0.007310378, 0.0015403364, 0.0007758229, -0.00018236361, 0.014934402, -0.006821813, -0.02369238, -0.011828095, -0.006731338, -0.0064900713, 0.006070871, -0.008233222, 0.014765516, -0.03667252, 0.008667503, -0.0034350331, -0.0016994216, -0.014065842, 0.0019708464, -0.010603667, 0.003564714, -0.018046742, -0.008570996, -0.009439556, 0.009234479, -0.0056396066, -0.01564614, 0.013728069, -0.014910276, 0.024995219, -0.019542594, 0.0122744385, 0.026684085, 0.0029117863, -0.0053138966, -0.010283989, -0.02188288, -0.0121719, 0.0110862, -0.02101432, -0.008402109, 0.011562701, 0.0034048748, -0.014150286, 0.00006328536, -0.00025691124, -0.006496103, -0.025236486, -0.0135712465, 0.014862022, 0.004394068, -0.0063995966, 0.0055129416, 0.00003458783, -0.018288009, 0.00075508904, -0.030496098, 0.012256343, 0.006007538, 0.011243023, -0.0063815014, 0.016828345, -0.0043458147, 0.017793411, 0.02177431, -0.003959788, 0.010971598, -0.028517712, -0.005160089, 0.016080419, -0.009023371, -0.0014189492, -0.012630306, 0.030206578, 0.011586828, 0.00035417182, 0.003513445, 0.0019497357, 0.032450356, -0.006104045, -0.010953504, 0.025936158, -0.03288464, 0.015235986, 0.015742645, -0.011598892, 0.0221724, 0.01191857, -0.011043979, -0.00684594, 0.010241767, 0.0054646884, -0.0276009, -0.016032165, -0.0028921834, 0.0020809243, 0.0069967313, -0.018927364, -0.0050485036, -0.009216384, 0.0029736108, -0.0054073874, 0.004789142, 0.029217385, -0.017045485, 0.012943953, -0.015127416, -0.02202764, 0.008263381, -0.0035707457, 0.012666496, 0.008070367, 0.007937671, 0.0055129416, 0.036069356, -0.010253831, 0.0102719255, 0.031919573, 0.008239254, -0.009493841, -0.0055219894, 0.0149223395, -0.0094274925, -0.009180194, -0.016466444, 0.014910276, -0.015790898, 0.0004810253, 0.0024186976, 0.005901984, -0.013426486, 0.00035379484, -0.005256596, 0.0059623006, 0.013884893, -0.012618243, 0.009192257, 0.005461673, -0.014451869, -0.011894443, 0.015923595, -0.01050716, -0.004053279, -0.00608595, -0.012214121, -0.019542594, 0.0032269408, 0.01032621, -0.011025883, -0.0030836887, 0.012376976, -0.0015169637, -0.0047951736, 0.0026720276, 0.0061824564, -0.0045327963, 0.0055431, 0.017250562, -0.016020102, 0.0022920326, 0.0046564452, -0.004104548, 0.03956772, -0.0027202808, 0.0020130682, -0.02293239, -0.014102032, 0.0051781842, 0.027842166, -0.019542594, 0.008124652, 0.0010826838, 0.0044905744, -0.00032212862, -0.006851971, -0.0155978855, -0.013619499, -0.0071897446, -0.00009014512, -0.0019829099, 0.019940684, 0.01076049, 0.032040205, 0.01029002, 0.013173156, -0.009300827, -0.03126815, 0.0026614722, 0.0060557914, 0.010712237, -0.0250676, -0.019590847, -0.002197034, -0.00050138216, 0.01343855, 0.026346313, -0.0022302081, 0.007817037, 0.0041497853, -0.010458907, -0.014343299, 0.013535056, -0.0012342294, 0.029338019, -0.0010351845, -0.010706205, 0.035345558, 0.022051767, -0.028059306, 0.007985924, 0.024138723, -0.0064056283, 0.024802206, -0.008390046, 0.0015410904, 0.0115687335, -0.00028499617, 0.013161093, 0.020821307, -0.005594369, 0.0155978855, 0.020543851, 0.004246292, 0.012859509, -0.023668254, -0.017467702, -0.023716506, 0.004394068, -0.007611961, 0.015911533, -0.024995219, 0.012618243, 0.0029027387, 0.025550133, -0.012268406, 0.016912788, -0.002806232, 0.0025634577, -0.00045425977, -0.009403366, -0.0077024363, -0.00054888154, 0.03329479, -0.020314647, -0.012594116, -0.023451114, 0.029169131, 0.014102032, -0.004369941, -0.020374965, -0.02365619, 0.016599141, -0.010404622, -0.0118220635, 0.016357874, -0.010416686, 0.009578284, -0.0036310623, -0.0185534, 0.0055280207, -0.013233473, 0.009867804, 0.03380145, 0.00967479, -0.007581803, -0.014584566, 0.009837645, -0.019228948, 0.0057723033, -0.0070691113, -0.02260668, 0.0062729316, 0.019663228, 0.020543851, -0.0077506895, 0.011128422, -0.0012696654, -0.0007102285, -0.013559183, 0.014777579, -0.003637094, -0.02293239, 0.006858003, 0.0042885137, 0.021738121, -0.0011316912, 0.02238954, 0.008776072, -0.014825832, 0.027214872, 0.008160843, -0.00006469903, -0.017757222, -0.026249805, 0.003579793, 0.006622768, 0.02394571, 0.009982405, -0.009517968, 0.01090525, -0.014391553, -0.024199039, 0.006574515, -0.00008072064, 0.013860766, 0.0051812003, -0.005612464, 0.019035935, 0.021436537, 0.004822316, 0.0029117863, 0.016430255, 0.01256999, -0.029241512, -0.012582053, 0.03271575, 0.016406128, -0.001565217, 0.0153445555, 0.0068881614, 0.005886905, -0.006604673, 0.017853728, 0.0044905744, 0.00064199534, -0.0049218386, -0.0028122638, 0.004068358, -0.023680316, 0.017250562, -0.027721532, -0.018758478, -0.019976875, 0.012606179, 0.009125909, 0.020821307, 0.012401103, 0.019614974, -0.02709424, 0.003869313, 0.017274689, -0.009469714, 0.008263381, 0.002754963, -0.009228447, 0.00832973, 0.0015373206, -0.013824576, -0.0076602143, -0.0050786617, 0.018203564, -0.017624525, 0.0031726558, -0.015416935, 0.010784617, 0.014825832, 0.011586828, 0.014041716, 0.0019829099, -0.03143704, 0.006067855, 0.033777323, -0.012346818, 0.0022060815, -0.018806731, -0.008353855, -0.0020929878, -0.0022090974, 0.006441818, -0.0005126915, 0.0064840396, -0.031244025, -0.016961042, 0.038313136, 0.0031515448, 0.007274188, -0.018239755, 0.0076300562, 0.00953003, 0.011767779, 0.0114481, -0.018710224, -0.0034048748, -0.018565465, 0.01206333, -0.0031817032, 0.025501879, -0.005494847, 0.0003168509, -0.018710224, 0.01152048, -0.0001783111, 0.0037396322, 0.0043277196, -0.013329979, 0.02141241, -0.0069364146, -0.00055302825, 0.0039808988, 0.006640863, 0.018975617, -0.017853728, 0.017503891, -0.016297558, -0.0031515448, -0.02079718, 0.014705199, 0.0021005273, 0.018698161, -0.0030912282, -0.000008493809, -0.011647145, 0.015320429, -0.025308866, -0.012376976, 0.0038662972, 0.039326455, 0.007991956, -0.025694892, 0.0142467925, -0.024283484, -0.008601154, 0.021931134, -0.0065564197, -0.010483034, 0.002646393, 0.019795924, 0.0053772293, -0.0067373696, 0.0020628294, 0.008655439, 0.0072199027, -0.0057813507, -0.0073164096, 0.018263882, 0.012039203, 0.011731588, -0.010109071, 0.03734807, 0.0008610202, 0.018589592, 0.007931639, -0.009723044, 0.006140235, 0.020157823, -0.0028710726, -0.005850715, 0.023704443, -0.0053380234, -0.0057180184, 0.011243023, 0.0008112589, 0.004252324, 0.001979894, 0.006212615, -0.007002763, 0.015018846, -0.0029901979, 0.028565966, 0.0012078409, -0.008884642, -0.0017838649, 0.017781349, -0.014295046, -0.0025332994, -0.009759234, 0.001655692, 0.014282983, -0.005241517, 0.019228948, 0.0040020095, -0.009596379, 0.0096506635, -0.01231666, 0.020133698, 0.023704443, -0.0037667747, 0.029844679, 0.0045327963, 0.0060738865, -0.011755715, 0.015585822, -0.016345812, -0.041353095, -0.002157828, 0.0075456128, -0.038433768, 0.012521736, -0.0042070863, 0.022305097, 0.0015064083, 0.0036039199, 0.011767779, -0.01604423, 0.023173656, 0.019313391, -0.004231213, 0.012346818, 0.009813519, -0.035345558, 0.013329979, 0.0025076647, -0.013100776, 0.01220809, -0.005310881, -0.00136768, 0.003615983, -0.025381247, -0.008486552, 0.014174413, -0.0017220404, 0.0019512436, 0.015115352, -0.009403366, 0.0005892183, -0.0066649895, 0.008004019, 0.010012563, -0.01246142, -0.023282226, 0.00035379484, -0.02025433, -0.0065564197, 0.031099264, 0.02007338, 0.00071512925, -0.01694898, -0.011695398, 0.007841164, -0.0042191497, -0.0009643124, -0.0029042466, 0.0013081173, 0.017986424, -0.019530531, -0.019976875, 0.0010872076, -0.008420204, -0.024319673, -0.020350838, -0.0020432265, 0.0022603665, 0.011900475, -0.0065624514, 0.011031915, 0.004378989, 0.033825576, 0.010109071, 0.009252574, 0.005169137, -0.0061311875, -0.0026388534, 0.029941184, -0.0025317913, 0.0026931383, -0.008884642, -0.027962798, 0.02387333, -0.0033777324, -0.019313391, -0.0036461414, -0.0015606933, 0.0119909495, -0.00036265387, 0.028469458, -0.011327467, -0.018432768, 0.0038934397, -0.012286501, -0.009318923, 0.007177681, -0.0139090195, -0.002967579, 0.004502638, 0.031075139, 0.007593866, 0.012189995, 0.01705755, -0.012968079, -0.0150067825, -0.008993212, -0.00056999235, 0.011562701, 0.018975617, -0.005359134, -0.002733852, -0.0150912255, 0.028348826, -0.0041678804, 0.018239755, 0.0077687846, 0.028710725, -0.017105801, -0.023680316, -0.00953003, 0.009095751, -0.00058544846, -0.0045629544, -0.02126765, 0.010368432, -0.00033551137, 0.009704948, 0.010006532, -0.017347068, -0.027214872, 0.015875341, -0.0053319917, -0.018770542, 0.028252319, 0.03464588, 0.028951991, 0.00013034052, -0.0033928116, 0.015561695, 0.02257049, 0.01079668, 0.0021804469, 0.018698161, -0.016900726, 0.021207334, 0.0079617975, -0.006393565, 0.007111333, 0.014150286, 0.01340236, 0.014102032, 0.0009907009, -0.0009982405, -0.017950235, -0.0046112076, -0.029531032, -0.024440307, 0.03553857, -0.002105051, 0.010555414, 0.0058989683, -0.013559183, 0.010826838, -0.011454131, -0.0049369177, -0.007563708, -0.0078773545, -0.012799193, 0.02293239, -0.00623071, -0.027190745, 0.015670266, 0.00992812, -0.024066344, -0.02485046, -0.0068881614, 0.005983412, 0.006393565, -0.00572405, 0.009692886, -0.019494342, 0.004701683, -0.01974767, 0.003401859, 0.017962297, 0.014596629, -0.019590847, -0.0016179942, -0.009439556, 0.016804218, 0.002501633, -0.033680815, -0.007328473, 0.014813769, -0.01029002, -0.025043473, 0.013112839, -0.0066589583, 0.015223922, -0.008733851, -0.011025883, 0.0066649895, 0.014862022, 0.005467704, 0.013824576, 0.029193258, -0.009150036, -0.0018758478, 0.0067916545, -0.0055038943, 0.00025804216, 0.012099519, 0.022341287, -0.012521736, 0.009608442, -0.008058304, -0.02658758, -0.0008346316, -0.0033536057, 0.0101331975, 0.0030912282, -0.011387783, 0.01282332, -0.0005074138, -0.016237242, 0.016273431, -0.00075546606, -0.02278763, 0.025477752, 0.021231461, -0.010772553, -0.012352849, -0.026563453, 0.0013088713, -0.004427242, -0.0041377223, 0.028831359, -0.0014574011, 0.0066287997, -0.0033355106, 0.0014807738, 0.005540084, -0.018215628, -0.019421961, 0.008950991, 0.016900726, -0.014886149, 0.0065322933, -0.012039203, -0.017781349, 0.012352849, 0.0013752196, -0.00060769025, 0.009487809, 0.03901281, 0.003923598, 0.011888412, 0.011797937, -0.007032921, -0.0010344306, 0.007358631, 0.01278713, 0.019771798, -0.027021859, 0.005998491, 0.008577027, -0.013884893, 0.0023990946, -0.01329379, -0.035080165, 0.005241517, -0.01380045, 0.0018019599, 0.00093716994, -0.014886149, -0.0053711976, 0.010965567, 0.031219898, -0.00047084683, 0.0040201047, 0.010591604, 0.0083840145, 0.012346818, -0.0046172393, -0.013716006, 0.009324954, -0.027118366, 0.0040593105, 0.008432267, -0.013209346, 0.0010562954, -0.002016084, -0.007829101, 0.019988937, 0.012630306, -0.0017371195, -0.0005096757, -0.015453125, -0.007979893, 0.0021713993, -0.010404622, 0.0025619497, 0.0108630285, -0.0043850206, -0.008438299, -0.02137622, -0.011550638, 0.008540837, -0.0059321425, -0.02887961, -0.00094395556, -0.0031575765, 0.0072138715, 0.0052656434, -0.017901981, 0.015742645, -0.0101030385, 0.015609949, 0.0012447848, 0.0004406885, 0.013703942, -0.008148779, 0.021508917, 0.0153445555, -0.016261369, 0.009825583, 0.014186476, -0.012907763, -0.012256343, 0.00096280454, -0.01644232, 0.0056999233, 0.0039386773, -0.0051812003, -0.009873835, -0.00019235356, 0.003006785, 0.0051238993, -0.005114852, 0.014705199, -0.0016692632, 0.018975617, -0.012093488, 0.0075033912, 0.023644127, 0.0025800446, -0.009885899, 0.013197283, 0.004179944, -0.023125403, 0.015670266, 0.028517712, -0.0070811748, 0.0076783095, 0.016900726, 0.013884893, 0.0374687, -0.0032480515, -0.022087958, -0.019180695, -0.0049369177, -0.018770542, 0.0028182955, 0.0113154035, 0.0070992694, 0.009668759, -0.021183208, 0.0041678804, -0.003003769, -0.0062548365, -0.012153805, -0.009861772, 0.0096506635, -0.014379489, -0.010917313, 0.008577027, 0.014222666, -0.0035405874, 0.02637044, 0.004029152, 0.0025332994, 0.00072945445, 0.02191907, 0.012322691, 0.015730582, -0.011532543, 0.016997231, -0.005075646, -0.011327467, 0.0013571247, 0.014789643, 0.038385514, 0.004053279, 0.017045485, 0.023813013, 0.010054786, -0.0056004007, 0.020230204, 0.016538825, -0.004948981, -0.018179437, 0.0081306845, 0.0038964555, -0.008902738, 0.007895449, 0.021617487, 0.011387783, 0.009077656, 0.014270919, -0.010856997, -0.003026388, 0.018963555, -0.011508416, -0.00659261, -0.002338778, -0.0037607432, 0.007871322, 0.001350339, -0.022739377, -0.012105552, -0.013366169, 0.0013752196, -0.026828846, -0.0052294536, 0.004608192, -0.0047800946, 0.020941941, -0.005525005, -0.006924351, -0.0064719766, 0.0015373206, 0.0017039453, -0.029965311, -0.025453625, -0.0029313893, 0.027070113, -0.000108664215, -0.0029132941, -0.008142748, 0.0084684575, 0.005286754, 0.026997732, -0.00013590095, 0.00320583, 0.007817037, -0.015694393, -0.028710725, -0.006206583, 0.020785118, 0.014174413, -0.0076059294, 0.008619249, -0.011689367, 0.019144505, 0.0067916545, 0.0016541841, 0.0000810505, -0.0005846945, 0.02007338, 0.027142493, -0.009240511, 0.0064719766, -0.018794667, 0.0011837143, 0.017250562, 0.0038301072, -0.012980143, -0.009216384, 0.004535812, -0.032377977, 0.0021834627, -0.007412916, -0.0031424975, -0.0039718514, 0.0033807482, 0.0051088203, 0.008088463, -0.0052053267, -0.0043337513, 0.009807487, 0.006387533, 0.027287252, -0.009493841, -0.012654433, 0.019253075, -0.0053380234, -0.004101532, 0.016261369, 0.014114096, -0.006242773, -0.008528775, -0.0012183964, 0.005681828, 0.012847446, -0.0049067596, -0.020845434, 0.010458907, 0.0033053523, 0.012485546, 0.0025438548, 0.007744658, 0.006514198, -0.0031002758, 0.001440814, 0.012401103, 0.0041316906, -0.017962297, -0.016900726, -0.023161594, -0.0035375715, 0.014693135, -0.014379489, -0.0013375217, -0.0221724, -0.024536813, 0.0038482023, -0.0027263125, 0.0021849705, 0.01206333, 0.0033596375, -0.014958529, 0.0045448593, -0.00659261, -0.026056793, 0.0057180184, -0.003941693, -0.016176926, -0.018263882, -0.04060517, -0.004771047, -0.0011233976, -0.014174413, -0.008414173, -0.0038843923, 0.044031154, -0.024319673, -0.008788136, 0.027190745, 0.011068105, -0.00411058, -0.013836639, 0.006640863, -0.000522493, 0.00089721015, -0.018155312, -0.018360388, -0.027190745, -0.0037577273, -0.008649408, 0.0058446834, 0.01593566, -0.00100955, 0.013728069, -0.008444331, -0.00010998364, -0.009011308, -0.008432267, -0.012413166, 0.0145001225, 0.017600399, -0.006212615, -0.0036039199, -0.0029615476, 0.004517717, 0.028348826, -0.015995976, -0.016309623, 0.00641166, 0.004774063, -0.0013488311, -0.0064780083, 0.014970592, 0.0058597624, 0.01314903, -0.013378233, 0.0048404112, -0.00034682074, 0.0017733095, -0.022293033, -0.011526512, 0.011436037, -0.001385021, 0.0022060815, -0.0052505643, -0.0088062305, 0.00086478994, 0.015042973, -0.020664483, -0.008926864, 0.018203564, 0.026635833, 0.008890674, 0.021122891, 0.02492284, 0.013498866, 0.011966824, -0.005199295, -0.00078185456, -0.013824576, -0.009457651, 0.0046745404, -0.009252574, 0.022087958, -0.0004893188, -0.013631563, 0.0069545098, 0.011683335, 0.00989193, 0.0154289985, 0.0022000498, -0.004496606, 0.027166618, 0.0076421197, -0.027407885, 0.022884136, -0.021870816, 0.012630306, -0.018541338, 0.011134453, 0.012364913, 0.01246142, -0.008703693, -0.019735608, -0.020302584, 0.0059291264, -0.0023372702, -0.0041075638, -0.007237998, 0.009644632, -0.0011927617, -0.00572405, 0.004955013, -0.0065262616, 0.0013194267, -0.01314903, 0.0043669255, 0.016755965, -0.019928621, -0.0049580284, -0.008275445, -0.007666246, 0.015211859, 0.028204065, -0.008148779, 0.0037155056, 0.034573503, 0.013305853, 0.032160837, 0.0035948723, -0.012606179, 0.030254832, 0.0078773545, -0.0032872574, 0.020411154, -0.0059080156, -0.010923346, 0.0062488047, -0.012177931, -0.010700174, 0.010627793, 0.0013488311, -0.00074453367, -0.018505149, 0.014126159, -0.012811257, -0.0151756685, -0.015320429, -0.00021506655, -0.009318923, -0.005057551, 0.018010551, 0.018999744, 0.028565966, 0.009729075, 0.0058205565, 0.0054586567, -0.008239254, -0.0077084675, -0.00608595, -0.017045485, 0.001709977, -0.022968581, 0.0045267646, 0.0062669, -0.012232216, 0.010428749, 0.019868305, 0.0064357864, 0.0073345047, -0.00528977, -0.0038270913, 0.005039456, -0.023523493, 0.017479764, -0.0016059307, -0.008625281, -0.0024066344, -0.012310628, 0.015416935, -0.00053229445, -0.010706205, 0.02068861, 0.0066529266, 0.015308365, -0.013161093, -0.012690623, -0.018119121, 0.00633928, 0.022630807, -0.0015049004, 0.02416285, -0.0049761236, 0.014789643, 0.008082431, 0.009753202, -0.020109572, 0.008727819, 0.011966824, 0.02376476, -0.008275445, -0.008022115, 0.0055370685, -0.02477808, 0.026201552, 0.016092481, 0.013221409, 0.007539581, -0.01369188, -0.0130887125, 0.004481527, -0.026611706, -0.0031545607, -0.0022211606, 0.027890418, -0.009222415, 0.021533044, 0.017503891, -0.021122891, 0.003257099, -0.0025544101, 0.0058265883 - ] - } - ] - }, - { - "name": "springfield-illinois-sangamon-county-seat", - "title": "Springfield: Illinois' Sangamon County Seat", - "content": "Springfield serves as the county seat of Sangamon County and is located in the U.S. state of Illinois.\n\n---", - "keywords": [ - "Springfield", - "county seat", - "Sangamon County", - "Illinois", - "United States" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.03381783, 0.03834294, -0.006908156, -0.029640809, 0.003065829, -0.028087812, 0.01140649, 0.005964309, -0.04364455, 0.019747151, -0.020349607, 0.015288981, -0.011935311, 0.005204546, 0.0035377522, 0.00076311023, -0.011868373, 0.021996317, -0.005338425, -0.0064429264, 0.039735284, 0.0016475483, -0.045652736, 0.031782873, -0.00293195, 0.017979948, -0.048276763, 0.02198293, -0.031836424, -0.0011814821, -0.016052091, -0.0011095221, 0.033228766, 0.034246244, -0.0037653465, -0.03818229, -0.004511722, -0.032211285, 0.0019696946, 0.019425841, -0.009699533, -0.0052078925, 0.015717393, 0.00463556, 0.019532945, -0.007530693, 0.0039126133, -0.02950693, 0.003952777, 0.010998159, 0.010516195, -0.009230956, 0.0012492583, 0.03944075, 0.032720026, -0.0015027917, 0.02051026, 0.037753876, -0.011533675, -0.053283837, -0.021514354, -0.010516195, -0.030444082, 0.005304955, 0.021675007, 0.020229114, -0.000713324, 0.0040498395, 0.034647882, 0.0127385855, 0.040029816, 0.036656067, 0.007798451, 0.04257352, 0.040324353, -0.005415405, -0.039360423, -0.0168152, -0.023468986, 0.020590588, 0.03836972, -0.03574569, -0.004090003, 0.02318784, 0.011781351, -0.020175563, 0.0062822714, -0.07272307, 0.0003252841, 0.010168109, 0.009699533, -0.0094987145, -0.021822276, -0.0139501905, -0.03906589, -0.036281206, 0.052962527, -0.013079977, -0.0028298672, 0.03887846, 0.0051945047, -0.008032739, -0.051382754, -0.0064529674, 0.016252909, 0.017926397, 0.008675358, -0.0226791, -0.019773927, -0.013561942, 0.010168109, -0.00012237376, 0.0060446365, -0.026173342, -0.01930535, -0.011895148, 0.0030256652, -0.02383046, 0.047768023, 0.009940515, -0.015195265, -0.00014465206, -0.012792137, 0.001655079, -0.013976966, 0.031461563, 0.016426953, -0.040029816, 0.028382346, -0.014927507, 0.017765742, -0.0035176703, 0.00974639, 0.008581643, -0.0020081848, -0.021688396, 0.02886431, 0.0503385, 0.030122772, 0.01981409, -0.07293727, 0.00017446105, 0.0153559195, -0.020563813, 0.015128326, -0.016600994, -0.0077850632, -0.015543351, -0.011045016, -0.031086702, -0.021969542, -0.018515464, -0.03732546, 0.019894417, 0.03446045, -0.00949202, 0.0015086489, -0.037566446, 0.039654955, -0.045706287, -0.022049868, -0.017645251, 0.029935341, 0.009485327, -0.020617364, -0.0067173783, -0.0046422537, -0.011654166, -0.007865391, 0.0069684014, 0.033683952, 0.00055476103, -0.016092254, -0.07475803, 0.0138297, -0.029988892, 0.004076615, -0.023214616, 0.0019295309, 0.036093775, -0.0051074834, 0.028302018, 0.026963228, -0.015114938, -0.019773927, -0.009512102, 0.001322055, 0.025088923, 0.045090444, -0.02313429, -0.024365976, 0.037566446, 0.036307983, 0.01401713, 0.026414324, -0.0031779525, -0.035638586, -0.031434786, 0.0016600995, 0.015918212, -0.05917451, 0.01885016, 0.030792167, -0.014753465, 0.006888074, 0.01937229, 0.0017538148, -0.031408012, 0.008099679, 0.04134183, 0.009143935, 0.0041335137, -0.016212745, 0.05976358, -0.02937305, 0.028944638, -0.013936803, -0.0028583163, -0.022264076, 0.05015107, 0.0327468, 0.014807016, -0.0055024265, -0.00075557956, -0.010951301, -0.006740807, -0.055854313, -0.027873605, 0.024419528, 0.00076018163, 0.014512482, -0.0038188982, 0.021059165, 0.039494302, -0.04286805, 0.04415329, 0.016855365, -0.0073767323, -0.042091552, 0.027498744, 0.0020550424, 0.043751653, -0.014204561, -0.02886431, -0.009773166, -0.021393862, 0.015770944, 0.015195265, 0.000923765, 0.015503187, -0.001325402, 0.0068278285, 0.032666475, 0.008193394, -0.016775038, 0.008233557, -0.017243614, -0.003323546, 0.018073663, 0.019158084, -0.007918942, 0.02701678, -0.011814821, 0.013227244, 0.0065266006, -0.024526631, -0.020055072, 0.0016609362, 0.0119553935, -0.015061386, -0.028061036, -0.009679451, -0.00076143676, 0.029721135, 0.041395385, 0.0024918227, 0.024365976, 0.025075534, -0.032345165, -0.020657528, -0.017029407, -0.004153596, 0.010723707, -0.043939084, 0.013039813, 0.019706987, -0.004632213, 0.049240693, 0.0063994154, -0.029721135, -0.0033252193, -0.01682859, 0.030685063, 0.0335233, 0.051115, 0.015208653, 0.021996317, 0.014418767, -0.044367496, 0.007497223, 0.026521428, 0.026200118, 0.0036046917, 0.0004248566, 0.019854253, -0.00481295, 0.010127946, 0.013802923, -0.029533705, -0.05671114, 0.006463008, -0.03976206, 0.0005279853, 0.026119791, -0.006874686, 0.022130197, 0.0028415816, -0.012524379, 0.051704064, -0.03338942, -0.016989244, 0.0168152, 0.020001521, -0.0013396266, -0.01510155, 0.008862789, -0.019961357, 0.0008275395, 0.026374161, 0.011366326, -0.03384461, 0.0040665744, -0.008226864, -0.043483894, -0.0077114296, -0.02905174, 0.021567905, 0.008494622, 0.022130197, 0.014686525, -0.018073663, -0.019961357, 0.004796215, 0.05052593, 0.03421947, 0.012002251, 0.0019797354, 0.00052421994, -0.026427712, 0.0035979978, 0.00070663006, 0.009900351, 0.030738616, -0.020483484, 0.018261094, 0.0071089743, -0.035531484, -0.040993746, 0.0016073846, -0.013468226, -0.029105293, 0.005532549, -0.0137895355, 0.0069483193, 0.010998159, 0.043885533, -0.017417656, 0.017578311, 0.035076294, 0.0037921222, -0.0070353406, 0.01708296, 0.0075775506, -0.0027294578, 0.019653436, 0.0010651747, -0.0064529674, 0.011379714, 0.00688138, -0.0065199067, 0.04134183, 0.016908916, -0.03866425, -0.016426953, 0.0017872845, -0.026655307, -0.012725198, -0.031140253, -0.023683192, 0.02313429, -0.039039113, -0.006730766, 0.005924145, -0.021059165, -0.017979948, 0.021005614, -0.062334057, 0.010502807, 0.054301318, 0.028435897, 0.014284888, -0.032827128, 0.010348846, 0.029132068, -0.031300906, -0.02886431, 0.00083548855, -0.054943938, 0.0056162234, -0.006088147, -0.030524408, -0.001994797, 0.0035410991, 0.007972494, -0.02459357, -0.053337388, -0.021581292, 0.030872494, 0.0168152, 0.012203069, 0.0060245544, -0.042038, -0.0019245105, -0.018435137, -0.00876238, 0.016226133, 0.02918562, 0.035504706, -0.028489448, -0.034701433, -0.030497633, -0.014405379, -0.0071022804, 0.03325554, 0.0020349605, 0.04262707, -0.0020366341, 0.013501696, 0.052319907, 0.013180386, -0.015463023, 0.017939784, 0.048598073, -0.04308226, -0.001778917, 0.009612511, 0.0184753, 0.011854985, 0.030122772, 0.035129845, -0.0097129205, 0.020778019, 0.012015639, 0.011145426, 0.0015998539, 0.021112716, -0.049267467, 0.016145807, -0.024767613, 0.045652736, 0.030952822, 0.002520272, -0.000594088, -0.04351067, -0.012859077, 0.019947968, 0.016975855, 0.014405379, -0.0015103223, -0.030631512, -0.012477522, -0.02771295, -0.018743059, 0.010663461, -0.042412862, -0.009217569, -0.004197106, 0.0041234726, 0.013816312, -0.009739696, -0.048089333, 0.007015259, -0.06276247, 0.008996668, 0.0051677288, 0.0040565333, -0.030711839, -0.042278983, 0.0031511767, -0.010281906, 0.08477218, 0.022920083, -0.012243234, -0.015543351, 0.02071108, 0.0128992405, -0.02032283, -0.032853905, 0.02554411, 0.035263725, -0.00739012, 0.057353757, 0.036415085, -0.019291962, 0.0037251827, -0.0058337767, -0.0131669985, -0.041582815, 0.011540369, 0.009150629, 0.054515526, 0.0076779597, -0.031193804, -0.0302031, 0.004505028, -0.014606197, -0.017364105, -0.0059007164, 0.037164807, -0.0009614185, -0.0013145243, 0.009505408, -0.0031779525, 0.0042941687, 0.0040833093, 0.012189682, 0.013495002, -0.0029553787, 0.0075775506, -0.032827128, -0.036656067, -0.0037854284, 0.002249167, 0.030095996, 0.038075183, -0.00525475, -0.019666823, -0.0047192345, 0.0002273851, 0.015422859, -0.03657574, -0.025892196, 0.016386788, 0.0069750953, 0.00075223256, 0.007122362, 0.001224156, 0.005853859, -0.008856095, -0.02515586, 0.042145107, -0.0072428533, -0.004769439, -0.005562672, -0.0022541874, -0.004153596, -0.0140572935, -0.011620696, -0.005820389, -0.004920053, 0.012591319, 0.0080394335, 0.03804841, 0.01169433, 0.005311649, 0.011620696, -0.0051509943, -0.0037251827, -0.0076779597, -0.0049367878, 0.006744154, 0.0061115758, 0.043242913, 0.004501681, 0.0192518, -0.0066504385, -0.028810758, -0.012129436, 0.008300497, -0.008568255, 0.013314266, -0.0033536686, 0.027097108, 0.014445542, 0.008869483, 0.013976966, -0.0024985166, -0.024165157, -0.026347386, 0.030256651, 0.020737855, 0.01491412, -0.0010266845, -0.0062722308, -0.0016358339, 0.0031879933, -0.02528974, 0.0028415816, -0.017310552, -0.017377492, 0.013963578, 0.0041703302, 0.02739164, -0.03930687, -0.01886355, 0.002987175, 0.032398716, 0.009391611, 0.017792517, 0.003303464, 0.019747151, 0.01727039, -0.005294914, -0.001619099, -0.010455949, -0.011185589, 0.030229876, 0.035852794, 0.016654545, -0.012832301, -0.010081088, -0.028194916, -0.0023545967, -0.015061386, 0.031193804, -0.021126105, 0.012557849, -0.028676879, -0.022893308, 0.013976966, -0.013307571, -0.0006459661, -0.025597662, 0.00020709405, 0.011895148, -0.010636685, -0.0037184888, -0.003828939, -0.008849401, 0.004230576, 0.003701754, 0.040377904, 0.013856475, 0.020215727, -0.023174454, -0.0032432184, -0.019037591, 0.04289483, -0.005298261, 0.0017287125, 0.059870683, -0.0030959516, -0.001994797, -0.0023211269, 0.0029687665, 0.008836013, 0.005950921, -0.035611812, 0.04179702, -0.019211635, -0.02516925, 0.010924526, -0.016346624, 0.007791757, 0.033603627, 0.014793628, -0.02364303, 0.0588532, 0.02115288, 0.0036615904, -0.008240252, 0.004545192, 0.003946083, -0.009010056, -0.0065332945, -0.011125344, -0.00311436, -0.008876177, -0.014579422, 0.005013768, -0.00825364, -0.00532169, -0.0057835723, -0.023629641, 0.028783983, -0.02306735, 0.03293423, 0.043537445, -0.019666823, -0.031274132, 0.016065478, 0.02982824, 0.012162906, -0.0096259, 0.031809647, 0.0017077938, 0.020055072, 0.034969192, -0.013019731, -0.010174803, -0.0012149519, 0.017096346, 0.025129085, -0.052587666, 0.0061684744, 0.025182638, -0.0011396449, -0.03700415, -0.031488337, -0.0064596613, 0.010435867, -0.0138966385, 0.020630753, 0.00056689384, 0.034942415, -0.036200877, 0.044099737, -0.036388308, -0.010917831, 0.02937305, 0.015208653, 0.0077047357, -0.038637474, -0.015650453, -0.019532945, 0.007999269, -0.031434786, 0.006245455, 0.009036832, 0.0053785886, -0.04262707, -0.0064194975, -0.014472319, -0.008427682, 0.024700673, 0.0069617075, -0.019091144, -0.023977727, -0.002160472, 0.00952549, 0.008307192, -0.017926397, 0.03834294, 0.0065098656, -0.009438469, 0.021554517, -0.0035678751, -0.00048656645, -0.05912096, 0.022103421, -0.019479392, -0.014043906, 0.036120553, -0.0067910114, 0.008300497, 0.010214967, 0.019412452, 0.01688214, 0.02593236, 0.036066998, 0.019332126, 0.024205321, 0.0015320777, 0.0013212182, 0.011419877, 0.030845718, 0.014981058, 0.00691485, 0.017752353, 0.012410582, 0.028757207, 0.0014919139, -0.01682859, -0.011366326, 0.017176675, 0.015342532, -0.0010810728, -0.007657878, -0.006379334, -0.009806636, 0.043885533, 0.0053317305, -0.031167028, -0.01057644, -0.0032750147, -0.025838645, -0.009967291, 0.007992576, -0.0001796907, -0.013133529, -0.009652675, -0.0013664024, -0.035076294, 0.014298276, 0.0021203086, -0.016400175, 0.0060245544, -0.018033499, 0.029212395, 0.014485707, 0.0014760159, 0.018046888, 0.010763871, 0.0054756505, 0.0022608815, -0.00927112, -0.013233938, 0.026240282, -0.028489448, 0.034754984, 0.043805204, -0.0061785155, 0.027605847, 0.014351827, -0.0042740866, -0.018180767, 0.03344297, -0.00012551155, -0.00344069, -0.017190062, 0.0231209, -0.014887343, 0.019358901, 0.0011212365, -0.000525475, 0.020925285, 0.017939784, -0.041047297, -0.0021019, 0.014967671, 0.009311284, -0.019466005, -0.016065478, -0.017859457, -0.0049669105, 0.008936423, 0.031595442, 0.0013028099, 0.02905174, 0.0108174225, -0.015422859, -0.007590939, 0.025892196, -0.0069416254, 0.0070353406, 0.043376792, 0.01156045, -0.0427074, 0.00942508, -0.043751653, -0.022290852, 0.013039813, -0.02108594, 0.027579071, -0.014244724, 0.009398305, 0.03828939, -0.00053844455, 0.00942508, 0.018328033, 0.014391991, -0.0023428823, 0.027820054, 0.006563417, 0.020028297, 0.030792167, 0.019466005, -0.003032359, -0.0014115865, -0.011540369, 0.023535926, 0.0028248467, -0.024781, -0.012176294, -0.013749372, 0.00311436, -0.017230226, 0.001224156, 0.0029470113, 0.016199358, -0.0327468, -0.006908156, -0.014258112, -0.0336304, 0.021193044, 0.023308331, -0.022103421, -0.016547443, 0.0116006145, -0.035183396, -0.005278179, 0.005646346, 0.008842708, -0.0062521487, 0.014311664, 0.009076996, -0.022545222, -0.01955972, -0.016252909, 0.044233616, 0.012236539, -0.0015923232, 0.0028717043, -0.022598773, -0.0096259, -0.0062722308, 0.023147676, -0.008106372, 0.0012040741, 0.018970653, 0.015704006, 0.034514003, -0.007262935, -0.020095237, -0.01032207, 0.0041167787, -0.033603627, 0.0050874017, -0.0034172612, 0.027605847, 0.035852794, 0.012571237, -0.020403158, -0.0036281205, -0.003457425, -0.009639287, 0.02071108, -0.016426953, -0.021755336, 0.0039260015, -0.0056061824, -0.00844107, -0.0067776237, 0.009251038, 0.03333587, -0.019144695, -0.029935341, -0.027686175, -0.010850892, 0.009164017, 0.013294184, 0.048972934, -0.034005262, -0.01828787, -0.0045083747, 0.03330909, -0.001119563, 0.021380475, 0.030711839, -0.0038958786, 0.020242503, 0.004786174, -0.005780225, -0.0053685475, 0.035022743, 0.015556739, -0.035263725, 0.032961007, 0.0011371346, 0.0057166326, -0.011767963, 0.0038992255, -0.015784333, 0.00463556, 0.004906665, -0.014512482, -0.008133149, -0.04294838, -0.009076996, 0.0017044469, 0.0048330314, -0.01083081, -0.019077756, -0.020737855, 0.009311284, 0.016641159, -0.0030926047, 0.0024985166, 0.011446654, 0.02127337, 0.0086084185, 0.028275242, -0.009478632, 0.030872494, 0.031006373, -0.014873955, -0.018876938, -0.010583134, -0.002465047, 0.0013413001, 0.041877348, -0.024044666, -0.0015421186, -0.0019445923, 0.037111256, 0.0039996346, 0.0029637462, 0.010275212, 0.0034021998, -0.0013555248, 0.019840866, -0.029988892, -0.01918486, 0.016868753, -0.02982824, 0.012591319, -0.042653844, 0.0053484654, -0.006446273, 0.010931219, 0.0013505043, -0.0036080386, 0.0141108455, -0.030256651, -0.026026076, -0.022290852, -0.005663081, -0.017002631, -0.014458931, 0.00056438363, -0.01133955, 0.008481234, 0.0053183427, -0.0154094715, 0.00063676195, -0.003018971, 0.017190062, 0.020590588, 0.035129845, 0.023562701, 0.020563813, 0.015931599, -0.0031980344, 0.023147676, -0.028114587, 0.004910012, -0.01918486, 0.049776208, 0.009010056, 0.0128456885, 0.022759428, -0.019091144, 0.013106753, 0.014311664, -0.007149138, 0.008601725, 0.00005600953, 0.0070755044, 0.009853493, 0.0023596173, -0.0023094125, -0.0068043997, -0.0004936788, 0.0066604796, -0.037566446, -0.012624788, 0.0043142503, 0.02515586, -0.00372853, -0.013722597, 0.019573107, -0.0048330314, 0.039494302, -0.008682053, 0.015757557, -0.01025513, -0.0171499, 0.022638937, 0.020483484, 0.0025085576, 0.016467115, -0.009632593, 0.023669805, -0.02905174, 0.0032013813, 0.007959106, -0.026106402, 0.00850801, -0.011767963, -0.029480154, 0.0073097926, 0.022692489, -0.033362642, 0.033416197, 0.025329905, -0.055961415, 0.01840836, -0.017123122, -0.017832682, 0.039922714, -0.018876938, -0.0019529597, 0.004946829, -0.012952792, 0.028141363, -0.026454488, -0.0007534877, 0.04377843, 0.0072093834, 0.005823736, -0.017230226, -0.012792137, 0.00006181444, -0.03242549, -0.019934582, -0.021849051, -0.025370069, -0.002111941, -0.033469748, -0.010469337, 0.003952777, 0.013575329, -0.019104531, 0.0126114, 0.009485327, 0.016239522, -0.012818913, 0.004853113, 0.0068077464, -0.031140253, -0.007517305, -0.0171499, -0.026842738, 0.022250688, 0.005693204, -0.043564223, -0.017618475, -0.022143584, -0.0034708127, 0.0045083747, 0.0060847998, 0.00952549, -0.009699533, -0.022237299, 0.022063257, -0.017497983, 0.022344403, -0.02574493, 0.047446713, -0.0213537, 0.0079323305, 0.00078988605, -0.021367086, -0.025517335, 0.031327683, -0.0050940956, 0.02835557, -0.012055803, -0.0047828266, -0.015436247, -0.024687285, -0.0010835831, 0.009759778, -0.015235429, -0.0056262645, -0.020697692, -0.012571237, -0.01510155, -0.02618673, 0.024473079, -0.021500966, 0.003035706, 0.001021664, 0.014499094, 0.0071959957, 0.012979568, 0.02286653, -0.0036850192, -0.014164397, 0.031836424, 0.011774657, 0.012912628, -0.0184753, -0.020911898, -0.0038222452, -0.013963578, 0.006121617, 0.0133276535, -0.014084069, 0.013655657, -0.00032967702, 0.005174423, -0.008668665, 0.0024884758, -0.02701678, -0.014766852, -0.0050740135, -0.030310202, -0.04166314, 0.009170711, -0.024151769, 0.0044180066, 0.017444432, 0.01484718, 0.018394973, -0.004210494, 0.00046104577, -0.008882871, 0.024245484, 0.03435335, -0.03467466, 0.03542438, -0.0040063285, 0.005786919, 0.021567905, -0.0105496645, 0.0058438177, -0.01955972, -0.0041335137, -0.00035122316, 0.008146537, 0.01840836, 0.007691348, -0.009679451, 0.0035076295, -0.024124993, -0.033924937, 0.004518416, 0.011219059, 0.0056061824, 0.021688396, 0.0041837185, -0.03333587, 0.010275212, -0.0062923124, 0.008889565, 0.010214967, 0.00026043647, 0.023428824, 0.0004903318, -0.00038260105, -0.0006434559, -0.0097664725, 0.0016216092, -0.026200118, -0.030336978, -0.032479044, 0.011687635, 0.0035143234, 0.027927157, 0.040029816, -0.005435487, 0.015503187, 0.016159194, 0.0034741596, 0.0037218358, -0.021996317, -0.02676241, 0.0044146595, -0.048464194, 0.00042276474, 0.0205772, 0.015998539, -0.021889215, -0.011185589, 0.0020801448, -0.028650103, 0.021300146, -0.024044666, 0.009599123, 0.014378604, 0.0189171, 0.0017521413, 0.014927507, 0.031889975, 0.013628881, -0.020871734, -0.00017404268, 0.015128326, -0.014151009, -0.0056195706, 0.017257001, -0.004997033, -0.022451507, 0.022237299, 0.029212395, -0.030979598, -0.012270009, 0.010596522, -0.016413564, 0.0034641188, 0.007249547, -0.011386408, 0.0029386438, 0.00039410626, -0.006245455, 0.02370997, 0.004545192, 0.020737855, -0.002578844, 0.022116808, -0.0032515859, 0.0013739332, 0.015476411, -0.014927507, -0.008454458, 0.039146215, -0.034835313, 0.0020249197, -0.0031879933, 0.019211635, -0.024285648, -0.00091288734, -0.034058813, -0.00872891, -0.003228157, -0.015717393, -0.0008861116, 0.0024750878, 0.0075842445, 0.028783983, -0.0011321142, -0.013856475, 0.00054974057, 0.032130957, -0.010208272, 0.026427712, -0.00036356514, -0.01347492, -0.010563052, -0.05671114, -0.008333967, -0.0020081848, -0.010181497, 0.013126834, 0.0024081483, 0.020858346, -0.006730766, 0.008420988, 0.015503187, -0.008072903, -0.0069215437, -0.009465245, 0.0058672465, -0.014860568, -0.035799243, -0.0071022804, -0.024071442, 0.034701433, 0.019720376, 0.004454823, 0.011024934, -0.00013429736, 0.02159468, -0.0077047357, 0.0026625183, -0.0039828997, 0.00096225523, 0.018113827, -0.029667584, -0.00942508, 0.03122058, -0.031729322, 0.009364835, -0.020416545, -0.0005422099, -0.017364105, 0.0058505116, 0.007021953, 0.024928268, 0.0021403902, 0.017805906, 0.03898556, 0.010081088, -0.0048430725, -0.001912796, -0.004769439, -0.023107514, -0.005264791, -0.014766852, -0.006600234, 0.018769834, 0.03237194, 0.027605847, -0.0009731329, -0.0068612983, 0.009431775, -0.020630753, -0.028114587, -0.007021953, -0.028221691, -0.008166619, 0.020590588, 0.008354049, 0.011834903, 0.024392752, 0.012336949, 0.0016977529, 0.017109735, 0.018957265, -0.0038188982, 0.0048598074, -0.016989244, 0.014552646, -0.0038088572, 0.001615752, 0.010455949, 0.0028449285, -0.01249091, -0.0034473839, -0.009003362, -0.0056396523, 0.008287109, -0.0094451625, 0.044822685, -0.009103771, 0.002928603, -0.0168152, -0.01510155, -0.009277814, -0.0027997443, -0.010589828, 0.011808127, -0.031916752, 0.01752476, -0.03330909, -0.01752476, 0.028998189, 0.0055927946, -0.021126105, 0.0076110205, -0.0252228, -0.008307192, 0.003701754, -0.002383046, -0.031113477, -0.01662777, -0.011004853, -0.0104961125, 0.010489419, -0.013568635, -0.011988863, -0.00060789427, -0.016935691, 0.0074704476, -0.0136824325, -0.0053819353, -0.00688138, 0.0025671297, -0.014900731, -0.016105643, -0.024392752, -0.0010434194, 0.01701602, -0.003959471, -0.011185589, 0.0003148248, -0.03815551, 0.00043008625, -0.0033955057, -0.02498182, 0.0120624965, -0.0369506, -0.0058739404, 0.03413914, 0.011111956, 0.012805525, 0.023228005, 0.021072553, -0.004100044, 0.014887343, 0.0009438469, 0.008226864, 0.015288981, 0.03708448, -0.008139842, 0.010971383, 0.00024223729, -0.007798451, 0.0034842007, -0.0061249635, 0.01745782, -0.0021270025, 0.014458931, 0.015824497, 0.02140725, 0.025343293, 0.013990354, -0.008956504, -0.00923765, -0.00438119, -0.0017471208, 0.0068947678, 0.005284873, -0.0064261914, 0.03759322, -0.018997429, -0.0013714229, -0.0020349605, 0.0060111666, -0.032023855, 0.0184753, 0.028783983, -0.008079597, -0.034380123, 0.010382315, 0.008882871, 0.006218679, 0.01086428, 0.024606958, 0.0059944317, 0.020055072, -0.002915215, -0.020282667, 0.0055927946, -0.006473049, 0.006238761, -0.03344297, -0.0021721865, -0.0112793045, 0.00930459, 0.033416197, -0.008320579, 0.013735984, -0.014632974, 0.018087052, 0.019010816, -0.006473049, 0.0122231515, 0.0051443, 0.013146916, 0.03957463, 0.004515069, -0.020362994, 0.009679451, -0.006988483, -0.00045853553, 0.00055768964, 0.00673746, -0.012437358, 0.005646346, 0.001465975, -0.020938674, -0.012604707, 0.0012157885, 0.022946859, -0.0012793811, 0.011607308, -0.026441101, -0.004153596, -0.019064369, 0.0042941687, 0.023535926, -0.004846419, -0.027190823, 0.02803426, 0.01561029, -0.011828208, -0.013776148, -0.01293271, -0.031943526, -0.0038523679, 0.0020935326, -0.0089297285, 0.009411693, -0.004371149, -0.007276323, 0.025316516, 0.0068043997, 0.01257793, -0.011419877, 0.018461913, 0.030551186, -0.007490529, -0.00032800352, -0.006888074, 0.0014651382, 0.014177785, 0.018394973, -0.001025011, 0.009485327, -0.0006643745, 0.01510155, -0.0028968067, -0.013300877, -0.010301989, -0.012892546, -0.0009061934, 0.015503187, 0.018635955, -0.003986247, 0.0012659932, -0.009619205, 0.0026139873, 0.010804035, -0.00067065004, -0.015021223, 0.006573458, 0.021434026, 0.0019027551, -0.010469337, 0.0014810363, -0.011192284, -0.013180386, -0.0071692197, -0.023562701, 0.008019351, -0.015931599, -0.0013329326, -0.015262204, -0.009940515, 0.012959486, -0.03976206, 0.014298276, -0.018073663, -0.0011923597, 0.009565653, -0.0031260743, -0.0057099387, -0.0061350046, -0.008882871, -0.020215727, -0.014432155, -0.010931219, -0.01376276, 0.031622216, -0.0011630737, -0.016520668, -0.006322435, 0.01184829, -0.017484596, -0.013856475, -0.012410582, 0.02318784, -0.020751243, 0.008059515, -0.008106372, -0.024553407, -0.011968781, 0.0068612983, 0.0051108305, 0.000101559766, 0.027793277, -0.0073499563, -0.0060446365, 0.012363724, 0.025878808, -0.015114938, 0.0119487, 0.01713651, 0.032104183, -0.011239141, 0.005529202, 0.029292723, 0.034058813, -0.00597435, 0.004756051, -0.0070688105, -0.008782461, -0.005693204, -0.009291202, 0.019399066, 0.011292692, 0.007590939, -0.013963578, -0.016011927, 0.0015956702, -0.017846068, -0.003107666, 0.004096697, 0.00532169, -0.00959243, 0.012444052, -0.009545572, 0.017216837, 0.012129436, 0.011219059, -0.00517777, 0.0014140968, -0.011312774, -0.0019696946, -0.006499825, -0.012504297, -0.004997033, 0.00020385168, 0.0075842445, -0.017042795, -0.0022090033, -0.0025805174, -0.009753084, 0.003594651, -0.012417276, 0.04771447, 0.007959106, -0.0057400614, -0.0045351507, 0.002043328, -0.0028181528, -0.017658638, -0.0060111666, 0.013963578, -0.0014894038, -0.0066772145, -0.0069416254, -0.0036850192, 0.005807001, 0.00082042714, -0.01893049, 0.020670915, -0.022438118, 0.0008794176, -0.015061386, -0.0024918227, -0.0026240281, -0.0057266736, -0.019666823, -0.02096545, 0.0070755044, 0.0140572935, -0.0013128509, 0.007912248, 0.001328749, 0.015784333, -0.008896259, 0.012042415, 0.022625549, 0.029024964, 0.017484596, -0.008059515, 0.024888104, -0.003457425, -0.0121695995, -0.013287489, 0.026722247, 0.009338059, -0.00074930396, -0.0046991524, 0.01369582, -0.026387548, -0.014860568, -0.0052279746, -0.004632213, -0.0077783694, -0.010971383, 0.0055158143, -0.014793628, 0.0029419907, -0.017939784, 0.00011275121, -0.016065478, -0.00026462018, 0.009900351, 0.0049367878, 0.0019211635, -0.024606958, 0.020992225, -0.0040230635, 0.015770944, -0.00077189604, 0.010790646, 0.012149518, -0.02364303, 0.008233557, -0.003979553, 0.0042038, -0.021675007, -0.027364865, 0.020429933, 0.0030741962, 0.004910012, 0.0037352238, -0.009036832, -0.009338059, -0.00065684377, 0.014338439, 0.013595412, 0.017765742, -0.016600994, -0.0037184888, 0.012698422, 0.013187081, 0.010844198, -0.006315741, 0.006476396, -0.0014283215, -0.008327273, 0.0040264106, 0.019733762, -0.033282317, 0.00094552035, -0.021367086, -0.028302018, -0.013816312, -0.01674826, -0.022170361, 0.015529962, 0.00077147764, 0.026200118, -0.017926397, 0.019278575, -0.0057501025, 0.0056597344, -0.031300906, -0.0070554228, 0.011085181, -0.013421369, -0.0034674658, 0.03727191, 0.01650728, 0.02198293, 0.008400907, -0.0138297, 0.02140725, -0.009371529, -0.024111606, -0.0065433355, 0.0072093834, 0.01054297, 0.012049109, 0.021567905, 0.0035979978, -0.008742298, 0.014151009, -0.010395704, 0.01162739, 0.0034105673, 0.009070301, 0.0061483923, -0.0010225008, 0.022786204, 0.006620316, 0.0022123503, -0.028489448, -0.015583514, -0.0027428458, -0.0075976327, 0.010770565, -0.010107864, 0.008581643, -0.009478632, 0.0044380883, -0.00002852773, 0.000048635728, -0.009619205, -0.017310552, -0.03786098, 0.011379714, 0.009579042, 0.0020450016, -0.031408012, 0.013635575, -0.004103391, 0.003959471, -0.0076043266, 0.013026426, -0.0039025724, -0.00018042285, -0.011426572, -0.018100439, 0.015637066, -0.0009087036, 0.0015061386, 0.011513593, 0.011259223, -0.011198977, 0.0055057732, -0.005264791, 0.011547063, 0.0011178895, 0.011178896, -0.0013797904, -0.006225373, -0.01860918, 0.0053752414, -0.007256241, -0.006606928, 0.0097664725, 0.0037251827, 0.014552646, -0.0189171, 0.00844107, -0.009157322, -0.015663842, -0.011614002, -0.015302368, -0.0031612176, -0.019519556, 0.00004845269, -0.029346274, 0.000018159952, 0.0012542788, 0.006355905, 0.004903318, 0.023268169, -0.030658288, 0.026708858, -0.014566034, -0.0062354137, -0.0066604796, 0.0026641919, -0.0023412087, -0.00576349, -0.021326922, -0.017283777, -0.0044180066, -0.016105643, -0.018314645, -0.0010785626, -0.017310552, -0.0012199723, 0.0053785886, 0.009538878, -0.0014007089, -0.027150659, -0.008387519, 0.0069550136, 0.022143584, 0.0028666838, 0.0021537782, -0.0066537857, -0.018033499, 0.032586146, -0.032961007, -0.014043906, 0.00474601, 0.017551536, -0.0030306855, 0.013401287, 0.0025939054, 0.0053685475, 0.024245484, -0.010636685, 0.010395704, -0.009913739, -0.010047618, 0.0062655364, -0.007925636, 0.003845674, -0.006740807, 0.01656083, 0.008869483, 0.004484946, -0.0000034581046, 0.012156212, 0.04739316, -0.0032633003, -0.0013396266, 0.0029654196, -0.020095237, -0.008628501, 0.026133178, -0.010409092, 0.014927507, -0.0016391808, 0.0012785443, 0.0123838065, 0.0011120323, 0.0066504385, -0.032211285, -0.015476411, -0.01191523, -0.011098568, 0.019144695, -0.016868753, 0.009344754, -0.013615493, 0.003127748, 0.017805906, 0.015864661, 0.021434026, -0.023535926, 0.014820404, -0.0069349315, -0.022652324, 0.0038624087, -0.009987373, -0.015074774, -0.0043309852, 0.022518447, 0.010221661, 0.008755686, 0.012450746, 0.0030608084, 0.035344053, 0.0057601435, 0.005589448, -0.021206431, -0.0013011364, -0.027793277, -0.028837534, -0.009050219, 0.020483484, -0.021259984, -0.010047618, -0.01140649, 0.0048598074, -0.011219059, 0.005827083, 0.0030290121, -0.007383426, -0.009572348, 0.001289422, 0.006867992, -0.021514354, -0.023281556, -0.010536277, 0.004484946, 0.0276594, -0.024272261, 0.007256241, 0.0058003073, -0.0050673196, -0.008240252, 0.007537387, -0.00105095, 0.012350337, 0.013976966, 0.020684304, 0.0119553935, 0.010850892, 0.012859077, 0.023790296, 0.006218679, 0.0013915048, -0.012792137, 0.009418387, -0.000063540225, -0.0029955423, 0.035317276, 0.015208653, -0.0033837913, -0.02574493, -0.0009639287, 0.011982169, 0.024379363, 0.014994446, -0.01618597, 0.0037419177, 0.019452617, -0.008976586, -0.010034231, -0.016868753, -0.024740838, -0.006633704, -0.0028733779, 0.010723707, 0.013548554, 0.020858346, 0.03365718, 0.024781, 0.029480154, -0.023107514, -0.011178896, 0.002210677, 0.015744168, 0.006466355, -0.006981789, 0.0040565333, 0.005120871, -0.008347355, 0.00046983158, 0.011359632, 0.006208638, 0.0010718687, 0.009619205, 0.000055486566, -0.012256621, 0.0013605452, 0.00078946765, 0.028971413, 0.006888074, -0.015663842, 0.022304239, 0.010094476, -0.0051911576, 0.001478526, 0.0063759866, 0.02835557, 0.013421369, -0.019412452, -0.007624408, 0.0054957327, 0.005412058, 0.009666063, -0.008113067, -0.013582023, -0.0058505116, 0.024098217, -0.0063759866, 0.018046888, -0.007403508, -0.021179656, -0.0073700384, 0.00463556, -0.0123838065, 0.007845309, -0.010783953, 0.032211285, 0.0061115758, 0.028114587, -0.012484215, -0.021045776, 0.003835633, -0.008300497, 0.0069416254, -0.007838614, -0.001563874, -0.02108594, -0.003815551, -0.015704006, -0.00920418, -0.019974746, 0.020603977, -0.009110465, 0.0043142503, -0.01872967, -0.018809998, 0.007316487, -0.00084762136, -0.016721485, 0.004494987, -0.020778019, -0.008059515, -0.0132205505, -0.02051026, -0.001916143, -0.009471938, -0.0002215279, 0.016333237, -0.006469702, -0.016654545, -0.020095237, 0.012209764, -0.018769834, 0.018394973, -0.008333967, -0.018421749, 0.023402046, 0.009344754, 0.0031997077, 0.0041837185, 0.0047024996, -0.004655642, 0.0008626827, -0.026561592, 0.014418767, -0.003825592, -0.012162906, -0.014124233, 0.02242473, 0.014244724, 0.013073283, 0.01727039, 0.0130063435, -0.020871734, 0.023241391, -0.0117947385, -0.003594651, -0.013802923, -0.012397194, 0.011158814, -0.0031963608, 0.027953932, -0.014994446, -0.006322435, -0.003104319, -0.0060680653, -0.0021671662, -0.004320944, -0.027311314, 0.0017839376, 0.012484215, -0.0003522691, 0.0050673196, 0.00594088, 0.0023211269, 0.0035678751, 0.025048759, 0.030818943, -0.012798831, -0.012972874, 0.010984771, 0.015891436, -0.014325052, -0.007845309, -0.0021270025, -0.0022458201, -0.0073365686, 0.018100439, -0.0052580973, 0.018716283, -0.008481234, 0.0053518126, 0.014043906, -0.030685063, 0.01981409, -0.032827128, -0.0055091204, -0.024486467, -0.0011028282, 0.018903714, 0.0052748322, 0.0079858815, 0.012089273, -0.01057644, -0.0045753145, 0.025918972, -0.007490529, -0.010743788, -0.009786554, -0.024473079, -0.0276594, 0.015556739, 0.007115668, -0.01086428, -0.018555628, 0.004002982, -0.0153559195, -0.01682859, -0.019773927, 0.017404268, 0.014619585, 0.007898861, 0.007550775, 0.0022659018, -0.015208653, -0.00056940404, 0.022879919, 0.0102283545, -0.0061651273, 0.001517853, -0.01618597, 0.020095237, -0.022076646, 0.00684791, -0.0020734507, -0.013548554, -0.018247707, -0.019318737, 0.024794389, 0.010971383, 0.0119553935, -0.0014643015, 0.014258112, -0.0029436643, -0.014391991, 0.0065701115, -0.023495762, -0.0025152515, -0.013254019, 0.010061006, 0.02561105, 0.027164046, -0.009793248, 0.01079734, -0.022063257, 0.017310552, 0.017230226, 0.0010291947, 0.012564543, -0.008795849, 0.029265948, 0.009907045, -0.0027344783, -0.019091144, 0.018180767, -0.0015312409, 0.018943876, 0.017605087, 0.0041770246, 0.0037218358, -0.007155832, -0.015891436, 0.0030206447, 0.020349607, 0.005147647, -0.000007851009, -0.007918942, 0.0077047357, -0.018301258, -0.0034507308, -0.0038389799, 0.013495002, 0.027257763, -0.018823385, 0.014378604, -0.013722597, -0.01057644, 0.026333997, -0.009197487, 0.0013421369, 0.014151009, 0.033924937, -0.0030608084, 0.0030792167, -0.0068612983, 0.007952412, -0.023107514, 0.014124233, -0.026159953, 0.010931219, 0.004478252, 0.006590193, -0.001785611, 0.036468636, 0.009907045, 0.0102283545, 0.021822276, 0.016775038, 0.013207162, -0.0038523679, -0.009920433, -0.0017370798, 0.0058438177, -0.00959243, 0.010583134, 0.0037519587, 0.00010386081, 0.016734874, -0.014740077, 0.0097664725, 0.01886355, 0.0205772, -0.025276354, 0.0030022364, -0.0092845075, 0.008641888, -0.010998159, 0.01656083, -0.018957265, -0.00481295, -0.00945855, 0.0015939967, -0.0010266845, 0.00474601, 0.010081088, -0.0022039828, 0.009137241, 0.02038977, -0.013407981, 0.004879889, 0.010790646, 0.007698042, 0.005120871, -0.0039326954, -0.010623298, -0.0049167057, -0.007490529, -0.0081867, -0.04096697, 0.008682053, 0.009117159, 0.00013774892, -0.0018809998, -0.021500966, 0.03574569, 0.0005802817, -0.005264791, -0.0022039828, -0.020456709, 0.01713651, 0.01727039, -0.010221661, 0.0020065114, 0.018020112, -0.016600994, 0.033469748, 0.00477948, -0.01089775, 0.012069191, -0.010174803, 0.00018000448, 0.006908156, -0.018528853, 0.00038009082, 0.009117159, 0.0103890095, 0.0005631285, 0.0024165157, -0.010235049, -0.0022876572, 0.036227655, -0.013274102, 0.011031629, -0.010201579, -0.02134031, 0.013100059, 0.008092985, -0.0022056564, 0.0154094715, 0.028409122, -0.018207543, -0.006600234, -0.008367437, 0.0070286468, 0.017056184, -0.00065684377, 0.010382315, 0.006995177, 0.011299387, -0.018381584, -0.0017253654, -0.011747881, -0.010616603, -0.009726308, -0.017645251, 0.010696931, -0.00093380595, -0.010583134, -0.016252909, 0.015904823, 0.012256621, 0.019318737, 0.008333967, 0.014244724, 0.0053685475, -0.024473079, -0.005669775, 0.0067776237, -0.005723327, 0.022183748, 0.007979188, -0.016868753, 0.011359632, -0.013501696, -0.031675767, 0.011265917, -0.003581263, 0.014619585, 0.015637066, 0.01886355, 0.011165507, 0.012825607, 0.017819293, -0.005278179, -0.0020701038, 0.011265917, -0.0052246274, -0.012136131, 0.008461152, 0.015007835, -0.0010066027, -0.011868373, 0.020148788, -0.013963578, -0.012504297, -0.00369506, -0.022411343, -0.0014433828, 0.022411343, 0.009806636, 0.006054677, -0.011821515, 0.009431775, -0.008487928, -0.008414295, 0.017042795, 0.023522539, -0.029319499, -0.031193804, -0.008173312, -0.004615478, 0.027953932, 0.0077783694, 0.0030089302, -0.0035377522, 0.009478632, -0.018448524, -0.0011036649, -0.017109735, -0.010188191, 0.02064414, -0.020242503, -0.014619585, -0.0192518, 0.01257793, 0.034567554, 0.01706957, -0.016614383, 0.022331014, 0.015114938, -0.0015755884, 0.00622872, -0.0036883662, -0.0022274116, 0.015811108, -0.0053350776, 0.0068244813, 0.01060991, 0.005656387, 0.0069282376, 0.0031093394, 0.0036615904, -0.023308331, 0.009050219, -0.0058705937, -0.011386408, -0.004869848, 0.030229876, -0.0054522217, 0.0006832012, 0.0064027626, -0.014699913, -0.006707337, -0.014391991, -0.011004853, 0.003039053, 0.0042807804, -0.004645601, 0.0231209, -0.007939024, -0.016038703, 0.007651184, 0.00006359252, -0.006312394, -0.013013038, -0.024473079, 0.012209764, -0.0016885487, -0.019867642, -0.004371149, 0.0029637462, -0.0048330314, -0.014619585, 0.0028717043, 0.023857236, 0.023535926, -0.007149138, 0.003104319, -0.006764236, 0.031167028, -0.0032967702, -0.023817072, -0.0104961125, 0.036093775, -0.015181878, -0.0018876938, 0.004863154, 0.00016713954, 0.028328793, 0.010007454, 0.010409092, 0.0012191355, 0.01840836, -0.0057534496, 0.0036883662, 0.003444037, -0.0061751683, -0.002026593, -0.012450746, 0.014766852, 0.005418752, -0.0013320959, 0.029667584, -0.006064718, 0.018515464, 0.010603216, -0.006606928, -0.0013990354, -0.007798451, 0.014298276, 0.007624408, -0.017497983, -0.008849401, -0.007671266, -0.014244724, 0.016052091, 0.008320579, -0.015329144, 0.021782111, -0.007155832, -0.0061115758, -0.010670155, -0.025785092, 0.01249091, 0.0021253289, 0.0021755334, 0.013561942, -0.0030959516, 0.013642269, 0.016426953, 0.0035076295, 0.0053652003, -0.019358901, 0.005817042, 0.005301608, -0.0011379714, -0.021313535, -0.007530693, -0.00029599806, -0.0029252558, 0.0011287673, -0.0110115465, 0.0070420345, 0.0025503947, 0.018756445, 0.016775038, 0.0043242914, -0.008200088, 0.010777258, 0.018153992, 0.010014148, 0.013173693, 0.013240632, -0.004873195, -0.011908536, 0.003976206, 0.0013337694, -0.0020968798, 0.018943876, -0.003678325, -0.0005957615, -0.0028164792, 0.025477171, 0.0018676119, -0.00060371053, -0.00067399704, 0.008815931, 0.020188952, 0.0045920494, -0.009358142, 0.015262204, 0.0017688761, 0.012055803, -0.004618825, -0.009030138, -0.0111722015, -0.00048070925, -0.010891056, 0.010281906, 0.005853859, -0.014740077, 0.022960246, -0.0013789536, 0.0019730416, 0.013300877, -0.0042941687, -0.002774642, -0.008735604, -0.029078515, 0.014284888, -0.014191173, 0.009471938, 0.0039996346, 0.003039053, 0.003812204, -0.011245835, -0.011553757, -0.005562672, 0.013428062, -0.028623328, -0.015704006, -0.00075223256, 0.00217386, 0.020563813, -0.014365216, -0.0036147328, 0.0006229557, 0.015342532, 0.002111941, 0.013501696, -0.020041684, -0.006476396, 0.0020868387, 0.027498744, -0.02338866, -0.0008421825, -0.0033218723, -0.006215332, -0.007798451, -0.0013262387, -0.016226133, -0.008534785, 0.018743059, 0.0028097853, -0.0038055102, -0.005549284, -0.0039059194, 0.003316852, -0.018997429, 0.000579445, -0.0024985166, 0.028676879, -0.025196025, 0.021206431, -0.014070681, 0.007249547, -0.012504297, 0.007129056, 0.0069349315, 0.0017504678, 0.0010827463, 0.015971763, -0.002650804, 0.013963578, 0.028168138, 0.00008273303, 0.03842327, -0.011553757, -0.024138382, 0.00080703927, -0.006489784, -0.021206431, 0.00087021344, -0.0059910845, -0.018582404, 0.022290852, -0.0005945064, 0.01047603, -0.002434924, 0.006288965, 0.00981333, -0.0071357503, 0.01860918, 0.0026206812, -0.00815323, 0.0042506577, 0.023094125, -0.014391991, 0.007845309, 0.013582023, 0.0061651273, -0.003976206, -0.0042573516, 0.016252909, 0.013153611, -0.016908916, 0.017765742, -0.011138732, -0.012484215, 0.007523999, 0.0141108455, 0.03657574, -0.009799942, -0.007122362, 0.013153611, 0.009070301, -0.013287489, -0.009672757, 0.017176675, 0.003231504, -0.01064338, 0.015128326, 0.013213856, -0.0036180797, 0.0073030987, 0.01035554, 0.0070755044, -0.016239522, 0.020456709, -0.009418387, -0.004337679, 0.016895529, 0.005147647, -0.003959471, -0.009907045, 0.0029838278, -0.002331168, -0.02128676, -0.020309443, 0.0016107316, -0.01162739, -0.0047259284, -0.017002631, -0.0123838065, 0.004223882, -0.0018056928, 0.0477948, -0.0035678751, -0.009840106, 0.0025872116, -0.00543214, 0.008568255, -0.027605847, -0.023495762, -0.002438271, 0.0064228442, -0.0116006145, 0.013702515, -0.029453378, -0.0023646376, -0.0030725228, 0.019653436, 0.023147676, 0.02160807, -0.00622872, 0.0033603625, -0.026815962, 0.0008333967, 0.021099329, 0.018461913, -0.015744168, 0.02918562, -0.00096811244, 0.012664952, 0.0019010816, 0.00691485, 0.0012425644, -0.016600994, 0.018502075, 0.018502075, -0.020791408, 0.015302368, -0.0053283838, 0.011453347, 0.008434377, 0.0046958057, -0.014981058, -0.035129845, 0.002598926, -0.015864661, -0.0030340326, -0.005422099, -0.01662777, -0.0036649373, 0.016011927, -0.004120126, 0.008735604, -0.013013038, -0.006362599, 0.017899621, 0.0014207908, 0.026133178, -0.01032207, 0.0064128037, 0.012229846, -0.009907045, 0.006255496, 0.031541888, 0.0069750953, -0.006747501, 0.007008565, -0.003172932, 0.030336978, 0.002722764, 0.007858696, -0.001916143, 0.027030168, -0.010623298, -0.0034172612, 0.0035210173, 0.016333237, -0.008601725, -0.014137621, 0.00427074, 0.022786204, -0.020737855, -0.01166086, 0.0014960977, -0.015396084, -0.00010867209, 0.010201579, -0.032077406, -0.0024365976, -0.010435867, -0.022264076, -0.0032499125, -0.009244344, -0.0065868464, -0.0037452646, -0.0117612695, -0.018073663, 0.020470098, -0.024419528, -0.008113067, 0.012430664, -0.001971368, -0.016333237, -0.006757542, -0.020242503, -0.0018241012, -0.0020249197, -0.0056162234, 0.0014157703, -0.0060044727, 0.042466413, -0.026829349, -0.014833792, 0.03644186, 0.013575329, -0.0156236775, -0.008514703, -0.017953172, 0.0010860933, -0.0015278939, -0.010208272, -0.033362642, -0.014405379, -0.014338439, -0.00077817164, 0.013347735, 0.03879813, -0.00062504754, 0.003849021, 0.007959106, -0.005003727, -0.0027545602, -0.003362036, -0.0031310949, -0.006774277, 0.0077448995, -0.0069684014, 0.010047618, 0.0192518, 0.0003401363, 0.03454078, -0.008200088, -0.03261292, 0.021367086, -0.00837413, -0.021059165, 0.007550775, 0.020443322, -0.015757557, 0.008180006, -0.014499094, 0.025088923, -0.029640809, 0.0025420273, -0.017953172, -0.0033285664, 0.012022333, 0.0039326954, -0.019706987, 0.0073030987, 0.0025754971, -0.016547443, 0.010241742, -0.0117278, 0.00033783526, 0.022411343, 0.020229114, -0.0037519587, 0.019091144, 0.022223912, 0.017752353, 0.024714062, 0.005790266, 0.0031310949, -0.010717013, -0.012310173, 0.0064496202, -0.0009831738, 0.0006388538, 0.010301989, -0.014967671, -0.0004443109, 0.038958784, 0.01866273, 0.018180767, 0.00822017, -0.003678325, 0.019358901, -0.00018282849, -0.018622568, -0.0061751683, 0.0013061569, 0.011152119, 0.000056323308, 0.02242473, -0.017190062, 0.016708098, -0.03325554, -0.02191599, -0.013561942, 0.0037251827, -0.010723707, 0.0035477933, 0.005040544, 0.015770944, 0.0065466827, -0.0021203086, 0.021581292, 0.017979948, 0.0050874017, -0.020416545, 0.005425446, 0.002115288, -0.015074774, -0.006037942, -0.011547063, -0.0029972158, 0.019519556, 0.0074838353, -0.0064228442, 0.024004502, 0.011018241, 0.0034306492, 0.025704766, -0.008534785, -0.004210494, 0.016574219, -0.003094278, -0.0069416254, 0.009110465, -0.028543001, -0.0135217775, -0.0045251097, -0.0076444903, -0.012270009, -0.0112793045, 0.008675358, 0.014070681, -0.0105496645, 0.006208638, 0.014740077, -0.022893308, 0.006081453, -0.003979553, 0.009538878, -0.013923415, 0.016547443, 0.014231336, 0.029453378, 0.00029934506, -0.0123838065, -0.0011521961, -0.005171076, -0.005027156, 0.026749022, -0.00061584334, -0.010529582, 0.0002063619, 0.0033536686, -0.0044581704, -0.009793248, 0.012336949, 0.01943923, -0.0077248174, 0.016413564, -0.0006836196, -0.0255575, 0.008387519, -0.01898404, -0.011399796, 0.008722216, -0.003090931, -0.017672027, -0.00095890823, 0.021527741, 0.0022541874, 0.0039393893, 0.015288981, 0.004618825, 0.00977986, -0.016775038, -0.016239522, -0.0039025724, -0.015369308, -0.00034850373, -0.0036649373, 0.010395704, 0.0005493222, 0.009840106, 0.0071022804, 0.0238974, -0.026293833, 0.02064414, 0.008320579, 0.0055024265, 0.0011438286, -0.00013628462, -0.017283777, -0.009471938, 0.007912248, 0.011446654, -0.008641888, 0.0019914499, 0.0068177874, -0.0128992405, 0.019747151, -0.027632624, -0.0050940956, -0.008267027, 0.021206431, -0.0029520318, 0.0027930504, 0.028221691, -0.01267834, -0.0017487942, -0.0027077026, -0.0016333236 - ] - } - ] - } + { + "name": "springfield-illinois-sixth-largest-city", + "title": "Springfield: Illinois' Sixth Largest City", + "content": "---\n\nSpringfield is the sixth most populous city in Illinois, with a population of 10,566 as of 2019.", + "keywords": [ + "Springfield", + "Illinois", + "population", + "sixth most populous", + "10", + "566", + "2019" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.019289264, 0.03618999, -0.0003673661, -0.017636588, 0.008607185, + 0.0005126915, 0.0048856484, 0.0038120123, -0.043162595, 0.025212359, + -0.020821307, 0.013740133, 0.000021711638, 0.016478509, 0.0023855234, + 0.018058805, -0.020857498, 0.03423573, 0.0003524754, -0.006858003, + 0.0070148264, -0.0144156795, -0.045816526, 0.035779838, 0.025598386, + 0.010627793, -0.027214872, 0.008353855, -0.025622513, -0.008191001, + -0.02275144, -0.0048886645, 0.05771097, -0.006640863, 0.0073887897, + -0.0066830846, 0.0035405874, -0.020121634, 0.018818794, 0.035152543, + 0.014295046, -0.004632319, 0.015139479, 0.0104709705, 0.038843922, + -0.039422963, 0.00031892429, -0.02738376, 0.02311334, 0.008745914, + 0.024355862, -0.018239755, -0.024898713, 0.042945456, -0.0009213368, + -0.027238999, 0.007684341, 0.024223166, -0.0077989427, -0.03980899, + -0.001979894, -0.007883386, 0.012871573, -0.005344055, -0.0017506907, + 0.0069967313, 0.008564964, -0.0053711976, 0.06441818, 0.012618243, + 0.010772553, 0.04658858, 0.011580797, 0.03322241, 0.058048744, + 0.0027217888, -0.017660715, -0.0023885393, -0.03271575, 0.03286051, + 0.015658202, -0.0064176917, -0.003401859, 0.0003402236, 0.009463682, + 0.029868804, 0.009771297, -0.06779592, -0.002287509, 0.010911282, + 0.025694892, -0.0035918565, -0.0012040711, -0.03611761, -0.043693382, + -0.023065086, 0.032957017, -0.0059080156, 0.009620505, 0.020616231, + 0.017359132, -0.0029796425, -0.010917313, 0.025212359, 0.00038244526, + 0.0096506635, -0.018456895, 0.0061522983, -0.027432011, 0.0050545353, + 0.010380495, 0.002035687, -0.012286501, 0.01206333, -0.010404622, + -0.0057572243, -0.02535712, -0.0075154547, 0.04198039, 0.02130384, + 0.00055189733, 0.021943197, -0.025622513, 0.018493084, -0.0070691113, + -0.010338274, 0.02101432, -0.013245536, 0.003293289, -0.0450686, + 0.03433224, 0.005509926, -0.020495597, 0.010615731, 0.012153805, + -0.039181694, -0.011840158, 0.016804218, 0.027769785, 0.020917814, + -0.059399836, -0.006731338, 0.019542594, 0.0049972343, -0.00008434906, + -0.014886149, -0.0125338, 0.008812263, 0.009656696, -0.029289765, + -0.014524249, -0.005923095, -0.02372857, 0.0044604163, 0.00053191744, + 0.031509418, -0.0018607687, -0.013848702, 0.047167618, -0.05259612, + -0.01293189, -0.021955261, 0.027456138, -0.03799949, -0.022630807, + -0.010766522, -0.0076964046, -0.00031609694, 0.0028755963, + 0.002898215, 0.022582553, -0.01369188, -0.063839145, -0.036310624, + -0.007358631, -0.010585572, -0.033560183, -0.016032165, -0.010911282, + 0.03039959, -0.0028484538, -0.00058997225, 0.030737365, -0.0038843923, + -0.019253075, -0.018300071, 0.0064900713, 0.007991956, 0.0268771, + -0.014331236, -0.003362653, 0.044586066, 0.0050485036, -0.0054827835, + 0.04017089, -0.00031628544, -0.009964311, -0.027528519, -0.0047559678, + 0.024042217, -0.044899713, 0.0054888153, -0.037854727, -0.024946965, + 0.0059954748, 0.018806731, -0.0014491075, -0.0018683082, 0.021279713, + 0.020314647, -0.006369438, -0.0076541826, -0.025308866, 0.026515199, + -0.011828095, 0.022015577, -0.009976374, -0.00009099332, -0.019047998, + 0.019940684, 0.0142467925, 0.038023617, -0.000017482404, -0.028903738, + -0.033367172, -0.020724801, -0.045599386, 0.034549378, -0.022003513, + -0.037637588, 0.026611706, 0.027335506, -0.0130887125, 0.054477997, + -0.011387783, 0.04743301, 0.013522993, 0.004249308, -0.014898213, + 0.034766518, 0.025164107, 0.0026569483, 0.0033686848, -0.015223922, + 0.0012817289, -0.019699417, 0.025260612, -0.014331236, 0.016225178, + 0.017274689, 0.0114842905, -0.008293539, 0.05462276, -0.005989443, + -0.008715756, 0.0145001225, -0.013209346, 0.021219397, 0.0002141241, + -0.0066891164, -0.022293033, -0.0021683834, -0.03387383, 0.03474239, + 0.0074189478, -0.012141742, 0.028807232, 0.00072870054, 0.005485799, + -0.030761492, -0.000269917, -0.012811257, 0.005344055, 0.0061432505, + -0.009692886, 0.030423718, 0.045454625, 0.0031877349, -0.029434524, + -0.032233216, -0.028445331, 0.04608192, 0.0029208336, -0.013872829, + 0.010392559, 0.0016647396, -0.018432768, -0.002213621, 0.011357625, + -0.030278958, -0.01039859, 0.00026897455, 0.044586066, 0.025912033, + -0.005160089, -0.014560439, 0.022437794, 0.0006725306, -0.042221654, + 0.03184719, 0.034404617, 0.036793157, -0.03112339, -0.028228192, + 0.007159586, -0.038868047, 0.016490571, -0.017528018, -0.03618999, + -0.06480421, -0.016804218, -0.050762493, -0.013607436, -0.00066762994, + 0.006206583, 0.0012447848, 0.025912033, 0.008812263, 0.029482778, + -0.015067099, -0.0033415423, 0.0071897446, 0.020049253, 0.02557426, + 0.0031364658, 0.054429743, -0.00158482, -0.0020387028, 0.01119477, + 0.0038482023, -0.011206834, 0.022763504, -0.0028198033, -0.033198282, + 0.017865792, 0.002376476, 0.030568479, 0.011725557, -0.0066167363, + 0.02300477, 0.001078914, 0.008564964, 0.009439556, 0.03626237, + 0.020097507, -0.013764259, -0.00072191487, -0.0070811748, + -0.018891174, 0.039422963, 0.0006216385, 0.016719775, 0.05829001, + -0.0071897446, 0.012473483, -0.009801456, -0.02267906, -0.0029238495, + 0.024753952, -0.050231706, -0.023535557, 0.0064900713, -0.018468957, + 0.004979139, -0.032257345, 0.021798437, -0.010525255, 0.0060889656, + 0.030013565, 0.003223925, 0.0071173646, 0.016755965, -0.02535712, + 0.014801705, -0.0014166873, 0.012919826, -0.018239755, -0.0017521987, + -0.021098765, -0.005721034, 0.02416285, 0.020785118, -0.010416686, + -0.02249811, -0.006212615, -0.007310378, -0.007057048, -0.013414423, + -0.036962043, 0.012594116, 0.021496853, -0.028927865, 0.011604924, + 0.015766772, 0.032402106, 0.029675791, -0.019494342, -0.01115858, + 0.017962297, -0.00044596623, 0.016888661, -0.022401603, + -0.00055566715, 0.02557426, -0.019035935, -0.0030007532, -0.01018145, + -0.05814525, 0.008275445, -0.011466195, -0.006827845, 0.026273932, + 0.013872829, -0.010428749, -0.020037191, -0.046685085, -0.010344305, + 0.039471216, 0.015549632, 0.015501379, -0.007020858, -0.024247292, + 0.002877104, 0.012943953, -0.015308365, 0.015959786, -0.012545863, + 0.027721532, -0.022763504, -0.030327212, -0.023885394, -0.0036491572, + -0.011864285, -0.015694393, -0.009125909, 0.051631052, -0.04781904, + 0.0022226686, 0.05027996, 0.030882124, -0.03061673, 0.009162099, + 0.043066088, -0.018999744, -0.004427242, 0.026563453, 0.017733095, + 0.024319673, 0.019868305, 0.048373953, -0.013100776, 0.018613718, + 0.009916057, 0.016526762, 0.028927865, -0.0028077401, -0.029820552, + 0.01032621, -0.007509423, 0.011731588, 0.037951235, -0.023825077, + 0.014632819, -0.031147517, -0.037661716, 0.03206433, 0.057662718, + 0.018348325, -0.0004195777, -0.0038270913, -0.0074189478, + -0.025936158, -0.049194258, 0.03647951, -0.03618999, 0.014439805, + 0.0014543852, 0.019663228, 0.012871573, -0.028228192, -0.030327212, + 0.018324198, -0.044972092, 0.004243276, -0.003223925, 0.056745905, + -0.018372452, -0.008770041, 0.037806477, -0.0052837385, 0.053464677, + 0.0018607687, -0.009916057, -0.0144156795, 0.0142467925, + -0.0037999488, -0.031244025, -0.05129328, 0.04058104, 0.038240757, + -0.0063754697, 0.01195476, 0.029241512, 0.020857498, -0.02318572, + -0.0114842905, -0.018493084, -0.029555159, 0.0034621756, 0.010531287, + 0.067120366, 0.013740133, -0.026346313, -0.020519724, 0.014560439, + -0.01278713, 0.0053319917, -0.02658758, 0.026949478, 0.0071414914, + -0.013607436, 0.02260668, 0.009554157, 0.004231213, -0.027673278, + 0.03604523, -0.002034179, -0.0034290014, 0.03913344, 0.0059653167, + -0.04521336, -0.0031093233, 0.00072191487, 0.025719019, 0.049146004, + -0.02629806, -0.021038447, -0.028469458, -0.0016587079, 0.035731584, + -0.011502385, -0.051872317, 0.0031937666, 0.036576018, 0.025453625, + 0.013957272, -0.011562701, -0.026249805, 0.011502385, -0.012835383, + 0.047915544, -0.0061824564, 0.010850965, -0.0060407124, -0.008770041, + -0.004164865, -0.010483034, -0.012015076, -0.035948724, -0.030592604, + 0.012039203, 0.011610955, 0.021508917, -0.0038120123, 0.00025295294, + 0.017745158, 0.007937671, 0.008938927, 0.007985924, 0.009560189, + 0.025815526, 0.024030153, 0.017129928, -0.006646895, 0.0032028141, + 0.016369939, 0.00284393, -0.015706455, -0.0035043973, -0.0010140736, + -0.006550388, 0.017540082, 0.030037692, 0.027962798, -0.0047137463, + 0.0144156795, -0.011683335, -0.04487559, -0.016104545, 0.0515828, + 0.050810747, 0.025188232, 0.007865291, 0.009505904, -0.00084066327, + 0.014222666, -0.007883386, 0.005057551, -0.02286001, -0.052789133, + 0.019023871, 0.0018140232, 0.010501129, -0.026225679, -0.020338774, + -0.003944709, 0.008106558, 0.0043850206, 0.02213621, -0.011285245, + -0.01934958, -0.00070268894, 0.01155667, -0.016780091, -0.035707455, + 0.006948478, 0.020640356, 0.027721532, 0.0014174413, -0.014463932, + 0.0031002758, -0.030496098, -0.027769785, -0.008203064, 0.014451869, + -0.01191857, 0.013197283, -0.010042722, -0.0003726438, -0.008426236, + -0.01130334, 0.026056793, -0.006924351, -0.0018200548, -0.007032921, + -0.004647398, 0.005096757, 0.0052596116, -0.03129228, 0.003525508, + 0.019325454, 0.061957262, 0.011496353, 0.02286001, -0.009662727, + -0.001421965, -0.026177425, 0.009035434, -0.0019874335, 0.017672779, + 0.023776824, -0.020604167, -0.0011671272, 0.0077144993, -0.011683335, + 0.02231716, 0.028276445, -0.031051012, 0.019735608, 0.0054254825, + -0.034501124, 0.004930886, 0.002700678, 0.01076049, 0.007563708, + 0.029096752, -0.044537812, 0.0232581, 0.037420448, 0.008776072, + 0.008341793, 0.0218105, -0.007509423, 0.014210602, -0.015139479, + 0.017974362, -0.007075143, -0.0021563203, -0.01329379, -0.009113845, + -0.008414173, -0.010205577, 0.0007237998, -0.010483034, -0.0014264887, + -0.026515199, 0.045261614, 0.025550133, 0.008824326, 0.0043850206, + 0.008764009, 0.03618999, -0.005814525, 0.0034621756, 0.03645538, + -0.0014325205, -0.012413166, 0.034983657, 0.011466195, -0.007684341, + -0.022075893, 0.022329224, -0.007093238, -0.041859753, 0.00619452, + 0.0062367413, -0.0025468706, -0.026901225, -0.05380245, -0.0012131186, + -0.0046715243, 0.0128957, 0.014331236, -0.033704944, 0.053995464, + -0.03768584, 0.019385772, -0.015609949, -0.0012583561, 0.036576018, + 0.030520225, 0.017093739, -0.045454625, -0.0039326455, -0.031026885, + 0.0043277196, -0.035273176, -0.038023617, 0.017443575, 0.009988437, + -0.031364657, 0.013486803, 0.037854727, -0.005775319, 0.0028167875, + 0.0028559933, 0.0003585071, -0.028927865, 0.025188232, -0.002194018, + -0.012702687, -0.02810756, 0.038409643, 0.006930383, 0.010078912, + 0.010893187, 0.0043820045, -0.0049821553, -0.058676038, -0.0018502132, + -0.031388786, -0.019759735, 0.05129328, 0.030882124, 0.00004311698, + -0.0139090195, -0.0014822816, -0.0035948723, 0.010386527, 0.049411397, + 0.029989438, 0.008703693, 0.007889418, 0.00858306, 0.015585822, + -0.005021361, 0.010084944, -0.0009688362, -0.0046896194, 0.0038421706, + 0.04354862, 0.0003566222, -0.016864534, 0.0018155312, 0.03322241, + 0.002643377, 0.009729075, -0.021364158, 0.021545107, -0.040556915, + 0.025622513, -0.010151292, 0.012075393, 0.011170643, 0.00023466947, + -0.02115908, -0.0043126405, -0.009101782, 0.008191001, -0.013317917, + -0.015790898, -0.0196391, -0.044706702, 0.0038089964, 0.009958278, + -0.014174413, 0.0034470966, -0.026225679, 0.05249961, 0.0059321425, + 0.015971849, 0.031509418, 0.00087760726, 0.01772103, -0.048205066, + 0.008257349, 0.002718773, -0.0016602158, -0.0058838893, 0.0030686096, + 0.032257345, -0.0326675, 0.027818039, -0.016261369, -0.031750683, + -0.025887907, 0.0150912255, -0.0243438, -0.0041920072, -0.029506905, + 0.025550133, -0.02383714, 0.027118366, -0.018312134, 0.023089213, + 0.010519223, 0.0016571999, -0.016647395, 0.007581803, 0.02485046, + -0.022039704, 0.00496406, -0.020604167, -0.010495097, -0.009813519, + 0.023451114, 0.00056094484, 0.01202714, 0.018782604, 0.005232469, + 0.007509423, -0.012799193, 0.02441618, -0.016249305, 0.027118366, + 0.03973661, -0.0060015065, -0.03980899, 0.024440307, -0.017757222, + -0.03143704, 0.015947722, -0.022739377, 0.017624525, -0.019204821, + 0.028155811, 0.021460664, 0.009119878, -0.017383259, 0.015706455, + 0.0019361645, 0.003778838, 0.029048499, -0.01354712, 0.0011942696, + 0.0070691113, 0.0044905744, 0.0073164096, -0.021098765, -0.01732294, + 0.0034832866, 0.002414174, -0.00070193503, -0.018782604, + -0.0010766522, 0.009620505, -0.001764262, -0.0013601404, -0.024090469, + 0.026008539, -0.036624268, -0.008154811, -0.015139479, -0.022124147, + 0.016538825, 0.020133698, -0.014126159, -0.028758978, 0.0059562693, + -0.005926111, 0.0030610699, 0.009903993, -0.0053531025, -0.016249305, + 0.006206583, 0.016321685, -0.029531032, -0.02887961, -0.0011120882, + 0.005090725, 0.017250562, -0.008414173, 0.014427743, -0.025719019, + -0.0011331991, 0.016997231, 0.026780592, 0.008420204, -0.009517968, + 0.011616986, -0.016466444, 0.022763504, 0.0049519967, -0.020833371, + -0.004315656, 0.020845434, -0.011852222, -0.011623018, -0.0032088459, + 0.018456895, 0.024223166, 0.007690373, -0.03901281, 0.007256093, + -0.0051721525, 0.0051962794, 0.01282332, -0.041353095, -0.009409397, + -0.0036310623, 0.01985624, 0.016599141, 0.0008798691, -0.004502638, + -0.00316964, -0.035948724, -0.036865536, 0.012449357, -0.008136716, + 0.028011052, -0.01014526, 0.008727819, -0.030737365, 0.0029826583, + -0.030496098, 0.037396323, -0.027456138, 0.02101432, 0.030278958, + -0.0009861772, -0.0020387028, 0.014065842, -0.009602411, 0.002286001, + 0.036383003, 0.022546364, -0.015983911, 0.027407885, -0.016188988, + 0.02608092, -0.0029434524, 0.019313391, -0.040774055, -0.0030218642, + 0.008341793, -0.015187732, -0.028831359, -0.05481577, -0.012256343, + 0.012449357, 0.014681072, -0.0038843923, 0.007153555, -0.019916557, + -0.0005183462, 0.030278958, -0.0081306845, -0.0005311635, 0.049749173, + 0.035200797, 0.010416686, 0.009481777, 0.0014099017, 0.058048744, + 0.016032165, -0.01264237, -0.0130887125, -0.008691629, -0.011562701, + -0.004243276, 0.02658758, -0.005793414, -0.010549382, -0.000053106924, + 0.057324942, -0.009150036, -0.01137572, 0.008178937, -0.016164862, + 0.023173656, 0.03126815, 0.010820807, -0.028059306, -0.002429253, + -0.009306859, 0.03242623, -0.04152198, -0.013945209, -0.009729075, + 0.0018155312, 0.020423217, -0.021352094, 0.03351193, -0.037323944, + -0.0022467952, -0.031726558, -0.016008038, -0.017986424, 0.005368182, + -0.008197033, 0.009343049, -0.003223925, -0.0019225932, -0.023378734, + -0.0026524246, -0.0013631562, 0.014379489, -0.01094144, 0.015272176, + 0.040774055, 0.021798437, -0.0221724, -0.0058537307, -0.0027926608, + -0.01318522, -0.0064176917, -0.008872579, 0.049990438, 0.021219397, + 0.015103289, 0.0038572496, -0.03481477, 0.031075139, 0.0027489313, + 0.00072455377, -0.0053953244, -0.007666246, 0.003618999, 0.032957017, + 0.03112339, -0.009337017, 0.0056486544, -0.0072500613, 0.0012651418, + -0.036793157, -0.004647398, 0.027407885, 0.016345812, -0.002684091, + -0.018710224, -0.003869313, -0.018251818, -0.004336767, 0.0062186467, + 0.026032666, -0.012847446, -0.012943953, 0.010350337, -0.013450612, + -0.003887408, 0.0036491572, -0.006948478, 0.034356363, -0.013764259, + 0.028734852, 0.0048826328, -0.009361144, -0.024609193, -0.0062729316, + -0.029338019, 0.013740133, 0.008070367, -0.021448601, 0.017624525, + -0.0021110827, -0.026925351, 0.017757222, 0.0006352097, -0.008209096, + 0.0028695646, -0.016394066, -0.01177381, 0.024753952, 0.010187482, + 0.04557526, -0.018312134, 0.028276445, 0.04933902, -0.005814525, + -0.00020960036, -0.009005276, 0.0007090976, 0.009451619, -0.04304196, + 0.007286251, 0.013836639, -0.041256588, -0.019590847, -0.055153545, + -0.009053529, -0.018324198, 0.021605425, -0.03242623, -0.0021291778, + 0.022908263, 0.019204821, -0.020929877, -0.007358631, -0.007002763, + -0.0150912255, -0.024705699, -0.012883636, -0.024874587, + -0.0029042466, 0.03310178, -0.012594116, -0.010332242, -0.0013088713, + 0.012714749, -0.0030490067, -0.020990195, 0.017805476, -0.004014073, + -0.042993706, 0.018722288, 0.011502385, 0.008172906, -0.0065986416, + 0.019421961, -0.007907513, 0.00576024, -0.0006514198, 0.0022603665, + -0.010169387, -0.0043910523, -0.0060738865, 0.017009296, -0.000396205, + 0.0055431, -0.016599141, -0.029000245, 0.0005424729, 0.016647395, + -0.006369438, -0.0014649406, -0.021822564, 0.0022407635, -0.010109071, + -0.022884136, 0.017901981, -0.015356619, -0.0037999488, -0.008046241, + 0.015356619, 0.012141742, 0.008709724, 0.007165618, 0.0026599641, + 0.0034139224, 0.027287252, 0.00044860507, 0.007183713, -0.023607936, + -0.025212359, 0.0122744385, -0.007358631, 0.013378233, -0.01705755, + 0.006031665, 0.0077989427, -0.0026298058, -0.0049369177, -0.008070367, + 0.009952247, -0.017443575, -0.014849959, -0.017238498, -0.022051767, + -0.03575571, -0.0015591853, 0.0064780083, -0.02267906, 0.026346313, + 0.02314953, -0.0026298058, -0.018143248, -0.0055159577, 0.004125659, + 0.017708968, 0.027021859, -0.030496098, 0.027359633, -0.016068356, + -0.011785873, 0.009463682, -0.006260868, 0.0011271674, 0.00087609934, + -0.0037728064, -0.01974767, 0.013414423, 0.006441818, 0.019759735, + -0.012811257, 0.017407386, -0.014475996, -0.017528018, 0.018915301, + 0.041232463, 0.010501129, -0.008227191, 0.0012327215, -0.026466945, + 0.00847449, 0.014439805, 0.008613218, -0.037155055, 0.004973108, + -0.029096752, -0.0013329979, -0.010663983, 0.020664483, 0.00009353793, + 0.009873835, 0.012968079, -0.027238999, -0.024367927, 0.03264337, + 0.010483034, 0.0155978855, 0.00021487806, -0.017901981, 0.020374965, + -0.004409147, -0.008070367, 0.024319673, -0.035056036, -0.03343955, + 0.011037947, -0.037806477, 0.020990195, 0.011236992, 0.0059683323, + -0.027021859, -0.043283228, -0.0011392307, -0.02130384, -0.0018818794, + -0.02430761, -0.012147773, 0.014946465, -0.0025091725, -0.009349081, + 0.020543851, 0.006719275, 0.017009296, -0.007925607, -0.0009522491, + 0.023535557, 0.006912288, 0.0034802707, 0.019820051, -0.024705699, + -0.010567477, 0.033487804, 0.024319673, -0.005163105, -0.009885899, + 0.020857498, 0.010223672, 0.014910276, -0.0045267646, 0.0035315398, + 0.018649908, -0.0057029394, -0.0025227438, 0.028179938, -0.015296302, + 0.020326711, 0.004590097, 0.02680472, 0.012226185, 0.011333498, + 0.0008368935, -0.031533543, -0.0013646642, 0.014319172, -0.038216628, + -0.004210102, -0.02278763, 0.026201552, -0.0261533, 0.008932896, + -0.012165868, -0.012847446, -0.013752196, -0.022992706, 0.012835383, + 0.002519728, 0.003148529, 0.011369688, 0.019832114, -0.035393808, + -0.013426486, 0.00702689, -0.005989443, 0.019590847, -0.00076187466, + -0.002862025, 0.0044061313, -0.039664228, -0.009916057, -0.02383714, + -0.0017069612, 0.008999244, -0.0006423723, 0.01003669, 0.0067434013, + -0.010066849, 0.008926864, -0.0010057801, 0.007020858, -0.017045485, + -0.0041135955, -0.009391302, -0.016056292, 0.02155717, -0.017202308, + 0.05056948, 0.0013156569, 0.018915301, 0.0098497085, -0.00010762752, + -0.00568786, 0.0039658197, 0.0017265641, -0.0016964058, -0.024404116, + 0.0015124399, -0.02058004, -0.004158833, 0.01166524, -0.019735608, + 0.020302584, -0.03329479, 0.00055905996, -0.0070691113, 0.0030655938, + 0.025260612, 0.009385271, 0.021315904, 0.031919573, 0.023607936, + -0.01665946, 0.005681828, -0.006731338, -0.010736364, -0.010718268, + 0.004900728, -0.02343905, 0.0228962, 0.02018195, 0.014102032, + 0.017371194, 0.008341793, 0.0065564197, -0.0072621247, -0.03341542, + -0.001242523, 0.0016240258, -0.0062186467, -0.0005202311, 0.003778838, + -0.0022814772, 0.02220859, 0.016635332, 0.035635076, 0.0019572752, + -0.004752952, -0.0050002504, 0.018589592, 0.009584315, 0.018541338, + 0.0021910022, -0.018143248, 0.008293539, 0.010603667, -0.015947722, + 0.0055431, -0.005202311, -0.00989193, -0.017226435, 0.0007434027, + -0.0073465677, 0.048132688, -0.0025468706, 0.013305853, -0.017986424, + -0.007919576, 0.019807987, 0.0045961286, -0.022558426, 0.013354106, + -0.02513998, -0.01181, -0.008528775, -0.010971598, 0.019759735, + 0.015042973, -0.035948724, 0.01665946, -0.014234729, -0.009005276, + 0.016828345, 0.002216637, -0.034259856, -0.021641614, -0.016357874, + 0.003347574, -0.002430761, 0.008064336, 0.013957272, -0.005199295, + 0.0044302577, 0.008004019, 0.0080764, 0.002860517, -0.0077325944, + -0.024186976, 0.0028002004, -0.0031213865, 0.008812263, 0.0039024872, + 0.01307665, -0.016213115, 0.013921083, -0.024295546, -0.024898713, + -0.0142467925, -0.017733095, -0.021038447, -0.0019889416, 0.009958278, + -0.009355112, 0.027745659, 0.010000501, 0.002773058, 0.013981399, + 0.009463682, -0.0015064083, 0.018493084, -0.009493841, 0.0035948723, + 0.011550638, -0.020748928, 0.006924351, 0.000737748, 0.01593566, + -0.008993212, 0.014463932, -0.006725306, -0.004357878, 0.02228097, + 0.029458651, 0.003796933, 0.029434524, 0.036407128, 0.0022241764, + -0.026491072, -0.00953003, -0.016285496, 0.019602912, 0.013535056, + 0.0069183195, 0.0065805465, 0.030471971, -0.009439556, 0.003365669, + 0.013703942, 0.031895444, -0.029724045, 0.013703942, 0.03271575, + -0.018589592, -0.015802963, -0.00044709718, 0.026225679, 0.011249055, + 0.0069605415, 0.026828846, -0.009638601, 0.010446844, -0.030206578, + -0.00953003, -0.0203629, 0.00702689, 0.011442068, -0.034911275, + -0.0124976095, -0.013957272, 0.020917814, 0.02170193, -0.0047046985, + 0.025936158, -0.04159436, 0.011743652, -0.0025302835, 0.0014159334, + 0.022872074, -0.011031915, 0.014825832, 0.06740989, 0.033608437, + -0.010066849, 0.017491829, -0.006284995, -0.004412163, 0.021894943, + -0.0028635329, -0.020302584, -0.014319172, 0.036358874, -0.018082932, + -0.003546619, -0.0033355106, 0.0014905751, 0.0029811503, 0.007581803, + -0.0089751175, -0.008070367, -0.014198539, 0.015995976, 0.014898213, + -0.0010397083, -0.0069967313, 0.020809244, 0.0027851213, -0.014524249, + -0.028011052, -0.013245536, -0.022727313, -0.0023734602, 0.014005526, + 0.0021080668, 0.003419954, 0.01916863, -0.0015388285, 0.027890418, + -0.006119124, 0.010869061, -0.0027866291, 0.015549632, -0.0016783108, + 0.019880367, 0.00033965815, -0.0074973595, 0.00032495594, + -0.017081674, 0.010887155, -0.012003013, 0.013390296, 0.0058235726, + 0.014898213, -0.0059803957, -0.022582553, -0.0058356356, -0.008377982, + -0.010844934, -0.0047619995, 0.008884642, -0.009861772, -0.0018668002, + -0.022112083, -0.022124147, 0.020628294, 0.014210602, 0.0067916545, + -0.010875092, 0.010452875, 0.00045237486, -0.009542094, 0.004572002, + 0.0032902732, 0.014849959, -0.02300477, -0.015887406, 0.02076099, + -0.002950992, -0.0082151275, -0.01039859, 0.003489318, 0.024102533, + -0.04342799, -0.005959285, -0.000121575744, 0.00020903489, + 0.008540837, -0.008685597, -0.0075154547, -0.015923595, 0.0013578786, + -0.015272176, 0.0035194764, 0.004463432, -0.012051267, 0.0002665242, + 0.0017069612, -0.015441062, -0.0023372702, 0.0000022795452, + -0.003818044, -0.012992206, -0.008450363, 0.035152543, -0.02608092, + 0.01985624, 0.0013058555, 0.0024910776, 0.007485296, 0.014729326, + 0.009596379, 0.011056041, 0.008733851, 0.006242773, -0.0067795916, + -0.008106558, 0.020833371, -0.0070811748, 0.011478258, -0.008498616, + 0.0457924, -0.008570996, -0.03083387, 0.029989438, 0.009041466, + -0.009596379, 0.0074973595, -0.0064900713, 0.00040374458, -0.01895149, + -0.00011582681, -0.016430255, 0.010621762, 0.022835884, 0.0014966069, + -0.024319673, 0.0024563957, -0.015863279, -0.0013774815, -0.004198039, + 0.0128957, 0.0070691113, 0.0027263125, -0.010368432, 0.005422467, + 0.021738121, 0.022329224, -0.010151292, -0.02333048, -0.0196391, + 0.010850965, 0.0016436287, 0.008981149, -0.0008391554, -0.004140738, + 0.009952247, -0.01593566, 0.001188238, 0.010458907, -0.00169339, + 0.00316964, -0.005310881, 0.026249805, 0.0072259344, 0.008607185, + 0.008667503, -0.008643376, -0.0026509168, -0.0066649895, -0.024078406, + 0.002287509, -0.0030701174, -0.0038632813, -0.0056637335, + -0.009125909, 0.01202714, 0.00028480767, -0.02079718, -0.008920833, + -0.02709424, -0.009023371, -0.0053078653, 0.008824326, -0.0150912255, + -0.010700174, -0.016273431, -0.01633375, 0.021569233, 0.014451869, + 0.005199295, -0.0032510674, -0.014512186, 0.00089947204, 0.0012930381, + 0.016164862, 0.021605425, 0.013486803, 0.017383259, -0.0004267403, + 0.009234479, 0.012232216, -0.012943953, -0.009861772, 0.027142493, + 0.015634075, -0.0037034422, -0.016804218, 0.0026659959, -0.0055038943, + -0.037396323, -0.02224478, -0.0039326455, 0.00010743903, -0.021364158, + -0.0147534525, -0.011580797, 0.006978636, -0.017431512, -0.004731841, + 0.010537319, -0.0036793156, 0.0075456128, 0.024753952, -0.007997988, + -0.022112083, 0.023921583, 0.0051570735, -0.011140485, 0.0056094485, + -0.00039695896, 0.0016466446, 0.0011746667, 0.008269412, 0.011616986, + -0.003920582, -0.033680815, -0.031581797, 0.022437794, 0.01014526, + 0.026732339, -0.004014073, 0.008782105, -0.0020085445, 0.0071233963, + 0.014403616, 0.011966824, 0.023270164, 0.002105051, 0.025526006, + 0.009554157, 0.014644883, 0.03503191, -0.009101782, 0.016538825, + 0.009946216, 0.0031274182, 0.007744658, 0.011562701, -0.031002758, + 0.021134954, -0.037854727, 0.0032390042, -0.012811257, -0.009005276, + -0.026008539, 0.003706458, -0.015477252, -0.012232216, -0.00266298, + 0.016514698, -0.0021125907, 0.007539581, -0.0006284241, -0.026346313, + 0.012334755, -0.006713243, -0.005350087, 0.01873435, 0.017817538, + 0.033704944, -0.017648652, -0.0018577528, -0.00026690116, + -0.017335005, -0.026442818, 0.0029856742, -0.002089972, 0.009246542, + -0.021605425, 0.025815526, -0.00038941938, -0.007020858, -0.007274188, + -0.016828345, 0.01665946, 0.0015169637, 0.008667503, -0.015561695, + -0.019434024, 0.006441818, -0.002284493, -0.0031274182, -0.040556915, + -0.001964815, 0.019409897, -0.016840408, -0.0036069357, 0.0028891675, + -0.0013789894, -0.031026885, 0.019602912, -0.00001858978, + -0.007835133, 0.010657952, -0.018915301, -0.012654433, 0.007690373, + -0.010024627, 0.011224928, -0.012919826, 0.0148378955, -0.00019357874, + 0.0046685087, 0.009288764, 0.019108314, 0.002789645, 0.012521736, + 0.010965567, -0.017757222, 0.027673278, 0.0092767, 0.008607185, + 0.0025604418, -0.0073887897, 0.0007679063, -0.0033174157, + -0.007310378, 0.0015403364, 0.0007758229, -0.00018236361, 0.014934402, + -0.006821813, -0.02369238, -0.011828095, -0.006731338, -0.0064900713, + 0.006070871, -0.008233222, 0.014765516, -0.03667252, 0.008667503, + -0.0034350331, -0.0016994216, -0.014065842, 0.0019708464, + -0.010603667, 0.003564714, -0.018046742, -0.008570996, -0.009439556, + 0.009234479, -0.0056396066, -0.01564614, 0.013728069, -0.014910276, + 0.024995219, -0.019542594, 0.0122744385, 0.026684085, 0.0029117863, + -0.0053138966, -0.010283989, -0.02188288, -0.0121719, 0.0110862, + -0.02101432, -0.008402109, 0.011562701, 0.0034048748, -0.014150286, + 0.00006328536, -0.00025691124, -0.006496103, -0.025236486, + -0.0135712465, 0.014862022, 0.004394068, -0.0063995966, 0.0055129416, + 0.00003458783, -0.018288009, 0.00075508904, -0.030496098, 0.012256343, + 0.006007538, 0.011243023, -0.0063815014, 0.016828345, -0.0043458147, + 0.017793411, 0.02177431, -0.003959788, 0.010971598, -0.028517712, + -0.005160089, 0.016080419, -0.009023371, -0.0014189492, -0.012630306, + 0.030206578, 0.011586828, 0.00035417182, 0.003513445, 0.0019497357, + 0.032450356, -0.006104045, -0.010953504, 0.025936158, -0.03288464, + 0.015235986, 0.015742645, -0.011598892, 0.0221724, 0.01191857, + -0.011043979, -0.00684594, 0.010241767, 0.0054646884, -0.0276009, + -0.016032165, -0.0028921834, 0.0020809243, 0.0069967313, -0.018927364, + -0.0050485036, -0.009216384, 0.0029736108, -0.0054073874, 0.004789142, + 0.029217385, -0.017045485, 0.012943953, -0.015127416, -0.02202764, + 0.008263381, -0.0035707457, 0.012666496, 0.008070367, 0.007937671, + 0.0055129416, 0.036069356, -0.010253831, 0.0102719255, 0.031919573, + 0.008239254, -0.009493841, -0.0055219894, 0.0149223395, -0.0094274925, + -0.009180194, -0.016466444, 0.014910276, -0.015790898, 0.0004810253, + 0.0024186976, 0.005901984, -0.013426486, 0.00035379484, -0.005256596, + 0.0059623006, 0.013884893, -0.012618243, 0.009192257, 0.005461673, + -0.014451869, -0.011894443, 0.015923595, -0.01050716, -0.004053279, + -0.00608595, -0.012214121, -0.019542594, 0.0032269408, 0.01032621, + -0.011025883, -0.0030836887, 0.012376976, -0.0015169637, + -0.0047951736, 0.0026720276, 0.0061824564, -0.0045327963, 0.0055431, + 0.017250562, -0.016020102, 0.0022920326, 0.0046564452, -0.004104548, + 0.03956772, -0.0027202808, 0.0020130682, -0.02293239, -0.014102032, + 0.0051781842, 0.027842166, -0.019542594, 0.008124652, 0.0010826838, + 0.0044905744, -0.00032212862, -0.006851971, -0.0155978855, + -0.013619499, -0.0071897446, -0.00009014512, -0.0019829099, + 0.019940684, 0.01076049, 0.032040205, 0.01029002, 0.013173156, + -0.009300827, -0.03126815, 0.0026614722, 0.0060557914, 0.010712237, + -0.0250676, -0.019590847, -0.002197034, -0.00050138216, 0.01343855, + 0.026346313, -0.0022302081, 0.007817037, 0.0041497853, -0.010458907, + -0.014343299, 0.013535056, -0.0012342294, 0.029338019, -0.0010351845, + -0.010706205, 0.035345558, 0.022051767, -0.028059306, 0.007985924, + 0.024138723, -0.0064056283, 0.024802206, -0.008390046, 0.0015410904, + 0.0115687335, -0.00028499617, 0.013161093, 0.020821307, -0.005594369, + 0.0155978855, 0.020543851, 0.004246292, 0.012859509, -0.023668254, + -0.017467702, -0.023716506, 0.004394068, -0.007611961, 0.015911533, + -0.024995219, 0.012618243, 0.0029027387, 0.025550133, -0.012268406, + 0.016912788, -0.002806232, 0.0025634577, -0.00045425977, -0.009403366, + -0.0077024363, -0.00054888154, 0.03329479, -0.020314647, -0.012594116, + -0.023451114, 0.029169131, 0.014102032, -0.004369941, -0.020374965, + -0.02365619, 0.016599141, -0.010404622, -0.0118220635, 0.016357874, + -0.010416686, 0.009578284, -0.0036310623, -0.0185534, 0.0055280207, + -0.013233473, 0.009867804, 0.03380145, 0.00967479, -0.007581803, + -0.014584566, 0.009837645, -0.019228948, 0.0057723033, -0.0070691113, + -0.02260668, 0.0062729316, 0.019663228, 0.020543851, -0.0077506895, + 0.011128422, -0.0012696654, -0.0007102285, -0.013559183, 0.014777579, + -0.003637094, -0.02293239, 0.006858003, 0.0042885137, 0.021738121, + -0.0011316912, 0.02238954, 0.008776072, -0.014825832, 0.027214872, + 0.008160843, -0.00006469903, -0.017757222, -0.026249805, 0.003579793, + 0.006622768, 0.02394571, 0.009982405, -0.009517968, 0.01090525, + -0.014391553, -0.024199039, 0.006574515, -0.00008072064, 0.013860766, + 0.0051812003, -0.005612464, 0.019035935, 0.021436537, 0.004822316, + 0.0029117863, 0.016430255, 0.01256999, -0.029241512, -0.012582053, + 0.03271575, 0.016406128, -0.001565217, 0.0153445555, 0.0068881614, + 0.005886905, -0.006604673, 0.017853728, 0.0044905744, 0.00064199534, + -0.0049218386, -0.0028122638, 0.004068358, -0.023680316, 0.017250562, + -0.027721532, -0.018758478, -0.019976875, 0.012606179, 0.009125909, + 0.020821307, 0.012401103, 0.019614974, -0.02709424, 0.003869313, + 0.017274689, -0.009469714, 0.008263381, 0.002754963, -0.009228447, + 0.00832973, 0.0015373206, -0.013824576, -0.0076602143, -0.0050786617, + 0.018203564, -0.017624525, 0.0031726558, -0.015416935, 0.010784617, + 0.014825832, 0.011586828, 0.014041716, 0.0019829099, -0.03143704, + 0.006067855, 0.033777323, -0.012346818, 0.0022060815, -0.018806731, + -0.008353855, -0.0020929878, -0.0022090974, 0.006441818, + -0.0005126915, 0.0064840396, -0.031244025, -0.016961042, 0.038313136, + 0.0031515448, 0.007274188, -0.018239755, 0.0076300562, 0.00953003, + 0.011767779, 0.0114481, -0.018710224, -0.0034048748, -0.018565465, + 0.01206333, -0.0031817032, 0.025501879, -0.005494847, 0.0003168509, + -0.018710224, 0.01152048, -0.0001783111, 0.0037396322, 0.0043277196, + -0.013329979, 0.02141241, -0.0069364146, -0.00055302825, 0.0039808988, + 0.006640863, 0.018975617, -0.017853728, 0.017503891, -0.016297558, + -0.0031515448, -0.02079718, 0.014705199, 0.0021005273, 0.018698161, + -0.0030912282, -0.000008493809, -0.011647145, 0.015320429, + -0.025308866, -0.012376976, 0.0038662972, 0.039326455, 0.007991956, + -0.025694892, 0.0142467925, -0.024283484, -0.008601154, 0.021931134, + -0.0065564197, -0.010483034, 0.002646393, 0.019795924, 0.0053772293, + -0.0067373696, 0.0020628294, 0.008655439, 0.0072199027, -0.0057813507, + -0.0073164096, 0.018263882, 0.012039203, 0.011731588, -0.010109071, + 0.03734807, 0.0008610202, 0.018589592, 0.007931639, -0.009723044, + 0.006140235, 0.020157823, -0.0028710726, -0.005850715, 0.023704443, + -0.0053380234, -0.0057180184, 0.011243023, 0.0008112589, 0.004252324, + 0.001979894, 0.006212615, -0.007002763, 0.015018846, -0.0029901979, + 0.028565966, 0.0012078409, -0.008884642, -0.0017838649, 0.017781349, + -0.014295046, -0.0025332994, -0.009759234, 0.001655692, 0.014282983, + -0.005241517, 0.019228948, 0.0040020095, -0.009596379, 0.0096506635, + -0.01231666, 0.020133698, 0.023704443, -0.0037667747, 0.029844679, + 0.0045327963, 0.0060738865, -0.011755715, 0.015585822, -0.016345812, + -0.041353095, -0.002157828, 0.0075456128, -0.038433768, 0.012521736, + -0.0042070863, 0.022305097, 0.0015064083, 0.0036039199, 0.011767779, + -0.01604423, 0.023173656, 0.019313391, -0.004231213, 0.012346818, + 0.009813519, -0.035345558, 0.013329979, 0.0025076647, -0.013100776, + 0.01220809, -0.005310881, -0.00136768, 0.003615983, -0.025381247, + -0.008486552, 0.014174413, -0.0017220404, 0.0019512436, 0.015115352, + -0.009403366, 0.0005892183, -0.0066649895, 0.008004019, 0.010012563, + -0.01246142, -0.023282226, 0.00035379484, -0.02025433, -0.0065564197, + 0.031099264, 0.02007338, 0.00071512925, -0.01694898, -0.011695398, + 0.007841164, -0.0042191497, -0.0009643124, -0.0029042466, + 0.0013081173, 0.017986424, -0.019530531, -0.019976875, 0.0010872076, + -0.008420204, -0.024319673, -0.020350838, -0.0020432265, 0.0022603665, + 0.011900475, -0.0065624514, 0.011031915, 0.004378989, 0.033825576, + 0.010109071, 0.009252574, 0.005169137, -0.0061311875, -0.0026388534, + 0.029941184, -0.0025317913, 0.0026931383, -0.008884642, -0.027962798, + 0.02387333, -0.0033777324, -0.019313391, -0.0036461414, -0.0015606933, + 0.0119909495, -0.00036265387, 0.028469458, -0.011327467, -0.018432768, + 0.0038934397, -0.012286501, -0.009318923, 0.007177681, -0.0139090195, + -0.002967579, 0.004502638, 0.031075139, 0.007593866, 0.012189995, + 0.01705755, -0.012968079, -0.0150067825, -0.008993212, -0.00056999235, + 0.011562701, 0.018975617, -0.005359134, -0.002733852, -0.0150912255, + 0.028348826, -0.0041678804, 0.018239755, 0.0077687846, 0.028710725, + -0.017105801, -0.023680316, -0.00953003, 0.009095751, -0.00058544846, + -0.0045629544, -0.02126765, 0.010368432, -0.00033551137, 0.009704948, + 0.010006532, -0.017347068, -0.027214872, 0.015875341, -0.0053319917, + -0.018770542, 0.028252319, 0.03464588, 0.028951991, 0.00013034052, + -0.0033928116, 0.015561695, 0.02257049, 0.01079668, 0.0021804469, + 0.018698161, -0.016900726, 0.021207334, 0.0079617975, -0.006393565, + 0.007111333, 0.014150286, 0.01340236, 0.014102032, 0.0009907009, + -0.0009982405, -0.017950235, -0.0046112076, -0.029531032, + -0.024440307, 0.03553857, -0.002105051, 0.010555414, 0.0058989683, + -0.013559183, 0.010826838, -0.011454131, -0.0049369177, -0.007563708, + -0.0078773545, -0.012799193, 0.02293239, -0.00623071, -0.027190745, + 0.015670266, 0.00992812, -0.024066344, -0.02485046, -0.0068881614, + 0.005983412, 0.006393565, -0.00572405, 0.009692886, -0.019494342, + 0.004701683, -0.01974767, 0.003401859, 0.017962297, 0.014596629, + -0.019590847, -0.0016179942, -0.009439556, 0.016804218, 0.002501633, + -0.033680815, -0.007328473, 0.014813769, -0.01029002, -0.025043473, + 0.013112839, -0.0066589583, 0.015223922, -0.008733851, -0.011025883, + 0.0066649895, 0.014862022, 0.005467704, 0.013824576, 0.029193258, + -0.009150036, -0.0018758478, 0.0067916545, -0.0055038943, + 0.00025804216, 0.012099519, 0.022341287, -0.012521736, 0.009608442, + -0.008058304, -0.02658758, -0.0008346316, -0.0033536057, 0.0101331975, + 0.0030912282, -0.011387783, 0.01282332, -0.0005074138, -0.016237242, + 0.016273431, -0.00075546606, -0.02278763, 0.025477752, 0.021231461, + -0.010772553, -0.012352849, -0.026563453, 0.0013088713, -0.004427242, + -0.0041377223, 0.028831359, -0.0014574011, 0.0066287997, + -0.0033355106, 0.0014807738, 0.005540084, -0.018215628, -0.019421961, + 0.008950991, 0.016900726, -0.014886149, 0.0065322933, -0.012039203, + -0.017781349, 0.012352849, 0.0013752196, -0.00060769025, 0.009487809, + 0.03901281, 0.003923598, 0.011888412, 0.011797937, -0.007032921, + -0.0010344306, 0.007358631, 0.01278713, 0.019771798, -0.027021859, + 0.005998491, 0.008577027, -0.013884893, 0.0023990946, -0.01329379, + -0.035080165, 0.005241517, -0.01380045, 0.0018019599, 0.00093716994, + -0.014886149, -0.0053711976, 0.010965567, 0.031219898, -0.00047084683, + 0.0040201047, 0.010591604, 0.0083840145, 0.012346818, -0.0046172393, + -0.013716006, 0.009324954, -0.027118366, 0.0040593105, 0.008432267, + -0.013209346, 0.0010562954, -0.002016084, -0.007829101, 0.019988937, + 0.012630306, -0.0017371195, -0.0005096757, -0.015453125, -0.007979893, + 0.0021713993, -0.010404622, 0.0025619497, 0.0108630285, -0.0043850206, + -0.008438299, -0.02137622, -0.011550638, 0.008540837, -0.0059321425, + -0.02887961, -0.00094395556, -0.0031575765, 0.0072138715, + 0.0052656434, -0.017901981, 0.015742645, -0.0101030385, 0.015609949, + 0.0012447848, 0.0004406885, 0.013703942, -0.008148779, 0.021508917, + 0.0153445555, -0.016261369, 0.009825583, 0.014186476, -0.012907763, + -0.012256343, 0.00096280454, -0.01644232, 0.0056999233, 0.0039386773, + -0.0051812003, -0.009873835, -0.00019235356, 0.003006785, + 0.0051238993, -0.005114852, 0.014705199, -0.0016692632, 0.018975617, + -0.012093488, 0.0075033912, 0.023644127, 0.0025800446, -0.009885899, + 0.013197283, 0.004179944, -0.023125403, 0.015670266, 0.028517712, + -0.0070811748, 0.0076783095, 0.016900726, 0.013884893, 0.0374687, + -0.0032480515, -0.022087958, -0.019180695, -0.0049369177, + -0.018770542, 0.0028182955, 0.0113154035, 0.0070992694, 0.009668759, + -0.021183208, 0.0041678804, -0.003003769, -0.0062548365, -0.012153805, + -0.009861772, 0.0096506635, -0.014379489, -0.010917313, 0.008577027, + 0.014222666, -0.0035405874, 0.02637044, 0.004029152, 0.0025332994, + 0.00072945445, 0.02191907, 0.012322691, 0.015730582, -0.011532543, + 0.016997231, -0.005075646, -0.011327467, 0.0013571247, 0.014789643, + 0.038385514, 0.004053279, 0.017045485, 0.023813013, 0.010054786, + -0.0056004007, 0.020230204, 0.016538825, -0.004948981, -0.018179437, + 0.0081306845, 0.0038964555, -0.008902738, 0.007895449, 0.021617487, + 0.011387783, 0.009077656, 0.014270919, -0.010856997, -0.003026388, + 0.018963555, -0.011508416, -0.00659261, -0.002338778, -0.0037607432, + 0.007871322, 0.001350339, -0.022739377, -0.012105552, -0.013366169, + 0.0013752196, -0.026828846, -0.0052294536, 0.004608192, -0.0047800946, + 0.020941941, -0.005525005, -0.006924351, -0.0064719766, 0.0015373206, + 0.0017039453, -0.029965311, -0.025453625, -0.0029313893, 0.027070113, + -0.000108664215, -0.0029132941, -0.008142748, 0.0084684575, + 0.005286754, 0.026997732, -0.00013590095, 0.00320583, 0.007817037, + -0.015694393, -0.028710725, -0.006206583, 0.020785118, 0.014174413, + -0.0076059294, 0.008619249, -0.011689367, 0.019144505, 0.0067916545, + 0.0016541841, 0.0000810505, -0.0005846945, 0.02007338, 0.027142493, + -0.009240511, 0.0064719766, -0.018794667, 0.0011837143, 0.017250562, + 0.0038301072, -0.012980143, -0.009216384, 0.004535812, -0.032377977, + 0.0021834627, -0.007412916, -0.0031424975, -0.0039718514, + 0.0033807482, 0.0051088203, 0.008088463, -0.0052053267, -0.0043337513, + 0.009807487, 0.006387533, 0.027287252, -0.009493841, -0.012654433, + 0.019253075, -0.0053380234, -0.004101532, 0.016261369, 0.014114096, + -0.006242773, -0.008528775, -0.0012183964, 0.005681828, 0.012847446, + -0.0049067596, -0.020845434, 0.010458907, 0.0033053523, 0.012485546, + 0.0025438548, 0.007744658, 0.006514198, -0.0031002758, 0.001440814, + 0.012401103, 0.0041316906, -0.017962297, -0.016900726, -0.023161594, + -0.0035375715, 0.014693135, -0.014379489, -0.0013375217, -0.0221724, + -0.024536813, 0.0038482023, -0.0027263125, 0.0021849705, 0.01206333, + 0.0033596375, -0.014958529, 0.0045448593, -0.00659261, -0.026056793, + 0.0057180184, -0.003941693, -0.016176926, -0.018263882, -0.04060517, + -0.004771047, -0.0011233976, -0.014174413, -0.008414173, + -0.0038843923, 0.044031154, -0.024319673, -0.008788136, 0.027190745, + 0.011068105, -0.00411058, -0.013836639, 0.006640863, -0.000522493, + 0.00089721015, -0.018155312, -0.018360388, -0.027190745, + -0.0037577273, -0.008649408, 0.0058446834, 0.01593566, -0.00100955, + 0.013728069, -0.008444331, -0.00010998364, -0.009011308, -0.008432267, + -0.012413166, 0.0145001225, 0.017600399, -0.006212615, -0.0036039199, + -0.0029615476, 0.004517717, 0.028348826, -0.015995976, -0.016309623, + 0.00641166, 0.004774063, -0.0013488311, -0.0064780083, 0.014970592, + 0.0058597624, 0.01314903, -0.013378233, 0.0048404112, -0.00034682074, + 0.0017733095, -0.022293033, -0.011526512, 0.011436037, -0.001385021, + 0.0022060815, -0.0052505643, -0.0088062305, 0.00086478994, + 0.015042973, -0.020664483, -0.008926864, 0.018203564, 0.026635833, + 0.008890674, 0.021122891, 0.02492284, 0.013498866, 0.011966824, + -0.005199295, -0.00078185456, -0.013824576, -0.009457651, + 0.0046745404, -0.009252574, 0.022087958, -0.0004893188, -0.013631563, + 0.0069545098, 0.011683335, 0.00989193, 0.0154289985, 0.0022000498, + -0.004496606, 0.027166618, 0.0076421197, -0.027407885, 0.022884136, + -0.021870816, 0.012630306, -0.018541338, 0.011134453, 0.012364913, + 0.01246142, -0.008703693, -0.019735608, -0.020302584, 0.0059291264, + -0.0023372702, -0.0041075638, -0.007237998, 0.009644632, + -0.0011927617, -0.00572405, 0.004955013, -0.0065262616, 0.0013194267, + -0.01314903, 0.0043669255, 0.016755965, -0.019928621, -0.0049580284, + -0.008275445, -0.007666246, 0.015211859, 0.028204065, -0.008148779, + 0.0037155056, 0.034573503, 0.013305853, 0.032160837, 0.0035948723, + -0.012606179, 0.030254832, 0.0078773545, -0.0032872574, 0.020411154, + -0.0059080156, -0.010923346, 0.0062488047, -0.012177931, -0.010700174, + 0.010627793, 0.0013488311, -0.00074453367, -0.018505149, 0.014126159, + -0.012811257, -0.0151756685, -0.015320429, -0.00021506655, + -0.009318923, -0.005057551, 0.018010551, 0.018999744, 0.028565966, + 0.009729075, 0.0058205565, 0.0054586567, -0.008239254, -0.0077084675, + -0.00608595, -0.017045485, 0.001709977, -0.022968581, 0.0045267646, + 0.0062669, -0.012232216, 0.010428749, 0.019868305, 0.0064357864, + 0.0073345047, -0.00528977, -0.0038270913, 0.005039456, -0.023523493, + 0.017479764, -0.0016059307, -0.008625281, -0.0024066344, -0.012310628, + 0.015416935, -0.00053229445, -0.010706205, 0.02068861, 0.0066529266, + 0.015308365, -0.013161093, -0.012690623, -0.018119121, 0.00633928, + 0.022630807, -0.0015049004, 0.02416285, -0.0049761236, 0.014789643, + 0.008082431, 0.009753202, -0.020109572, 0.008727819, 0.011966824, + 0.02376476, -0.008275445, -0.008022115, 0.0055370685, -0.02477808, + 0.026201552, 0.016092481, 0.013221409, 0.007539581, -0.01369188, + -0.0130887125, 0.004481527, -0.026611706, -0.0031545607, + -0.0022211606, 0.027890418, -0.009222415, 0.021533044, 0.017503891, + -0.021122891, 0.003257099, -0.0025544101, 0.0058265883 + ] + } + ] + }, + { + "name": "springfield-illinois-sangamon-county-seat", + "title": "Springfield: Illinois' Sangamon County Seat", + "content": "Springfield serves as the county seat of Sangamon County and is located in the U.S. state of Illinois.\n\n---", + "keywords": [ + "Springfield", + "county seat", + "Sangamon County", + "Illinois", + "United States" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.03381783, 0.03834294, -0.006908156, -0.029640809, 0.003065829, + -0.028087812, 0.01140649, 0.005964309, -0.04364455, 0.019747151, + -0.020349607, 0.015288981, -0.011935311, 0.005204546, 0.0035377522, + 0.00076311023, -0.011868373, 0.021996317, -0.005338425, -0.0064429264, + 0.039735284, 0.0016475483, -0.045652736, 0.031782873, -0.00293195, + 0.017979948, -0.048276763, 0.02198293, -0.031836424, -0.0011814821, + -0.016052091, -0.0011095221, 0.033228766, 0.034246244, -0.0037653465, + -0.03818229, -0.004511722, -0.032211285, 0.0019696946, 0.019425841, + -0.009699533, -0.0052078925, 0.015717393, 0.00463556, 0.019532945, + -0.007530693, 0.0039126133, -0.02950693, 0.003952777, 0.010998159, + 0.010516195, -0.009230956, 0.0012492583, 0.03944075, 0.032720026, + -0.0015027917, 0.02051026, 0.037753876, -0.011533675, -0.053283837, + -0.021514354, -0.010516195, -0.030444082, 0.005304955, 0.021675007, + 0.020229114, -0.000713324, 0.0040498395, 0.034647882, 0.0127385855, + 0.040029816, 0.036656067, 0.007798451, 0.04257352, 0.040324353, + -0.005415405, -0.039360423, -0.0168152, -0.023468986, 0.020590588, + 0.03836972, -0.03574569, -0.004090003, 0.02318784, 0.011781351, + -0.020175563, 0.0062822714, -0.07272307, 0.0003252841, 0.010168109, + 0.009699533, -0.0094987145, -0.021822276, -0.0139501905, -0.03906589, + -0.036281206, 0.052962527, -0.013079977, -0.0028298672, 0.03887846, + 0.0051945047, -0.008032739, -0.051382754, -0.0064529674, 0.016252909, + 0.017926397, 0.008675358, -0.0226791, -0.019773927, -0.013561942, + 0.010168109, -0.00012237376, 0.0060446365, -0.026173342, -0.01930535, + -0.011895148, 0.0030256652, -0.02383046, 0.047768023, 0.009940515, + -0.015195265, -0.00014465206, -0.012792137, 0.001655079, -0.013976966, + 0.031461563, 0.016426953, -0.040029816, 0.028382346, -0.014927507, + 0.017765742, -0.0035176703, 0.00974639, 0.008581643, -0.0020081848, + -0.021688396, 0.02886431, 0.0503385, 0.030122772, 0.01981409, + -0.07293727, 0.00017446105, 0.0153559195, -0.020563813, 0.015128326, + -0.016600994, -0.0077850632, -0.015543351, -0.011045016, -0.031086702, + -0.021969542, -0.018515464, -0.03732546, 0.019894417, 0.03446045, + -0.00949202, 0.0015086489, -0.037566446, 0.039654955, -0.045706287, + -0.022049868, -0.017645251, 0.029935341, 0.009485327, -0.020617364, + -0.0067173783, -0.0046422537, -0.011654166, -0.007865391, + 0.0069684014, 0.033683952, 0.00055476103, -0.016092254, -0.07475803, + 0.0138297, -0.029988892, 0.004076615, -0.023214616, 0.0019295309, + 0.036093775, -0.0051074834, 0.028302018, 0.026963228, -0.015114938, + -0.019773927, -0.009512102, 0.001322055, 0.025088923, 0.045090444, + -0.02313429, -0.024365976, 0.037566446, 0.036307983, 0.01401713, + 0.026414324, -0.0031779525, -0.035638586, -0.031434786, 0.0016600995, + 0.015918212, -0.05917451, 0.01885016, 0.030792167, -0.014753465, + 0.006888074, 0.01937229, 0.0017538148, -0.031408012, 0.008099679, + 0.04134183, 0.009143935, 0.0041335137, -0.016212745, 0.05976358, + -0.02937305, 0.028944638, -0.013936803, -0.0028583163, -0.022264076, + 0.05015107, 0.0327468, 0.014807016, -0.0055024265, -0.00075557956, + -0.010951301, -0.006740807, -0.055854313, -0.027873605, 0.024419528, + 0.00076018163, 0.014512482, -0.0038188982, 0.021059165, 0.039494302, + -0.04286805, 0.04415329, 0.016855365, -0.0073767323, -0.042091552, + 0.027498744, 0.0020550424, 0.043751653, -0.014204561, -0.02886431, + -0.009773166, -0.021393862, 0.015770944, 0.015195265, 0.000923765, + 0.015503187, -0.001325402, 0.0068278285, 0.032666475, 0.008193394, + -0.016775038, 0.008233557, -0.017243614, -0.003323546, 0.018073663, + 0.019158084, -0.007918942, 0.02701678, -0.011814821, 0.013227244, + 0.0065266006, -0.024526631, -0.020055072, 0.0016609362, 0.0119553935, + -0.015061386, -0.028061036, -0.009679451, -0.00076143676, 0.029721135, + 0.041395385, 0.0024918227, 0.024365976, 0.025075534, -0.032345165, + -0.020657528, -0.017029407, -0.004153596, 0.010723707, -0.043939084, + 0.013039813, 0.019706987, -0.004632213, 0.049240693, 0.0063994154, + -0.029721135, -0.0033252193, -0.01682859, 0.030685063, 0.0335233, + 0.051115, 0.015208653, 0.021996317, 0.014418767, -0.044367496, + 0.007497223, 0.026521428, 0.026200118, 0.0036046917, 0.0004248566, + 0.019854253, -0.00481295, 0.010127946, 0.013802923, -0.029533705, + -0.05671114, 0.006463008, -0.03976206, 0.0005279853, 0.026119791, + -0.006874686, 0.022130197, 0.0028415816, -0.012524379, 0.051704064, + -0.03338942, -0.016989244, 0.0168152, 0.020001521, -0.0013396266, + -0.01510155, 0.008862789, -0.019961357, 0.0008275395, 0.026374161, + 0.011366326, -0.03384461, 0.0040665744, -0.008226864, -0.043483894, + -0.0077114296, -0.02905174, 0.021567905, 0.008494622, 0.022130197, + 0.014686525, -0.018073663, -0.019961357, 0.004796215, 0.05052593, + 0.03421947, 0.012002251, 0.0019797354, 0.00052421994, -0.026427712, + 0.0035979978, 0.00070663006, 0.009900351, 0.030738616, -0.020483484, + 0.018261094, 0.0071089743, -0.035531484, -0.040993746, 0.0016073846, + -0.013468226, -0.029105293, 0.005532549, -0.0137895355, 0.0069483193, + 0.010998159, 0.043885533, -0.017417656, 0.017578311, 0.035076294, + 0.0037921222, -0.0070353406, 0.01708296, 0.0075775506, -0.0027294578, + 0.019653436, 0.0010651747, -0.0064529674, 0.011379714, 0.00688138, + -0.0065199067, 0.04134183, 0.016908916, -0.03866425, -0.016426953, + 0.0017872845, -0.026655307, -0.012725198, -0.031140253, -0.023683192, + 0.02313429, -0.039039113, -0.006730766, 0.005924145, -0.021059165, + -0.017979948, 0.021005614, -0.062334057, 0.010502807, 0.054301318, + 0.028435897, 0.014284888, -0.032827128, 0.010348846, 0.029132068, + -0.031300906, -0.02886431, 0.00083548855, -0.054943938, 0.0056162234, + -0.006088147, -0.030524408, -0.001994797, 0.0035410991, 0.007972494, + -0.02459357, -0.053337388, -0.021581292, 0.030872494, 0.0168152, + 0.012203069, 0.0060245544, -0.042038, -0.0019245105, -0.018435137, + -0.00876238, 0.016226133, 0.02918562, 0.035504706, -0.028489448, + -0.034701433, -0.030497633, -0.014405379, -0.0071022804, 0.03325554, + 0.0020349605, 0.04262707, -0.0020366341, 0.013501696, 0.052319907, + 0.013180386, -0.015463023, 0.017939784, 0.048598073, -0.04308226, + -0.001778917, 0.009612511, 0.0184753, 0.011854985, 0.030122772, + 0.035129845, -0.0097129205, 0.020778019, 0.012015639, 0.011145426, + 0.0015998539, 0.021112716, -0.049267467, 0.016145807, -0.024767613, + 0.045652736, 0.030952822, 0.002520272, -0.000594088, -0.04351067, + -0.012859077, 0.019947968, 0.016975855, 0.014405379, -0.0015103223, + -0.030631512, -0.012477522, -0.02771295, -0.018743059, 0.010663461, + -0.042412862, -0.009217569, -0.004197106, 0.0041234726, 0.013816312, + -0.009739696, -0.048089333, 0.007015259, -0.06276247, 0.008996668, + 0.0051677288, 0.0040565333, -0.030711839, -0.042278983, 0.0031511767, + -0.010281906, 0.08477218, 0.022920083, -0.012243234, -0.015543351, + 0.02071108, 0.0128992405, -0.02032283, -0.032853905, 0.02554411, + 0.035263725, -0.00739012, 0.057353757, 0.036415085, -0.019291962, + 0.0037251827, -0.0058337767, -0.0131669985, -0.041582815, 0.011540369, + 0.009150629, 0.054515526, 0.0076779597, -0.031193804, -0.0302031, + 0.004505028, -0.014606197, -0.017364105, -0.0059007164, 0.037164807, + -0.0009614185, -0.0013145243, 0.009505408, -0.0031779525, + 0.0042941687, 0.0040833093, 0.012189682, 0.013495002, -0.0029553787, + 0.0075775506, -0.032827128, -0.036656067, -0.0037854284, 0.002249167, + 0.030095996, 0.038075183, -0.00525475, -0.019666823, -0.0047192345, + 0.0002273851, 0.015422859, -0.03657574, -0.025892196, 0.016386788, + 0.0069750953, 0.00075223256, 0.007122362, 0.001224156, 0.005853859, + -0.008856095, -0.02515586, 0.042145107, -0.0072428533, -0.004769439, + -0.005562672, -0.0022541874, -0.004153596, -0.0140572935, + -0.011620696, -0.005820389, -0.004920053, 0.012591319, 0.0080394335, + 0.03804841, 0.01169433, 0.005311649, 0.011620696, -0.0051509943, + -0.0037251827, -0.0076779597, -0.0049367878, 0.006744154, + 0.0061115758, 0.043242913, 0.004501681, 0.0192518, -0.0066504385, + -0.028810758, -0.012129436, 0.008300497, -0.008568255, 0.013314266, + -0.0033536686, 0.027097108, 0.014445542, 0.008869483, 0.013976966, + -0.0024985166, -0.024165157, -0.026347386, 0.030256651, 0.020737855, + 0.01491412, -0.0010266845, -0.0062722308, -0.0016358339, 0.0031879933, + -0.02528974, 0.0028415816, -0.017310552, -0.017377492, 0.013963578, + 0.0041703302, 0.02739164, -0.03930687, -0.01886355, 0.002987175, + 0.032398716, 0.009391611, 0.017792517, 0.003303464, 0.019747151, + 0.01727039, -0.005294914, -0.001619099, -0.010455949, -0.011185589, + 0.030229876, 0.035852794, 0.016654545, -0.012832301, -0.010081088, + -0.028194916, -0.0023545967, -0.015061386, 0.031193804, -0.021126105, + 0.012557849, -0.028676879, -0.022893308, 0.013976966, -0.013307571, + -0.0006459661, -0.025597662, 0.00020709405, 0.011895148, -0.010636685, + -0.0037184888, -0.003828939, -0.008849401, 0.004230576, 0.003701754, + 0.040377904, 0.013856475, 0.020215727, -0.023174454, -0.0032432184, + -0.019037591, 0.04289483, -0.005298261, 0.0017287125, 0.059870683, + -0.0030959516, -0.001994797, -0.0023211269, 0.0029687665, 0.008836013, + 0.005950921, -0.035611812, 0.04179702, -0.019211635, -0.02516925, + 0.010924526, -0.016346624, 0.007791757, 0.033603627, 0.014793628, + -0.02364303, 0.0588532, 0.02115288, 0.0036615904, -0.008240252, + 0.004545192, 0.003946083, -0.009010056, -0.0065332945, -0.011125344, + -0.00311436, -0.008876177, -0.014579422, 0.005013768, -0.00825364, + -0.00532169, -0.0057835723, -0.023629641, 0.028783983, -0.02306735, + 0.03293423, 0.043537445, -0.019666823, -0.031274132, 0.016065478, + 0.02982824, 0.012162906, -0.0096259, 0.031809647, 0.0017077938, + 0.020055072, 0.034969192, -0.013019731, -0.010174803, -0.0012149519, + 0.017096346, 0.025129085, -0.052587666, 0.0061684744, 0.025182638, + -0.0011396449, -0.03700415, -0.031488337, -0.0064596613, 0.010435867, + -0.0138966385, 0.020630753, 0.00056689384, 0.034942415, -0.036200877, + 0.044099737, -0.036388308, -0.010917831, 0.02937305, 0.015208653, + 0.0077047357, -0.038637474, -0.015650453, -0.019532945, 0.007999269, + -0.031434786, 0.006245455, 0.009036832, 0.0053785886, -0.04262707, + -0.0064194975, -0.014472319, -0.008427682, 0.024700673, 0.0069617075, + -0.019091144, -0.023977727, -0.002160472, 0.00952549, 0.008307192, + -0.017926397, 0.03834294, 0.0065098656, -0.009438469, 0.021554517, + -0.0035678751, -0.00048656645, -0.05912096, 0.022103421, -0.019479392, + -0.014043906, 0.036120553, -0.0067910114, 0.008300497, 0.010214967, + 0.019412452, 0.01688214, 0.02593236, 0.036066998, 0.019332126, + 0.024205321, 0.0015320777, 0.0013212182, 0.011419877, 0.030845718, + 0.014981058, 0.00691485, 0.017752353, 0.012410582, 0.028757207, + 0.0014919139, -0.01682859, -0.011366326, 0.017176675, 0.015342532, + -0.0010810728, -0.007657878, -0.006379334, -0.009806636, 0.043885533, + 0.0053317305, -0.031167028, -0.01057644, -0.0032750147, -0.025838645, + -0.009967291, 0.007992576, -0.0001796907, -0.013133529, -0.009652675, + -0.0013664024, -0.035076294, 0.014298276, 0.0021203086, -0.016400175, + 0.0060245544, -0.018033499, 0.029212395, 0.014485707, 0.0014760159, + 0.018046888, 0.010763871, 0.0054756505, 0.0022608815, -0.00927112, + -0.013233938, 0.026240282, -0.028489448, 0.034754984, 0.043805204, + -0.0061785155, 0.027605847, 0.014351827, -0.0042740866, -0.018180767, + 0.03344297, -0.00012551155, -0.00344069, -0.017190062, 0.0231209, + -0.014887343, 0.019358901, 0.0011212365, -0.000525475, 0.020925285, + 0.017939784, -0.041047297, -0.0021019, 0.014967671, 0.009311284, + -0.019466005, -0.016065478, -0.017859457, -0.0049669105, 0.008936423, + 0.031595442, 0.0013028099, 0.02905174, 0.0108174225, -0.015422859, + -0.007590939, 0.025892196, -0.0069416254, 0.0070353406, 0.043376792, + 0.01156045, -0.0427074, 0.00942508, -0.043751653, -0.022290852, + 0.013039813, -0.02108594, 0.027579071, -0.014244724, 0.009398305, + 0.03828939, -0.00053844455, 0.00942508, 0.018328033, 0.014391991, + -0.0023428823, 0.027820054, 0.006563417, 0.020028297, 0.030792167, + 0.019466005, -0.003032359, -0.0014115865, -0.011540369, 0.023535926, + 0.0028248467, -0.024781, -0.012176294, -0.013749372, 0.00311436, + -0.017230226, 0.001224156, 0.0029470113, 0.016199358, -0.0327468, + -0.006908156, -0.014258112, -0.0336304, 0.021193044, 0.023308331, + -0.022103421, -0.016547443, 0.0116006145, -0.035183396, -0.005278179, + 0.005646346, 0.008842708, -0.0062521487, 0.014311664, 0.009076996, + -0.022545222, -0.01955972, -0.016252909, 0.044233616, 0.012236539, + -0.0015923232, 0.0028717043, -0.022598773, -0.0096259, -0.0062722308, + 0.023147676, -0.008106372, 0.0012040741, 0.018970653, 0.015704006, + 0.034514003, -0.007262935, -0.020095237, -0.01032207, 0.0041167787, + -0.033603627, 0.0050874017, -0.0034172612, 0.027605847, 0.035852794, + 0.012571237, -0.020403158, -0.0036281205, -0.003457425, -0.009639287, + 0.02071108, -0.016426953, -0.021755336, 0.0039260015, -0.0056061824, + -0.00844107, -0.0067776237, 0.009251038, 0.03333587, -0.019144695, + -0.029935341, -0.027686175, -0.010850892, 0.009164017, 0.013294184, + 0.048972934, -0.034005262, -0.01828787, -0.0045083747, 0.03330909, + -0.001119563, 0.021380475, 0.030711839, -0.0038958786, 0.020242503, + 0.004786174, -0.005780225, -0.0053685475, 0.035022743, 0.015556739, + -0.035263725, 0.032961007, 0.0011371346, 0.0057166326, -0.011767963, + 0.0038992255, -0.015784333, 0.00463556, 0.004906665, -0.014512482, + -0.008133149, -0.04294838, -0.009076996, 0.0017044469, 0.0048330314, + -0.01083081, -0.019077756, -0.020737855, 0.009311284, 0.016641159, + -0.0030926047, 0.0024985166, 0.011446654, 0.02127337, 0.0086084185, + 0.028275242, -0.009478632, 0.030872494, 0.031006373, -0.014873955, + -0.018876938, -0.010583134, -0.002465047, 0.0013413001, 0.041877348, + -0.024044666, -0.0015421186, -0.0019445923, 0.037111256, 0.0039996346, + 0.0029637462, 0.010275212, 0.0034021998, -0.0013555248, 0.019840866, + -0.029988892, -0.01918486, 0.016868753, -0.02982824, 0.012591319, + -0.042653844, 0.0053484654, -0.006446273, 0.010931219, 0.0013505043, + -0.0036080386, 0.0141108455, -0.030256651, -0.026026076, -0.022290852, + -0.005663081, -0.017002631, -0.014458931, 0.00056438363, -0.01133955, + 0.008481234, 0.0053183427, -0.0154094715, 0.00063676195, -0.003018971, + 0.017190062, 0.020590588, 0.035129845, 0.023562701, 0.020563813, + 0.015931599, -0.0031980344, 0.023147676, -0.028114587, 0.004910012, + -0.01918486, 0.049776208, 0.009010056, 0.0128456885, 0.022759428, + -0.019091144, 0.013106753, 0.014311664, -0.007149138, 0.008601725, + 0.00005600953, 0.0070755044, 0.009853493, 0.0023596173, -0.0023094125, + -0.0068043997, -0.0004936788, 0.0066604796, -0.037566446, + -0.012624788, 0.0043142503, 0.02515586, -0.00372853, -0.013722597, + 0.019573107, -0.0048330314, 0.039494302, -0.008682053, 0.015757557, + -0.01025513, -0.0171499, 0.022638937, 0.020483484, 0.0025085576, + 0.016467115, -0.009632593, 0.023669805, -0.02905174, 0.0032013813, + 0.007959106, -0.026106402, 0.00850801, -0.011767963, -0.029480154, + 0.0073097926, 0.022692489, -0.033362642, 0.033416197, 0.025329905, + -0.055961415, 0.01840836, -0.017123122, -0.017832682, 0.039922714, + -0.018876938, -0.0019529597, 0.004946829, -0.012952792, 0.028141363, + -0.026454488, -0.0007534877, 0.04377843, 0.0072093834, 0.005823736, + -0.017230226, -0.012792137, 0.00006181444, -0.03242549, -0.019934582, + -0.021849051, -0.025370069, -0.002111941, -0.033469748, -0.010469337, + 0.003952777, 0.013575329, -0.019104531, 0.0126114, 0.009485327, + 0.016239522, -0.012818913, 0.004853113, 0.0068077464, -0.031140253, + -0.007517305, -0.0171499, -0.026842738, 0.022250688, 0.005693204, + -0.043564223, -0.017618475, -0.022143584, -0.0034708127, 0.0045083747, + 0.0060847998, 0.00952549, -0.009699533, -0.022237299, 0.022063257, + -0.017497983, 0.022344403, -0.02574493, 0.047446713, -0.0213537, + 0.0079323305, 0.00078988605, -0.021367086, -0.025517335, 0.031327683, + -0.0050940956, 0.02835557, -0.012055803, -0.0047828266, -0.015436247, + -0.024687285, -0.0010835831, 0.009759778, -0.015235429, -0.0056262645, + -0.020697692, -0.012571237, -0.01510155, -0.02618673, 0.024473079, + -0.021500966, 0.003035706, 0.001021664, 0.014499094, 0.0071959957, + 0.012979568, 0.02286653, -0.0036850192, -0.014164397, 0.031836424, + 0.011774657, 0.012912628, -0.0184753, -0.020911898, -0.0038222452, + -0.013963578, 0.006121617, 0.0133276535, -0.014084069, 0.013655657, + -0.00032967702, 0.005174423, -0.008668665, 0.0024884758, -0.02701678, + -0.014766852, -0.0050740135, -0.030310202, -0.04166314, 0.009170711, + -0.024151769, 0.0044180066, 0.017444432, 0.01484718, 0.018394973, + -0.004210494, 0.00046104577, -0.008882871, 0.024245484, 0.03435335, + -0.03467466, 0.03542438, -0.0040063285, 0.005786919, 0.021567905, + -0.0105496645, 0.0058438177, -0.01955972, -0.0041335137, + -0.00035122316, 0.008146537, 0.01840836, 0.007691348, -0.009679451, + 0.0035076295, -0.024124993, -0.033924937, 0.004518416, 0.011219059, + 0.0056061824, 0.021688396, 0.0041837185, -0.03333587, 0.010275212, + -0.0062923124, 0.008889565, 0.010214967, 0.00026043647, 0.023428824, + 0.0004903318, -0.00038260105, -0.0006434559, -0.0097664725, + 0.0016216092, -0.026200118, -0.030336978, -0.032479044, 0.011687635, + 0.0035143234, 0.027927157, 0.040029816, -0.005435487, 0.015503187, + 0.016159194, 0.0034741596, 0.0037218358, -0.021996317, -0.02676241, + 0.0044146595, -0.048464194, 0.00042276474, 0.0205772, 0.015998539, + -0.021889215, -0.011185589, 0.0020801448, -0.028650103, 0.021300146, + -0.024044666, 0.009599123, 0.014378604, 0.0189171, 0.0017521413, + 0.014927507, 0.031889975, 0.013628881, -0.020871734, -0.00017404268, + 0.015128326, -0.014151009, -0.0056195706, 0.017257001, -0.004997033, + -0.022451507, 0.022237299, 0.029212395, -0.030979598, -0.012270009, + 0.010596522, -0.016413564, 0.0034641188, 0.007249547, -0.011386408, + 0.0029386438, 0.00039410626, -0.006245455, 0.02370997, 0.004545192, + 0.020737855, -0.002578844, 0.022116808, -0.0032515859, 0.0013739332, + 0.015476411, -0.014927507, -0.008454458, 0.039146215, -0.034835313, + 0.0020249197, -0.0031879933, 0.019211635, -0.024285648, + -0.00091288734, -0.034058813, -0.00872891, -0.003228157, -0.015717393, + -0.0008861116, 0.0024750878, 0.0075842445, 0.028783983, -0.0011321142, + -0.013856475, 0.00054974057, 0.032130957, -0.010208272, 0.026427712, + -0.00036356514, -0.01347492, -0.010563052, -0.05671114, -0.008333967, + -0.0020081848, -0.010181497, 0.013126834, 0.0024081483, 0.020858346, + -0.006730766, 0.008420988, 0.015503187, -0.008072903, -0.0069215437, + -0.009465245, 0.0058672465, -0.014860568, -0.035799243, -0.0071022804, + -0.024071442, 0.034701433, 0.019720376, 0.004454823, 0.011024934, + -0.00013429736, 0.02159468, -0.0077047357, 0.0026625183, + -0.0039828997, 0.00096225523, 0.018113827, -0.029667584, -0.00942508, + 0.03122058, -0.031729322, 0.009364835, -0.020416545, -0.0005422099, + -0.017364105, 0.0058505116, 0.007021953, 0.024928268, 0.0021403902, + 0.017805906, 0.03898556, 0.010081088, -0.0048430725, -0.001912796, + -0.004769439, -0.023107514, -0.005264791, -0.014766852, -0.006600234, + 0.018769834, 0.03237194, 0.027605847, -0.0009731329, -0.0068612983, + 0.009431775, -0.020630753, -0.028114587, -0.007021953, -0.028221691, + -0.008166619, 0.020590588, 0.008354049, 0.011834903, 0.024392752, + 0.012336949, 0.0016977529, 0.017109735, 0.018957265, -0.0038188982, + 0.0048598074, -0.016989244, 0.014552646, -0.0038088572, 0.001615752, + 0.010455949, 0.0028449285, -0.01249091, -0.0034473839, -0.009003362, + -0.0056396523, 0.008287109, -0.0094451625, 0.044822685, -0.009103771, + 0.002928603, -0.0168152, -0.01510155, -0.009277814, -0.0027997443, + -0.010589828, 0.011808127, -0.031916752, 0.01752476, -0.03330909, + -0.01752476, 0.028998189, 0.0055927946, -0.021126105, 0.0076110205, + -0.0252228, -0.008307192, 0.003701754, -0.002383046, -0.031113477, + -0.01662777, -0.011004853, -0.0104961125, 0.010489419, -0.013568635, + -0.011988863, -0.00060789427, -0.016935691, 0.0074704476, + -0.0136824325, -0.0053819353, -0.00688138, 0.0025671297, -0.014900731, + -0.016105643, -0.024392752, -0.0010434194, 0.01701602, -0.003959471, + -0.011185589, 0.0003148248, -0.03815551, 0.00043008625, -0.0033955057, + -0.02498182, 0.0120624965, -0.0369506, -0.0058739404, 0.03413914, + 0.011111956, 0.012805525, 0.023228005, 0.021072553, -0.004100044, + 0.014887343, 0.0009438469, 0.008226864, 0.015288981, 0.03708448, + -0.008139842, 0.010971383, 0.00024223729, -0.007798451, 0.0034842007, + -0.0061249635, 0.01745782, -0.0021270025, 0.014458931, 0.015824497, + 0.02140725, 0.025343293, 0.013990354, -0.008956504, -0.00923765, + -0.00438119, -0.0017471208, 0.0068947678, 0.005284873, -0.0064261914, + 0.03759322, -0.018997429, -0.0013714229, -0.0020349605, 0.0060111666, + -0.032023855, 0.0184753, 0.028783983, -0.008079597, -0.034380123, + 0.010382315, 0.008882871, 0.006218679, 0.01086428, 0.024606958, + 0.0059944317, 0.020055072, -0.002915215, -0.020282667, 0.0055927946, + -0.006473049, 0.006238761, -0.03344297, -0.0021721865, -0.0112793045, + 0.00930459, 0.033416197, -0.008320579, 0.013735984, -0.014632974, + 0.018087052, 0.019010816, -0.006473049, 0.0122231515, 0.0051443, + 0.013146916, 0.03957463, 0.004515069, -0.020362994, 0.009679451, + -0.006988483, -0.00045853553, 0.00055768964, 0.00673746, -0.012437358, + 0.005646346, 0.001465975, -0.020938674, -0.012604707, 0.0012157885, + 0.022946859, -0.0012793811, 0.011607308, -0.026441101, -0.004153596, + -0.019064369, 0.0042941687, 0.023535926, -0.004846419, -0.027190823, + 0.02803426, 0.01561029, -0.011828208, -0.013776148, -0.01293271, + -0.031943526, -0.0038523679, 0.0020935326, -0.0089297285, 0.009411693, + -0.004371149, -0.007276323, 0.025316516, 0.0068043997, 0.01257793, + -0.011419877, 0.018461913, 0.030551186, -0.007490529, -0.00032800352, + -0.006888074, 0.0014651382, 0.014177785, 0.018394973, -0.001025011, + 0.009485327, -0.0006643745, 0.01510155, -0.0028968067, -0.013300877, + -0.010301989, -0.012892546, -0.0009061934, 0.015503187, 0.018635955, + -0.003986247, 0.0012659932, -0.009619205, 0.0026139873, 0.010804035, + -0.00067065004, -0.015021223, 0.006573458, 0.021434026, 0.0019027551, + -0.010469337, 0.0014810363, -0.011192284, -0.013180386, -0.0071692197, + -0.023562701, 0.008019351, -0.015931599, -0.0013329326, -0.015262204, + -0.009940515, 0.012959486, -0.03976206, 0.014298276, -0.018073663, + -0.0011923597, 0.009565653, -0.0031260743, -0.0057099387, + -0.0061350046, -0.008882871, -0.020215727, -0.014432155, -0.010931219, + -0.01376276, 0.031622216, -0.0011630737, -0.016520668, -0.006322435, + 0.01184829, -0.017484596, -0.013856475, -0.012410582, 0.02318784, + -0.020751243, 0.008059515, -0.008106372, -0.024553407, -0.011968781, + 0.0068612983, 0.0051108305, 0.000101559766, 0.027793277, + -0.0073499563, -0.0060446365, 0.012363724, 0.025878808, -0.015114938, + 0.0119487, 0.01713651, 0.032104183, -0.011239141, 0.005529202, + 0.029292723, 0.034058813, -0.00597435, 0.004756051, -0.0070688105, + -0.008782461, -0.005693204, -0.009291202, 0.019399066, 0.011292692, + 0.007590939, -0.013963578, -0.016011927, 0.0015956702, -0.017846068, + -0.003107666, 0.004096697, 0.00532169, -0.00959243, 0.012444052, + -0.009545572, 0.017216837, 0.012129436, 0.011219059, -0.00517777, + 0.0014140968, -0.011312774, -0.0019696946, -0.006499825, -0.012504297, + -0.004997033, 0.00020385168, 0.0075842445, -0.017042795, + -0.0022090033, -0.0025805174, -0.009753084, 0.003594651, -0.012417276, + 0.04771447, 0.007959106, -0.0057400614, -0.0045351507, 0.002043328, + -0.0028181528, -0.017658638, -0.0060111666, 0.013963578, + -0.0014894038, -0.0066772145, -0.0069416254, -0.0036850192, + 0.005807001, 0.00082042714, -0.01893049, 0.020670915, -0.022438118, + 0.0008794176, -0.015061386, -0.0024918227, -0.0026240281, + -0.0057266736, -0.019666823, -0.02096545, 0.0070755044, 0.0140572935, + -0.0013128509, 0.007912248, 0.001328749, 0.015784333, -0.008896259, + 0.012042415, 0.022625549, 0.029024964, 0.017484596, -0.008059515, + 0.024888104, -0.003457425, -0.0121695995, -0.013287489, 0.026722247, + 0.009338059, -0.00074930396, -0.0046991524, 0.01369582, -0.026387548, + -0.014860568, -0.0052279746, -0.004632213, -0.0077783694, + -0.010971383, 0.0055158143, -0.014793628, 0.0029419907, -0.017939784, + 0.00011275121, -0.016065478, -0.00026462018, 0.009900351, + 0.0049367878, 0.0019211635, -0.024606958, 0.020992225, -0.0040230635, + 0.015770944, -0.00077189604, 0.010790646, 0.012149518, -0.02364303, + 0.008233557, -0.003979553, 0.0042038, -0.021675007, -0.027364865, + 0.020429933, 0.0030741962, 0.004910012, 0.0037352238, -0.009036832, + -0.009338059, -0.00065684377, 0.014338439, 0.013595412, 0.017765742, + -0.016600994, -0.0037184888, 0.012698422, 0.013187081, 0.010844198, + -0.006315741, 0.006476396, -0.0014283215, -0.008327273, 0.0040264106, + 0.019733762, -0.033282317, 0.00094552035, -0.021367086, -0.028302018, + -0.013816312, -0.01674826, -0.022170361, 0.015529962, 0.00077147764, + 0.026200118, -0.017926397, 0.019278575, -0.0057501025, 0.0056597344, + -0.031300906, -0.0070554228, 0.011085181, -0.013421369, -0.0034674658, + 0.03727191, 0.01650728, 0.02198293, 0.008400907, -0.0138297, + 0.02140725, -0.009371529, -0.024111606, -0.0065433355, 0.0072093834, + 0.01054297, 0.012049109, 0.021567905, 0.0035979978, -0.008742298, + 0.014151009, -0.010395704, 0.01162739, 0.0034105673, 0.009070301, + 0.0061483923, -0.0010225008, 0.022786204, 0.006620316, 0.0022123503, + -0.028489448, -0.015583514, -0.0027428458, -0.0075976327, 0.010770565, + -0.010107864, 0.008581643, -0.009478632, 0.0044380883, -0.00002852773, + 0.000048635728, -0.009619205, -0.017310552, -0.03786098, 0.011379714, + 0.009579042, 0.0020450016, -0.031408012, 0.013635575, -0.004103391, + 0.003959471, -0.0076043266, 0.013026426, -0.0039025724, + -0.00018042285, -0.011426572, -0.018100439, 0.015637066, + -0.0009087036, 0.0015061386, 0.011513593, 0.011259223, -0.011198977, + 0.0055057732, -0.005264791, 0.011547063, 0.0011178895, 0.011178896, + -0.0013797904, -0.006225373, -0.01860918, 0.0053752414, -0.007256241, + -0.006606928, 0.0097664725, 0.0037251827, 0.014552646, -0.0189171, + 0.00844107, -0.009157322, -0.015663842, -0.011614002, -0.015302368, + -0.0031612176, -0.019519556, 0.00004845269, -0.029346274, + 0.000018159952, 0.0012542788, 0.006355905, 0.004903318, 0.023268169, + -0.030658288, 0.026708858, -0.014566034, -0.0062354137, -0.0066604796, + 0.0026641919, -0.0023412087, -0.00576349, -0.021326922, -0.017283777, + -0.0044180066, -0.016105643, -0.018314645, -0.0010785626, + -0.017310552, -0.0012199723, 0.0053785886, 0.009538878, -0.0014007089, + -0.027150659, -0.008387519, 0.0069550136, 0.022143584, 0.0028666838, + 0.0021537782, -0.0066537857, -0.018033499, 0.032586146, -0.032961007, + -0.014043906, 0.00474601, 0.017551536, -0.0030306855, 0.013401287, + 0.0025939054, 0.0053685475, 0.024245484, -0.010636685, 0.010395704, + -0.009913739, -0.010047618, 0.0062655364, -0.007925636, 0.003845674, + -0.006740807, 0.01656083, 0.008869483, 0.004484946, -0.0000034581046, + 0.012156212, 0.04739316, -0.0032633003, -0.0013396266, 0.0029654196, + -0.020095237, -0.008628501, 0.026133178, -0.010409092, 0.014927507, + -0.0016391808, 0.0012785443, 0.0123838065, 0.0011120323, 0.0066504385, + -0.032211285, -0.015476411, -0.01191523, -0.011098568, 0.019144695, + -0.016868753, 0.009344754, -0.013615493, 0.003127748, 0.017805906, + 0.015864661, 0.021434026, -0.023535926, 0.014820404, -0.0069349315, + -0.022652324, 0.0038624087, -0.009987373, -0.015074774, -0.0043309852, + 0.022518447, 0.010221661, 0.008755686, 0.012450746, 0.0030608084, + 0.035344053, 0.0057601435, 0.005589448, -0.021206431, -0.0013011364, + -0.027793277, -0.028837534, -0.009050219, 0.020483484, -0.021259984, + -0.010047618, -0.01140649, 0.0048598074, -0.011219059, 0.005827083, + 0.0030290121, -0.007383426, -0.009572348, 0.001289422, 0.006867992, + -0.021514354, -0.023281556, -0.010536277, 0.004484946, 0.0276594, + -0.024272261, 0.007256241, 0.0058003073, -0.0050673196, -0.008240252, + 0.007537387, -0.00105095, 0.012350337, 0.013976966, 0.020684304, + 0.0119553935, 0.010850892, 0.012859077, 0.023790296, 0.006218679, + 0.0013915048, -0.012792137, 0.009418387, -0.000063540225, + -0.0029955423, 0.035317276, 0.015208653, -0.0033837913, -0.02574493, + -0.0009639287, 0.011982169, 0.024379363, 0.014994446, -0.01618597, + 0.0037419177, 0.019452617, -0.008976586, -0.010034231, -0.016868753, + -0.024740838, -0.006633704, -0.0028733779, 0.010723707, 0.013548554, + 0.020858346, 0.03365718, 0.024781, 0.029480154, -0.023107514, + -0.011178896, 0.002210677, 0.015744168, 0.006466355, -0.006981789, + 0.0040565333, 0.005120871, -0.008347355, 0.00046983158, 0.011359632, + 0.006208638, 0.0010718687, 0.009619205, 0.000055486566, -0.012256621, + 0.0013605452, 0.00078946765, 0.028971413, 0.006888074, -0.015663842, + 0.022304239, 0.010094476, -0.0051911576, 0.001478526, 0.0063759866, + 0.02835557, 0.013421369, -0.019412452, -0.007624408, 0.0054957327, + 0.005412058, 0.009666063, -0.008113067, -0.013582023, -0.0058505116, + 0.024098217, -0.0063759866, 0.018046888, -0.007403508, -0.021179656, + -0.0073700384, 0.00463556, -0.0123838065, 0.007845309, -0.010783953, + 0.032211285, 0.0061115758, 0.028114587, -0.012484215, -0.021045776, + 0.003835633, -0.008300497, 0.0069416254, -0.007838614, -0.001563874, + -0.02108594, -0.003815551, -0.015704006, -0.00920418, -0.019974746, + 0.020603977, -0.009110465, 0.0043142503, -0.01872967, -0.018809998, + 0.007316487, -0.00084762136, -0.016721485, 0.004494987, -0.020778019, + -0.008059515, -0.0132205505, -0.02051026, -0.001916143, -0.009471938, + -0.0002215279, 0.016333237, -0.006469702, -0.016654545, -0.020095237, + 0.012209764, -0.018769834, 0.018394973, -0.008333967, -0.018421749, + 0.023402046, 0.009344754, 0.0031997077, 0.0041837185, 0.0047024996, + -0.004655642, 0.0008626827, -0.026561592, 0.014418767, -0.003825592, + -0.012162906, -0.014124233, 0.02242473, 0.014244724, 0.013073283, + 0.01727039, 0.0130063435, -0.020871734, 0.023241391, -0.0117947385, + -0.003594651, -0.013802923, -0.012397194, 0.011158814, -0.0031963608, + 0.027953932, -0.014994446, -0.006322435, -0.003104319, -0.0060680653, + -0.0021671662, -0.004320944, -0.027311314, 0.0017839376, 0.012484215, + -0.0003522691, 0.0050673196, 0.00594088, 0.0023211269, 0.0035678751, + 0.025048759, 0.030818943, -0.012798831, -0.012972874, 0.010984771, + 0.015891436, -0.014325052, -0.007845309, -0.0021270025, -0.0022458201, + -0.0073365686, 0.018100439, -0.0052580973, 0.018716283, -0.008481234, + 0.0053518126, 0.014043906, -0.030685063, 0.01981409, -0.032827128, + -0.0055091204, -0.024486467, -0.0011028282, 0.018903714, 0.0052748322, + 0.0079858815, 0.012089273, -0.01057644, -0.0045753145, 0.025918972, + -0.007490529, -0.010743788, -0.009786554, -0.024473079, -0.0276594, + 0.015556739, 0.007115668, -0.01086428, -0.018555628, 0.004002982, + -0.0153559195, -0.01682859, -0.019773927, 0.017404268, 0.014619585, + 0.007898861, 0.007550775, 0.0022659018, -0.015208653, -0.00056940404, + 0.022879919, 0.0102283545, -0.0061651273, 0.001517853, -0.01618597, + 0.020095237, -0.022076646, 0.00684791, -0.0020734507, -0.013548554, + -0.018247707, -0.019318737, 0.024794389, 0.010971383, 0.0119553935, + -0.0014643015, 0.014258112, -0.0029436643, -0.014391991, 0.0065701115, + -0.023495762, -0.0025152515, -0.013254019, 0.010061006, 0.02561105, + 0.027164046, -0.009793248, 0.01079734, -0.022063257, 0.017310552, + 0.017230226, 0.0010291947, 0.012564543, -0.008795849, 0.029265948, + 0.009907045, -0.0027344783, -0.019091144, 0.018180767, -0.0015312409, + 0.018943876, 0.017605087, 0.0041770246, 0.0037218358, -0.007155832, + -0.015891436, 0.0030206447, 0.020349607, 0.005147647, -0.000007851009, + -0.007918942, 0.0077047357, -0.018301258, -0.0034507308, + -0.0038389799, 0.013495002, 0.027257763, -0.018823385, 0.014378604, + -0.013722597, -0.01057644, 0.026333997, -0.009197487, 0.0013421369, + 0.014151009, 0.033924937, -0.0030608084, 0.0030792167, -0.0068612983, + 0.007952412, -0.023107514, 0.014124233, -0.026159953, 0.010931219, + 0.004478252, 0.006590193, -0.001785611, 0.036468636, 0.009907045, + 0.0102283545, 0.021822276, 0.016775038, 0.013207162, -0.0038523679, + -0.009920433, -0.0017370798, 0.0058438177, -0.00959243, 0.010583134, + 0.0037519587, 0.00010386081, 0.016734874, -0.014740077, 0.0097664725, + 0.01886355, 0.0205772, -0.025276354, 0.0030022364, -0.0092845075, + 0.008641888, -0.010998159, 0.01656083, -0.018957265, -0.00481295, + -0.00945855, 0.0015939967, -0.0010266845, 0.00474601, 0.010081088, + -0.0022039828, 0.009137241, 0.02038977, -0.013407981, 0.004879889, + 0.010790646, 0.007698042, 0.005120871, -0.0039326954, -0.010623298, + -0.0049167057, -0.007490529, -0.0081867, -0.04096697, 0.008682053, + 0.009117159, 0.00013774892, -0.0018809998, -0.021500966, 0.03574569, + 0.0005802817, -0.005264791, -0.0022039828, -0.020456709, 0.01713651, + 0.01727039, -0.010221661, 0.0020065114, 0.018020112, -0.016600994, + 0.033469748, 0.00477948, -0.01089775, 0.012069191, -0.010174803, + 0.00018000448, 0.006908156, -0.018528853, 0.00038009082, 0.009117159, + 0.0103890095, 0.0005631285, 0.0024165157, -0.010235049, -0.0022876572, + 0.036227655, -0.013274102, 0.011031629, -0.010201579, -0.02134031, + 0.013100059, 0.008092985, -0.0022056564, 0.0154094715, 0.028409122, + -0.018207543, -0.006600234, -0.008367437, 0.0070286468, 0.017056184, + -0.00065684377, 0.010382315, 0.006995177, 0.011299387, -0.018381584, + -0.0017253654, -0.011747881, -0.010616603, -0.009726308, -0.017645251, + 0.010696931, -0.00093380595, -0.010583134, -0.016252909, 0.015904823, + 0.012256621, 0.019318737, 0.008333967, 0.014244724, 0.0053685475, + -0.024473079, -0.005669775, 0.0067776237, -0.005723327, 0.022183748, + 0.007979188, -0.016868753, 0.011359632, -0.013501696, -0.031675767, + 0.011265917, -0.003581263, 0.014619585, 0.015637066, 0.01886355, + 0.011165507, 0.012825607, 0.017819293, -0.005278179, -0.0020701038, + 0.011265917, -0.0052246274, -0.012136131, 0.008461152, 0.015007835, + -0.0010066027, -0.011868373, 0.020148788, -0.013963578, -0.012504297, + -0.00369506, -0.022411343, -0.0014433828, 0.022411343, 0.009806636, + 0.006054677, -0.011821515, 0.009431775, -0.008487928, -0.008414295, + 0.017042795, 0.023522539, -0.029319499, -0.031193804, -0.008173312, + -0.004615478, 0.027953932, 0.0077783694, 0.0030089302, -0.0035377522, + 0.009478632, -0.018448524, -0.0011036649, -0.017109735, -0.010188191, + 0.02064414, -0.020242503, -0.014619585, -0.0192518, 0.01257793, + 0.034567554, 0.01706957, -0.016614383, 0.022331014, 0.015114938, + -0.0015755884, 0.00622872, -0.0036883662, -0.0022274116, 0.015811108, + -0.0053350776, 0.0068244813, 0.01060991, 0.005656387, 0.0069282376, + 0.0031093394, 0.0036615904, -0.023308331, 0.009050219, -0.0058705937, + -0.011386408, -0.004869848, 0.030229876, -0.0054522217, 0.0006832012, + 0.0064027626, -0.014699913, -0.006707337, -0.014391991, -0.011004853, + 0.003039053, 0.0042807804, -0.004645601, 0.0231209, -0.007939024, + -0.016038703, 0.007651184, 0.00006359252, -0.006312394, -0.013013038, + -0.024473079, 0.012209764, -0.0016885487, -0.019867642, -0.004371149, + 0.0029637462, -0.0048330314, -0.014619585, 0.0028717043, 0.023857236, + 0.023535926, -0.007149138, 0.003104319, -0.006764236, 0.031167028, + -0.0032967702, -0.023817072, -0.0104961125, 0.036093775, -0.015181878, + -0.0018876938, 0.004863154, 0.00016713954, 0.028328793, 0.010007454, + 0.010409092, 0.0012191355, 0.01840836, -0.0057534496, 0.0036883662, + 0.003444037, -0.0061751683, -0.002026593, -0.012450746, 0.014766852, + 0.005418752, -0.0013320959, 0.029667584, -0.006064718, 0.018515464, + 0.010603216, -0.006606928, -0.0013990354, -0.007798451, 0.014298276, + 0.007624408, -0.017497983, -0.008849401, -0.007671266, -0.014244724, + 0.016052091, 0.008320579, -0.015329144, 0.021782111, -0.007155832, + -0.0061115758, -0.010670155, -0.025785092, 0.01249091, 0.0021253289, + 0.0021755334, 0.013561942, -0.0030959516, 0.013642269, 0.016426953, + 0.0035076295, 0.0053652003, -0.019358901, 0.005817042, 0.005301608, + -0.0011379714, -0.021313535, -0.007530693, -0.00029599806, + -0.0029252558, 0.0011287673, -0.0110115465, 0.0070420345, + 0.0025503947, 0.018756445, 0.016775038, 0.0043242914, -0.008200088, + 0.010777258, 0.018153992, 0.010014148, 0.013173693, 0.013240632, + -0.004873195, -0.011908536, 0.003976206, 0.0013337694, -0.0020968798, + 0.018943876, -0.003678325, -0.0005957615, -0.0028164792, 0.025477171, + 0.0018676119, -0.00060371053, -0.00067399704, 0.008815931, + 0.020188952, 0.0045920494, -0.009358142, 0.015262204, 0.0017688761, + 0.012055803, -0.004618825, -0.009030138, -0.0111722015, + -0.00048070925, -0.010891056, 0.010281906, 0.005853859, -0.014740077, + 0.022960246, -0.0013789536, 0.0019730416, 0.013300877, -0.0042941687, + -0.002774642, -0.008735604, -0.029078515, 0.014284888, -0.014191173, + 0.009471938, 0.0039996346, 0.003039053, 0.003812204, -0.011245835, + -0.011553757, -0.005562672, 0.013428062, -0.028623328, -0.015704006, + -0.00075223256, 0.00217386, 0.020563813, -0.014365216, -0.0036147328, + 0.0006229557, 0.015342532, 0.002111941, 0.013501696, -0.020041684, + -0.006476396, 0.0020868387, 0.027498744, -0.02338866, -0.0008421825, + -0.0033218723, -0.006215332, -0.007798451, -0.0013262387, + -0.016226133, -0.008534785, 0.018743059, 0.0028097853, -0.0038055102, + -0.005549284, -0.0039059194, 0.003316852, -0.018997429, 0.000579445, + -0.0024985166, 0.028676879, -0.025196025, 0.021206431, -0.014070681, + 0.007249547, -0.012504297, 0.007129056, 0.0069349315, 0.0017504678, + 0.0010827463, 0.015971763, -0.002650804, 0.013963578, 0.028168138, + 0.00008273303, 0.03842327, -0.011553757, -0.024138382, 0.00080703927, + -0.006489784, -0.021206431, 0.00087021344, -0.0059910845, + -0.018582404, 0.022290852, -0.0005945064, 0.01047603, -0.002434924, + 0.006288965, 0.00981333, -0.0071357503, 0.01860918, 0.0026206812, + -0.00815323, 0.0042506577, 0.023094125, -0.014391991, 0.007845309, + 0.013582023, 0.0061651273, -0.003976206, -0.0042573516, 0.016252909, + 0.013153611, -0.016908916, 0.017765742, -0.011138732, -0.012484215, + 0.007523999, 0.0141108455, 0.03657574, -0.009799942, -0.007122362, + 0.013153611, 0.009070301, -0.013287489, -0.009672757, 0.017176675, + 0.003231504, -0.01064338, 0.015128326, 0.013213856, -0.0036180797, + 0.0073030987, 0.01035554, 0.0070755044, -0.016239522, 0.020456709, + -0.009418387, -0.004337679, 0.016895529, 0.005147647, -0.003959471, + -0.009907045, 0.0029838278, -0.002331168, -0.02128676, -0.020309443, + 0.0016107316, -0.01162739, -0.0047259284, -0.017002631, -0.0123838065, + 0.004223882, -0.0018056928, 0.0477948, -0.0035678751, -0.009840106, + 0.0025872116, -0.00543214, 0.008568255, -0.027605847, -0.023495762, + -0.002438271, 0.0064228442, -0.0116006145, 0.013702515, -0.029453378, + -0.0023646376, -0.0030725228, 0.019653436, 0.023147676, 0.02160807, + -0.00622872, 0.0033603625, -0.026815962, 0.0008333967, 0.021099329, + 0.018461913, -0.015744168, 0.02918562, -0.00096811244, 0.012664952, + 0.0019010816, 0.00691485, 0.0012425644, -0.016600994, 0.018502075, + 0.018502075, -0.020791408, 0.015302368, -0.0053283838, 0.011453347, + 0.008434377, 0.0046958057, -0.014981058, -0.035129845, 0.002598926, + -0.015864661, -0.0030340326, -0.005422099, -0.01662777, -0.0036649373, + 0.016011927, -0.004120126, 0.008735604, -0.013013038, -0.006362599, + 0.017899621, 0.0014207908, 0.026133178, -0.01032207, 0.0064128037, + 0.012229846, -0.009907045, 0.006255496, 0.031541888, 0.0069750953, + -0.006747501, 0.007008565, -0.003172932, 0.030336978, 0.002722764, + 0.007858696, -0.001916143, 0.027030168, -0.010623298, -0.0034172612, + 0.0035210173, 0.016333237, -0.008601725, -0.014137621, 0.00427074, + 0.022786204, -0.020737855, -0.01166086, 0.0014960977, -0.015396084, + -0.00010867209, 0.010201579, -0.032077406, -0.0024365976, + -0.010435867, -0.022264076, -0.0032499125, -0.009244344, + -0.0065868464, -0.0037452646, -0.0117612695, -0.018073663, + 0.020470098, -0.024419528, -0.008113067, 0.012430664, -0.001971368, + -0.016333237, -0.006757542, -0.020242503, -0.0018241012, + -0.0020249197, -0.0056162234, 0.0014157703, -0.0060044727, + 0.042466413, -0.026829349, -0.014833792, 0.03644186, 0.013575329, + -0.0156236775, -0.008514703, -0.017953172, 0.0010860933, + -0.0015278939, -0.010208272, -0.033362642, -0.014405379, -0.014338439, + -0.00077817164, 0.013347735, 0.03879813, -0.00062504754, 0.003849021, + 0.007959106, -0.005003727, -0.0027545602, -0.003362036, -0.0031310949, + -0.006774277, 0.0077448995, -0.0069684014, 0.010047618, 0.0192518, + 0.0003401363, 0.03454078, -0.008200088, -0.03261292, 0.021367086, + -0.00837413, -0.021059165, 0.007550775, 0.020443322, -0.015757557, + 0.008180006, -0.014499094, 0.025088923, -0.029640809, 0.0025420273, + -0.017953172, -0.0033285664, 0.012022333, 0.0039326954, -0.019706987, + 0.0073030987, 0.0025754971, -0.016547443, 0.010241742, -0.0117278, + 0.00033783526, 0.022411343, 0.020229114, -0.0037519587, 0.019091144, + 0.022223912, 0.017752353, 0.024714062, 0.005790266, 0.0031310949, + -0.010717013, -0.012310173, 0.0064496202, -0.0009831738, 0.0006388538, + 0.010301989, -0.014967671, -0.0004443109, 0.038958784, 0.01866273, + 0.018180767, 0.00822017, -0.003678325, 0.019358901, -0.00018282849, + -0.018622568, -0.0061751683, 0.0013061569, 0.011152119, + 0.000056323308, 0.02242473, -0.017190062, 0.016708098, -0.03325554, + -0.02191599, -0.013561942, 0.0037251827, -0.010723707, 0.0035477933, + 0.005040544, 0.015770944, 0.0065466827, -0.0021203086, 0.021581292, + 0.017979948, 0.0050874017, -0.020416545, 0.005425446, 0.002115288, + -0.015074774, -0.006037942, -0.011547063, -0.0029972158, 0.019519556, + 0.0074838353, -0.0064228442, 0.024004502, 0.011018241, 0.0034306492, + 0.025704766, -0.008534785, -0.004210494, 0.016574219, -0.003094278, + -0.0069416254, 0.009110465, -0.028543001, -0.0135217775, + -0.0045251097, -0.0076444903, -0.012270009, -0.0112793045, + 0.008675358, 0.014070681, -0.0105496645, 0.006208638, 0.014740077, + -0.022893308, 0.006081453, -0.003979553, 0.009538878, -0.013923415, + 0.016547443, 0.014231336, 0.029453378, 0.00029934506, -0.0123838065, + -0.0011521961, -0.005171076, -0.005027156, 0.026749022, + -0.00061584334, -0.010529582, 0.0002063619, 0.0033536686, + -0.0044581704, -0.009793248, 0.012336949, 0.01943923, -0.0077248174, + 0.016413564, -0.0006836196, -0.0255575, 0.008387519, -0.01898404, + -0.011399796, 0.008722216, -0.003090931, -0.017672027, -0.00095890823, + 0.021527741, 0.0022541874, 0.0039393893, 0.015288981, 0.004618825, + 0.00977986, -0.016775038, -0.016239522, -0.0039025724, -0.015369308, + -0.00034850373, -0.0036649373, 0.010395704, 0.0005493222, 0.009840106, + 0.0071022804, 0.0238974, -0.026293833, 0.02064414, 0.008320579, + 0.0055024265, 0.0011438286, -0.00013628462, -0.017283777, + -0.009471938, 0.007912248, 0.011446654, -0.008641888, 0.0019914499, + 0.0068177874, -0.0128992405, 0.019747151, -0.027632624, -0.0050940956, + -0.008267027, 0.021206431, -0.0029520318, 0.0027930504, 0.028221691, + -0.01267834, -0.0017487942, -0.0027077026, -0.0016333236 + ] + } + ] + } ] diff --git a/src/scrapers/document-legacy/examples/10-simple.rtf.knowledge.json b/src/scrapers/document-legacy/examples/10-simple.rtf.knowledge.json index e50460a7d4..614a2182f8 100644 --- a/src/scrapers/document-legacy/examples/10-simple.rtf.knowledge.json +++ b/src/scrapers/document-legacy/examples/10-simple.rtf.knowledge.json @@ -1,24 +1,645 @@ [ - { - "name": "untitled", - "title": "Untitled", - "content": "---\n\nSpringfield is the county seat of Sangamon County in Illinois, United States. As of 2019, it had a population of 10,566, ranking it as the sixth most populous city in the state.", - "keywords": [ - "Springfield", - "county seat", - "Sangamon County", - "Illinois", - "United States", - "population", - "2019" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.026481831, 0.04921357, -0.002366284, -0.021460302, -0.027380828, -0.0036216667, 0.012605198, -0.006658986, -0.06981341, 0.021896956, -0.03893934, 0.025441566, -0.013189545, -0.0004924407, 0.0061902246, 0.013844527, -0.015963588, 0.04998414, 0.0066975146, -0.0013460849, 0.0354461, -0.0061163786, -0.060566597, 0.044179197, 0.012682254, 0.00824507, -0.030694267, 0.0064470805, -0.026327718, 0.0017578568, -0.003660195, -0.001222473, 0.045232303, 0.001574847, 0.009728413, -0.019392611, 0.0015339106, -0.020548463, 0.029872328, 0.034881018, 0.011256705, 0.0020708998, 0.018943114, 0.010145804, 0.028896276, -0.02853668, 0.0003991298, -0.036550578, 0.021190602, 0.023232607, 0.016002117, -0.014949007, -0.021293346, 0.041456524, 0.011834631, -0.004276649, 0.011937373, 0.019135756, -0.007423132, -0.050857447, 0.00016474891, -0.014332553, -0.024452671, 0.009426608, 0.012438241, 0.0215502, 0.017299237, -0.002600665, 0.04433331, 0.020317294, 0.029666845, 0.06642291, 0.0055320314, 0.029332932, 0.054299314, -0.008938582, -0.03149052, 0.01201443, -0.025955278, 0.013651885, 0.02559568, -0.028100023, -0.020188866, 0.0044949763, -0.0021094282, 0.0000010096138, 0.011256705, -0.073666245, 0.0030164502, 0.0036056133, 0.010672358, -0.013728942, -0.011584196, -0.020908061, -0.044487424, -0.027791796, 0.045566216, -0.010299917, -0.005217383, 0.024298558, 0.011956637, -0.002674511, -0.0404291, 0.01731208, 0.0005526413, 0.016438771, -0.0006758518, -0.006209489, -0.016143387, 0.013112488, 0.025955278, -0.003226751, -0.0071662767, -0.0046651433, -0.027201029, 0.0028061494, -0.009728413, -0.017800106, 0.044127826, 0.015115963, -0.0037211983, 0.014743523, -0.03226109, 0.006071429, -0.021601573, 0.010254968, 0.020432878, -0.017453352, 0.01961094, -0.032158345, 0.02591675, 0.011353026, -0.0068516284, 0.011333762, 0.009336708, -0.04225278, -0.0065401904, 0.031798746, 0.028100023, 0.048263203, -0.06390572, -0.007159855, 0.019790739, -0.00056187203, -0.0013733758, -0.025467252, -0.016952483, 0.01606633, 0.012733626, -0.026327718, -0.024195816, -0.013780314, -0.027509255, 0.00446608, 0.018095491, 0.020047594, 0.0060842717, -0.024414143, 0.05676513, -0.046901867, -0.018801844, -0.013587671, 0.028228452, -0.036807436, -0.011166805, -0.0064246054, -0.020201707, -0.00026046782, 0.0016093621, 0.0034482891, 0.042278465, -0.0054389215, -0.04572033, -0.07489915, 0.0060329004, -0.025261767, -0.014204125, -0.014846264, -0.005731095, 0.024375614, 0.008501926, -0.0031464836, 0.028022967, -0.009779784, -0.020908061, -0.022128126, 0.006890157, 0.023964645, 0.038554054, -0.021575887, 0.00079223956, 0.051448215, 0.013883055, 0.0060040043, 0.02905039, 0.0014680914, -0.011725467, -0.033365566, 0.015937902, 0.026032334, -0.054196574, 0.015385662, -0.02205107, -0.014782051, 0.0024593943, 0.015937902, -0.009535771, -0.0095743, 0.020869533, 0.030642897, -0.01035771, -0.017607465, -0.017787263, 0.042612378, -0.014936164, 0.02817708, -0.017183652, -0.007352497, -0.017389137, 0.0459515, 0.026404776, 0.037115663, 0.007513032, -0.026404776, -0.018557832, -0.009304601, -0.06303241, 0.0059751077, 0.004613772, -0.026918488, 0.032158345, 0.01790285, 0.0072625973, 0.056919243, -0.0379376, 0.06133716, 0.024555413, 0.0073268116, -0.01717081, 0.034315933, 0.027406514, 0.02899902, -0.00980547, -0.02463247, -0.007904737, -0.028074337, 0.023733474, 0.0059141046, 0.0075708246, 0.028716477, 0.00035177203, 0.0030084234, 0.051936243, 0.00994674, -0.011930952, 0.009709149, -0.005750359, 0.006633301, -0.008906474, 0.0073139686, 0.0019874217, 0.010370553, -0.017157966, 0.017479036, 0.005731095, -0.035112187, 0.0005357851, 0.0032652793, 0.0019537094, -0.029153133, -0.001703275, -0.004225278, 0.007288283, 0.017466193, 0.028613735, 0.010903528, 0.042612378, 0.02301428, -0.03125935, -0.04623404, -0.021113547, 0.009054166, 0.0051242732, -0.025518622, 0.0045559797, 0.01243182, -0.007943265, 0.027098287, 0.0022137759, -0.046799127, -0.011423661, 0.0044436054, 0.030899752, 0.025711264, 0.030129185, -0.0028848117, 0.0030662161, -0.0033487573, -0.046901867, 0.019508198, 0.018146861, 0.02661026, -0.007866208, -0.015231549, 0.0122648645, -0.015128806, 0.005050427, -0.00395879, -0.026160762, -0.05404246, -0.009221123, -0.053374633, 0.0028334404, 0.025698422, -0.011995166, -0.00033009984, 0.017222181, -0.014139911, 0.038502682, -0.017440509, -0.020009067, 0.0029875538, 0.018429402, 0.019906323, -0.012348342, 0.023399564, -0.000100685465, -0.012727205, 0.01592506, 0.01201443, -0.03457279, 0.004626615, -0.017812949, -0.05106293, 0.011051221, -0.012759311, 0.018763315, -0.008296442, 0.0010033429, 0.019970538, -0.004976581, 0.0057664127, 0.009908212, 0.037192717, 0.026507517, -0.02149883, -0.009818313, -0.0053971824, -0.013163859, 0.01740198, 0.000666621, 0.003351968, 0.05501851, -0.0039620004, 0.017363451, -0.006283335, -0.02499207, -0.02283448, 0.01606633, -0.039581478, -0.024439828, 0.0067809927, -0.009458714, -0.0022812006, -0.019482512, 0.019495355, 0.00073284167, 0.009400922, 0.030617211, 0.0062191207, 0.013240916, 0.014011484, -0.0122648645, 0.011956637, 0.013497772, 0.006960792, -0.017363451, -0.008604669, -0.012386871, -0.00061725656, 0.032440886, 0.025531465, -0.019944852, 0.00086769095, -0.0011991955, -0.0030260824, -0.0017899638, -0.017838635, -0.017337766, 0.011089749, -0.008084536, -0.013382187, -0.00469725, 0.0030517678, -0.0038464156, 0.017517565, -0.044281937, -0.0033583895, 0.028896276, 0.0041642743, 0.023232607, -0.03056584, 0.002181669, 0.031336408, -0.020753948, -0.0052976506, -0.0019970539, -0.057124726, 0.012662991, -0.014011484, -0.024786584, -0.0030180556, 0.003393707, -0.012322657, -0.02899902, -0.05008688, -0.016541513, 0.03714135, 0.015488404, 0.034059078, -0.01449951, -0.037911914, 0.0063314955, 0.0039555794, -0.015411347, 0.0030276875, 0.012740048, 0.025698422, -0.0381174, -0.021357559, -0.031464834, -0.009477979, 0.0027917014, 0.010306339, -0.015000379, 0.050985876, -0.03688449, 0.0045591905, 0.03978696, 0.013356501, -0.02946136, 0.005962265, 0.032338142, -0.018095491, -0.013574828, 0.022487724, 0.02684143, 0.01873763, 0.024337087, 0.056919243, -0.014165597, 0.03434162, 0.016567199, 0.011622725, 0.007416711, 0.0011245467, -0.042920604, 0.012496035, -0.007988215, 0.031798746, 0.023399564, -0.023181235, 0.014345396, -0.0352663, -0.013972955, 0.02513334, 0.0461313, 0.00971557, 0.000971236, -0.017877163, -0.0072561763, -0.025582837, -0.039581478, 0.02541588, -0.045925815, -0.0031063498, 0.004482134, 0.012996903, 0.022962907, -0.015848001, -0.040814385, 0.022718895, -0.044924077, 0.015950745, -0.0050632698, 0.026147919, -0.025929593, -0.027663369, 0.0106274085, -0.015167334, 0.052295838, 0.0053522326, 0.008765204, -0.020227393, 0.010807208, 0.0070378487, -0.03724409, -0.055223994, 0.02370779, 0.033005968, -0.0033969178, 0.03084838, 0.037629373, -0.0029153132, 0.003204276, -0.0039427364, -0.02030445, -0.03521493, 0.013472087, 0.016323186, 0.061491277, 0.0035125029, -0.028279822, -0.020009067, 0.016913954, -0.015475562, -0.012386871, -0.010210018, 0.029409988, -0.009895369, -0.009580721, 0.011622725, -0.0025525044, 0.006543401, -0.019790739, 0.012579513, 0.0033038077, -0.0009977242, 0.029872328, -0.021241974, -0.033262827, 0.0057696234, 0.0056090886, 0.028022967, 0.06405983, -0.02513334, -0.02126766, -0.019238498, -0.0017867531, 0.042715117, -0.03351968, -0.057535697, 0.012412556, 0.03190149, 0.01850646, 0.006154907, -0.0047807284, -0.012207071, 0.0028703634, -0.03167032, 0.053888347, -0.014949007, 0.005888419, 0.0030581893, -0.014653623, -0.007121327, -0.014692151, -0.02007328, -0.037038606, -0.013330815, 0.02140893, 0.011243862, 0.021845585, -0.004456448, 0.0062865457, 0.015475562, 0.016002117, -0.011282391, 0.005410025, 0.0017562514, 0.011218177, 0.012451084, 0.026327718, 0.0064663445, 0.019906323, 0.011719045, -0.018660573, -0.015398505, 0.012367606, 0.011423661, -0.008360656, 0.019649468, 0.031387776, 0.025390195, 0.0005686948, 0.0066782506, -0.00814875, -0.030077813, -0.0034033393, 0.044769965, 0.036627635, 0.023450933, -0.0029265506, 0.015115963, 0.0010563195, 0.009060588, -0.027534941, 0.015681047, -0.028665107, -0.029435674, 0.0068387855, -0.0041450104, 0.015732417, -0.020458564, -0.024388457, 0.008463399, 0.023643576, 0.0153086055, 0.018904585, 0.010511823, 0.0024465513, 0.020831006, 0.0017129071, -0.008026743, -0.032286774, -0.006315442, 0.03403339, 0.029076075, 0.010800786, -0.022359297, -0.014294025, -0.024414143, -0.0074552395, -0.009766942, 0.025377352, -0.021421773, 0.012451084, -0.016323186, -0.0031175872, 0.0018814687, -0.001079597, 0.009914634, -0.015526933, 0.006459923, -0.01473068, -0.019687997, 0.0061805923, -0.008418448, -0.02080532, -0.012784997, 0.0026488253, 0.058306266, 0.016926797, 0.03624235, -0.004639458, 0.00016755828, -0.021100704, 0.024902169, 0.007191962, 0.004899524, 0.055994563, -0.028048653, 0.009477979, -0.0011646805, -0.00091825944, 0.01800559, 0.02776611, -0.030257612, 0.028048653, -0.003241199, -0.016836897, 0.014897636, 0.0004274241, 0.006877314, 0.012412556, 0.020908061, -0.03737252, 0.038091715, 0.017928535, -0.0034771853, -0.014332553, 0.017915692, -0.004116114, 0.011449347, -0.010132961, 0.0067938357, 0.0032797274, -0.0069351066, -0.010486138, 0.0020548464, 0.010486138, -0.0164773, -0.02946136, -0.005731095, 0.013818841, -0.021190602, 0.044025082, 0.03673038, -0.005506346, -0.01758178, 0.019366927, 0.033493996, -0.0018959168, -0.007827681, 0.031053865, 0.004337652, -0.0011686939, 0.031233665, 0.01150714, -0.014846264, 0.0020773213, 0.023373878, 0.01740198, -0.039966762, 0.010736573, 0.009766942, -0.0018333083, -0.024619628, -0.05224447, -0.008643198, 0.006703936, -0.004626615, 0.030694267, -0.011076906, 0.053888347, -0.035112187, 0.040583216, -0.035137873, 0.008495505, 0.039966762, 0.02375916, 0.008707412, -0.053785603, -0.012117172, -0.03236383, 0.008309285, -0.029230189, -0.025788322, 0.01473068, 0.0060650073, -0.038399942, 0.009599986, 0.026250662, -0.009266073, 0.026738688, -0.001855783, -0.013818841, -0.02499207, 0.022962907, -0.007172698, 0.0067424644, -0.032081287, 0.04374254, 0.006652565, 0.0006770558, 0.017055225, -0.0025187922, -0.003660195, -0.059179574, 0.0028510992, -0.038656797, -0.019816425, 0.038785223, 0.016271815, 0.0037500947, 0.00022956486, 0.012142858, 0.0070378487, 0.01588653, 0.0358057, 0.023656419, 0.010858579, 0.00202595, 0.0025492937, 0.0077827303, 0.0022137759, 0.004816046, -0.010113697, -0.0027130393, 0.0065947725, 0.03323714, -0.0023261502, -0.005262333, -0.0018188601, 0.022397825, -0.00060922984, 0.010967743, -0.011988744, 0.019662311, -0.03398202, 0.034649845, -0.006071429, 0.002319729, -0.0023405985, -0.007577246, -0.019392611, 0.009978848, -0.002969895, 0.0044725016, -0.013818841, -0.008444134, -0.004597719, -0.017684521, 0.010537509, -0.0093945, -0.016541513, 0.0115199825, -0.012194228, 0.044410367, 0.006601194, 0.012309814, 0.019598097, -0.010017375, 0.010909949, -0.030334668, 0.0030533732, 0.007179119, 0.013985798, -0.016708469, 0.018211076, 0.040994186, -0.022269398, 0.020009067, -0.0081102215, -0.036422152, -0.027534941, 0.016567199, -0.0024096284, 0.00019906324, -0.02003475, 0.034778275, -0.021421773, 0.0432802, -0.010794365, 0.020908061, 0.02306565, 0.023720633, -0.026481831, 0.00043344416, 0.018955957, -0.010730151, -0.0024609996, -0.013703257, -0.009734835, -0.00971557, 0.011564932, 0.0070185843, 0.0067681503, 0.02928156, -0.009323865, -0.0076157744, -0.015462719, 0.027971596, -0.0072947047, 0.010210018, 0.04199592, 0.0072112265, -0.054813027, 0.015038907, -0.04148221, -0.012984061, 0.017890006, -0.01510312, 0.019726524, -0.025467252, 0.02666163, 0.024683842, 0.0083542345, 0.0038110977, 0.008450556, 0.007943265, -0.012573091, 0.03102818, 0.0044500264, 0.004931631, 0.004074375, 0.0023839427, 0.012001587, -0.018724788, -0.0052655437, 0.015090277, 0.0019248131, -0.013960112, -0.024645314, -0.007493768, 0.014692151, -0.009291758, -0.01937977, -0.012252022, 0.013446401, -0.025390195, -0.004116114, -0.010556773, -0.033262827, 0.024285715, 0.03084838, -0.019970538, -0.012194228, 0.012149279, -0.021794215, 0.0012915031, 0.004543137, -0.0024064176, -0.006090693, 0.012457506, 0.025801165, -0.018917428, -0.020086123, -0.015629675, 0.027689055, 0.020381507, -0.008668883, 0.0035574527, -0.017235024, -0.0040165824, 0.00022976553, 0.023656419, 0.0075066104, -0.0037372517, -0.0025974542, -0.00833497, 0.012374028, 0.0052591222, -0.03369948, -0.00865604, 0.001333242, -0.02840825, -0.0136262, -0.00015060177, 0.031002494, 0.03231246, 0.020060437, -0.02817708, 0.0046651433, -0.00045351102, -0.0055866134, 0.014396767, -0.023502305, -0.01597643, -0.0005907683, 0.015603989, 0.0076992526, 0.0073717614, 0.006068218, 0.01864773, -0.021370402, -0.038887966, 0.0018943115, -0.01072373, 0.028973334, -0.008540455, 0.016567199, -0.023001436, -0.01836519, -0.0067103575, 0.044949763, -0.01868626, 0.025081968, 0.038476996, -0.0025043439, 0.010909949, 0.013202388, -0.010800786, 0.011410818, 0.047030296, 0.008277178, -0.025030596, 0.03218403, -0.01946967, 0.019135756, 0.004957317, 0.0035478207, -0.030899752, -0.008636776, 0.013651885, -0.008822996, -0.018390875, -0.06477903, 0.000050116978, -0.0009913028, 0.024157288, -0.0140500115, -0.015655361, -0.015205863, 0.00051050086, 0.03601118, -0.007724938, 0.0053522326, 0.024773741, 0.02559568, 0.015642518, 0.021627259, 0.007564403, 0.042021606, 0.02858805, -0.017568937, -0.022115285, -0.016053487, -0.004209224, 0.0015001984, 0.022256555, -0.025608523, -0.0017000643, -0.0069543705, 0.052655436, -0.010614566, 0.0023405985, 0.0005803336, -0.0041321674, 0.014704994, 0.035291985, -0.003554242, -0.018609202, -0.00071518286, -0.01634887, 0.02905039, -0.044179197, 0.0028832064, -0.014627937, -0.0056861453, 0.021370402, 0.0010282259, 0.032517944, -0.025801165, -0.008360656, -0.025069125, -0.0075001894, -0.016657097, -0.018519303, -0.00478715, 0.00285431, 0.018095491, 0.0045367153, -0.012733626, -0.0042284885, 0.010832893, 0.014627937, 0.010351288, 0.02324545, 0.02393896, 0.021807056, 0.00014438105, -0.0044949763, 0.014062854, -0.034290247, -0.007930422, -0.008135907, 0.05589182, 0.0023887588, 0.017697364, 0.012746469, -0.035985496, 0.021896956, 0.0022009332, 0.010473295, -0.014011484, -0.008084536, 0.003798255, 0.018172547, 0.01772305, 0.00055344397, 0.0025075546, -0.00028073534, 0.0032973862, -0.033622425, -0.008501926, 0.02858805, 0.02545441, 0.0018509671, -0.017568937, 0.005644406, -0.012528141, 0.006684672, -0.0027290927, 0.02066405, -0.00060481514, -0.023990331, 0.017607465, 0.0037372517, 0.009580721, 0.0019681575, -0.009709149, 0.035189245, -0.020766791, 0.019816425, 0.016104858, -0.01505175, -0.01431971, -0.0036762485, -0.0329546, -0.0017369873, 0.018211076, -0.018532146, 0.031875804, 0.015770946, -0.03816877, 0.012560248, 0.006251228, -0.013003324, 0.015501247, -0.016284658, -0.014576566, 0.0028575207, -0.009182595, 0.05332326, -0.024837956, 0.0155783035, 0.049316313, -0.013549143, -0.014704994, -0.016913954, -0.0056989877, 0.0055577173, -0.053374633, 0.0032973862, 0.004032636, -0.03940168, -0.0002867554, -0.034906704, -0.012964796, -0.008938582, 0.032517944, -0.021473145, 0.014846264, 0.013330815, 0.021241974, -0.017979905, -0.0019392612, -0.0050632698, -0.020959433, -0.011012692, -0.014692151, -0.029384302, 0.0020291607, 0.023104178, -0.023669261, -0.012849211, -0.0074295537, 0.018339504, -0.0155783035, -0.0077313595, 0.019777896, -0.016246129, -0.03770643, 0.0075322962, 0.0030854803, 0.01615623, -0.024593942, 0.028742163, -0.021768529, 0.015038907, 0.0031962493, -0.013138174, -0.020291608, 0.02158873, -0.01026781, 0.034007706, -0.0040390575, 0.0032829382, -0.013998641, -0.028433936, -0.005721463, 0.00814875, -0.0011510351, -0.016888268, -0.008784468, -0.0067360434, -0.01054393, -0.03411045, 0.022295082, -0.021935485, -0.001362941, -0.008168014, 0.0105182445, -0.0005365878, -0.0017626729, 0.021396087, 0.00015551815, 0.0028302297, 0.03246657, 0.0010458847, 0.0007224069, -0.018172547, -0.029332932, 0.008437713, -0.026687317, 0.00598474, -0.0076157744, -0.00059116964, 0.011744731, -0.008945003, -0.01164841, -0.020895218, 0.013151017, -0.013048274, -0.026353404, -0.008296442, -0.019199971, -0.031978548, -0.0028607314, 0.010132961, -0.01675984, 0.037834857, 0.023605047, 0.0048642065, -0.01449951, -0.000033310986, -0.0012296971, 0.012829946, 0.026481831, -0.017915692, 0.031233665, -0.011372291, -0.009728413, 0.014615094, -0.017967062, 0.0014416032, -0.011038378, -0.002725882, -0.0219098, 0.013163859, 0.012271285, 0.014473824, -0.02518471, 0.012842789, -0.008893631, -0.038143083, 0.014127068, 0.03246657, -0.0035799276, 0.0042702276, 0.0012642121, -0.0350865, 0.024131602, 0.0062769135, 0.016554356, -0.019816425, 0.010710887, -0.007269019, 0.005853101, -0.010100854, 0.013844527, -0.012817104, -0.0036955127, -0.0057888874, -0.031362094, -0.025403038, 0.029589787, 0.0043665483, 0.02292438, 0.020792477, -0.006228753, 0.028254136, -0.0032235403, -0.0017385926, 0.020086123, -0.035780013, -0.032029916, 0.015398505, -0.037860543, 0.0049348418, 0.0086110905, 0.011385133, -0.019122913, -0.02541588, -0.011622725, -0.029101761, 0.011808945, -0.020753948, 0.000802273, 0.019559568, -0.004616983, -0.005695777, 0.029076075, 0.021485986, 0.008168014, -0.013189545, -0.0007500992, 0.011320919, -0.0008291626, -0.0012401319, 0.025325982, -0.015270077, -0.015180177, 0.018198233, 0.019546725, -0.019803582, -0.007737781, 0.03310871, 0.0123997135, -0.0016583252, 0.008078115, -0.0016647466, -0.0054710284, -0.002546083, -0.00216401, 0.025980964, -0.008283599, 0.024439828, 0.00644387, 0.03195286, 0.011494297, 0.012862054, 0.003083875, -0.02545441, -0.00893216, 0.023797689, -0.04723578, -0.005422868, -0.001841335, 0.025801165, -0.02587822, -0.0097476775, -0.019996224, -0.008033165, -0.010601723, -0.019456826, -0.0005947817, -0.009182595, 0.003157721, 0.020895218, 0.0045463475, -0.022256555, -0.006036111, 0.004379391, -0.0069543705, 0.018082649, -0.004783939, -0.020497093, 0.0016181915, -0.050035506, -0.0018172547, -0.013523458, 0.0016872215, 0.01449951, -0.009831156, 0.02035582, 0.0033327038, -0.00019735756, 0.02205107, -0.0037725694, 0.0074102897, -0.020728262, -0.0037308303, -0.012303392, -0.02518471, 0.023258291, -0.02481227, 0.04538642, 0.0064213946, 0.014255497, 0.02853668, -0.00010665535, 0.016104858, 0.01173831, 0.004915578, -0.00045190565, -0.019122913, 0.01288774, -0.027278084, -0.008501926, 0.027534941, -0.023874747, 0.0032540418, -0.031362094, 0.0115199825, -0.005833837, 0.013112488, 0.015783789, 0.012720783, 0.0072625973, 0.02789454, 0.023052808, -0.0028880222, -0.0057664127, -0.015629675, -0.00547745, -0.017658835, -0.010807208, -0.011044799, 0.019443983, 0.015347133, 0.015038907, 0.021139232, 0.0032700952, -0.0034258142, -0.0051949085, -0.021884114, -0.0034129713, -0.0071470123, -0.0029650792, -0.0012634094, 0.012027272, 0.004658722, 0.013587671, 0.032286774, 0.025017753, 0.0057439376, 0.01279784, 0.0076029315, 0.0023213343, 0.0012834762, 0.011083327, 0.0011735099, -0.010768679, 0.0066268793, 0.0047261463, -0.0077056736, 0.004061532, 0.0025974542, -0.0019023382, -0.0061805923, 0.001878258, -0.011275969, 0.04949611, -0.0046940395, 0.0021014016, -0.007423132, -0.013960112, 0.013728942, 0.004581665, -0.015526933, 0.028279822, -0.035112187, -0.008142329, -0.023810532, -0.010280653, 0.031798746, 0.007236912, -0.016271815, 0.009028481, -0.01072373, -0.019803582, 0.009978848, -0.00018702312, -0.036807436, -0.016772684, -0.02066405, -0.007757045, -0.0012987271, -0.008816575, 0.011828209, -0.015899373, -0.004485344, 0.009535771, -0.00024240765, -0.003586349, -0.0023373878, -0.013080381, -0.0056604594, -0.014294025, -0.010640251, -0.0023839427, 0.026353404, -0.011712624, 0.0014335763, -0.0077827303, -0.033648107, -0.011719045, -0.0059911613, -0.027046915, 0.0020548464, -0.012618041, -0.009567878, 0.023861904, 0.012425399, 0.01666994, 0.02434993, 0.006344338, -0.017157966, 0.005288019, -0.010967743, 0.011796102, 0.015539776, 0.0049284203, -0.02016318, -0.00012391285, 0.015372819, -0.008822996, 0.0064984513, -0.0065530334, 0.0005153169, 0.0048513636, 0.022808794, 0.013960112, 0.025467252, 0.032158345, 0.014024327, -0.019212814, -0.014460981, 0.0033969178, 0.016258972, 0.0043858127, 0.0030662161, -0.003522135, 0.02408023, -0.006658986, 0.0034129713, 0.0012481586, 0.026918488, -0.02794591, 0.022436354, 0.02237214, -0.012772154, -0.019944852, -0.0022763845, 0.016734155, 0.015822317, 0.017055225, 0.028870592, -0.0021190604, 0.01473068, -0.013780314, -0.014512353, 0.0013284261, 0.013215231, 0.012528141, -0.034726903, -0.0023807322, -0.011866737, 0.011886002, 0.028485307, -0.007757045, 0.020586992, -0.03269774, 0.02173, 0.020753948, -0.001878258, 0.018416561, 0.0023887588, 0.021832742, 0.045823075, 0.029692529, -0.01763315, 0.014101383, -0.021370402, 0.0040904284, 0.007204805, -0.0015828739, -0.028921962, -0.0025942435, 0.017979905, -0.02748357, -0.010858579, -0.0037147768, 0.011430083, -0.011385133, 0.0054581854, -0.0244912, -0.008668883, -0.008758782, 0.0032620686, 0.016284658, -0.0020692945, -0.011051221, 0.018275289, 0.0015347133, -0.007879051, -0.021010803, -0.0072754403, -0.042201407, -0.007885473, 0.004254174, -0.0069094207, -0.0030967176, 0.019418297, -0.007795573, 0.034906704, 0.000084631974, 0.009433029, -0.00446608, 0.02481227, 0.010210018, -0.0062865457, -0.0019039436, -0.011173227, -0.0022812006, -0.011622725, 0.00704427, -0.014525195, 0.0077827303, 0.007821259, 0.0076221954, 0.003907419, -0.0242215, -0.007288283, -0.0145637235, -0.0011422056, 0.012040115, 0.013099645, -0.024735212, -0.0011767206, -0.02269321, -0.004135378, 0.014371081, 0.013870213, -0.013016167, -0.001703275, 0.02518471, -0.0021624046, -0.0053811288, 0.0071662767, 0.0016230076, 0.0006746478, -0.011038378, -0.013433558, 0.02375916, -0.014717837, -0.009638513, -0.025428724, -0.00584668, 0.01588653, -0.033802222, 0.0022908326, -0.009587143, -0.0016422718, 0.009683464, -0.004539926, -0.002513976, -0.0036698272, -0.00855972, -0.019366927, -0.0048642065, -0.005994372, -0.02223087, 0.016143387, -0.0007814035, -0.013433558, -0.008521191, 0.006652565, -0.012059379, -0.010813629, -0.012354763, 0.039324623, -0.022590468, 0.011597039, -0.007140591, -0.014717837, 0.0066654077, 0.01003664, -0.0013252153, 0.013497772, 0.021049332, 0.0014761181, -0.0072433334, -0.005088955, 0.019790739, -0.011918109, 0.007814838, 0.0047807284, 0.041816123, -0.0072754403, -0.013041853, 0.03560021, 0.025197553, -0.009336708, 0.0014464192, -0.0005991964, -0.00024702304, -0.0116548315, 0.0018525724, -0.004437184, 0.0035413993, 0.010145804, -0.007994637, -0.027278084, -0.0058402587, -0.010563195, 0.00013474896, -0.00088454713, 0.016284658, 0.0065209265, 0.008964268, -0.0103833955, 0.009240387, 0.020291608, 0.012008009, -0.016400242, -0.0127914185, -0.024439828, 0.009176173, -0.0039555794, 0.0022539096, -0.011468611, -0.0060168467, 0.0060842717, -0.020728262, 0.0143068675, 0.011301654, -0.0064470805, 0.009214701, -0.009099117, 0.039941076, 0.004722936, 0.0064759767, 0.0021431404, -0.01449951, -0.0075451387, -0.0040390575, -0.015488404, 0.0099916905, -0.0032941755, -0.00019805989, -0.009696306, -0.00038347766, 0.011571353, 0.0046779863, -0.01649014, 0.005419657, -0.025300296, 0.004010161, -0.013844527, 0.007692831, -0.0009856841, -0.0058434694, -0.019559568, -0.018121175, 0.010486138, 0.0014600647, 0.0013220047, 0.00095598516, -0.004957317, 0.006556244, -0.0012706335, 0.016708469, 0.015398505, 0.023489462, 0.013279445, -0.0001430767, 0.016130544, 0.0035349778, -0.022436354, -0.011346605, 0.026995543, 0.005496714, 0.004411498, -0.009811891, 0.008758782, -0.008970688, -0.026533203, -0.012085065, -0.010858579, -0.0001264212, -0.015732417, -0.0062640705, -0.013253759, 0.0017787263, -0.00575357, -0.003917051, 0.006081061, -0.007005742, 0.0060200575, 0.015950745, 0.00382073, -0.028151395, 0.027149657, -0.008906474, 0.007679988, 0.0033583895, 0.012226336, 0.004170696, -0.008842261, 0.0023951803, 0.0048353104, -0.0008765204, -0.032903228, -0.029358618, 0.014576566, 0.0054581854, 0.0143068675, 0.0026311665, 0.0047582537, 0.0013838105, 0.004437184, 0.012695097, 0.01464078, 0.029332932, -0.016258972, 0.019187128, 0.008534034, 0.01464078, 0.030488782, -0.007224069, 0.016926797, 0.0056604594, -0.004828889, 0.0049637384, 0.008765204, -0.023515148, 0.01440961, -0.03369948, -0.014062854, -0.018557832, -0.008142329, -0.030411726, 0.012926267, -0.012457506, 0.0070506916, -0.005262333, 0.025711264, -0.0008395974, 0.00066341035, -0.014345396, -0.013587671, 0.012515298, -0.017183652, -0.010409081, 0.03863111, 0.019456826, 0.03701292, -0.00023919696, -0.012727205, 0.015719574, -0.012733626, -0.025287453, -0.0035895596, -0.000015940612, 0.006440659, -0.0045367153, 0.024452671, 0.00043665484, 0.009831156, 0.009555035, -0.017877163, 0.0111411195, 0.007814838, 0.023181235, 0.006684672, -0.01141724, 0.010556773, -0.0057439376, -0.00029177213, -0.046259727, -0.00013384596, 0.014242654, -0.004915578, 0.014139911, -0.0013693625, 0.010254968, -0.027586313, 0.010935635, -0.000024381234, 0.0048674173, 0.0008516375, -0.009818313, -0.020908061, 0.013703257, 0.0039106295, 0.008578983, -0.02454257, 0.010338445, -0.0054838713, -0.0071277483, 0.005593035, 0.015411347, -0.00013163859, -0.0010306338, -0.003364811, -0.016310344, 0.030617211, 0.0008267546, 0.013960112, 0.0068516284, 0.0017947798, -0.006748886, 0.0009961189, -0.008778047, -0.0033198611, 0.006357181, 0.0009616039, 0.004254174, -0.011160384, -0.027046915, -0.0011285602, -0.012136436, 0.009587143, 0.011308076, -0.02375916, 0.017466193, -0.031875804, 0.0040294253, -0.0069672135, -0.010742993, -0.015758103, -0.0012385265, -0.0070506916, -0.0070250058, -0.003586349, -0.023219764, -0.00589484, 0.008309285, -0.001671168, -0.0031705636, 0.022282239, -0.028896276, 0.017812949, -0.026507517, 0.008450556, 0.017761579, 0.009831156, -0.017299237, -0.008482663, -0.024439828, -0.009696306, 0.006270492, -0.018583516, -0.0127785755, -0.00028595273, -0.0065369797, -0.0029843433, 0.0051371157, 0.0061452747, -0.0055962456, -0.028690793, -0.013998641, 0.008052429, 0.0162076, -0.0019039436, 0.0016270209, -0.008964268, -0.026815744, 0.009644935, -0.030000757, -0.0063475487, -0.0029201293, 0.010287074, -0.00014347804, 0.012444663, 0.0059718974, 0.007879051, 0.013484929, -0.008277178, -0.0037789908, -0.021845585, -0.015706731, 0.013472087, -0.010158647, 0.00031605302, -0.01542419, 0.025120497, 0.010595301, 0.008970688, -0.0067809927, 0.006257649, 0.039735593, -0.0028478885, -0.0040390575, 0.031130921, -0.032492258, -0.0009118381, 0.025248924, -0.011539247, 0.022885852, 0.007667145, -0.01040266, 0.0024786585, 0.004986213, 0.009002795, -0.02053562, -0.021896956, -0.0066268793, 0.00014387938, 0.009233966, -0.015706731, 0.00072280824, -0.022577625, -0.007159855, 0.004498187, 0.01772305, 0.03195286, -0.011937373, 0.021858428, -0.012219914, -0.021922642, 0.004713304, -0.007140591, 0.0076864096, 0.0014175229, 0.02357936, 0.013125331, 0.023283977, -0.0003373239, 0.011198913, 0.030180555, 0.008514769, 0.00069351064, -0.0034001286, 0.011153962, -0.019649468, -0.016554356, -0.020599835, 0.021575887, -0.021485986, -0.00589484, 0.014062854, 0.00007720724, -0.019161442, 0.002610297, -0.005326547, -0.0006112365, 0.00076133665, -0.016708469, 0.0022169866, -0.008367077, -0.024786584, -0.011340183, 0.010030218, 0.005763202, -0.013960112, -0.0067553073, 0.0026761163, -0.013960112, -0.0027066178, 0.0055127675, -0.0052527008, 0.004860996, 0.015154492, 0.0148077365, 0.00003150497, 0.002297254, 0.0067553073, 0.011385133, 0.0038945759, 0.020407192, -0.02509481, 0.015732417, 0.009433029, -0.006055375, 0.040994186, 0.0023100968, 0.00244013, -0.020098966, -0.00855972, 0.01615623, 0.026404776, -0.0014857502, -0.0016101648, 0.0021158496, 0.010871422, -0.012720783, -0.006819521, -0.009933898, -0.023052808, -0.018108333, 0.0015917033, 0.0031785904, 0.013844527, 0.019854954, 0.028254136, 0.023656419, 0.015963588, -0.01882753, -0.012091487, 0.007891894, 0.011873159, 0.0073589184, -0.019906323, -0.02016318, 0.004337652, 0.00034193927, 0.006052165, 0.015668204, 0.002435314, 0.008399185, 0.010306339, 0.0010266205, -0.015437033, 0.015205863, -0.0013525062, 0.038682483, 0.00063772476, -0.013613357, 0.026263505, 0.015437033, -0.016130544, 0.007236912, 0.010980585, 0.012945532, 0.00870099, -0.008187278, 0.0048963134, 0.014846264, 0.00009371223, 0.013908741, 0.012425399, -0.005069691, 0.006800257, 0.018673416, -0.0005787282, 0.026327718, -0.020612678, -0.024889326, -0.0035991918, 0.00019655489, -0.020009067, 0.017247867, -0.022012541, 0.023682104, 0.0069029992, 0.023027122, -0.012130015, 0.00515638, -0.0032556471, -0.0038656797, 0.013266602, -0.0029056813, -0.00764146, -0.012123593, 0.011532825, -0.011218177, -0.008232228, -0.019880638, 0.023463776, -0.00070153736, 0.009927477, -0.013780314, -0.0299237, 0.023258291, -0.0045784544, -0.0044628694, 0.005943001, -0.011590618, -0.004915578, -0.010010955, -0.009677042, 0.0016302316, -0.016952483, 0.008212964, 0.03190149, -0.001942472, -0.007352497, -0.021832742, 0.0034611318, -0.01542419, 0.020060437, -0.00354461, -0.017003853, 0.015000379, 0.0107622575, -0.0011036772, 0.0069543705, 0.00025886248, -0.0019986592, -0.0008460188, -0.02881922, 0.009047746, -0.005981529, -0.013960112, 0.0029538416, 0.0071020625, 0.022539096, -0.0022346454, 0.011083327, 0.019302713, -0.02301428, 0.022539096, -0.0038271514, -0.0019167863, -0.018814687, -0.023283977, 0.01675984, 0.0004017385, 0.029204503, -0.003692302, 0.0013059512, 0.005647617, -0.011622725, -0.01786432, 0.008585405, -0.019867796, 0.0038817332, 0.01330513, -0.0017113017, 0.016644256, 0.018609202, 0.0017899638, 0.0047614644, 0.019765053, 0.019803582, -0.020599835, -0.006469555, 0.020766791, 0.020240236, -0.011333762, 0.0098696835, 0.0020741106, -0.0051082196, -0.00418996, 0.016002117, 0.000589163, 0.009876105, 0.0019216024, 0.0006646144, 0.0029265506, -0.037321147, 0.015013221, -0.04397371, -0.012919846, -0.027201029, 0.006524137, 0.019598097, 0.012990482, 0.013343658, 0.016965326, -0.015912216, -0.0012192624, 0.024465514, -0.010184332, -0.003393707, 0.00034033394, -0.01542419, -0.004138589, 0.020381507, 0.0002448157, -0.0015892952, -0.01473068, 0.011391554, -0.026790058, 0.0024979226, -0.014435296, 0.015270077, 0.008456977, 0.0067553073, 0.0148077365, -0.0043023345, -0.021164916, -0.0035638742, 0.025492938, -0.014666466, 0.0034290247, -0.0013444795, -0.0136262, 0.0101265395, -0.01196948, 0.005464607, -0.0047582537, -0.0033230719, -0.023309663, -0.01606633, 0.04068596, 0.005853101, 0.009612828, -0.018378032, 0.009375236, 0.0071983836, 0.0012738442, 0.02067689, -0.027611997, -0.011532825, -0.0127914185, 0.021049332, 0.0039523686, 0.028202767, -0.012046536, -0.003480396, -0.019443983, 0.014204125, 0.007583667, 0.0020468195, 0.0034418677, -0.023733474, 0.02187127, -0.011834631, -0.0043472843, -0.0034482891, 0.0020933747, 0.016014958, -0.01178968, 0.009105538, -0.011430083, -0.0010563195, -0.017003853, -0.0036955127, -0.0072947047, 0.025261767, 0.009047746, -0.0000065593545, -0.004902735, 0.001979395, -0.033853594, -0.004860996, -0.0010466874, 0.034958072, 0.02158873, -0.031233665, 0.025467252, -0.019765053, -0.011686939, 0.031053865, -0.015437033, -0.011545667, -0.0046490896, 0.021216288, 0.007468082, -0.008822996, -0.0016109674, 0.018018434, 0.0021030067, 0.0075194533, -0.017825792, 0.022179497, 0.009368815, 0.00319946, -0.016811213, 0.044769965, 0.0038817332, 0.015603989, 0.007435975, 0.008322128, 0.0150645925, 0.01155851, -0.0010121723, -0.0043440736, 0.01758178, -0.00056187203, 0.005474239, 0.0032668847, -0.009330287, 0.0092468085, -0.0070378487, 0.009657778, -0.0021656153, 0.01684974, -0.016323186, 0.01952104, 0.005474239, -0.0023999964, -0.0065626656, 0.021742843, -0.0046940395, -0.0005245477, -0.017055225, 0.0011967875, -0.0029570523, 0.0003947151, 0.013754628, 0.00065377826, -0.0018028066, 0.01408854, -0.0062897564, 0.013106067, 0.017196495, -0.004912367, 0.021254817, 0.00010986605, -0.0061934353, -0.016978167, 0.0026985912, -0.013420715, -0.045412105, -0.00053177174, 0.009407343, -0.016169071, -0.00051772495, -0.008302864, 0.026083706, 0.0028494939, -0.00621591, 0.014859107, -0.02269321, 0.02361789, 0.018275289, -0.000034916335, 0.0053426004, 0.0116548315, -0.03310871, 0.023322506, -0.0008917712, -0.012457506, 0.010794365, -0.004254174, -0.012457506, 0.008925739, -0.029307246, -0.0070185843, 0.014435296, 0.003968422, 0.003711566, 0.007063534, -0.010068747, 0.0070506916, 0.0111411195, -0.0052366476, 0.013343658, -0.009426608, -0.014949007, 0.0093559725, -0.008675304, -0.0064502913, 0.016785527, 0.023630733, 0.0028783903, -0.006832364, -0.016130544, 0.00354461, 0.0021479565, -0.0043344414, 0.0011751152, 0.006832364, 0.0075451387, -0.015488404, -0.015642518, -0.0053104935, -0.0076350383, -0.014139911, -0.020497093, 0.011776838, 0.0062897564, 0.011109013, -0.010531087, 0.009028481, 0.012181386, 0.030052127, 0.009786205, 0.00759651, 0.008341392, -0.01708091, 0.000027265844, 0.020098966, -0.005349022, 0.013093224, 0.003250831, -0.02794591, 0.023425248, 0.0053104935, -0.030694267, 0.008174435, -0.00064575154, -0.0052462798, 0.009144066, 0.026456147, 0.00040213985, 0.00068026653, 0.0035638742, -0.012207071, -0.0095743, 0.016888268, -0.001979395, -0.0030036075, -0.0012730415, 0.02509481, 0.006414973, -0.0009455504, 0.026212133, -0.006716779, -0.016913954, -0.010646673, -0.015180177, 0.018146861, 0.03177306, 0.001321202, -0.0024931065, -0.03038604, 0.014011484, -0.012907004, -0.0016872215, 0.015501247, 0.030000757, -0.03359674, -0.02002191, -0.006652565, 0.0081102215, 0.022218026, 0.006620458, -0.017915692, 0.0053233365, 0.005159591, 0.0006810692, 0.0137931565, -0.020111809, -0.0056283525, 0.022757424, -0.019983381, -0.02545441, 0.005689356, 0.030976808, 0.0358057, 0.0145508805, -0.0162076, 0.02219234, 0.016053487, 0.0069543705, 0.005644406, -0.00009918045, -0.010710887, 0.022076756, -0.0016398637, -0.000030702293, 0.004382602, 0.0028623368, 0.014846264, 0.014486667, 0.0011221387, -0.020522777, -0.006992899, -0.003978054, -0.027098287, -0.007435975, 0.04107124, 0.002536451, 0.012130015, 0.0067360434, -0.016079172, 0.0061902246, -0.014627937, -0.012091487, -0.0048192567, -0.0060425326, -0.004655511, 0.03020624, -0.011173227, -0.027637683, 0.009131223, 0.016464457, -0.014152754, -0.01708091, -0.008810153, 0.006980056, 0.0068580494, -0.0102613885, 0.0036248774, -0.009799048, 0.000051346076, -0.017761579, 0.008553298, 0.021884114, 0.028562363, -0.01864773, -0.00018270875, -0.004658722, 0.029101761, -0.0026568521, -0.019803582, -0.013446401, 0.032672055, -0.013317972, -0.003024477, 0.013228074, -0.0023518358, 0.023361035, -0.013690414, -0.0015186599, -0.0068580494, 0.017658835, 0.0043119667, 0.018390875, 0.016040644, -0.0050568483, -0.0041032713, -0.004084007, -0.0132922875, 0.0014624727, 0.0065658763, 0.018442245, -0.0041995924, 0.004816046, -0.010203596, -0.021293346, -0.007744202, 0.00621591, 0.0030726374, 0.0022442774, -0.012412556, 0.013202388, 0.0033327038, -0.013131753, 0.021100704, -0.002896049, -0.017222181, 0.023078494, -0.00488026, -0.007821259, -0.011391554, -0.020098966, -0.0025252134, 0.0012425398, -0.012643727, 0.026250662, -0.012393292, 0.005451764, 0.007269019, 0.010742993, 0.009792627, -0.020227393, -0.013574828, 0.0045046085, 0.00980547, -0.017196495, 0.0014424059, -0.0070121633, -0.013754628, 0.0072818617, 0.0036505628, -0.00188789, 0.0050568483, 0.024504043, 0.0033391253, 0.0043729697, 0.0014175229, 0.0021286923, 0.0066268793, 0.011391554, 0.0023502305, 0.010768679, -0.01510312, -0.004774307, 0.00529444, -0.00446608, -0.0021880902, 0.0033070184, -0.022641838, -0.00024060164, -0.0072433334, 0.009966005, 0.0039844755, -0.0071855406, 0.00036381214, 0.011950215, 0.026186448, 0.0044403947, -0.0008893632, 0.021601573, 0.013407872, 0.010935635, 0.002385548, -0.009272494, -0.0122777065, -0.009728413, -0.00042622007, 0.00704427, 0.0065498226, -0.008887211, 0.016862582, -0.0067745717, 0.015989274, 0.013151017, -0.015501247, -0.011076906, -0.012547405, -0.010113697, 0.005763202, -0.015681047, 0.0036730377, -0.0021431404, 0.0002586618, -0.0034771853, -0.0140500115, -0.006556244, -0.0041450104, -0.005612299, -0.028279822, -0.019174285, -0.005538453, 0.008636776, 0.0136262, -0.022320768, 0.009856841, 0.0020981908, 0.017620306, 0.0076543028, 0.0067103575, -0.003933104, -0.0038496263, 0.00870099, 0.035934124, -0.0137931565, 0.00043705618, 0.0036377201, -0.0065755085, -0.01649014, -0.0010370553, -0.022115285, 0.0018429403, 0.012136436, 0.00056387874, -0.005349022, 0.0011863527, -0.0030501625, 0.0015170545, -0.012829946, 0.01408854, 0.0016310343, 0.022205183, -0.017183652, 0.02016318, 0.003310229, 0.008135907, -0.004369759, 0.01026781, 0.008270756, -0.008649618, 0.011635567, 0.031747375, -0.013549143, 0.0191486, 0.021357559, 0.0074038683, 0.03665332, -0.00976052, -0.016580041, -0.012894161, -0.013240916, -0.018596359, -0.0014351817, 0.0082193855, -0.0053650755, 0.008546877, -0.0015981246, 0.009856841, 0.005650827, 0.0070250058, -0.0028783903, -0.0081937, 0.022474881, -0.006100325, -0.007525875, 0.007468082, 0.010479717, -0.0060457434, 0.017389137, 0.015693888, 0.0093559725, -0.001772305, 0.0072754403, 0.015796632, 0.012380449, -0.005666881, 0.014037169, -0.0077506234, -0.023219764, 0.009041324, 0.013106067, 0.04084007, 0.006418184, 0.009182595, 0.021563044, 0.009458714, -0.010768679, 0.0021351138, 0.009362394, -0.0031224033, -0.021665785, 0.015899373, 0.009009217, 0.0010587275, 0.0037436732, 0.014679309, 0.0025123707, -0.005782466, 0.016721312, -0.01082005, -0.0031159818, 0.011404397, 0.008675304, -0.009144066, -0.0038014657, -0.0032379883, 0.0040679537, -0.004523873, -0.01961094, -0.0005590627, -0.014974693, -0.0055641388, -0.026635945, -0.006703936, -0.0011759179, -0.008412027, 0.030540153, -0.0023550466, -0.0067938357, -0.0011052826, 0.0007472898, 0.0070956415, -0.024915012, -0.029076075, 0.0044275518, 0.026289191, 0.004568822, 0.001528292, -0.026083706, -0.00708922, -0.0025717686, 0.018544989, 0.008026743, -0.0024240764, -0.0009038113, -0.011340183, -0.033365566, -0.00055665465, 0.025839692, 0.02748357, -0.009933898, 0.008604669, -0.009471557, 0.014627937, -0.009477979, 0.0064053414, 0.0010571221, -0.003618456, 0.014846264, 0.022423511, -0.013818841, 0.0059044724, -0.014974693, 0.019212814, 0.018211076, 0.0023775215, -0.018416561, -0.023271134, 0.011905266, -0.021190602, 0.0028992598, -0.0032973862, 0.004084007, -0.019508198, -0.001153443, 0.00041377862, 0.011731888, -0.011609882, -0.009150487, 0.015115963, 0.008386342, 0.027509255, -0.011757574, -0.006411763, 0.019919166, -0.005085745, 0.0041000606, 0.021062175, 0.016284658, -0.009458714, 0.008142329, 0.0012634094, 0.024118759, 0.002284411, -0.0056925663, -0.021563044, 0.023399564, -0.0043986556, 0.008200121, 0.004568822, 0.015475562, -0.00055665465, -0.0018204654, -0.0012682255, 0.020638363, -0.0022860165, -0.015693888, -0.018557832, -0.013960112, 0.0028238085, 0.009426608, -0.015732417, -0.007493768, -0.015372819, -0.02591675, -0.0047454108, -0.011564932, 0.002412839, 0.017106596, -0.0061805923, -0.013382187, 0.0162975, -0.0093816575, -0.021845585, -0.00038668836, -0.0027852799, -0.024324244, -0.025236081, -0.036756065, -0.010698044, 0.00003102838, 0.0014014695, -0.00071839354, -0.0140500115, 0.044769965, -0.018442245, -0.023643576, 0.0358057, 0.009966005, -0.0071662767, -0.003949158, 0.0013725731, -0.010415503, -0.0020548464, -0.025724107, -0.036910176, -0.039838333, -0.011211756, -0.0043729697, 0.003945947, 0.02702123, 0.0037083556, 0.012977639, 0.002859126, -0.012065801, -0.010190753, 0.0020933747, -0.0097476775, 0.007757045, 0.021357559, -0.0061324323, 0.0030373198, 0.010370553, 0.008347813, 0.031644635, -0.025608523, -0.02108786, 0.011629146, 0.0081102215, -0.008168014, 0.0014424059, 0.020432878, -0.0059879506, 0.0005377918, -0.015270077, 0.0143068675, -0.007583667, -0.005589824, -0.01173831, -0.010004533, 0.022115285, -0.003907419, -0.00093110226, -0.0013525062, -0.013690414, 0.0017674889, 0.017273553, -0.016310344, -0.00515638, 0.021357559, 0.02661026, 0.0034258142, 0.023232607, 0.026687317, 0.017761579, 0.023836218, -0.009921055, 0.00069110264, -0.0065658763, -0.010685201, 0.006440659, 0.001574847, 0.011975901, 0.00037906296, -0.003711566, 0.0037372517, 0.013356501, 0.005888419, 0.016836897, -0.0022956487, -0.006819521, 0.023258291, -0.002827019, -0.017684521, 0.011841052, -0.018018434, 0.011173227, 0.009580721, 0.021614416, -0.004469291, 0.01758178, -0.02062552, -0.026867116, -0.016400242, 0.0022812006, 0.0026167184, -0.0036890914, 0.0059751077, 0.016592884, 0.010511823, 0.0051114303, 0.018878901, 0.0015395294, -0.0076029315, -0.015128806, 0.0024369194, 0.019212814, -0.025287453, -0.007988215, -0.0031384567, -0.005082534, 0.016258972, 0.022616152, -0.0055448744, 0.010325603, 0.020548463, 0.0024850797, 0.029641159, -0.00092387816, -0.005952633, 0.022847323, 0.0047935713, -0.002550899, 0.010441188, -0.016220443, -0.009137644, -0.015026064, -0.0075387177, -0.016284658, 0.000010064784, 0.00032026708, 0.006559455, -0.022577625, 0.0017498301, -0.0059590545, -0.023091335, -0.011545667, 0.012329078, -0.0025541098, -0.013446401, 0.029692529, 0.025569994, 0.023797689, 0.004138589, -0.008052429, 0.010088012, -0.003300597, -0.008071694, -0.0041995924, -0.010100854, -0.0056411955, -0.01496185, 0.008534034, -0.0018670205, -0.0155783035, 0.008347813, 0.0027001966, -0.0087202545, 0.028716477, -0.002127087, -0.013677571, 0.0031641424, -0.032851856, -0.0003864877, 0.001675984, -0.003329493, -0.004751832, -0.015488404, 0.022140969, -0.012123593, -0.006501662, 0.0069543705, 0.013459244, 0.010017375, -0.008437713, -0.010152225, -0.007936844, 0.005377918, 0.024902169, 0.0060939034, 0.019187128, 0.0040711644, 0.016181914, 0.0011076906, 0.005878787, -0.03899071, 0.007879051, 0.01058888, 0.024889326, 0.0021977224, 0.00082434656, -0.004751832, -0.012091487, 0.016567199, 0.019970538, 0.011012692, 0.0046908287, 0.0014536432, -0.010332025, 0.010120118, -0.0358057, -0.008161593, -0.00049725675, 0.016772684, -0.017055225, 0.013459244, 0.037732117, -0.014371081, 0.0140500115, -0.009522929, -0.0024882904 - ] - } - ] - } + { + "name": "untitled", + "title": "Untitled", + "content": "---\n\nSpringfield is the county seat of Sangamon County in Illinois, United States. As of 2019, it had a population of 10,566, ranking it as the sixth most populous city in the state.", + "keywords": [ + "Springfield", + "county seat", + "Sangamon County", + "Illinois", + "United States", + "population", + "2019" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.026481831, 0.04921357, -0.002366284, -0.021460302, -0.027380828, + -0.0036216667, 0.012605198, -0.006658986, -0.06981341, 0.021896956, + -0.03893934, 0.025441566, -0.013189545, -0.0004924407, 0.0061902246, + 0.013844527, -0.015963588, 0.04998414, 0.0066975146, -0.0013460849, + 0.0354461, -0.0061163786, -0.060566597, 0.044179197, 0.012682254, + 0.00824507, -0.030694267, 0.0064470805, -0.026327718, 0.0017578568, + -0.003660195, -0.001222473, 0.045232303, 0.001574847, 0.009728413, + -0.019392611, 0.0015339106, -0.020548463, 0.029872328, 0.034881018, + 0.011256705, 0.0020708998, 0.018943114, 0.010145804, 0.028896276, + -0.02853668, 0.0003991298, -0.036550578, 0.021190602, 0.023232607, + 0.016002117, -0.014949007, -0.021293346, 0.041456524, 0.011834631, + -0.004276649, 0.011937373, 0.019135756, -0.007423132, -0.050857447, + 0.00016474891, -0.014332553, -0.024452671, 0.009426608, 0.012438241, + 0.0215502, 0.017299237, -0.002600665, 0.04433331, 0.020317294, + 0.029666845, 0.06642291, 0.0055320314, 0.029332932, 0.054299314, + -0.008938582, -0.03149052, 0.01201443, -0.025955278, 0.013651885, + 0.02559568, -0.028100023, -0.020188866, 0.0044949763, -0.0021094282, + 0.0000010096138, 0.011256705, -0.073666245, 0.0030164502, + 0.0036056133, 0.010672358, -0.013728942, -0.011584196, -0.020908061, + -0.044487424, -0.027791796, 0.045566216, -0.010299917, -0.005217383, + 0.024298558, 0.011956637, -0.002674511, -0.0404291, 0.01731208, + 0.0005526413, 0.016438771, -0.0006758518, -0.006209489, -0.016143387, + 0.013112488, 0.025955278, -0.003226751, -0.0071662767, -0.0046651433, + -0.027201029, 0.0028061494, -0.009728413, -0.017800106, 0.044127826, + 0.015115963, -0.0037211983, 0.014743523, -0.03226109, 0.006071429, + -0.021601573, 0.010254968, 0.020432878, -0.017453352, 0.01961094, + -0.032158345, 0.02591675, 0.011353026, -0.0068516284, 0.011333762, + 0.009336708, -0.04225278, -0.0065401904, 0.031798746, 0.028100023, + 0.048263203, -0.06390572, -0.007159855, 0.019790739, -0.00056187203, + -0.0013733758, -0.025467252, -0.016952483, 0.01606633, 0.012733626, + -0.026327718, -0.024195816, -0.013780314, -0.027509255, 0.00446608, + 0.018095491, 0.020047594, 0.0060842717, -0.024414143, 0.05676513, + -0.046901867, -0.018801844, -0.013587671, 0.028228452, -0.036807436, + -0.011166805, -0.0064246054, -0.020201707, -0.00026046782, + 0.0016093621, 0.0034482891, 0.042278465, -0.0054389215, -0.04572033, + -0.07489915, 0.0060329004, -0.025261767, -0.014204125, -0.014846264, + -0.005731095, 0.024375614, 0.008501926, -0.0031464836, 0.028022967, + -0.009779784, -0.020908061, -0.022128126, 0.006890157, 0.023964645, + 0.038554054, -0.021575887, 0.00079223956, 0.051448215, 0.013883055, + 0.0060040043, 0.02905039, 0.0014680914, -0.011725467, -0.033365566, + 0.015937902, 0.026032334, -0.054196574, 0.015385662, -0.02205107, + -0.014782051, 0.0024593943, 0.015937902, -0.009535771, -0.0095743, + 0.020869533, 0.030642897, -0.01035771, -0.017607465, -0.017787263, + 0.042612378, -0.014936164, 0.02817708, -0.017183652, -0.007352497, + -0.017389137, 0.0459515, 0.026404776, 0.037115663, 0.007513032, + -0.026404776, -0.018557832, -0.009304601, -0.06303241, 0.0059751077, + 0.004613772, -0.026918488, 0.032158345, 0.01790285, 0.0072625973, + 0.056919243, -0.0379376, 0.06133716, 0.024555413, 0.0073268116, + -0.01717081, 0.034315933, 0.027406514, 0.02899902, -0.00980547, + -0.02463247, -0.007904737, -0.028074337, 0.023733474, 0.0059141046, + 0.0075708246, 0.028716477, 0.00035177203, 0.0030084234, 0.051936243, + 0.00994674, -0.011930952, 0.009709149, -0.005750359, 0.006633301, + -0.008906474, 0.0073139686, 0.0019874217, 0.010370553, -0.017157966, + 0.017479036, 0.005731095, -0.035112187, 0.0005357851, 0.0032652793, + 0.0019537094, -0.029153133, -0.001703275, -0.004225278, 0.007288283, + 0.017466193, 0.028613735, 0.010903528, 0.042612378, 0.02301428, + -0.03125935, -0.04623404, -0.021113547, 0.009054166, 0.0051242732, + -0.025518622, 0.0045559797, 0.01243182, -0.007943265, 0.027098287, + 0.0022137759, -0.046799127, -0.011423661, 0.0044436054, 0.030899752, + 0.025711264, 0.030129185, -0.0028848117, 0.0030662161, -0.0033487573, + -0.046901867, 0.019508198, 0.018146861, 0.02661026, -0.007866208, + -0.015231549, 0.0122648645, -0.015128806, 0.005050427, -0.00395879, + -0.026160762, -0.05404246, -0.009221123, -0.053374633, 0.0028334404, + 0.025698422, -0.011995166, -0.00033009984, 0.017222181, -0.014139911, + 0.038502682, -0.017440509, -0.020009067, 0.0029875538, 0.018429402, + 0.019906323, -0.012348342, 0.023399564, -0.000100685465, -0.012727205, + 0.01592506, 0.01201443, -0.03457279, 0.004626615, -0.017812949, + -0.05106293, 0.011051221, -0.012759311, 0.018763315, -0.008296442, + 0.0010033429, 0.019970538, -0.004976581, 0.0057664127, 0.009908212, + 0.037192717, 0.026507517, -0.02149883, -0.009818313, -0.0053971824, + -0.013163859, 0.01740198, 0.000666621, 0.003351968, 0.05501851, + -0.0039620004, 0.017363451, -0.006283335, -0.02499207, -0.02283448, + 0.01606633, -0.039581478, -0.024439828, 0.0067809927, -0.009458714, + -0.0022812006, -0.019482512, 0.019495355, 0.00073284167, 0.009400922, + 0.030617211, 0.0062191207, 0.013240916, 0.014011484, -0.0122648645, + 0.011956637, 0.013497772, 0.006960792, -0.017363451, -0.008604669, + -0.012386871, -0.00061725656, 0.032440886, 0.025531465, -0.019944852, + 0.00086769095, -0.0011991955, -0.0030260824, -0.0017899638, + -0.017838635, -0.017337766, 0.011089749, -0.008084536, -0.013382187, + -0.00469725, 0.0030517678, -0.0038464156, 0.017517565, -0.044281937, + -0.0033583895, 0.028896276, 0.0041642743, 0.023232607, -0.03056584, + 0.002181669, 0.031336408, -0.020753948, -0.0052976506, -0.0019970539, + -0.057124726, 0.012662991, -0.014011484, -0.024786584, -0.0030180556, + 0.003393707, -0.012322657, -0.02899902, -0.05008688, -0.016541513, + 0.03714135, 0.015488404, 0.034059078, -0.01449951, -0.037911914, + 0.0063314955, 0.0039555794, -0.015411347, 0.0030276875, 0.012740048, + 0.025698422, -0.0381174, -0.021357559, -0.031464834, -0.009477979, + 0.0027917014, 0.010306339, -0.015000379, 0.050985876, -0.03688449, + 0.0045591905, 0.03978696, 0.013356501, -0.02946136, 0.005962265, + 0.032338142, -0.018095491, -0.013574828, 0.022487724, 0.02684143, + 0.01873763, 0.024337087, 0.056919243, -0.014165597, 0.03434162, + 0.016567199, 0.011622725, 0.007416711, 0.0011245467, -0.042920604, + 0.012496035, -0.007988215, 0.031798746, 0.023399564, -0.023181235, + 0.014345396, -0.0352663, -0.013972955, 0.02513334, 0.0461313, + 0.00971557, 0.000971236, -0.017877163, -0.0072561763, -0.025582837, + -0.039581478, 0.02541588, -0.045925815, -0.0031063498, 0.004482134, + 0.012996903, 0.022962907, -0.015848001, -0.040814385, 0.022718895, + -0.044924077, 0.015950745, -0.0050632698, 0.026147919, -0.025929593, + -0.027663369, 0.0106274085, -0.015167334, 0.052295838, 0.0053522326, + 0.008765204, -0.020227393, 0.010807208, 0.0070378487, -0.03724409, + -0.055223994, 0.02370779, 0.033005968, -0.0033969178, 0.03084838, + 0.037629373, -0.0029153132, 0.003204276, -0.0039427364, -0.02030445, + -0.03521493, 0.013472087, 0.016323186, 0.061491277, 0.0035125029, + -0.028279822, -0.020009067, 0.016913954, -0.015475562, -0.012386871, + -0.010210018, 0.029409988, -0.009895369, -0.009580721, 0.011622725, + -0.0025525044, 0.006543401, -0.019790739, 0.012579513, 0.0033038077, + -0.0009977242, 0.029872328, -0.021241974, -0.033262827, 0.0057696234, + 0.0056090886, 0.028022967, 0.06405983, -0.02513334, -0.02126766, + -0.019238498, -0.0017867531, 0.042715117, -0.03351968, -0.057535697, + 0.012412556, 0.03190149, 0.01850646, 0.006154907, -0.0047807284, + -0.012207071, 0.0028703634, -0.03167032, 0.053888347, -0.014949007, + 0.005888419, 0.0030581893, -0.014653623, -0.007121327, -0.014692151, + -0.02007328, -0.037038606, -0.013330815, 0.02140893, 0.011243862, + 0.021845585, -0.004456448, 0.0062865457, 0.015475562, 0.016002117, + -0.011282391, 0.005410025, 0.0017562514, 0.011218177, 0.012451084, + 0.026327718, 0.0064663445, 0.019906323, 0.011719045, -0.018660573, + -0.015398505, 0.012367606, 0.011423661, -0.008360656, 0.019649468, + 0.031387776, 0.025390195, 0.0005686948, 0.0066782506, -0.00814875, + -0.030077813, -0.0034033393, 0.044769965, 0.036627635, 0.023450933, + -0.0029265506, 0.015115963, 0.0010563195, 0.009060588, -0.027534941, + 0.015681047, -0.028665107, -0.029435674, 0.0068387855, -0.0041450104, + 0.015732417, -0.020458564, -0.024388457, 0.008463399, 0.023643576, + 0.0153086055, 0.018904585, 0.010511823, 0.0024465513, 0.020831006, + 0.0017129071, -0.008026743, -0.032286774, -0.006315442, 0.03403339, + 0.029076075, 0.010800786, -0.022359297, -0.014294025, -0.024414143, + -0.0074552395, -0.009766942, 0.025377352, -0.021421773, 0.012451084, + -0.016323186, -0.0031175872, 0.0018814687, -0.001079597, 0.009914634, + -0.015526933, 0.006459923, -0.01473068, -0.019687997, 0.0061805923, + -0.008418448, -0.02080532, -0.012784997, 0.0026488253, 0.058306266, + 0.016926797, 0.03624235, -0.004639458, 0.00016755828, -0.021100704, + 0.024902169, 0.007191962, 0.004899524, 0.055994563, -0.028048653, + 0.009477979, -0.0011646805, -0.00091825944, 0.01800559, 0.02776611, + -0.030257612, 0.028048653, -0.003241199, -0.016836897, 0.014897636, + 0.0004274241, 0.006877314, 0.012412556, 0.020908061, -0.03737252, + 0.038091715, 0.017928535, -0.0034771853, -0.014332553, 0.017915692, + -0.004116114, 0.011449347, -0.010132961, 0.0067938357, 0.0032797274, + -0.0069351066, -0.010486138, 0.0020548464, 0.010486138, -0.0164773, + -0.02946136, -0.005731095, 0.013818841, -0.021190602, 0.044025082, + 0.03673038, -0.005506346, -0.01758178, 0.019366927, 0.033493996, + -0.0018959168, -0.007827681, 0.031053865, 0.004337652, -0.0011686939, + 0.031233665, 0.01150714, -0.014846264, 0.0020773213, 0.023373878, + 0.01740198, -0.039966762, 0.010736573, 0.009766942, -0.0018333083, + -0.024619628, -0.05224447, -0.008643198, 0.006703936, -0.004626615, + 0.030694267, -0.011076906, 0.053888347, -0.035112187, 0.040583216, + -0.035137873, 0.008495505, 0.039966762, 0.02375916, 0.008707412, + -0.053785603, -0.012117172, -0.03236383, 0.008309285, -0.029230189, + -0.025788322, 0.01473068, 0.0060650073, -0.038399942, 0.009599986, + 0.026250662, -0.009266073, 0.026738688, -0.001855783, -0.013818841, + -0.02499207, 0.022962907, -0.007172698, 0.0067424644, -0.032081287, + 0.04374254, 0.006652565, 0.0006770558, 0.017055225, -0.0025187922, + -0.003660195, -0.059179574, 0.0028510992, -0.038656797, -0.019816425, + 0.038785223, 0.016271815, 0.0037500947, 0.00022956486, 0.012142858, + 0.0070378487, 0.01588653, 0.0358057, 0.023656419, 0.010858579, + 0.00202595, 0.0025492937, 0.0077827303, 0.0022137759, 0.004816046, + -0.010113697, -0.0027130393, 0.0065947725, 0.03323714, -0.0023261502, + -0.005262333, -0.0018188601, 0.022397825, -0.00060922984, 0.010967743, + -0.011988744, 0.019662311, -0.03398202, 0.034649845, -0.006071429, + 0.002319729, -0.0023405985, -0.007577246, -0.019392611, 0.009978848, + -0.002969895, 0.0044725016, -0.013818841, -0.008444134, -0.004597719, + -0.017684521, 0.010537509, -0.0093945, -0.016541513, 0.0115199825, + -0.012194228, 0.044410367, 0.006601194, 0.012309814, 0.019598097, + -0.010017375, 0.010909949, -0.030334668, 0.0030533732, 0.007179119, + 0.013985798, -0.016708469, 0.018211076, 0.040994186, -0.022269398, + 0.020009067, -0.0081102215, -0.036422152, -0.027534941, 0.016567199, + -0.0024096284, 0.00019906324, -0.02003475, 0.034778275, -0.021421773, + 0.0432802, -0.010794365, 0.020908061, 0.02306565, 0.023720633, + -0.026481831, 0.00043344416, 0.018955957, -0.010730151, -0.0024609996, + -0.013703257, -0.009734835, -0.00971557, 0.011564932, 0.0070185843, + 0.0067681503, 0.02928156, -0.009323865, -0.0076157744, -0.015462719, + 0.027971596, -0.0072947047, 0.010210018, 0.04199592, 0.0072112265, + -0.054813027, 0.015038907, -0.04148221, -0.012984061, 0.017890006, + -0.01510312, 0.019726524, -0.025467252, 0.02666163, 0.024683842, + 0.0083542345, 0.0038110977, 0.008450556, 0.007943265, -0.012573091, + 0.03102818, 0.0044500264, 0.004931631, 0.004074375, 0.0023839427, + 0.012001587, -0.018724788, -0.0052655437, 0.015090277, 0.0019248131, + -0.013960112, -0.024645314, -0.007493768, 0.014692151, -0.009291758, + -0.01937977, -0.012252022, 0.013446401, -0.025390195, -0.004116114, + -0.010556773, -0.033262827, 0.024285715, 0.03084838, -0.019970538, + -0.012194228, 0.012149279, -0.021794215, 0.0012915031, 0.004543137, + -0.0024064176, -0.006090693, 0.012457506, 0.025801165, -0.018917428, + -0.020086123, -0.015629675, 0.027689055, 0.020381507, -0.008668883, + 0.0035574527, -0.017235024, -0.0040165824, 0.00022976553, 0.023656419, + 0.0075066104, -0.0037372517, -0.0025974542, -0.00833497, 0.012374028, + 0.0052591222, -0.03369948, -0.00865604, 0.001333242, -0.02840825, + -0.0136262, -0.00015060177, 0.031002494, 0.03231246, 0.020060437, + -0.02817708, 0.0046651433, -0.00045351102, -0.0055866134, 0.014396767, + -0.023502305, -0.01597643, -0.0005907683, 0.015603989, 0.0076992526, + 0.0073717614, 0.006068218, 0.01864773, -0.021370402, -0.038887966, + 0.0018943115, -0.01072373, 0.028973334, -0.008540455, 0.016567199, + -0.023001436, -0.01836519, -0.0067103575, 0.044949763, -0.01868626, + 0.025081968, 0.038476996, -0.0025043439, 0.010909949, 0.013202388, + -0.010800786, 0.011410818, 0.047030296, 0.008277178, -0.025030596, + 0.03218403, -0.01946967, 0.019135756, 0.004957317, 0.0035478207, + -0.030899752, -0.008636776, 0.013651885, -0.008822996, -0.018390875, + -0.06477903, 0.000050116978, -0.0009913028, 0.024157288, + -0.0140500115, -0.015655361, -0.015205863, 0.00051050086, 0.03601118, + -0.007724938, 0.0053522326, 0.024773741, 0.02559568, 0.015642518, + 0.021627259, 0.007564403, 0.042021606, 0.02858805, -0.017568937, + -0.022115285, -0.016053487, -0.004209224, 0.0015001984, 0.022256555, + -0.025608523, -0.0017000643, -0.0069543705, 0.052655436, -0.010614566, + 0.0023405985, 0.0005803336, -0.0041321674, 0.014704994, 0.035291985, + -0.003554242, -0.018609202, -0.00071518286, -0.01634887, 0.02905039, + -0.044179197, 0.0028832064, -0.014627937, -0.0056861453, 0.021370402, + 0.0010282259, 0.032517944, -0.025801165, -0.008360656, -0.025069125, + -0.0075001894, -0.016657097, -0.018519303, -0.00478715, 0.00285431, + 0.018095491, 0.0045367153, -0.012733626, -0.0042284885, 0.010832893, + 0.014627937, 0.010351288, 0.02324545, 0.02393896, 0.021807056, + 0.00014438105, -0.0044949763, 0.014062854, -0.034290247, -0.007930422, + -0.008135907, 0.05589182, 0.0023887588, 0.017697364, 0.012746469, + -0.035985496, 0.021896956, 0.0022009332, 0.010473295, -0.014011484, + -0.008084536, 0.003798255, 0.018172547, 0.01772305, 0.00055344397, + 0.0025075546, -0.00028073534, 0.0032973862, -0.033622425, + -0.008501926, 0.02858805, 0.02545441, 0.0018509671, -0.017568937, + 0.005644406, -0.012528141, 0.006684672, -0.0027290927, 0.02066405, + -0.00060481514, -0.023990331, 0.017607465, 0.0037372517, 0.009580721, + 0.0019681575, -0.009709149, 0.035189245, -0.020766791, 0.019816425, + 0.016104858, -0.01505175, -0.01431971, -0.0036762485, -0.0329546, + -0.0017369873, 0.018211076, -0.018532146, 0.031875804, 0.015770946, + -0.03816877, 0.012560248, 0.006251228, -0.013003324, 0.015501247, + -0.016284658, -0.014576566, 0.0028575207, -0.009182595, 0.05332326, + -0.024837956, 0.0155783035, 0.049316313, -0.013549143, -0.014704994, + -0.016913954, -0.0056989877, 0.0055577173, -0.053374633, 0.0032973862, + 0.004032636, -0.03940168, -0.0002867554, -0.034906704, -0.012964796, + -0.008938582, 0.032517944, -0.021473145, 0.014846264, 0.013330815, + 0.021241974, -0.017979905, -0.0019392612, -0.0050632698, -0.020959433, + -0.011012692, -0.014692151, -0.029384302, 0.0020291607, 0.023104178, + -0.023669261, -0.012849211, -0.0074295537, 0.018339504, -0.0155783035, + -0.0077313595, 0.019777896, -0.016246129, -0.03770643, 0.0075322962, + 0.0030854803, 0.01615623, -0.024593942, 0.028742163, -0.021768529, + 0.015038907, 0.0031962493, -0.013138174, -0.020291608, 0.02158873, + -0.01026781, 0.034007706, -0.0040390575, 0.0032829382, -0.013998641, + -0.028433936, -0.005721463, 0.00814875, -0.0011510351, -0.016888268, + -0.008784468, -0.0067360434, -0.01054393, -0.03411045, 0.022295082, + -0.021935485, -0.001362941, -0.008168014, 0.0105182445, -0.0005365878, + -0.0017626729, 0.021396087, 0.00015551815, 0.0028302297, 0.03246657, + 0.0010458847, 0.0007224069, -0.018172547, -0.029332932, 0.008437713, + -0.026687317, 0.00598474, -0.0076157744, -0.00059116964, 0.011744731, + -0.008945003, -0.01164841, -0.020895218, 0.013151017, -0.013048274, + -0.026353404, -0.008296442, -0.019199971, -0.031978548, -0.0028607314, + 0.010132961, -0.01675984, 0.037834857, 0.023605047, 0.0048642065, + -0.01449951, -0.000033310986, -0.0012296971, 0.012829946, 0.026481831, + -0.017915692, 0.031233665, -0.011372291, -0.009728413, 0.014615094, + -0.017967062, 0.0014416032, -0.011038378, -0.002725882, -0.0219098, + 0.013163859, 0.012271285, 0.014473824, -0.02518471, 0.012842789, + -0.008893631, -0.038143083, 0.014127068, 0.03246657, -0.0035799276, + 0.0042702276, 0.0012642121, -0.0350865, 0.024131602, 0.0062769135, + 0.016554356, -0.019816425, 0.010710887, -0.007269019, 0.005853101, + -0.010100854, 0.013844527, -0.012817104, -0.0036955127, -0.0057888874, + -0.031362094, -0.025403038, 0.029589787, 0.0043665483, 0.02292438, + 0.020792477, -0.006228753, 0.028254136, -0.0032235403, -0.0017385926, + 0.020086123, -0.035780013, -0.032029916, 0.015398505, -0.037860543, + 0.0049348418, 0.0086110905, 0.011385133, -0.019122913, -0.02541588, + -0.011622725, -0.029101761, 0.011808945, -0.020753948, 0.000802273, + 0.019559568, -0.004616983, -0.005695777, 0.029076075, 0.021485986, + 0.008168014, -0.013189545, -0.0007500992, 0.011320919, -0.0008291626, + -0.0012401319, 0.025325982, -0.015270077, -0.015180177, 0.018198233, + 0.019546725, -0.019803582, -0.007737781, 0.03310871, 0.0123997135, + -0.0016583252, 0.008078115, -0.0016647466, -0.0054710284, + -0.002546083, -0.00216401, 0.025980964, -0.008283599, 0.024439828, + 0.00644387, 0.03195286, 0.011494297, 0.012862054, 0.003083875, + -0.02545441, -0.00893216, 0.023797689, -0.04723578, -0.005422868, + -0.001841335, 0.025801165, -0.02587822, -0.0097476775, -0.019996224, + -0.008033165, -0.010601723, -0.019456826, -0.0005947817, -0.009182595, + 0.003157721, 0.020895218, 0.0045463475, -0.022256555, -0.006036111, + 0.004379391, -0.0069543705, 0.018082649, -0.004783939, -0.020497093, + 0.0016181915, -0.050035506, -0.0018172547, -0.013523458, 0.0016872215, + 0.01449951, -0.009831156, 0.02035582, 0.0033327038, -0.00019735756, + 0.02205107, -0.0037725694, 0.0074102897, -0.020728262, -0.0037308303, + -0.012303392, -0.02518471, 0.023258291, -0.02481227, 0.04538642, + 0.0064213946, 0.014255497, 0.02853668, -0.00010665535, 0.016104858, + 0.01173831, 0.004915578, -0.00045190565, -0.019122913, 0.01288774, + -0.027278084, -0.008501926, 0.027534941, -0.023874747, 0.0032540418, + -0.031362094, 0.0115199825, -0.005833837, 0.013112488, 0.015783789, + 0.012720783, 0.0072625973, 0.02789454, 0.023052808, -0.0028880222, + -0.0057664127, -0.015629675, -0.00547745, -0.017658835, -0.010807208, + -0.011044799, 0.019443983, 0.015347133, 0.015038907, 0.021139232, + 0.0032700952, -0.0034258142, -0.0051949085, -0.021884114, + -0.0034129713, -0.0071470123, -0.0029650792, -0.0012634094, + 0.012027272, 0.004658722, 0.013587671, 0.032286774, 0.025017753, + 0.0057439376, 0.01279784, 0.0076029315, 0.0023213343, 0.0012834762, + 0.011083327, 0.0011735099, -0.010768679, 0.0066268793, 0.0047261463, + -0.0077056736, 0.004061532, 0.0025974542, -0.0019023382, + -0.0061805923, 0.001878258, -0.011275969, 0.04949611, -0.0046940395, + 0.0021014016, -0.007423132, -0.013960112, 0.013728942, 0.004581665, + -0.015526933, 0.028279822, -0.035112187, -0.008142329, -0.023810532, + -0.010280653, 0.031798746, 0.007236912, -0.016271815, 0.009028481, + -0.01072373, -0.019803582, 0.009978848, -0.00018702312, -0.036807436, + -0.016772684, -0.02066405, -0.007757045, -0.0012987271, -0.008816575, + 0.011828209, -0.015899373, -0.004485344, 0.009535771, -0.00024240765, + -0.003586349, -0.0023373878, -0.013080381, -0.0056604594, + -0.014294025, -0.010640251, -0.0023839427, 0.026353404, -0.011712624, + 0.0014335763, -0.0077827303, -0.033648107, -0.011719045, + -0.0059911613, -0.027046915, 0.0020548464, -0.012618041, -0.009567878, + 0.023861904, 0.012425399, 0.01666994, 0.02434993, 0.006344338, + -0.017157966, 0.005288019, -0.010967743, 0.011796102, 0.015539776, + 0.0049284203, -0.02016318, -0.00012391285, 0.015372819, -0.008822996, + 0.0064984513, -0.0065530334, 0.0005153169, 0.0048513636, 0.022808794, + 0.013960112, 0.025467252, 0.032158345, 0.014024327, -0.019212814, + -0.014460981, 0.0033969178, 0.016258972, 0.0043858127, 0.0030662161, + -0.003522135, 0.02408023, -0.006658986, 0.0034129713, 0.0012481586, + 0.026918488, -0.02794591, 0.022436354, 0.02237214, -0.012772154, + -0.019944852, -0.0022763845, 0.016734155, 0.015822317, 0.017055225, + 0.028870592, -0.0021190604, 0.01473068, -0.013780314, -0.014512353, + 0.0013284261, 0.013215231, 0.012528141, -0.034726903, -0.0023807322, + -0.011866737, 0.011886002, 0.028485307, -0.007757045, 0.020586992, + -0.03269774, 0.02173, 0.020753948, -0.001878258, 0.018416561, + 0.0023887588, 0.021832742, 0.045823075, 0.029692529, -0.01763315, + 0.014101383, -0.021370402, 0.0040904284, 0.007204805, -0.0015828739, + -0.028921962, -0.0025942435, 0.017979905, -0.02748357, -0.010858579, + -0.0037147768, 0.011430083, -0.011385133, 0.0054581854, -0.0244912, + -0.008668883, -0.008758782, 0.0032620686, 0.016284658, -0.0020692945, + -0.011051221, 0.018275289, 0.0015347133, -0.007879051, -0.021010803, + -0.0072754403, -0.042201407, -0.007885473, 0.004254174, -0.0069094207, + -0.0030967176, 0.019418297, -0.007795573, 0.034906704, 0.000084631974, + 0.009433029, -0.00446608, 0.02481227, 0.010210018, -0.0062865457, + -0.0019039436, -0.011173227, -0.0022812006, -0.011622725, 0.00704427, + -0.014525195, 0.0077827303, 0.007821259, 0.0076221954, 0.003907419, + -0.0242215, -0.007288283, -0.0145637235, -0.0011422056, 0.012040115, + 0.013099645, -0.024735212, -0.0011767206, -0.02269321, -0.004135378, + 0.014371081, 0.013870213, -0.013016167, -0.001703275, 0.02518471, + -0.0021624046, -0.0053811288, 0.0071662767, 0.0016230076, + 0.0006746478, -0.011038378, -0.013433558, 0.02375916, -0.014717837, + -0.009638513, -0.025428724, -0.00584668, 0.01588653, -0.033802222, + 0.0022908326, -0.009587143, -0.0016422718, 0.009683464, -0.004539926, + -0.002513976, -0.0036698272, -0.00855972, -0.019366927, -0.0048642065, + -0.005994372, -0.02223087, 0.016143387, -0.0007814035, -0.013433558, + -0.008521191, 0.006652565, -0.012059379, -0.010813629, -0.012354763, + 0.039324623, -0.022590468, 0.011597039, -0.007140591, -0.014717837, + 0.0066654077, 0.01003664, -0.0013252153, 0.013497772, 0.021049332, + 0.0014761181, -0.0072433334, -0.005088955, 0.019790739, -0.011918109, + 0.007814838, 0.0047807284, 0.041816123, -0.0072754403, -0.013041853, + 0.03560021, 0.025197553, -0.009336708, 0.0014464192, -0.0005991964, + -0.00024702304, -0.0116548315, 0.0018525724, -0.004437184, + 0.0035413993, 0.010145804, -0.007994637, -0.027278084, -0.0058402587, + -0.010563195, 0.00013474896, -0.00088454713, 0.016284658, + 0.0065209265, 0.008964268, -0.0103833955, 0.009240387, 0.020291608, + 0.012008009, -0.016400242, -0.0127914185, -0.024439828, 0.009176173, + -0.0039555794, 0.0022539096, -0.011468611, -0.0060168467, + 0.0060842717, -0.020728262, 0.0143068675, 0.011301654, -0.0064470805, + 0.009214701, -0.009099117, 0.039941076, 0.004722936, 0.0064759767, + 0.0021431404, -0.01449951, -0.0075451387, -0.0040390575, -0.015488404, + 0.0099916905, -0.0032941755, -0.00019805989, -0.009696306, + -0.00038347766, 0.011571353, 0.0046779863, -0.01649014, 0.005419657, + -0.025300296, 0.004010161, -0.013844527, 0.007692831, -0.0009856841, + -0.0058434694, -0.019559568, -0.018121175, 0.010486138, 0.0014600647, + 0.0013220047, 0.00095598516, -0.004957317, 0.006556244, -0.0012706335, + 0.016708469, 0.015398505, 0.023489462, 0.013279445, -0.0001430767, + 0.016130544, 0.0035349778, -0.022436354, -0.011346605, 0.026995543, + 0.005496714, 0.004411498, -0.009811891, 0.008758782, -0.008970688, + -0.026533203, -0.012085065, -0.010858579, -0.0001264212, -0.015732417, + -0.0062640705, -0.013253759, 0.0017787263, -0.00575357, -0.003917051, + 0.006081061, -0.007005742, 0.0060200575, 0.015950745, 0.00382073, + -0.028151395, 0.027149657, -0.008906474, 0.007679988, 0.0033583895, + 0.012226336, 0.004170696, -0.008842261, 0.0023951803, 0.0048353104, + -0.0008765204, -0.032903228, -0.029358618, 0.014576566, 0.0054581854, + 0.0143068675, 0.0026311665, 0.0047582537, 0.0013838105, 0.004437184, + 0.012695097, 0.01464078, 0.029332932, -0.016258972, 0.019187128, + 0.008534034, 0.01464078, 0.030488782, -0.007224069, 0.016926797, + 0.0056604594, -0.004828889, 0.0049637384, 0.008765204, -0.023515148, + 0.01440961, -0.03369948, -0.014062854, -0.018557832, -0.008142329, + -0.030411726, 0.012926267, -0.012457506, 0.0070506916, -0.005262333, + 0.025711264, -0.0008395974, 0.00066341035, -0.014345396, -0.013587671, + 0.012515298, -0.017183652, -0.010409081, 0.03863111, 0.019456826, + 0.03701292, -0.00023919696, -0.012727205, 0.015719574, -0.012733626, + -0.025287453, -0.0035895596, -0.000015940612, 0.006440659, + -0.0045367153, 0.024452671, 0.00043665484, 0.009831156, 0.009555035, + -0.017877163, 0.0111411195, 0.007814838, 0.023181235, 0.006684672, + -0.01141724, 0.010556773, -0.0057439376, -0.00029177213, -0.046259727, + -0.00013384596, 0.014242654, -0.004915578, 0.014139911, -0.0013693625, + 0.010254968, -0.027586313, 0.010935635, -0.000024381234, 0.0048674173, + 0.0008516375, -0.009818313, -0.020908061, 0.013703257, 0.0039106295, + 0.008578983, -0.02454257, 0.010338445, -0.0054838713, -0.0071277483, + 0.005593035, 0.015411347, -0.00013163859, -0.0010306338, -0.003364811, + -0.016310344, 0.030617211, 0.0008267546, 0.013960112, 0.0068516284, + 0.0017947798, -0.006748886, 0.0009961189, -0.008778047, -0.0033198611, + 0.006357181, 0.0009616039, 0.004254174, -0.011160384, -0.027046915, + -0.0011285602, -0.012136436, 0.009587143, 0.011308076, -0.02375916, + 0.017466193, -0.031875804, 0.0040294253, -0.0069672135, -0.010742993, + -0.015758103, -0.0012385265, -0.0070506916, -0.0070250058, + -0.003586349, -0.023219764, -0.00589484, 0.008309285, -0.001671168, + -0.0031705636, 0.022282239, -0.028896276, 0.017812949, -0.026507517, + 0.008450556, 0.017761579, 0.009831156, -0.017299237, -0.008482663, + -0.024439828, -0.009696306, 0.006270492, -0.018583516, -0.0127785755, + -0.00028595273, -0.0065369797, -0.0029843433, 0.0051371157, + 0.0061452747, -0.0055962456, -0.028690793, -0.013998641, 0.008052429, + 0.0162076, -0.0019039436, 0.0016270209, -0.008964268, -0.026815744, + 0.009644935, -0.030000757, -0.0063475487, -0.0029201293, 0.010287074, + -0.00014347804, 0.012444663, 0.0059718974, 0.007879051, 0.013484929, + -0.008277178, -0.0037789908, -0.021845585, -0.015706731, 0.013472087, + -0.010158647, 0.00031605302, -0.01542419, 0.025120497, 0.010595301, + 0.008970688, -0.0067809927, 0.006257649, 0.039735593, -0.0028478885, + -0.0040390575, 0.031130921, -0.032492258, -0.0009118381, 0.025248924, + -0.011539247, 0.022885852, 0.007667145, -0.01040266, 0.0024786585, + 0.004986213, 0.009002795, -0.02053562, -0.021896956, -0.0066268793, + 0.00014387938, 0.009233966, -0.015706731, 0.00072280824, -0.022577625, + -0.007159855, 0.004498187, 0.01772305, 0.03195286, -0.011937373, + 0.021858428, -0.012219914, -0.021922642, 0.004713304, -0.007140591, + 0.0076864096, 0.0014175229, 0.02357936, 0.013125331, 0.023283977, + -0.0003373239, 0.011198913, 0.030180555, 0.008514769, 0.00069351064, + -0.0034001286, 0.011153962, -0.019649468, -0.016554356, -0.020599835, + 0.021575887, -0.021485986, -0.00589484, 0.014062854, 0.00007720724, + -0.019161442, 0.002610297, -0.005326547, -0.0006112365, 0.00076133665, + -0.016708469, 0.0022169866, -0.008367077, -0.024786584, -0.011340183, + 0.010030218, 0.005763202, -0.013960112, -0.0067553073, 0.0026761163, + -0.013960112, -0.0027066178, 0.0055127675, -0.0052527008, 0.004860996, + 0.015154492, 0.0148077365, 0.00003150497, 0.002297254, 0.0067553073, + 0.011385133, 0.0038945759, 0.020407192, -0.02509481, 0.015732417, + 0.009433029, -0.006055375, 0.040994186, 0.0023100968, 0.00244013, + -0.020098966, -0.00855972, 0.01615623, 0.026404776, -0.0014857502, + -0.0016101648, 0.0021158496, 0.010871422, -0.012720783, -0.006819521, + -0.009933898, -0.023052808, -0.018108333, 0.0015917033, 0.0031785904, + 0.013844527, 0.019854954, 0.028254136, 0.023656419, 0.015963588, + -0.01882753, -0.012091487, 0.007891894, 0.011873159, 0.0073589184, + -0.019906323, -0.02016318, 0.004337652, 0.00034193927, 0.006052165, + 0.015668204, 0.002435314, 0.008399185, 0.010306339, 0.0010266205, + -0.015437033, 0.015205863, -0.0013525062, 0.038682483, 0.00063772476, + -0.013613357, 0.026263505, 0.015437033, -0.016130544, 0.007236912, + 0.010980585, 0.012945532, 0.00870099, -0.008187278, 0.0048963134, + 0.014846264, 0.00009371223, 0.013908741, 0.012425399, -0.005069691, + 0.006800257, 0.018673416, -0.0005787282, 0.026327718, -0.020612678, + -0.024889326, -0.0035991918, 0.00019655489, -0.020009067, 0.017247867, + -0.022012541, 0.023682104, 0.0069029992, 0.023027122, -0.012130015, + 0.00515638, -0.0032556471, -0.0038656797, 0.013266602, -0.0029056813, + -0.00764146, -0.012123593, 0.011532825, -0.011218177, -0.008232228, + -0.019880638, 0.023463776, -0.00070153736, 0.009927477, -0.013780314, + -0.0299237, 0.023258291, -0.0045784544, -0.0044628694, 0.005943001, + -0.011590618, -0.004915578, -0.010010955, -0.009677042, 0.0016302316, + -0.016952483, 0.008212964, 0.03190149, -0.001942472, -0.007352497, + -0.021832742, 0.0034611318, -0.01542419, 0.020060437, -0.00354461, + -0.017003853, 0.015000379, 0.0107622575, -0.0011036772, 0.0069543705, + 0.00025886248, -0.0019986592, -0.0008460188, -0.02881922, 0.009047746, + -0.005981529, -0.013960112, 0.0029538416, 0.0071020625, 0.022539096, + -0.0022346454, 0.011083327, 0.019302713, -0.02301428, 0.022539096, + -0.0038271514, -0.0019167863, -0.018814687, -0.023283977, 0.01675984, + 0.0004017385, 0.029204503, -0.003692302, 0.0013059512, 0.005647617, + -0.011622725, -0.01786432, 0.008585405, -0.019867796, 0.0038817332, + 0.01330513, -0.0017113017, 0.016644256, 0.018609202, 0.0017899638, + 0.0047614644, 0.019765053, 0.019803582, -0.020599835, -0.006469555, + 0.020766791, 0.020240236, -0.011333762, 0.0098696835, 0.0020741106, + -0.0051082196, -0.00418996, 0.016002117, 0.000589163, 0.009876105, + 0.0019216024, 0.0006646144, 0.0029265506, -0.037321147, 0.015013221, + -0.04397371, -0.012919846, -0.027201029, 0.006524137, 0.019598097, + 0.012990482, 0.013343658, 0.016965326, -0.015912216, -0.0012192624, + 0.024465514, -0.010184332, -0.003393707, 0.00034033394, -0.01542419, + -0.004138589, 0.020381507, 0.0002448157, -0.0015892952, -0.01473068, + 0.011391554, -0.026790058, 0.0024979226, -0.014435296, 0.015270077, + 0.008456977, 0.0067553073, 0.0148077365, -0.0043023345, -0.021164916, + -0.0035638742, 0.025492938, -0.014666466, 0.0034290247, -0.0013444795, + -0.0136262, 0.0101265395, -0.01196948, 0.005464607, -0.0047582537, + -0.0033230719, -0.023309663, -0.01606633, 0.04068596, 0.005853101, + 0.009612828, -0.018378032, 0.009375236, 0.0071983836, 0.0012738442, + 0.02067689, -0.027611997, -0.011532825, -0.0127914185, 0.021049332, + 0.0039523686, 0.028202767, -0.012046536, -0.003480396, -0.019443983, + 0.014204125, 0.007583667, 0.0020468195, 0.0034418677, -0.023733474, + 0.02187127, -0.011834631, -0.0043472843, -0.0034482891, 0.0020933747, + 0.016014958, -0.01178968, 0.009105538, -0.011430083, -0.0010563195, + -0.017003853, -0.0036955127, -0.0072947047, 0.025261767, 0.009047746, + -0.0000065593545, -0.004902735, 0.001979395, -0.033853594, + -0.004860996, -0.0010466874, 0.034958072, 0.02158873, -0.031233665, + 0.025467252, -0.019765053, -0.011686939, 0.031053865, -0.015437033, + -0.011545667, -0.0046490896, 0.021216288, 0.007468082, -0.008822996, + -0.0016109674, 0.018018434, 0.0021030067, 0.0075194533, -0.017825792, + 0.022179497, 0.009368815, 0.00319946, -0.016811213, 0.044769965, + 0.0038817332, 0.015603989, 0.007435975, 0.008322128, 0.0150645925, + 0.01155851, -0.0010121723, -0.0043440736, 0.01758178, -0.00056187203, + 0.005474239, 0.0032668847, -0.009330287, 0.0092468085, -0.0070378487, + 0.009657778, -0.0021656153, 0.01684974, -0.016323186, 0.01952104, + 0.005474239, -0.0023999964, -0.0065626656, 0.021742843, -0.0046940395, + -0.0005245477, -0.017055225, 0.0011967875, -0.0029570523, + 0.0003947151, 0.013754628, 0.00065377826, -0.0018028066, 0.01408854, + -0.0062897564, 0.013106067, 0.017196495, -0.004912367, 0.021254817, + 0.00010986605, -0.0061934353, -0.016978167, 0.0026985912, + -0.013420715, -0.045412105, -0.00053177174, 0.009407343, -0.016169071, + -0.00051772495, -0.008302864, 0.026083706, 0.0028494939, -0.00621591, + 0.014859107, -0.02269321, 0.02361789, 0.018275289, -0.000034916335, + 0.0053426004, 0.0116548315, -0.03310871, 0.023322506, -0.0008917712, + -0.012457506, 0.010794365, -0.004254174, -0.012457506, 0.008925739, + -0.029307246, -0.0070185843, 0.014435296, 0.003968422, 0.003711566, + 0.007063534, -0.010068747, 0.0070506916, 0.0111411195, -0.0052366476, + 0.013343658, -0.009426608, -0.014949007, 0.0093559725, -0.008675304, + -0.0064502913, 0.016785527, 0.023630733, 0.0028783903, -0.006832364, + -0.016130544, 0.00354461, 0.0021479565, -0.0043344414, 0.0011751152, + 0.006832364, 0.0075451387, -0.015488404, -0.015642518, -0.0053104935, + -0.0076350383, -0.014139911, -0.020497093, 0.011776838, 0.0062897564, + 0.011109013, -0.010531087, 0.009028481, 0.012181386, 0.030052127, + 0.009786205, 0.00759651, 0.008341392, -0.01708091, 0.000027265844, + 0.020098966, -0.005349022, 0.013093224, 0.003250831, -0.02794591, + 0.023425248, 0.0053104935, -0.030694267, 0.008174435, -0.00064575154, + -0.0052462798, 0.009144066, 0.026456147, 0.00040213985, 0.00068026653, + 0.0035638742, -0.012207071, -0.0095743, 0.016888268, -0.001979395, + -0.0030036075, -0.0012730415, 0.02509481, 0.006414973, -0.0009455504, + 0.026212133, -0.006716779, -0.016913954, -0.010646673, -0.015180177, + 0.018146861, 0.03177306, 0.001321202, -0.0024931065, -0.03038604, + 0.014011484, -0.012907004, -0.0016872215, 0.015501247, 0.030000757, + -0.03359674, -0.02002191, -0.006652565, 0.0081102215, 0.022218026, + 0.006620458, -0.017915692, 0.0053233365, 0.005159591, 0.0006810692, + 0.0137931565, -0.020111809, -0.0056283525, 0.022757424, -0.019983381, + -0.02545441, 0.005689356, 0.030976808, 0.0358057, 0.0145508805, + -0.0162076, 0.02219234, 0.016053487, 0.0069543705, 0.005644406, + -0.00009918045, -0.010710887, 0.022076756, -0.0016398637, + -0.000030702293, 0.004382602, 0.0028623368, 0.014846264, 0.014486667, + 0.0011221387, -0.020522777, -0.006992899, -0.003978054, -0.027098287, + -0.007435975, 0.04107124, 0.002536451, 0.012130015, 0.0067360434, + -0.016079172, 0.0061902246, -0.014627937, -0.012091487, -0.0048192567, + -0.0060425326, -0.004655511, 0.03020624, -0.011173227, -0.027637683, + 0.009131223, 0.016464457, -0.014152754, -0.01708091, -0.008810153, + 0.006980056, 0.0068580494, -0.0102613885, 0.0036248774, -0.009799048, + 0.000051346076, -0.017761579, 0.008553298, 0.021884114, 0.028562363, + -0.01864773, -0.00018270875, -0.004658722, 0.029101761, -0.0026568521, + -0.019803582, -0.013446401, 0.032672055, -0.013317972, -0.003024477, + 0.013228074, -0.0023518358, 0.023361035, -0.013690414, -0.0015186599, + -0.0068580494, 0.017658835, 0.0043119667, 0.018390875, 0.016040644, + -0.0050568483, -0.0041032713, -0.004084007, -0.0132922875, + 0.0014624727, 0.0065658763, 0.018442245, -0.0041995924, 0.004816046, + -0.010203596, -0.021293346, -0.007744202, 0.00621591, 0.0030726374, + 0.0022442774, -0.012412556, 0.013202388, 0.0033327038, -0.013131753, + 0.021100704, -0.002896049, -0.017222181, 0.023078494, -0.00488026, + -0.007821259, -0.011391554, -0.020098966, -0.0025252134, 0.0012425398, + -0.012643727, 0.026250662, -0.012393292, 0.005451764, 0.007269019, + 0.010742993, 0.009792627, -0.020227393, -0.013574828, 0.0045046085, + 0.00980547, -0.017196495, 0.0014424059, -0.0070121633, -0.013754628, + 0.0072818617, 0.0036505628, -0.00188789, 0.0050568483, 0.024504043, + 0.0033391253, 0.0043729697, 0.0014175229, 0.0021286923, 0.0066268793, + 0.011391554, 0.0023502305, 0.010768679, -0.01510312, -0.004774307, + 0.00529444, -0.00446608, -0.0021880902, 0.0033070184, -0.022641838, + -0.00024060164, -0.0072433334, 0.009966005, 0.0039844755, + -0.0071855406, 0.00036381214, 0.011950215, 0.026186448, 0.0044403947, + -0.0008893632, 0.021601573, 0.013407872, 0.010935635, 0.002385548, + -0.009272494, -0.0122777065, -0.009728413, -0.00042622007, 0.00704427, + 0.0065498226, -0.008887211, 0.016862582, -0.0067745717, 0.015989274, + 0.013151017, -0.015501247, -0.011076906, -0.012547405, -0.010113697, + 0.005763202, -0.015681047, 0.0036730377, -0.0021431404, 0.0002586618, + -0.0034771853, -0.0140500115, -0.006556244, -0.0041450104, + -0.005612299, -0.028279822, -0.019174285, -0.005538453, 0.008636776, + 0.0136262, -0.022320768, 0.009856841, 0.0020981908, 0.017620306, + 0.0076543028, 0.0067103575, -0.003933104, -0.0038496263, 0.00870099, + 0.035934124, -0.0137931565, 0.00043705618, 0.0036377201, + -0.0065755085, -0.01649014, -0.0010370553, -0.022115285, 0.0018429403, + 0.012136436, 0.00056387874, -0.005349022, 0.0011863527, -0.0030501625, + 0.0015170545, -0.012829946, 0.01408854, 0.0016310343, 0.022205183, + -0.017183652, 0.02016318, 0.003310229, 0.008135907, -0.004369759, + 0.01026781, 0.008270756, -0.008649618, 0.011635567, 0.031747375, + -0.013549143, 0.0191486, 0.021357559, 0.0074038683, 0.03665332, + -0.00976052, -0.016580041, -0.012894161, -0.013240916, -0.018596359, + -0.0014351817, 0.0082193855, -0.0053650755, 0.008546877, + -0.0015981246, 0.009856841, 0.005650827, 0.0070250058, -0.0028783903, + -0.0081937, 0.022474881, -0.006100325, -0.007525875, 0.007468082, + 0.010479717, -0.0060457434, 0.017389137, 0.015693888, 0.0093559725, + -0.001772305, 0.0072754403, 0.015796632, 0.012380449, -0.005666881, + 0.014037169, -0.0077506234, -0.023219764, 0.009041324, 0.013106067, + 0.04084007, 0.006418184, 0.009182595, 0.021563044, 0.009458714, + -0.010768679, 0.0021351138, 0.009362394, -0.0031224033, -0.021665785, + 0.015899373, 0.009009217, 0.0010587275, 0.0037436732, 0.014679309, + 0.0025123707, -0.005782466, 0.016721312, -0.01082005, -0.0031159818, + 0.011404397, 0.008675304, -0.009144066, -0.0038014657, -0.0032379883, + 0.0040679537, -0.004523873, -0.01961094, -0.0005590627, -0.014974693, + -0.0055641388, -0.026635945, -0.006703936, -0.0011759179, + -0.008412027, 0.030540153, -0.0023550466, -0.0067938357, + -0.0011052826, 0.0007472898, 0.0070956415, -0.024915012, -0.029076075, + 0.0044275518, 0.026289191, 0.004568822, 0.001528292, -0.026083706, + -0.00708922, -0.0025717686, 0.018544989, 0.008026743, -0.0024240764, + -0.0009038113, -0.011340183, -0.033365566, -0.00055665465, + 0.025839692, 0.02748357, -0.009933898, 0.008604669, -0.009471557, + 0.014627937, -0.009477979, 0.0064053414, 0.0010571221, -0.003618456, + 0.014846264, 0.022423511, -0.013818841, 0.0059044724, -0.014974693, + 0.019212814, 0.018211076, 0.0023775215, -0.018416561, -0.023271134, + 0.011905266, -0.021190602, 0.0028992598, -0.0032973862, 0.004084007, + -0.019508198, -0.001153443, 0.00041377862, 0.011731888, -0.011609882, + -0.009150487, 0.015115963, 0.008386342, 0.027509255, -0.011757574, + -0.006411763, 0.019919166, -0.005085745, 0.0041000606, 0.021062175, + 0.016284658, -0.009458714, 0.008142329, 0.0012634094, 0.024118759, + 0.002284411, -0.0056925663, -0.021563044, 0.023399564, -0.0043986556, + 0.008200121, 0.004568822, 0.015475562, -0.00055665465, -0.0018204654, + -0.0012682255, 0.020638363, -0.0022860165, -0.015693888, -0.018557832, + -0.013960112, 0.0028238085, 0.009426608, -0.015732417, -0.007493768, + -0.015372819, -0.02591675, -0.0047454108, -0.011564932, 0.002412839, + 0.017106596, -0.0061805923, -0.013382187, 0.0162975, -0.0093816575, + -0.021845585, -0.00038668836, -0.0027852799, -0.024324244, + -0.025236081, -0.036756065, -0.010698044, 0.00003102838, 0.0014014695, + -0.00071839354, -0.0140500115, 0.044769965, -0.018442245, + -0.023643576, 0.0358057, 0.009966005, -0.0071662767, -0.003949158, + 0.0013725731, -0.010415503, -0.0020548464, -0.025724107, -0.036910176, + -0.039838333, -0.011211756, -0.0043729697, 0.003945947, 0.02702123, + 0.0037083556, 0.012977639, 0.002859126, -0.012065801, -0.010190753, + 0.0020933747, -0.0097476775, 0.007757045, 0.021357559, -0.0061324323, + 0.0030373198, 0.010370553, 0.008347813, 0.031644635, -0.025608523, + -0.02108786, 0.011629146, 0.0081102215, -0.008168014, 0.0014424059, + 0.020432878, -0.0059879506, 0.0005377918, -0.015270077, 0.0143068675, + -0.007583667, -0.005589824, -0.01173831, -0.010004533, 0.022115285, + -0.003907419, -0.00093110226, -0.0013525062, -0.013690414, + 0.0017674889, 0.017273553, -0.016310344, -0.00515638, 0.021357559, + 0.02661026, 0.0034258142, 0.023232607, 0.026687317, 0.017761579, + 0.023836218, -0.009921055, 0.00069110264, -0.0065658763, -0.010685201, + 0.006440659, 0.001574847, 0.011975901, 0.00037906296, -0.003711566, + 0.0037372517, 0.013356501, 0.005888419, 0.016836897, -0.0022956487, + -0.006819521, 0.023258291, -0.002827019, -0.017684521, 0.011841052, + -0.018018434, 0.011173227, 0.009580721, 0.021614416, -0.004469291, + 0.01758178, -0.02062552, -0.026867116, -0.016400242, 0.0022812006, + 0.0026167184, -0.0036890914, 0.0059751077, 0.016592884, 0.010511823, + 0.0051114303, 0.018878901, 0.0015395294, -0.0076029315, -0.015128806, + 0.0024369194, 0.019212814, -0.025287453, -0.007988215, -0.0031384567, + -0.005082534, 0.016258972, 0.022616152, -0.0055448744, 0.010325603, + 0.020548463, 0.0024850797, 0.029641159, -0.00092387816, -0.005952633, + 0.022847323, 0.0047935713, -0.002550899, 0.010441188, -0.016220443, + -0.009137644, -0.015026064, -0.0075387177, -0.016284658, + 0.000010064784, 0.00032026708, 0.006559455, -0.022577625, + 0.0017498301, -0.0059590545, -0.023091335, -0.011545667, 0.012329078, + -0.0025541098, -0.013446401, 0.029692529, 0.025569994, 0.023797689, + 0.004138589, -0.008052429, 0.010088012, -0.003300597, -0.008071694, + -0.0041995924, -0.010100854, -0.0056411955, -0.01496185, 0.008534034, + -0.0018670205, -0.0155783035, 0.008347813, 0.0027001966, + -0.0087202545, 0.028716477, -0.002127087, -0.013677571, 0.0031641424, + -0.032851856, -0.0003864877, 0.001675984, -0.003329493, -0.004751832, + -0.015488404, 0.022140969, -0.012123593, -0.006501662, 0.0069543705, + 0.013459244, 0.010017375, -0.008437713, -0.010152225, -0.007936844, + 0.005377918, 0.024902169, 0.0060939034, 0.019187128, 0.0040711644, + 0.016181914, 0.0011076906, 0.005878787, -0.03899071, 0.007879051, + 0.01058888, 0.024889326, 0.0021977224, 0.00082434656, -0.004751832, + -0.012091487, 0.016567199, 0.019970538, 0.011012692, 0.0046908287, + 0.0014536432, -0.010332025, 0.010120118, -0.0358057, -0.008161593, + -0.00049725675, 0.016772684, -0.017055225, 0.013459244, 0.037732117, + -0.014371081, 0.0140500115, -0.009522929, -0.0024882904 + ] + } + ] + } ] diff --git a/src/scrapers/document-legacy/playground/legacy-document-scraper-playground.ts b/src/scrapers/document-legacy/playground/legacy-document-scraper-playground.ts index 05d63bf8d4..725181da4a 100644 --- a/src/scrapers/document-legacy/playground/legacy-document-scraper-playground.ts +++ b/src/scrapers/document-legacy/playground/legacy-document-scraper-playground.ts @@ -1,77 +1,79 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { writeFile } from 'fs/promises'; -import { join } from 'path'; -import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode'; -import { usageToHuman } from '../../../execution/utils/usageToHuman'; -import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground'; -import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson'; -import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode'; -import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler'; -import { LegacyDocumentScraper } from '../LegacyDocumentScraper'; +import { join } from "path"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { writeFile } from "fs/promises"; +import { $provideExecutablesForNode } from "../../../executables/$provideExecutablesForNode"; +import { usageToHuman } from "../../../execution/utils/usageToHuman"; +import { $provideLlmToolsForTestingAndScriptsAndPlayground } from "../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground"; +import { stringifyPipelineJson } from "../../../utils/editable/utils/stringifyPipelineJson"; +import { $provideFilesystemForNode } from "../../_common/register/$provideFilesystemForNode"; +import { makeKnowledgeSourceHandler } from "../../_common/utils/makeKnowledgeSourceHandler"; +import { LegacyDocumentScraper } from "../LegacyDocumentScraper"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 Scrape knowledge from legacy documents (playground)`); + console.info(`🧸 Scrape knowledge from legacy documents (playground)`); - // Do here stuff you want to test - //========================================> + // Do here stuff you want to test + //========================================> - //const example = '10-simple.doc'; - const example = '10-simple.rtf'; - // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them + //const example = '10-simple.doc'; + const example = "10-simple.rtf"; + // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them - const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true }); - const rootDirname = join(__dirname, '..', 'examples'); + const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ + isCacheReloaded: true, + }); + const rootDirname = join(__dirname, "..", "examples"); - const legacyDocumentScraper = new LegacyDocumentScraper( - { - fs: $provideFilesystemForNode(), - llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), - executables: await $provideExecutablesForNode(), - }, - { - rootDirname, - }, - ); + const legacyDocumentScraper = new LegacyDocumentScraper( + { + fs: $provideFilesystemForNode(), + llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), + executables: await $provideExecutablesForNode(), + }, + { + rootDirname, + }, + ); - const knowledge = await legacyDocumentScraper.scrape( - await makeKnowledgeSourceHandler( - { knowledgeSourceContent: example }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ); + const knowledge = await legacyDocumentScraper.scrape( + await makeKnowledgeSourceHandler( + { knowledgeSourceContent: example }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ); - console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); - console.info(colors.bgGreen(' Knowledge: ')); - console.info(knowledge); + console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); + console.info(colors.bgGreen(" Knowledge: ")); + console.info(knowledge); - await writeFile( - join( - __dirname, - `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, - ), - stringifyPipelineJson(knowledge), - 'utf-8', - ); - /**/ + await writeFile( + join( + __dirname, + `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, + ), + stringifyPipelineJson(knowledge), + "utf-8", + ); + /**/ - //========================================/ + //========================================/ } /** diff --git a/src/scrapers/document-legacy/playground/tsconfig.json b/src/scrapers/document-legacy/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/scrapers/document-legacy/playground/tsconfig.json +++ b/src/scrapers/document-legacy/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/scrapers/document-legacy/register-constructor.ts b/src/scrapers/document-legacy/register-constructor.ts index bf2c5a2e00..18ee373ab6 100644 --- a/src/scrapers/document-legacy/register-constructor.ts +++ b/src/scrapers/document-legacy/register-constructor.ts @@ -1,6 +1,6 @@ -import type { Registration } from '../../utils/$Register'; -import { $scrapersRegister } from '../_common/register/$scrapersRegister'; -import { createLegacyDocumentScraper } from './createLegacyDocumentScraper'; +import type { Registration } from "../../utils/$Register"; +import { $scrapersRegister } from "../_common/register/$scrapersRegister"; +import { createLegacyDocumentScraper } from "./createLegacyDocumentScraper"; /** * Registration of known scraper @@ -11,7 +11,8 @@ import { createLegacyDocumentScraper } from './createLegacyDocumentScraper'; * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _LegacyDocumentScraperRegistration: Registration = $scrapersRegister.register(createLegacyDocumentScraper); +export const _LegacyDocumentScraperRegistration: Registration = + $scrapersRegister.register(createLegacyDocumentScraper); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/scrapers/document-legacy/register-metadata.ts b/src/scrapers/document-legacy/register-metadata.ts index dfaed589cd..1c39705d76 100644 --- a/src/scrapers/document-legacy/register-metadata.ts +++ b/src/scrapers/document-legacy/register-metadata.ts @@ -1,8 +1,8 @@ -import type { Registration } from '../../utils/$Register'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import { $deepFreeze } from '../../utils/serialization/$deepFreeze'; -import { $scrapersMetadataRegister } from '../_common/register/$scrapersMetadataRegister'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; +import type { Registration } from "../../utils/$Register"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import { $deepFreeze } from "../../utils/serialization/$deepFreeze"; +import { $scrapersMetadataRegister } from "../_common/register/$scrapersMetadataRegister"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; keepTypeImported(); @@ -12,18 +12,18 @@ keepTypeImported(); * @private within the scraper directory */ export const legacyDocumentScraperMetadata = $deepFreeze({ - title: 'LegacyDocument scraper', - packageName: '@promptbook/legacy-documents', - className: 'LegacyDocumentScraper', - mimeTypes: ['application/msword', 'text/rtf'], - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@', - isAvilableInBrowser: false, - // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server - requiredExecutables: [ - 'Pandoc', - 'LibreOffice', - // <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency - ], + title: "LegacyDocument scraper", + packageName: "@promptbook/legacy-documents", + className: "LegacyDocumentScraper", + mimeTypes: ["application/msword", "text/rtf"], + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@", + isAvilableInBrowser: false, + // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server + requiredExecutables: [ + "Pandoc", + "LibreOffice", + // <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency + ], }) satisfies ScraperAndConverterMetadata; /* <- Note: [🤛] */ /** @@ -36,7 +36,7 @@ export const legacyDocumentScraperMetadata = $deepFreeze({ * @public exported from `@promptbook/cli` */ export const _LegacyDocumentScraperMetadataRegistration: Registration = - $scrapersMetadataRegister.register(legacyDocumentScraperMetadata); + $scrapersMetadataRegister.register(legacyDocumentScraperMetadata); /** * Note: [💞] Ignore a discrepancy between file name and entity name diff --git a/src/scrapers/document/DocumentScraper.test.ts b/src/scrapers/document/DocumentScraper.test.ts index fb83d95125..2f0fedc5d9 100644 --- a/src/scrapers/document/DocumentScraper.test.ts +++ b/src/scrapers/document/DocumentScraper.test.ts @@ -1,88 +1,94 @@ -import { describe, expect, it } from '@jest/globals'; -import { join } from 'path'; -import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode'; -import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground'; -import { $provideFilesystemForNode } from '../_common/register/$provideFilesystemForNode'; -import { makeKnowledgeSourceHandler } from '../_common/utils/makeKnowledgeSourceHandler'; -import { DocumentScraper } from './DocumentScraper'; +import { join } from "path"; +import { describe, expect, it } from "@jest/globals"; +import { $provideExecutablesForNode } from "../../executables/$provideExecutablesForNode"; +import { $provideLlmToolsForTestingAndScriptsAndPlayground } from "../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground"; +import { $provideFilesystemForNode } from "../_common/register/$provideFilesystemForNode"; +import { makeKnowledgeSourceHandler } from "../_common/utils/makeKnowledgeSourceHandler"; +import { DocumentScraper } from "./DocumentScraper"; -describe('how creating knowledge from docx works', () => { - const rootDirname = join(__dirname, 'examples'); +describe("how creating knowledge from docx works", () => { + const rootDirname = join(__dirname, "examples"); - const documentScraperPromise = (async () => - new DocumentScraper( - { - fs: $provideFilesystemForNode(), - llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), - executables: await $provideExecutablesForNode(), - }, - { - rootDirname, - }, - ))(); + const documentScraperPromise = (async () => + new DocumentScraper( + { + fs: $provideFilesystemForNode(), + llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), + executables: await $provideExecutablesForNode(), + }, + { + rootDirname, + }, + ))(); - it('should scrape simple information from a .docx file', () => - expect( - Promise.all([ - documentScraperPromise, - makeKnowledgeSourceHandler( - { - knowledgeSourceContent: '10-simple.docx', - }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ]) - .then(([documentScraper, sourceHandler]) => documentScraper.scrape(sourceHandler)) - .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) - .then((knowledge) => knowledge?.slice(0, 1)), - ).resolves.toMatchObject([ - { - content: expect.stringMatching(/Springfield (is )?.*/i), - }, - ])); + it("should scrape simple information from a .docx file", () => + expect( + Promise.all([ + documentScraperPromise, + makeKnowledgeSourceHandler( + { + knowledgeSourceContent: "10-simple.docx", + }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ]) + .then(([documentScraper, sourceHandler]) => + documentScraper.scrape(sourceHandler), + ) + .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) + .then((knowledge) => knowledge?.slice(0, 1)), + ).resolves.toMatchObject([ + { + content: expect.stringMatching(/Springfield (is )?.*/i), + }, + ])); - it('should scrape simple information from a .odt file', () => - expect( - Promise.all([ - documentScraperPromise, - makeKnowledgeSourceHandler( - { - knowledgeSourceContent: '10-simple.odt', - }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ]) - .then(([documentScraper, sourceHandler]) => documentScraper.scrape(sourceHandler)) - .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) - .then((knowledge) => knowledge?.slice(0, 1)), - ).resolves.toMatchObject([ - { - content: expect.stringMatching(/Springfield (is )?.*/i), - }, - ])); + it("should scrape simple information from a .odt file", () => + expect( + Promise.all([ + documentScraperPromise, + makeKnowledgeSourceHandler( + { + knowledgeSourceContent: "10-simple.odt", + }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ]) + .then(([documentScraper, sourceHandler]) => + documentScraper.scrape(sourceHandler), + ) + .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) + .then((knowledge) => knowledge?.slice(0, 1)), + ).resolves.toMatchObject([ + { + content: expect.stringMatching(/Springfield (is )?.*/i), + }, + ])); - it('should NOT scrape irrelevant information', () => - expect( - Promise.all([ - documentScraperPromise, - makeKnowledgeSourceHandler( - { - knowledgeSourceContent: '10-simple.docx', - }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ]) - .then(([documentScraper, sourceHandler]) => documentScraper.scrape(sourceHandler)) - .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) - .then((knowledge) => knowledge?.slice(0, 1)), - ).resolves.toMatchObject([ - { - content: expect.not.stringMatching(/London (is )?.*/i), - }, - ])); + it("should NOT scrape irrelevant information", () => + expect( + Promise.all([ + documentScraperPromise, + makeKnowledgeSourceHandler( + { + knowledgeSourceContent: "10-simple.docx", + }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ]) + .then(([documentScraper, sourceHandler]) => + documentScraper.scrape(sourceHandler), + ) + .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) + .then((knowledge) => knowledge?.slice(0, 1)), + ).resolves.toMatchObject([ + { + content: expect.not.stringMatching(/London (is )?.*/i), + }, + ])); }); /** diff --git a/src/scrapers/document/DocumentScraper.ts b/src/scrapers/document/DocumentScraper.ts index 4b7a686c70..ee29b18139 100644 --- a/src/scrapers/document/DocumentScraper.ts +++ b/src/scrapers/document/DocumentScraper.ts @@ -1,29 +1,29 @@ -import type { KnowledgePiecePreparedJson } from '../../pipeline/PipelineJson/KnowledgePieceJson'; // TODO: [🏳‍🌈] Finally take pick of .json vs .ts // import PipelineCollection from '../../../books/books'; -import { readFile } from 'fs/promises'; -import spaceTrim from 'spacetrim'; -import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from '../../config'; -import { DEFAULT_IS_VERBOSE } from '../../config'; -import { DEFAULT_SCRAPE_CACHE_DIRNAME } from '../../config'; -import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError'; -import { KnowledgeScrapeError } from '../../errors/KnowledgeScrapeError'; -import { MissingToolsError } from '../../errors/MissingToolsError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { $isRunningInNode } from '../../utils/environment/$isRunningInNode'; -import { $execCommand } from '../../utils/execCommand/$execCommand'; -import { getFileExtension } from '../../utils/files/getFileExtension'; -import { isFileExisting } from '../../utils/files/isFileExisting'; -import type { Converter } from '../_common/Converter'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; -import type { Scraper } from '../_common/Scraper'; -import type { ScraperSourceHandler } from '../_common/Scraper'; -import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource'; -import { getScraperIntermediateSource } from '../_common/utils/getScraperIntermediateSource'; -import { MarkdownScraper } from '../markdown/MarkdownScraper'; -import { documentScraperMetadata } from './register-metadata'; +import { readFile } from "fs/promises"; +import spaceTrim from "spacetrim"; +import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from "../../config"; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import { DEFAULT_SCRAPE_CACHE_DIRNAME } from "../../config"; +import { EnvironmentMismatchError } from "../../errors/EnvironmentMismatchError"; +import { KnowledgeScrapeError } from "../../errors/KnowledgeScrapeError"; +import { MissingToolsError } from "../../errors/MissingToolsError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { KnowledgePiecePreparedJson } from "../../pipeline/PipelineJson/KnowledgePieceJson"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { $isRunningInNode } from "../../utils/environment/$isRunningInNode"; +import { $execCommand } from "../../utils/execCommand/$execCommand"; +import { getFileExtension } from "../../utils/files/getFileExtension"; +import { isFileExisting } from "../../utils/files/isFileExisting"; +import type { Converter } from "../_common/Converter"; +import type { Scraper } from "../_common/Scraper"; +import type { ScraperSourceHandler } from "../_common/Scraper"; +import type { ScraperIntermediateSource } from "../_common/ScraperIntermediateSource"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; +import { getScraperIntermediateSource } from "../_common/utils/getScraperIntermediateSource"; +import { MarkdownScraper } from "../markdown/MarkdownScraper"; +import { documentScraperMetadata } from "./register-metadata"; /** * Scraper of .docx and .odt files @@ -32,77 +32,87 @@ import { documentScraperMetadata } from './register-metadata'; * @public exported from `@promptbook/documents` */ export class DocumentScraper implements Converter, Scraper { - /** - * Metadata of the scraper which includes title, mime types, etc. - */ - public get metadata(): ScraperAndConverterMetadata { - return documentScraperMetadata; - } - - /** - * Markdown scraper is used internally - */ - private readonly markdownScraper: MarkdownScraper; - - public constructor( - private readonly tools: Pick, - private readonly options: PrepareAndScrapeOptions, - ) { - this.markdownScraper = new MarkdownScraper(tools, options); - } - - /** - * Convert the `.docx` or `.odt` to `.md` file and returns intermediate source - * - * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object - */ - public async $convert(source: ScraperSourceHandler): Promise { - const { - rootDirname = process.cwd(), - cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, - intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, - isVerbose = DEFAULT_IS_VERBOSE, - } = this.options; - - if (!$isRunningInNode()) { - throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment'); - } - - if (this.tools.fs === undefined) { - throw new EnvironmentMismatchError('Can not scrape documents without filesystem tools'); - // <- TODO: [🧠] What is the best error type here` - } - - if (this.tools.executables?.pandocPath === undefined) { - throw new MissingToolsError('Pandoc is required for scraping .docx files'); - } - - if (source.filename === null) { - // TODO: [🧠] Maybe save file as temporary - throw new KnowledgeScrapeError('When parsing .docx file, it must be real file in the file system'); - } - - const extension = getFileExtension(source.filename); - - const cacheFilehandler = await getScraperIntermediateSource(source, { - rootDirname, - cacheDirname, - intermediateFilesStrategy, - extension: 'md', - isVerbose, - }); - - // Note: Running Pandoc ONLY if the file in the cache does not exist - if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { - const command = `"${this.tools.executables.pandocPath}" -f ${extension} -t markdown "${source.filename}" -o "${cacheFilehandler.filename}"`; - - await $execCommand(command); - - // Note: [0] - if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + /** + * Metadata of the scraper which includes title, mime types, etc. + */ + public get metadata(): ScraperAndConverterMetadata { + return documentScraperMetadata; + } + + /** + * Markdown scraper is used internally + */ + private readonly markdownScraper: MarkdownScraper; + + public constructor( + private readonly tools: Pick, + private readonly options: PrepareAndScrapeOptions, + ) { + this.markdownScraper = new MarkdownScraper(tools, options); + } + + /** + * Convert the `.docx` or `.odt` to `.md` file and returns intermediate source + * + * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object + */ + public async $convert( + source: ScraperSourceHandler, + ): Promise { + const { + rootDirname = process.cwd(), + cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, + intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, + isVerbose = DEFAULT_IS_VERBOSE, + } = this.options; + + if (!$isRunningInNode()) { + throw new KnowledgeScrapeError( + "Scraping .docx files is only supported in Node environment", + ); + } + + if (this.tools.fs === undefined) { + throw new EnvironmentMismatchError( + "Can not scrape documents without filesystem tools", + ); + // <- TODO: [🧠] What is the best error type here` + } + + if (this.tools.executables?.pandocPath === undefined) { + throw new MissingToolsError( + "Pandoc is required for scraping .docx files", + ); + } + + if (source.filename === null) { + // TODO: [🧠] Maybe save file as temporary + throw new KnowledgeScrapeError( + "When parsing .docx file, it must be real file in the file system", + ); + } + + const extension = getFileExtension(source.filename); + + const cacheFilehandler = await getScraperIntermediateSource(source, { + rootDirname, + cacheDirname, + intermediateFilesStrategy, + extension: "md", + isVerbose, + }); + + // Note: Running Pandoc ONLY if the file in the cache does not exist + if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { + const command = `"${this.tools.executables.pandocPath}" -f ${extension} -t markdown "${source.filename}" -o "${cacheFilehandler.filename}"`; + + await $execCommand(command); + + // Note: [0] + if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { + throw new UnexpectedError( + spaceTrim( + (block) => ` File that was supposed to be created by Pandoc does not exist for unknown reason Expected file: @@ -112,37 +122,39 @@ export class DocumentScraper implements Converter, Scraper { > ${block(command)} `, - ), - ); - } - } - - return cacheFilehandler; - } - - /** - * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it - */ - public async scrape( - source: ScraperSourceHandler, - ): Promise> | null> { - const cacheFilehandler = await this.$convert(source); - - const markdownSource = { - source: source.source, - filename: cacheFilehandler.filename, - url: null, - mimeType: 'text/markdown', - async asText() { - // Note: [0] In $convert we check that the file exists - return await readFile(cacheFilehandler.filename, 'utf-8'); - }, - asJson() { - throw new UnexpectedError( - 'Did not expect that `markdownScraper` would need to get the content `asJson`', - ); - }, - /* + ), + ); + } + } + + return cacheFilehandler; + } + + /** + * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it + */ + public async scrape( + source: ScraperSourceHandler, + ): Promise + > | null> { + const cacheFilehandler = await this.$convert(source); + + const markdownSource = { + source: source.source, + filename: cacheFilehandler.filename, + url: null, + mimeType: "text/markdown", + async asText() { + // Note: [0] In $convert we check that the file exists + return await readFile(cacheFilehandler.filename, "utf-8"); + }, + asJson() { + throw new UnexpectedError( + "Did not expect that `markdownScraper` would need to get the content `asJson`", + ); + }, + /* TODO: [🥽] > asBlob() { > throw new UnexpectedError( @@ -150,14 +162,14 @@ export class DocumentScraper implements Converter, Scraper { > ); > }, */ - } satisfies ScraperSourceHandler; + } satisfies ScraperSourceHandler; - const knowledge = this.markdownScraper.scrape(markdownSource); + const knowledge = this.markdownScraper.scrape(markdownSource); - await cacheFilehandler.destroy(); + await cacheFilehandler.destroy(); - return knowledge; - } + return knowledge; + } } /** diff --git a/src/scrapers/document/createDocumentScraper.ts b/src/scrapers/document/createDocumentScraper.ts index 2b0d97ce84..83cfe1fcec 100644 --- a/src/scrapers/document/createDocumentScraper.ts +++ b/src/scrapers/document/createDocumentScraper.ts @@ -1,9 +1,9 @@ -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import type { ScraperConstructor } from '../_common/register/ScraperConstructor'; -import { DocumentScraper } from './DocumentScraper'; -import { documentScraperMetadata } from './register-metadata'; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import type { ScraperConstructor } from "../_common/register/ScraperConstructor"; +import { DocumentScraper } from "./DocumentScraper"; +import { documentScraperMetadata } from "./register-metadata"; keepTypeImported(); @@ -13,10 +13,13 @@ keepTypeImported(); * @public exported from `@promptbook/documents` */ export const createDocumentScraper = Object.assign( - (tools: Pick, options: PrepareAndScrapeOptions): DocumentScraper => { - return new DocumentScraper(tools, options); - }, - documentScraperMetadata, + ( + tools: Pick, + options: PrepareAndScrapeOptions, + ): DocumentScraper => { + return new DocumentScraper(tools, options); + }, + documentScraperMetadata, ) satisfies ScraperConstructor; /* <- Note: [🤛] */ /** diff --git a/src/scrapers/document/examples/10-simple.docx.knowledge.json b/src/scrapers/document/examples/10-simple.docx.knowledge.json index 2d0efd7a62..94aa70b1f0 100644 --- a/src/scrapers/document/examples/10-simple.docx.knowledge.json +++ b/src/scrapers/document/examples/10-simple.docx.knowledge.json @@ -1,44 +1,1286 @@ [ - { - "name": "springfield-illinois-sixth-largest-city", - "title": "Springfield: Illinois' Sixth-Largest City", - "content": "---\n\nSpringfield is the sixth most populous city in Illinois, with a population of 10,566 as of 2019.", - "keywords": [ - "Springfield", - "Illinois", - "population", - "sixth most populous", - "10", - "566", - "2019" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.019281438, 0.03617381, -0.00036933212, -0.017628398, 0.008584945, 0.00047773743, 0.0048957714, 0.0038249157, -0.043123815, 0.025193768, -0.020825882, 0.013731086, 0.0000029840915, 0.016445931, 0.002387556, 0.018050706, -0.02086208, 0.03421912, 0.00035802028, -0.006817279, 0.006998269, -0.014418847, -0.045802463, 0.035787698, 0.025604011, 0.0106059965, -0.027220853, 0.008307427, -0.025628142, -0.008192801, -0.022744372, -0.004910854, 0.057723653, -0.0066423225, 0.0073843803, -0.006720751, 0.0035654972, -0.020138122, 0.018822929, 0.03520853, 0.014225791, -0.0046061883, 0.015106607, 0.010467238, 0.038852457, -0.03940749, 0.00032559293, -0.027389776, 0.02311842, 0.008759902, 0.024337083, -0.018219631, -0.024904184, 0.042979024, -0.00092606404, -0.02719672, 0.0076980954, 0.024216423, -0.0077825575, -0.039866, -0.0019878703, -0.007873053, 0.012874401, -0.0053391964, -0.0017209104, 0.006992236, 0.00855478, -0.005390477, 0.06438407, 0.012645148, 0.010768888, 0.046550553, 0.011577308, 0.03322971, 0.0580615, 0.0027631095, -0.017664596, -0.0023739818, -0.03272294, 0.032843597, 0.01569784, -0.0064311675, -0.003375458, 0.00035670056, 0.009477828, 0.029851235, 0.009785511, -0.06781081, -0.0022563383, 0.010907646, 0.025676407, -0.0035956623, -0.0011651212, -0.036077283, -0.043702982, -0.023033956, 0.032940127, -0.005897248, 0.009628653, 0.020596629, 0.017326748, -0.0029772806, -0.010961943, 0.0252179, 0.00039855443, 0.009628653, -0.018448884, 0.0061204685, -0.027462171, 0.0050586625, 0.01037071, 0.0020406588, -0.0122711025, 0.012072014, -0.0104008755, -0.00574944, -0.025386823, -0.007517106, 0.04194135, 0.021260258, 0.0005467397, 0.021935953, -0.025604011, 0.018497149, -0.0070404997, -0.010340545, 0.02104307, -0.013260513, 0.0032879796, -0.045054372, 0.03433978, 0.0054990705, -0.020500101, 0.010636162, 0.012150442, -0.03916617, -0.011848793, 0.016856175, 0.02780002, 0.020934476, -0.05941289, -0.006720751, 0.019546889, 0.0050043655, -0.0000790416, -0.0149135515, -0.012536554, 0.008802133, 0.009616586, -0.029272068, -0.014515375, -0.005915347, -0.023709651, 0.0044372645, 0.00055842864, 0.03151634, -0.0018596692, -0.013863811, 0.04712972, -0.052607674, -0.012922665, -0.021935953, 0.027462171, -0.03800784, -0.022647845, -0.010750788, -0.0077342936, -0.0003295521, 0.0028717034, 0.0028792445, 0.022611648, -0.013694888, -0.06385317, -0.0363186, -0.0074085123, -0.010587898, -0.033519294, -0.01605982, -0.010931779, 0.030430403, -0.002846063, -0.00054447737, 0.030719986, -0.003909378, -0.019281438, -0.01827996, 0.0065337284, 0.008023877, 0.026883004, -0.014346451, -0.0033845075, 0.044619996, 0.0050526294, -0.005483988, 0.040155582, -0.00033313417, -0.009948401, -0.027534567, -0.004769079, 0.024035433, -0.044837184, 0.0054658893, -0.03788718, -0.024952447, 0.0059847264, 0.0187626, -0.0014675249, -0.0018822929, 0.02128439, 0.020319112, -0.0063587716, -0.0076317326, -0.025314428, 0.026521025, -0.011806562, 0.022020414, -0.009972533, -0.00007545951, -0.019040117, 0.019969197, 0.014237857, 0.03800784, -0.000021139032, -0.028934222, -0.033374503, -0.02071729, -0.045512877, 0.034508705, -0.021960085, -0.03762173, 0.02656929, 0.027341511, -0.013091589, 0.054441705, -0.011378219, 0.047443435, 0.013525964, 0.004253258, -0.014853222, 0.034750026, 0.025169635, 0.0026846807, 0.0033483095, -0.015215201, 0.0012767316, -0.019727878, 0.02524203, -0.014274055, 0.016204612, 0.01729055, 0.01148078, -0.008295362, 0.054683022, -0.005987743, -0.00873577, 0.014515375, -0.013188117, 0.021272324, 0.00019531806, -0.0066543883, -0.022261733, -0.0021658435, -0.03385714, 0.034725893, 0.007438677, -0.012168542, 0.028789429, 0.00074582844, 0.0054658893, -0.030768251, -0.00030504307, -0.012838203, 0.0053210976, 0.006141584, -0.009707081, 0.030430403, 0.045464616, 0.003161287, -0.029465124, -0.03221617, -0.028451582, 0.04601965, 0.0029410827, -0.01383968, 0.010364678, 0.0016832043, -0.018424753, -0.00222919, 0.011354088, -0.03026148, -0.010406909, 0.000268091, 0.044595864, 0.025917726, -0.005118992, -0.014527441, 0.022418592, 0.00063384103, -0.042279195, 0.03185419, 0.034388043, 0.03680124, -0.031106098, -0.028258527, 0.0071792584, -0.03890072, 0.016457997, -0.017543936, -0.03619794, -0.06481845, -0.01680791, -0.050773647, -0.013562162, -0.00065118587, 0.006183815, 0.0012480748, 0.025917726, 0.008802133, 0.029513389, -0.015082476, -0.0033603755, 0.0071792584, 0.020017462, 0.025628142, 0.0031100065, 0.054441705, -0.0016062837, -0.0020436754, 0.011209296, 0.0038701633, -0.011221362, 0.022744372, -0.0028294725, -0.033181448, 0.01783352, 0.002381523, 0.030551063, 0.011740199, -0.0066302563, 0.02299776, 0.0010384284, 0.008566846, 0.009411464, 0.03627034, 0.02011399, -0.013743152, -0.0007477137, -0.007070665, -0.018907392, 0.039455757, 0.0006210209, 0.016687252, 0.058254555, -0.0071852915, 0.012452092, -0.009803609, -0.022684043, -0.0028958353, 0.024759391, -0.05019448, -0.02356486, 0.0064914976, -0.018485082, 0.004953085, -0.032264434, 0.02179116, -0.01049137, 0.0061204685, 0.029996028, 0.0032186003, 0.0071370276, 0.016771713, -0.025338558, 0.014804958, -0.0014268022, 0.012910599, -0.018219631, -0.0017465507, -0.021079268, -0.005758489, 0.02416816, 0.020801751, -0.010437073, -0.022503054, -0.006226046, -0.0073300833, -0.0070646317, -0.013429436, -0.036946032, 0.01260895, 0.021489512, -0.028982485, 0.011625572, 0.015794368, 0.032385092, 0.029682312, -0.019474493, -0.011173098, 0.017942114, -0.0004464413, 0.016892374, -0.022454789, -0.0005527727, 0.025579879, -0.01902805, -0.0030059374, -0.010195754, -0.058158025, 0.008265196, -0.01148078, -0.006811246, 0.026255574, 0.01383968, -0.010412941, -0.019933, -0.046719477, -0.010316414, 0.03947989, 0.015553049, 0.015492719, -0.0070344666, -0.024276752, 0.002882261, 0.012922665, -0.015335862, 0.015963292, -0.012536554, 0.027703492, -0.022768505, -0.030333875, -0.02389064, -0.003662025, -0.011848793, -0.015685774, -0.009115849, 0.051642396, -0.04778128, 0.0021975166, 0.050242744, 0.03088891, -0.03064759, 0.009152046, 0.043051418, -0.01900392, -0.0044463137, 0.02656929, 0.017749058, 0.024325017, 0.019860605, 0.04833632, -0.013103655, 0.01858161, 0.00990617, 0.016506262, 0.028958354, -0.0027857332, -0.029802972, 0.010382776, -0.0075352048, 0.0117522655, 0.037959576, -0.023830311, 0.014611903, -0.031154362, -0.037645858, 0.032071378, 0.057627123, 0.018352356, -0.00039704618, -0.0038339654, -0.007450743, -0.02596599, -0.049180936, 0.036487523, -0.03619794, 0.014418847, 0.0014645084, 0.019643417, 0.01285027, -0.028210262, -0.030333875, 0.018352356, -0.044981975, 0.0042502414, -0.0032035178, 0.056758374, -0.018316159, -0.008814199, 0.037863046, -0.0052909325, 0.053476427, 0.0018596692, -0.009912203, -0.0143947145, 0.014261989, -0.0038369817, -0.031202625, -0.05130455, 0.040541694, 0.038225025, -0.0063889367, 0.0119573865, 0.029272068, 0.02086208, -0.023178749, -0.011474747, -0.01846095, -0.029585784, 0.003447854, 0.010527568, 0.06718338, 0.013743152, -0.026352102, -0.0205363, 0.0145515725, -0.012777873, 0.00533618, -0.02659342, 0.0269554, 0.0071309945, -0.013622492, 0.022587515, 0.009556256, 0.004220077, -0.027703492, 0.036101412, -0.0020361342, -0.003423722, 0.039142042, 0.005945512, -0.045247428, -0.00310699, 0.00072358176, 0.025748802, 0.049156804, -0.02632797, -0.021018937, -0.028475715, -0.0016575641, 0.03581183, -0.01149888, -0.051883716, 0.0031703364, 0.03655992, 0.025435086, 0.013984472, -0.011547144, -0.026255574, 0.011504913, -0.012814071, 0.047926076, -0.0061566667, 0.0108171515, -0.006045056, -0.008741803, -0.00415673, -0.010461205, -0.012017717, -0.03595662, -0.030575195, 0.01200565, 0.011589374, 0.021477446, -0.0037977675, 0.0002394343, 0.01771286, 0.007951481, 0.008934858, 0.007975613, 0.009598488, 0.025797067, 0.024059566, 0.017133692, -0.0066362894, 0.0032246334, 0.016385602, 0.0028415385, -0.015685774, -0.0034900848, -0.0009961976, -0.0065216627, 0.017543936, 0.029996028, 0.027920678, -0.0046876334, 0.014382649, -0.011679869, -0.044861317, -0.016083952, 0.051594134, 0.050773647, 0.025145503, 0.007921317, 0.009507992, -0.00087704597, 0.014225791, -0.00786702, 0.00509486, -0.022852967, -0.05275247, 0.01906425, 0.0018068806, 0.010503436, -0.026183179, -0.020307045, -0.003909378, 0.00815057, 0.0044070994, 0.022104876, -0.0112816915, -0.0193659, -0.0007141552, 0.01154111, -0.016771713, -0.035763565, 0.0069560376, 0.020596629, 0.027655227, 0.0013936207, -0.014479177, 0.0031160396, -0.030454535, -0.027751755, -0.0082229655, 0.014455045, -0.011897056, 0.013212249, -0.0100388955, -0.0003776275, -0.008422054, -0.011299791, 0.026038386, -0.0069379387, -0.0018204548, -0.0070344666, -0.0046273037, 0.0051069264, 0.005281883, -0.031275023, 0.0035202499, 0.019305568, 0.06201914, 0.011510946, 0.022804704, -0.00966485, -0.0014049326, -0.026183179, 0.009031386, -0.0019592135, 0.01765253, 0.023806179, -0.020608695, -0.0011734166, 0.0077222274, -0.011685902, 0.022297932, 0.02830679, -0.031081965, 0.01975201, 0.005429691, -0.034532838, 0.004904821, 0.0026907136, 0.010774921, 0.00756537, 0.029103145, -0.044523466, 0.023275277, 0.03738041, 0.008753869, 0.008355691, 0.021815293, -0.007499007, 0.0141895935, -0.015154871, 0.017954178, -0.0070827305, -0.002156794, -0.013320843, -0.009091716, -0.008409988, -0.010225918, 0.0007164176, -0.01049137, -0.00143736, -0.026472762, 0.045223296, 0.025555747, 0.0087961, 0.004349786, 0.0087900665, 0.03619794, -0.005836918, 0.0034448374, 0.036463395, -0.0014124739, -0.01242796, 0.034991343, 0.0114928465, -0.007667931, -0.02209281, 0.022346197, -0.0071008294, -0.041893084, 0.0061747655, 0.006244145, -0.002527823, -0.026907137, -0.053814273, -0.0012284677, -0.0046695345, 0.012886467, 0.014322319, -0.03371235, 0.05400733, -0.03766999, 0.0193659, -0.015601313, -0.0012608949, 0.03655992, 0.030502798, 0.01708543, -0.045440484, -0.0039365264, -0.031033702, 0.004325654, -0.035280928, -0.03800784, 0.017435342, 0.009978566, -0.03137155, 0.013525964, 0.037863046, -0.005773572, 0.0028551125, 0.002870195, 0.0003363392, -0.02891009, 0.025193768, -0.0021703683, -0.012729609, -0.028113734, 0.038442213, 0.00691984, 0.010099226, 0.010889548, 0.004379951, -0.0049561015, -0.05868893, -0.0018506198, -0.031443946, -0.019788207, 0.05130455, 0.030913042, 0.000030329917, -0.013912075, -0.0014705413, -0.0035594644, 0.010388809, 0.049422257, 0.029947763, 0.008693539, 0.007891151, 0.008560813, 0.015601313, -0.005016431, 0.010069061, -0.0009826233, -0.0047177984, 0.0038611137, 0.04358232, 0.00035161022, -0.016892374, 0.0017721909, 0.03320558, 0.0026695982, 0.00972518, -0.021356786, 0.021537775, -0.04051756, 0.025628142, -0.0101414565, 0.012053914, 0.011154999, 0.00024150814, -0.02116373, -0.0043286704, -0.009097749, 0.008192801, -0.013344974, -0.015746105, -0.019643417, -0.044740655, 0.0037917343, 0.009960467, -0.014177527, 0.0034327714, -0.02620731, 0.05251115, 0.0059515447, 0.015975358, 0.03146808, 0.000858947, 0.017724926, -0.04823979, 0.008253131, 0.0027284198, -0.001687729, -0.005873116, 0.0030632508, 0.0322403, -0.03265054, 0.02780002, -0.016252875, -0.031781793, -0.025917726, 0.015082476, -0.02434915, -0.0042140437, -0.029489256, 0.025531614, -0.023842378, 0.02707606, -0.018304093, 0.02311842, 0.010515502, 0.0016515311, -0.016651053, 0.00756537, 0.024807656, -0.022020414, 0.0049561015, -0.020608695, -0.010503436, -0.009839807, 0.023468332, 0.000548248, 0.012053914, 0.018786732, 0.0052245697, 0.007511073, -0.012802006, 0.024421545, -0.016289074, 0.027124325, 0.03974534, -0.0059907595, -0.039817736, 0.024506006, -0.017761124, -0.031443946, 0.015939161, -0.022720242, 0.017604265, -0.019196976, 0.028137866, 0.021489512, 0.009127914, -0.017375013, 0.01569784, 0.0019441311, 0.0038068169, 0.029054882, -0.013574228, 0.0011771872, 0.0070827305, 0.0044794953, 0.0073119844, -0.021139598, -0.017302616, 0.0034900848, 0.002425262, -0.0007054828, -0.018810865, -0.0010655769, 0.009616586, -0.0017797322, -0.0013755218, -0.024095763, 0.025990123, -0.036632318, -0.008120405, -0.01513074, -0.022129009, 0.016530393, 0.020150188, -0.014153395, -0.028717034, 0.005957578, -0.0059243967, 0.003088891, 0.009924269, -0.00533618, -0.016264942, 0.006226046, 0.01630114, -0.029561652, -0.028861826, -0.0011342021, 0.0050677117, 0.017254353, -0.008409988, 0.014455045, -0.02570054, -0.0011228903, 0.017000966, 0.026738213, 0.008397923, -0.009507992, 0.011619539, -0.016470063, 0.022792637, 0.004953085, -0.020825882, -0.0042985054, 0.020850014, -0.011878958, -0.011607474, -0.0031824023, 0.018436817, 0.02419229, 0.0077161947, -0.038973115, 0.0072637205, -0.0051491573, 0.0052034543, 0.012862335, -0.04136218, -0.009405432, -0.0036167777, 0.019836472, 0.016554525, 0.0008695047, -0.0045006108, -0.0031643035, -0.035980754, -0.036897767, 0.012440026, -0.008132471, 0.027993076, -0.010087159, 0.008741803, -0.030768251, 0.0029757724, -0.030502798, 0.03738041, -0.027438039, 0.021079268, 0.030285612, -0.00097206555, -0.002058758, 0.014056867, -0.009586422, 0.0022819785, 0.036366865, 0.022563383, -0.016011557, 0.027389776, -0.01618048, 0.026110781, -0.0029214756, 0.0193297, -0.04075888, -0.002990855, 0.008349659, -0.015215201, -0.028861826, -0.054827817, -0.012259036, 0.012464158, 0.014684298, -0.0039033447, 0.0071309945, -0.019920934, -0.0005101647, 0.030285612, -0.008126438, -0.0005320343, 0.049760103, 0.0351844, 0.010425007, 0.009514025, 0.0014094573, 0.0580615, 0.01605982, -0.012681345, -0.013103655, -0.008699572, -0.011553177, -0.0042472254, 0.02659342, -0.005803737, -0.010581865, -0.0000472977, 0.05733754, -0.009182211, -0.011414418, 0.0081686685, -0.01618048, 0.023178749, 0.03125089, 0.010799052, -0.028089603, -0.0024267703, -0.009302871, 0.032385092, -0.041506972, -0.013996538, -0.00972518, 0.0018325208, 0.02041564, -0.021332653, 0.033519294, -0.03730801, -0.0022563383, -0.031733528, -0.015987424, -0.01797831, 0.005384444, -0.008174702, 0.00932097, -0.0032065343, -0.0019049167, -0.023359738, -0.002656024, -0.0013747676, 0.014406781, -0.010937812, 0.015251399, 0.04073475, 0.021803226, -0.022189338, -0.005852001, -0.002790258, -0.0132001825, -0.006401003, -0.008886594, 0.04995316, 0.021211993, 0.015106607, 0.0038550808, -0.03482242, 0.031081965, 0.0027616012, 0.00071868, -0.005375394, -0.0076498315, 0.0036318603, 0.03296426, 0.03113023, -0.009339069, 0.005637829, -0.0072516543, 0.0012684362, -0.03680124, -0.004618254, 0.027413908, 0.016337337, -0.0026786476, -0.018714337, -0.0038701633, -0.018255828, -0.004331687, 0.006207947, 0.026038386, -0.012826137, -0.012958863, 0.01037071, -0.013429436, -0.0038852456, 0.0036469426, -0.0069379387, 0.034363914, -0.01377935, 0.028741166, 0.004880689, -0.009387333, -0.024566336, -0.006268277, -0.0292962, 0.0137190195, 0.008048009, -0.021441247, 0.017640464, -0.0021100384, -0.0269554, 0.017749058, 0.0006549565, -0.008228999, 0.0028566208, -0.016421799, -0.011758298, 0.02473526, 0.010213853, 0.045609407, -0.018316159, 0.028258527, 0.04934986, -0.005794687, -0.00020870376, -0.0090012215, 0.0007092534, 0.009477828, -0.043003153, 0.0073059513, 0.013863811, -0.04121739, -0.019571021, -0.055165663, -0.009061552, -0.01834029, 0.02161017, -0.032409225, -0.0021522692, 0.022925364, 0.01920904, -0.020934476, -0.0073542153, -0.006992236, -0.0150704095, -0.02467493, -0.012862335, -0.024904184, -0.0029018684, 0.033060785, -0.012572751, -0.010334512, -0.001297847, 0.012717543, -0.0030723002, -0.021006873, 0.017809387, -0.004027021, -0.043003153, 0.018714337, 0.011523011, 0.008138504, -0.0065880255, 0.019402096, -0.007921317, 0.005794687, -0.0006255457, 0.002277454, -0.01018972, -0.0043829675, -0.006105386, 0.017013032, -0.0004057186, 0.0055714664, -0.016578658, -0.028982485, 0.0005373132, 0.016651053, -0.0063829035, -0.0014712955, -0.021803226, 0.0022518137, -0.010123358, -0.022913298, 0.017905915, -0.015359993, -0.0038038003, -0.008035943, 0.015372059, 0.012150442, 0.008687506, 0.0071611595, 0.0026771394, 0.0034327714, 0.027293248, 0.00043211292, 0.0071973572, -0.023613123, -0.02524203, 0.012265069, -0.007366281, 0.013393238, -0.017073363, 0.0060480726, 0.0077885906, -0.0026364168, -0.004941019, -0.008054042, 0.009954434, -0.017447408, -0.014841156, -0.017266419, -0.022044547, -0.035763565, -0.0015670692, 0.0064794314, -0.02269611, 0.02632797, 0.02311842, -0.0026183177, -0.018147234, -0.0055141533, 0.0041295816, 0.01771286, 0.027003665, -0.030502798, 0.027365644, -0.016071886, -0.011776397, 0.009453696, -0.006232079, 0.0011115784, 0.0008695047, -0.0037977675, -0.019727878, 0.013417371, 0.006425135, 0.019788207, -0.012814071, 0.017423276, -0.014479177, -0.017543936, 0.018919457, 0.041193258, 0.010485337, -0.008216932, 0.001218664, -0.026496893, 0.008488418, 0.014418847, 0.008609077, -0.03716322, 0.004977217, -0.029054882, -0.0013257496, -0.010678393, 0.020656958, 0.000072348754, 0.009882038, 0.012958863, -0.027244983, -0.024385346, 0.03265054, 0.01049137, 0.015601313, 0.00018853095, -0.017930048, 0.020355308, -0.0043950332, -0.008066108, 0.024325017, -0.035039607, -0.03347103, 0.011058471, -0.03776652, 0.020970674, 0.011233428, 0.005987743, -0.026979532, -0.043268606, -0.0011432517, -0.02130852, -0.0018747518, -0.02431295, -0.012156475, 0.014961815, -0.0025082156, -0.009327003, 0.020572497, 0.00679918, 0.017013032, -0.007897184, -0.0009509501, 0.023492465, 0.0069077737, 0.0034931013, 0.019800274, -0.024686996, -0.0105517, 0.03349516, 0.024325017, -0.0051310584, -0.009857906, 0.020850014, 0.010213853, 0.014925618, -0.0045428416, 0.0035232664, 0.018629873, -0.005701176, -0.0025534632, 0.028161999, -0.015299663, 0.020343244, 0.0045820563, 0.026810609, 0.012180608, 0.01131789, 0.0008544222, -0.031540472, -0.0013536522, 0.014310253, -0.038200896, -0.004235159, -0.022756439, 0.026231442, -0.026134914, 0.008952958, -0.012156475, -0.012814071, -0.013743152, -0.022985693, 0.01285027, 0.0025293313, 0.0031401715, 0.011372186, 0.019836472, -0.035377454, -0.013405304, 0.0070404997, -0.0059907595, 0.019583086, -0.0007480908, -0.002870195, 0.004410116, -0.039672945, -0.00990617, -0.023818245, -0.0017163857, 0.008989155, -0.00067456375, 0.010032862, 0.006714718, -0.010087159, 0.008940891, -0.0010090176, 0.0070163677, -0.01704923, -0.004099417, -0.009393366, -0.01605982, 0.021549841, -0.017218154, 0.05058059, 0.0013393238, 0.018895326, 0.00984584, -0.00010746264, -0.005677044, 0.0039455756, 0.0017344847, -0.0017269434, -0.024397412, 0.001522576, -0.020632828, -0.0041778455, 0.01166177, -0.01975201, 0.020307045, -0.03325384, 0.0005769047, -0.0070525655, 0.0030919076, 0.025290295, 0.009357168, 0.021320587, 0.031926583, 0.023613123, -0.016638987, 0.005661961, -0.0067086853, -0.010714591, -0.010708557, 0.004886722, -0.0234442, 0.022937428, 0.020162253, 0.014117197, 0.017375013, 0.008307427, 0.0065638935, -0.0072637205, -0.033398636, -0.0013325367, 0.0016394651, -0.006220013, -0.0005218537, 0.0037977675, -0.0022472888, 0.022165207, 0.016651053, 0.035642907, 0.0019501641, -0.00475098, -0.0050103986, 0.018569544, 0.009550224, 0.018533345, 0.0021884672, -0.01813517, 0.008301395, 0.010593931, -0.015975358, 0.005544318, -0.0051793223, -0.009912203, -0.01723022, 0.00072094233, -0.0073300833, 0.04811913, -0.002564021, 0.013284645, -0.018026575, -0.007915284, 0.01981234, 0.0045911055, -0.022539252, 0.0133570405, -0.025145503, -0.011806562, -0.0085366815, -0.010967976, 0.019776141, 0.015058343, -0.03593249, 0.016651053, -0.014249923, -0.009019321, 0.016844109, 0.0022005332, -0.03431565, -0.021658435, -0.016373536, 0.0033211613, -0.0024297868, 0.008048009, 0.013948274, -0.005167256, 0.004428215, 0.008048009, 0.008072141, 0.0028520962, -0.0077282605, -0.024156094, 0.00281439, -0.0031039736, 0.0087900665, 0.0039274767, 0.013067457, -0.016216679, 0.013924141, -0.024252621, -0.024880052, -0.014237857, -0.017773189, -0.021031003, -0.001998428, 0.009954434, -0.0093632005, 0.027775887, 0.010014764, 0.0027812086, 0.014008603, 0.009459728, -0.0015059852, 0.018485082, -0.009514025, 0.0036016952, 0.011565242, -0.020777619, 0.006943972, 0.0007254671, 0.015963292, -0.008989155, 0.014455045, -0.0067086853, -0.004367885, 0.0222738, 0.029440993, 0.0037977675, 0.029440993, 0.036390997, 0.0021975166, -0.026472762, -0.009514025, -0.016289074, 0.01963135, 0.01353803, 0.0069077737, 0.0065699266, 0.030454535, -0.009435597, 0.003369425, 0.013670756, 0.031878322, -0.029730575, 0.0137190195, 0.032674674, -0.01858161, -0.0158185, -0.0004502119, 0.026279707, 0.011251527, 0.0069620707, 0.026810609, -0.009646751, 0.010425007, -0.030189084, -0.009526092, -0.020355308, 0.007022401, 0.011474747, -0.034870684, -0.012524487, -0.013984472, 0.02092241, 0.021682568, -0.004702716, 0.025941858, -0.041603502, 0.011746232, -0.0025323476, 0.0013875877, 0.0228409, -0.011028307, 0.01482909, 0.067376435, 0.03359169, -0.010032862, 0.017495671, -0.0063105077, -0.00439805, 0.021875624, -0.0028852776, -0.020282913, -0.014358517, 0.036366865, -0.01809897, -0.0035323158, -0.0033121116, 0.0014886403, 0.0029938715, 0.0075593367, -0.008965024, -0.008084207, -0.014201659, 0.015987424, 0.014901486, -0.001029379, -0.0069801696, 0.020837948, 0.0027736672, -0.014539506, -0.027993076, -0.013248446, -0.022732308, -0.002393589, 0.013984472, 0.0021055136, 0.0034418209, 0.019148711, -0.0015504785, 0.027896548, -0.0061265016, 0.010859382, -0.0028038323, 0.015565115, -0.0016560558, 0.01987267, 0.00033407685, -0.007492974, 0.00035311846, -0.017073363, 0.010871449, -0.012011684, 0.013393238, 0.005803737, 0.0149135515, -0.0059786933, -0.02257545, -0.0058489842, -0.008355691, -0.010841284, -0.0047720955, 0.008910727, -0.009888071, -0.0018626858, -0.02209281, -0.022116942, 0.020608695, 0.014213725, 0.006787114, -0.010865415, 0.010449139, 0.00043701474, -0.009538158, 0.004576023, 0.0033060787, 0.014841156, -0.023009825, -0.015902963, 0.020777619, -0.0029561652, -0.008216932, -0.010437073, 0.0035112004, 0.024107829, -0.04346166, -0.0059515447, -0.00012056554, 0.00019965427, 0.008566846, -0.008699572, -0.007517106, -0.015915029, 0.0013679805, -0.015287598, 0.0035202499, 0.004476479, -0.012053914, 0.00024659847, 0.0017013033, -0.015456521, -0.0023438167, 0.000011565196, -0.0038249157, -0.013007127, -0.008440154, 0.0351844, -0.02608665, 0.019884735, 0.0012669279, 0.0024886085, 0.007468842, 0.014744628, 0.00960452, 0.011052438, 0.00873577, 0.006250178, -0.006762982, -0.008108339, 0.020837948, -0.0070947967, 0.01148078, -0.008494451, 0.045802463, -0.008530648, -0.030840646, 0.029996028, 0.009043452, -0.009586422, 0.007511073, -0.0065035634, 0.00042004694, -0.018931523, -0.00010991354, -0.016409734, 0.010630129, 0.02281677, 0.0014841156, -0.02431295, 0.0024539188, -0.01587883, -0.0013913583, -0.0042140437, 0.012874401, 0.0070766974, 0.0027254033, -0.010334512, 0.0054085758, 0.021718765, 0.022370327, -0.010159556, -0.02332354, -0.019643417, 0.010865415, 0.0016319238, 0.0090012215, -0.00085366814, -0.0041265655, 0.009954434, -0.015927095, 0.0012088604, 0.010467238, -0.0016786795, 0.0031763695, -0.005318081, 0.026255574, 0.0072335554, 0.008621143, 0.008657341, -0.008633209, -0.0026786476, -0.006672487, -0.024095763, 0.0022985693, -0.003094924, -0.0038520643, -0.0056800605, -0.009115849, 0.012047881, 0.00028637852, -0.020777619, -0.008910727, -0.027124325, -0.009019321, -0.005312048, 0.008808166, -0.015094542, -0.010696492, -0.016264942, -0.016337337, 0.021561908, 0.014430913, 0.0052004377, -0.0032578148, -0.014503309, 0.00089363666, 0.0013001094, 0.016168414, 0.021634303, 0.0134777, 0.017375013, -0.00040609567, 0.009236508, 0.0122168055, -0.012946798, -0.009869972, 0.027172588, 0.015637511, -0.0037072725, -0.016795846, 0.002662057, -0.005502087, -0.037356276, -0.022261733, -0.003948592, 0.00010680278, -0.021368852, -0.014744628, -0.011589374, 0.0069801696, -0.017435342, -0.0047479635, 0.010533601, -0.0036559922, 0.007541238, 0.02473526, -0.007993712, -0.022116942, 0.023938905, 0.005118992, -0.011142933, 0.005616714, -0.00038592285, 0.0016439898, 0.0011553176, 0.008289329, 0.011577308, -0.003918427, -0.033688217, -0.031564604, 0.02239446, 0.01012939, 0.026714081, -0.004020988, 0.0087900665, -0.0020044611, 0.0071370276, 0.014418847, 0.011969453, 0.023251144, 0.0020889228, 0.025555747, 0.009574356, 0.014636034, 0.034991343, -0.009097749, 0.016542459, 0.009930302, 0.003125089, 0.0077584255, 0.011535077, -0.030961307, 0.021151664, -0.037838914, 0.0032276497, -0.012874401, -0.009031386, -0.026038386, 0.003716322, -0.015504785, -0.0122168055, -0.0026514991, 0.016506262, -0.0021296456, 0.007541238, -0.0006017908, -0.026376234, 0.012331432, -0.006726784, -0.0053452295, 0.018738467, 0.01785765, 0.033664085, -0.017664596, -0.0018370455, -0.00025282, -0.017314682, -0.026472762, 0.0029938715, -0.002070824, 0.009254607, -0.02161017, 0.025869463, -0.00038422606, -0.0070103346, -0.0072576874, -0.016856175, 0.01666312, 0.001516543, 0.008669407, -0.015528917, -0.01945036, 0.0064613326, -0.0023136518, -0.0031160396, -0.04058996, -0.001954689, 0.019426228, -0.016856175, -0.0036077283, 0.0028837693, -0.0013958832, -0.031057835, 0.01963135, -0.000018758828, -0.007830821, 0.01067236, -0.018931523, -0.012645148, 0.0076799965, -0.010020797, 0.011221362, -0.012922665, 0.014853222, -0.00021586793, 0.0046695345, 0.009296838, 0.019088382, 0.0027450107, 0.012512422, 0.0109740095, -0.017773189, 0.02767936, 0.009284772, 0.00861511, 0.0025791035, -0.0073843803, 0.0007529926, -0.0033000456, -0.0073119844, 0.0015376584, 0.0008054042, -0.00018796536, 0.014925618, -0.006811246, -0.023721717, -0.011794496, -0.006726784, -0.006473399, 0.0060722046, -0.0082229655, 0.014780826, -0.03665645, 0.008651308, -0.0034327714, -0.0017013033, -0.014093066, 0.00196223, -0.010587898, 0.0035504147, -0.018050706, -0.008548747, -0.00944163, 0.009218409, -0.005646879, -0.015649578, 0.013731086, -0.014925618, 0.025024844, -0.019522756, 0.012277136, 0.026689949, 0.002894327, -0.0053150645, -0.010286248, -0.021899754, -0.012198706, 0.011100702, -0.021031003, -0.008385857, 0.011553177, 0.003417689, -0.014165461, 0.000054721106, -0.0002601727, -0.0065035634, -0.025193768, -0.013610426, 0.014865288, 0.0043890006, -0.0064070355, 0.005520186, 0.000033912005, -0.01827996, 0.00075035315, -0.030502798, 0.012259036, 0.0060088583, 0.011245494, -0.006376871, 0.016819976, -0.004343753, 0.017821454, 0.021754963, -0.00393351, 0.01095591, -0.028523978, -0.005158207, 0.016071886, -0.009043452, -0.0014034243, -0.0126330815, 0.030213214, 0.0115954075, 0.00038837374, 0.0035021508, 0.0019516724, 0.03245749, -0.006105386, -0.010925746, 0.025917726, -0.032915995, 0.0152272675, 0.015746105, -0.011601441, 0.022141075, 0.011909123, -0.011040372, -0.006853477, 0.010231951, 0.0054658893, -0.027606964, -0.016011557, -0.0028958353, 0.0020843982, 0.0070163677, -0.018907392, -0.005037547, -0.009224442, 0.002984822, -0.0054085758, 0.004784161, 0.029223805, -0.017037164, 0.012946798, -0.01513074, -0.022008348, 0.008271229, -0.0035685138, 0.012669279, 0.008066108, 0.007933382, 0.0055141533, 0.036101412, -0.010225918, 0.010280215, 0.031926583, 0.008228999, -0.009514025, -0.0055473344, 0.014937684, -0.009429564, -0.009182211, -0.016457997, 0.014901486, -0.015782302, 0.00047999978, 0.0024071631, 0.0058942316, -0.013417371, 0.0003338883, -0.0052728336, 0.005969644, 0.0138758775, -0.0126330815, 0.009206343, 0.005444774, -0.014455045, -0.011909123, 0.015927095, -0.010497403, -0.004033054, -0.006084271, -0.01218664, -0.019546889, 0.0032246334, 0.010316414, -0.011004174, -0.0030617425, 0.012379696, -0.0014886403, -0.0047660624, 0.0026695982, 0.006171749, -0.0045277593, 0.005544318, 0.017242286, -0.01605982, 0.0022819785, 0.0046333363, -0.004093384, 0.039576415, -0.002710321, 0.0020150188, -0.022913298, -0.014080999, 0.005197421, 0.027848283, -0.019522756, 0.008114372, 0.0010889547, 0.004491561, -0.00031446962, -0.0068595097, -0.015577181, -0.013622492, -0.0071973572, -0.000084367595, -0.0019637384, 0.019908868, 0.010780954, 0.032023113, 0.010310381, 0.013176051, -0.009296838, -0.031275023, 0.0026454662, 0.006051089, 0.010714591, -0.025073107, -0.019571021, -0.0021914837, -0.0004999841, 0.013441502, 0.026352102, -0.0022442723, 0.0077885906, 0.0041687964, -0.010449139, -0.014334385, 0.013550096, -0.0012156476, 0.029320333, -0.0010437074, -0.010690459, 0.03530506, 0.02209281, -0.028065471, 0.007987679, 0.024131961, -0.0063829035, 0.024807656, -0.008397923, 0.001540675, 0.011577308, -0.00028166524, 0.013151919, 0.020825882, -0.005610681, 0.0155892465, 0.02056043, 0.0042381757, 0.012862335, -0.023649322, -0.017495671, -0.023757916, 0.0043950332, -0.0076076007, 0.015939161, -0.025024844, 0.0126330815, 0.0029169507, 0.025531614, -0.012283168, 0.01692857, -0.0028098652, 0.0025806115, -0.0004404083, -0.009399399, -0.0077161947, -0.00055201864, 0.033277974, -0.020331178, -0.012572751, -0.023432134, 0.02915141, 0.014093066, -0.0044010663, -0.020391507, -0.023673454, 0.01660279, -0.010437073, -0.011818628, 0.016349403, -0.01043104, 0.009568322, -0.0036348766, -0.018557478, 0.0055292356, -0.013248446, 0.009857906, 0.033808876, 0.00968295, -0.007595535, -0.014575705, 0.00984584, -0.019221107, 0.0057856375, -0.0070947967, -0.022599582, 0.00627431, 0.019667549, 0.020572497, -0.0077101616, 0.011130867, -0.0012669279, -0.00072471297, -0.013586294, 0.014780826, -0.0036318603, -0.022913298, 0.0068595097, 0.0043105716, 0.021718765, -0.001120628, 0.022406526, 0.008771968, -0.014817025, 0.02719672, 0.0081686685, -0.00006650429, -0.017761124, -0.026279707, 0.0035624807, 0.0066302563, 0.023950972, 0.0099665, -0.009507992, 0.010907646, -0.014406781, -0.024180224, 0.0065699266, -0.000070699105, 0.0138758775, 0.005176306, -0.0056076646, 0.019052183, 0.021429181, 0.0048113097, 0.0029169507, 0.016421799, 0.012536554, -0.029247938, -0.012584818, 0.032771204, 0.016385602, -0.0015851682, 0.015372059, 0.006895708, 0.005879149, -0.0065940586, 0.017869717, 0.004506644, 0.0006440217, -0.0049259365, -0.0028158983, 0.0041024336, -0.023697587, 0.01723022, -0.027751755, -0.018774666, -0.01999333, 0.012572751, 0.009133947, 0.020837948, 0.012379696, 0.019595152, -0.027100192, 0.0038912788, 0.01729055, -0.009471795, 0.008271229, 0.002746519, -0.009248574, 0.0083255265, 0.0015444455, -0.013815547, -0.007643799, -0.005085811, 0.018219631, -0.017628398, 0.003173353, -0.015396191, 0.010799052, 0.01482909, 0.011607474, 0.0140327355, 0.0019773126, -0.031419814, 0.006084271, 0.033808876, -0.012349531, 0.002192992, -0.018798798, -0.00833156, -0.0020949559, -0.0022125992, 0.0064311675, -0.0005143124, 0.0064613326, -0.03125089, -0.016964769, 0.038321555, 0.0031522375, 0.0072818194, -0.018255828, 0.0076256995, 0.009538158, 0.011746232, 0.011468714, -0.018714337, -0.003393557, -0.018569544, 0.012072014, -0.0032005014, 0.025507484, -0.0055051036, 0.00030051832, -0.018726401, 0.011504913, -0.00014592295, 0.0037495035, 0.0043226373, -0.013320843, 0.021368852, -0.006950005, -0.0005708717, 0.0039847903, 0.0066181906, 0.018955655, -0.01785765, 0.017507738, -0.016289074, -0.0031914518, -0.020813817, 0.014684298, 0.002114563, 0.018738467, -0.00310699, -0.0000085487, -0.011649705, 0.015311729, -0.025290295, -0.01236763, 0.0038309488, 0.039335098, 0.007993712, -0.02572467, 0.014225791, -0.024300884, -0.008584945, 0.021935953, -0.0065699266, -0.010479304, 0.0026213343, 0.019764077, 0.0053934935, -0.006702652, 0.0020723322, 0.008681473, 0.0072033904, -0.005770555, -0.0072878525, 0.01827996, 0.012041848, 0.011734166, -0.010111292, 0.037356276, 0.00087327533, 0.018605743, 0.007933382, -0.009719147, 0.006147617, 0.020150188, -0.0028520962, -0.0058339015, 0.023709651, -0.00533618, -0.005725308, 0.01125756, 0.0008235032, 0.004265324, 0.0020120023, 0.006195881, -0.0070043015, 0.015034212, -0.002996888, 0.02854811, 0.001194532, -0.008874529, -0.0017827486, 0.017785255, -0.014298187, -0.0025579878, -0.009749312, 0.0016454981, 0.014274055, -0.0052275863, 0.019196976, 0.003990823, -0.009598488, 0.009634686, -0.0123073, 0.02011399, 0.023721717, -0.003746487, 0.029827103, 0.0045096604, 0.0060872873, -0.011770364, 0.015565115, -0.016337337, -0.04136218, -0.002138695, 0.0075352048, -0.038393952, 0.012524487, -0.0042140437, 0.022309998, 0.0014833615, 0.0036077283, 0.011758298, -0.01605982, 0.023166683, 0.0193297, -0.004229126, 0.012349531, 0.0098337745, -0.035329193, 0.013332909, 0.0025112322, -0.013103655, 0.01218664, -0.00533618, -0.0013755218, 0.0036077283, -0.025410956, -0.008518582, 0.014129263, -0.001717894, 0.0019758043, 0.015106607, -0.009399399, 0.0005795441, -0.0066785202, 0.007975613, 0.009996665, -0.01248829, -0.02326321, 0.0003333227, -0.02023465, -0.0065638935, 0.031057835, 0.020089857, 0.0006904003, -0.01692857, -0.011673836, 0.007836854, -0.004204994, -0.00097206555, -0.0028988519, 0.0012789939, 0.017990377, -0.019522756, -0.01999333, 0.0010693475, -0.008428087, -0.024325017, -0.020331178, -0.0020421671, 0.0022623714, 0.011897056, -0.0065638935, 0.01103434, 0.00439805, 0.033808876, 0.010117325, 0.009242541, 0.0051461407, -0.006159683, -0.0026334003, 0.0298995, -0.0025610044, 0.0027012713, -0.008874529, -0.027968943, 0.023902707, -0.003381491, -0.0193297, -0.0036288437, -0.0015836599, 0.011987552, -0.00035255286, 0.028403318, -0.011323922, -0.018436817, 0.003918427, -0.012295234, -0.009290805, 0.0071913246, -0.013924141, -0.0029637064, 0.0045006108, 0.031081965, 0.0075533036, 0.012180608, 0.017061297, -0.012970929, -0.015022146, -0.008989155, -0.00059915136, 0.011553177, 0.018967722, -0.00533618, -0.002722387, -0.015094542, 0.028330922, -0.004186895, 0.018219631, 0.0077765244, 0.028717034, -0.017121626, -0.023661388, -0.009544191, 0.009097749, -0.0005497562, -0.004554908, -0.021272324, 0.010388809, -0.00032860943, 0.009713114, 0.010014764, -0.01735088, -0.02719672, 0.015842633, -0.0053391964, -0.0187626, 0.02828266, 0.034629364, 0.02891009, 0.0001245247, -0.003405623, 0.015565115, 0.02257545, 0.010805085, 0.0021899755, 0.018690204, -0.016904438, 0.021187862, 0.0079635475, -0.0063949698, 0.0070887636, 0.014129263, 0.013381172, 0.014080999, 0.0009856398, -0.0010037387, -0.017954178, -0.0046333363, -0.029537521, -0.024445677, 0.035498116, -0.002102497, 0.0105517, 0.005885182, -0.013574228, 0.010829218, -0.011468714, -0.0049168873, -0.0075834687, -0.0078609865, -0.012802006, 0.022937428, -0.006232079, -0.027220853, 0.015661642, 0.009918236, -0.024083696, -0.02485592, -0.0068836417, 0.00597266, 0.0063889367, -0.0057132416, 0.009707081, -0.01951069, 0.004705732, -0.01975201, 0.003387524, 0.017954178, 0.014623969, -0.019619284, -0.0016077919, -0.009429564, 0.01678378, 0.0024840836, -0.03371235, -0.00732405, 0.014792892, -0.01026815, -0.025024844, 0.013091589, -0.006672487, 0.0152272675, -0.00873577, -0.011028307, 0.0066664545, 0.014877354, 0.0054598562, 0.013827614, 0.02917554, -0.0091580795, -0.0018657022, 0.006787114, -0.0055141533, 0.0002552709, 0.012102178, 0.022358261, -0.012512422, 0.009616586, -0.008048009, -0.026617553, -0.00085819286, -0.0033271941, 0.010117325, 0.0030798416, -0.01142045, 0.012838203, -0.0004977217, -0.016252875, 0.016252875, -0.0007775016, -0.022792637, 0.025507484, 0.021236125, -0.010774921, -0.012337465, -0.026545158, 0.0013325367, -0.0044131326, -0.004123549, 0.028813561, -0.0014599836, 0.0066181906, -0.0033332272, 0.0014848697, 0.005541302, -0.018195499, -0.019426228, 0.008977089, 0.01692857, -0.014901486, 0.0065638935, -0.012041848, -0.017761124, 0.0123254, 0.0013649641, -0.0006176274, 0.009489894, 0.038997248, 0.0039214436, 0.011891024, 0.011788463, -0.0070344666, -0.0010203294, 0.007366281, 0.0127899395, 0.019764077, -0.027027797, 0.005996792, 0.008584945, -0.013863811, 0.0023905723, -0.013272579, -0.035087872, 0.0052366355, -0.013791416, 0.0018023559, 0.00092003105, -0.01488942, -0.0053814272, 0.01095591, 0.031202625, -0.0004894263, 0.0040360703, 0.010575832, 0.0083798235, 0.012355564, -0.004624287, -0.013743152, 0.009314937, -0.027100192, 0.0040300377, 0.008434121, -0.013188117, 0.0010587898, -0.0020044611, -0.007836854, 0.01999333, 0.012621015, -0.0017375012, -0.00053165725, -0.015444455, -0.007993712, 0.0021703683, -0.010418974, 0.002551955, 0.010847316, -0.004367885, -0.008446187, -0.021356786, -0.011547144, 0.008524615, -0.005915347, -0.028885957, -0.00095547485, -0.0031643035, 0.0071973572, 0.005293949, -0.017930048, 0.01575817, -0.010111292, 0.015625445, 0.001248829, 0.00045586782, 0.013694888, -0.00815057, 0.021525709, 0.015347927, -0.016264942, 0.009797576, 0.014201659, -0.012910599, -0.012222839, 0.0009622619, -0.016433865, 0.005683077, 0.0039455756, -0.005176306, -0.009882038, -0.00020757258, 0.003022528, 0.005128042, -0.005128042, 0.014720497, -0.001699795, 0.018955655, -0.012090112, 0.0075050397, 0.023661388, 0.0025821198, -0.009894104, 0.013212249, 0.0041687964, -0.02314255, 0.015685774, 0.02854811, -0.0070827305, 0.0076739634, 0.016880307, 0.0138758775, 0.037476934, -0.0032366994, -0.022104876, -0.019221107, -0.0049621346, -0.018750533, 0.002826456, 0.011311857, 0.0071008294, 0.00966485, -0.021187862, 0.004180862, -0.0030074457, -0.006256211, -0.012150442, -0.009876005, 0.009658817, -0.014358517, -0.010913679, 0.008578912, 0.014213725, -0.0035172333, 0.026376234, 0.004051153, 0.0025263147, 0.000737156, 0.021923887, 0.012331432, 0.015721973, -0.011547144, 0.016988901, -0.0050767614, -0.011305824, 0.0013551605, 0.01482909, 0.038442213, 0.00404512, 0.017013032, 0.023830311, 0.010069061, -0.005604648, 0.02023465, 0.016530393, -0.00492292, -0.018207565, 0.008138504, 0.0038942953, -0.008898661, 0.007903217, 0.021598104, 0.011378219, 0.009067585, 0.014286121, -0.010859382, -0.0030240363, 0.018967722, -0.01149888, -0.0066061243, -0.0023423084, -0.0037645858, 0.007885118, 0.0013287661, -0.022732308, -0.012102178, -0.013381172, 0.0013679805, -0.026858872, -0.0052275863, 0.0046061883, -0.004763046, 0.020934476, -0.005532252, -0.006901741, -0.0064794314, 0.0015489702, 0.0017028115, -0.029996028, -0.025459219, -0.0029410827, 0.02707606, -0.00014120968, -0.0029214756, -0.0081686685, 0.008464285, 0.0053090313, 0.027003665, -0.00014733693, 0.0032155837, 0.007812723, -0.01569784, -0.028692901, -0.006220013, 0.020777619, 0.014177527, -0.0076377657, 0.008597011, -0.011710034, 0.019148711, 0.006793147, 0.0016394651, 0.00007484679, -0.00060518435, 0.02005366, 0.027148455, -0.009230475, 0.0064673657, -0.018786732, 0.0012020733, 0.017254353, 0.0038369817, -0.012970929, -0.009230475, 0.0045187096, -0.032385092, 0.0021764012, -0.007420578, -0.0031582704, -0.003954625, 0.003393557, 0.005085811, 0.00809024, -0.005221553, -0.004307555, 0.009797576, 0.0064070355, 0.027293248, -0.009495927, -0.012645148, 0.019293502, -0.005342213, -0.0041265655, 0.01630114, 0.01414133, -0.006244145, -0.008524615, -0.0012284677, 0.0056800605, 0.012838203, -0.0049199034, -0.020837948, 0.010485337, 0.0032940127, 0.01248829, 0.0025459218, 0.0077704913, 0.0064914976, -0.003076825, 0.0014222774, 0.012391762, 0.0041386313, -0.017966244, -0.016904438, -0.023154616, -0.0035292993, 0.014684298, -0.014358517, -0.0013385697, -0.022165207, -0.024530139, 0.0038158663, -0.0027042879, 0.0021839426, 0.012035816, 0.0033603755, -0.01494975, 0.004545858, -0.0066121574, -0.026062518, 0.005692126, -0.00392446, -0.016156347, -0.018255828, -0.040638223, -0.0047600293, -0.0011357104, -0.01414133, -0.008385857, -0.0039123944, 0.04404083, -0.024337083, -0.008808166, 0.02719672, 0.011058471, -0.004132598, -0.01383968, 0.006624223, -0.0005252472, 0.0009019321, -0.018171366, -0.018328225, -0.027148455, -0.0037645858, -0.008657341, 0.005867083, 0.015951226, -0.0010180671, 0.013694888, -0.008434121, -0.00014092689, -0.009013288, -0.008452219, -0.012403828, 0.014503309, 0.0175922, -0.006220013, -0.0036077283, -0.00296069, 0.0045247427, 0.028355055, -0.01599949, -0.01632527, 0.0064191017, 0.0047720955, -0.0013310285, -0.0064552994, 0.014961815, 0.005855017, 0.013176051, -0.013393238, 0.0048233755, -0.00034406898, 0.0017978312, -0.022285866, -0.011516978, 0.01142045, -0.0013740135, 0.0022125992, -0.005263784, -0.0087900665, 0.00086120935, 0.0150704095, -0.020632828, -0.008958991, 0.018219631, 0.026641686, 0.008898661, 0.0211034, 0.024928316, 0.0134777, 0.0119573865, -0.0052004377, -0.00078278047, -0.01383968, -0.0094657615, 0.0046695345, -0.0092606405, 0.022104876, -0.00048867223, -0.013646624, 0.006925873, 0.011710034, 0.009912203, 0.015444455, 0.0022005332, -0.004491561, 0.027172588, 0.0076136338, -0.027413908, 0.022889165, -0.021875624, 0.0126330815, -0.018521281, 0.0111369, 0.012379696, 0.012464158, -0.008729737, -0.01975201, -0.020282913, 0.0059424955, -0.0023588992, -0.0041205324, -0.0072275223, 0.009646751, -0.001200565, -0.0057222913, 0.004971184, -0.0065156296, 0.0013008636, -0.013127787, 0.004361852, 0.016759647, -0.019920934, -0.0049440353, -0.008259163, -0.0076799965, 0.015203135, 0.028210262, -0.008156603, 0.003710289, 0.03460523, 0.01329671, 0.032192037, 0.0035866126, -0.012621015, 0.03026148, 0.007873053, -0.0032970293, 0.020367375, -0.0059123305, -0.010913679, 0.006232079, -0.012168542, -0.010696492, 0.01061203, 0.0013611935, -0.0007526155, -0.018485082, 0.01414133, -0.012802006, -0.015166937, -0.015299663, -0.00021360556, -0.009296838, -0.0050345305, 0.018002443, 0.019052183, 0.028572243, 0.00972518, 0.005827869, 0.005453823, -0.008247098, -0.0077342936, -0.006066172, -0.017013032, 0.0017163857, -0.022973627, 0.0045337924, 0.006298442, -0.012265069, 0.010418974, 0.019884735, 0.0064191017, 0.007366281, -0.005257751, -0.0038399983, 0.0050586625, -0.023540728, 0.017495671, -0.0016093001, -0.008603044, -0.00240113, -0.012319366, 0.015420323, -0.00053995266, -0.010714591, 0.020693157, 0.0066604214, 0.015323795, -0.013151919, -0.012681345, -0.018111037, 0.0063105077, 0.022623714, -0.0015022146, 0.02416816, -0.0050103986, 0.014780826, 0.008066108, 0.009737247, -0.020101923, 0.00873577, 0.011993584, 0.023794113, -0.008253131, -0.008011811, 0.005556384, -0.024759391, 0.026231442, 0.016108084, 0.013236381, 0.0075533036, -0.013694888, -0.013091589, 0.004467429, -0.026641686, -0.0031884355, -0.0022322063, 0.027872415, -0.009218409, 0.021513643, 0.017495671, -0.021115465, 0.0032638477, -0.002527823, 0.005836918 - ] - } - ] - }, - { - "name": "springfield-illinois-sangamon-county-seat", - "title": "Springfield: Illinois' Sangamon County Seat", - "content": "Springfield serves as the county seat of Sangamon County in the U.S. state of Illinois.\n\n---", - "keywords": [ - "Springfield", - "county seat", - "Sangamon County", - "Illinois", - "U.S. state" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.0343243, 0.03184899, -0.00958495, -0.02962121, 0.009942495, -0.025578203, 0.011001378, 0.01197775, -0.0375422, 0.0131053915, -0.021686466, 0.010190026, -0.009413054, 0.005342544, 0.013456061, -0.006195151, -0.012844109, 0.019197404, -0.0012668774, -0.0040464443, 0.04590325, 0.0017739722, -0.04799351, 0.022415308, -0.0046171406, 0.016117018, -0.04408802, 0.021383928, -0.03135393, 0.00081307057, -0.01596575, -0.0018908618, 0.032041512, 0.03822979, -0.009639957, -0.036139525, 0.004441806, -0.034214284, -0.002770972, 0.012163399, -0.014728094, -0.0055694478, 0.01805601, 0.0000574779, 0.018949874, -0.009495564, -0.0014009568, -0.020902619, 0.0048681097, 0.0109669985, 0.011970874, -0.013531695, 0.008560447, 0.037899747, 0.03058383, 0.0012496877, 0.021768976, 0.033251666, -0.012740971, -0.05151395, -0.02427179, -0.011792102, -0.03377423, 0.005995751, 0.022415308, 0.016584577, -0.004163334, 0.009406178, 0.037734725, 0.005345982, 0.030803857, 0.035011884, 0.012947246, 0.048901126, 0.04488562, -0.008993627, -0.03690962, -0.021466438, -0.026870865, 0.022126522, 0.03443431, -0.033966754, -0.009323668, 0.028136024, 0.0129403705, -0.0227591, 0.003508408, -0.070848875, 0.00008428302, 0.009426805, 0.006989313, -0.013758598, -0.023584204, -0.01709339, -0.032949127, -0.03872485, 0.05715216, -0.012417805, 0.0012591421, 0.02890612, 0.005397551, -0.008752971, -0.049121153, -0.008842357, 0.01892237, 0.019059887, 0.011475813, -0.022937873, -0.020957625, -0.009935619, 0.012507192, 0.0020730721, 0.00014385591, -0.029566202, -0.0239555, -0.011193902, -0.002671272, -0.020517569, 0.048433565, 0.010595702, -0.016075764, -0.006133268, -0.016185777, -0.000042034073, -0.016598329, 0.035561953, 0.01357295, -0.044995636, 0.031271417, -0.011565198, 0.0131053915, -0.0014353361, 0.004462434, 0.008285413, -0.003121641, -0.018234784, 0.029236162, 0.04903864, 0.026073266, 0.019857487, -0.07519442, 0.0015951998, 0.020847611, -0.01701088, 0.018344797, -0.005641644, -0.004696213, -0.022979127, -0.010348171, -0.031491444, -0.018262288, -0.014714343, -0.03536943, 0.019995004, 0.031188907, -0.01109764, 0.0006467607, -0.034351803, 0.034379307, -0.04136518, -0.022415308, -0.010808853, 0.028012259, 0.011902115, -0.01652957, -0.008010378, 0.002336074, -0.005995751, -0.007467185, 0.007914117, 0.029428685, 0.0045346306, -0.013593578, -0.074754365, 0.017285915, -0.03041881, 0.005242844, -0.022291541, 0.0016837266, 0.038284793, -0.0077559715, 0.029401183, 0.021177653, -0.016240785, -0.017780976, -0.008684212, 0.0020679152, 0.026073266, 0.042355306, -0.016405804, -0.015745722, 0.037322175, 0.038064767, 0.0140061295, 0.026238287, -0.014033632, -0.034709346, -0.030886369, -0.0013536852, 0.014659336, -0.05924242, 0.022539074, 0.029896244, -0.01932117, 0.007487813, 0.022869114, 0.0028087893, -0.028988631, 0.0026489254, 0.040347554, 0.008003502, 0.0071646473, -0.007322792, 0.06171773, -0.031628963, 0.031078892, -0.014425557, -0.0046412065, -0.024808107, 0.051376436, 0.031436436, 0.00882173, -0.009976874, 0.00077783177, -0.005397551, -0.007900365, -0.052669097, -0.03303164, 0.029676218, 0.004658396, 0.013064136, -0.004885299, 0.01580073, 0.04408802, -0.035781983, 0.04007252, 0.016213281, -0.007020254, -0.033251666, 0.030308796, -0.003719841, 0.04879111, -0.013483563, -0.027049638, -0.005552258, -0.01564946, 0.0075015645, 0.01948619, 0.0013046947, 0.013600454, 0.00069102406, 0.004307727, 0.0343243, 0.0062707854, -0.01177835, 0.013820481, -0.022786604, -0.0038917374, 0.010245034, 0.019101143, -0.007838482, 0.027847238, -0.010355047, 0.017272163, 0.007694089, -0.023460438, -0.023432935, 0.00043231976, 0.011922743, -0.014838109, -0.023680465, -0.0066180164, 0.005706965, 0.028933624, 0.041970257, 0.0040808236, 0.018977378, 0.031106396, -0.034269292, -0.015841983, -0.015181902, -0.012527819, 0.0116820885, -0.04854358, 0.012823481, 0.020215033, -0.009351172, 0.051348932, 0.0152506605, -0.028713597, -0.0012514067, -0.02267659, 0.032921623, 0.031683967, 0.050001264, 0.010960123, 0.02154895, 0.009096764, -0.03877986, 0.0013313387, 0.02370797, 0.024079265, 0.010348171, 0.00001035408, 0.01619953, -0.00825791, 0.01018315, 0.014961874, -0.03226154, -0.055749483, 0.0025475065, -0.03825729, -0.0018135083, 0.018798605, -0.00941993, 0.029016133, -0.003912365, -0.01229404, 0.04967122, -0.035039388, -0.019197404, 0.017863488, 0.022030259, -0.004359296, -0.016117018, 0.0069480576, -0.022896618, 0.004259596, 0.022662839, 0.0145355705, -0.029648714, 0.0006360172, -0.0059269923, -0.04191525, -0.014728094, -0.027517196, 0.017698467, 0.009234281, 0.020627584, 0.012706592, -0.023652961, -0.020710094, -0.0003661396, 0.04576573, 0.03160146, 0.010561323, 0.004242406, 0.0010193464, -0.027805982, 0.0056107026, 0.00074173353, 0.009399302, 0.035176907, -0.0195687, 0.02362546, 0.0054113027, -0.033471692, -0.03938493, -0.0070477575, -0.010361923, -0.028603582, 0.0062742233, -0.010196902, 0.0058169784, 0.014398053, 0.045160655, -0.017574701, 0.018908618, 0.036634587, 0.004912803, -0.010368799, 0.013964874, 0.013951123, -0.0074053025, 0.021892741, 0.0014714343, -0.004565572, 0.013270412, 0.005538506, -0.009117392, 0.040980134, 0.010341295, -0.038422313, -0.017602205, -0.00063558744, -0.029951252, -0.015910743, -0.027750976, -0.026059514, 0.018166026, -0.042740352, -0.007920992, 0.0063601714, -0.030171279, -0.026018258, 0.028851114, -0.05979249, 0.013751722, 0.058967385, 0.030941375, 0.015181902, -0.030501321, 0.011984626, 0.028273541, -0.03239906, -0.027695969, 0.0059476197, -0.060397565, 0.008931744, -0.0060645095, -0.026967127, 0.0002116476, 0.00893862, 0.007281537, -0.019651212, -0.05013878, -0.017698467, 0.032949127, 0.013786102, 0.008725468, 0.0039708097, -0.042382807, -0.003661396, -0.018908618, 0.00267471, 0.014205529, 0.02649957, 0.03734968, -0.033471692, -0.036634587, -0.034296796, -0.009014254, -0.0058444818, 0.035479445, -0.0030322548, 0.044995636, -0.002568134, 0.012452184, 0.046370808, 0.014590577, -0.017272163, 0.017107142, 0.04928617, -0.05046882, 0.00004904423, 0.004242406, 0.012555323, 0.0043352307, 0.028521072, 0.039219912, -0.013737971, 0.019541197, 0.0049678097, 0.012080888, -0.0011731938, 0.027737224, -0.045545705, 0.021438936, -0.02689837, 0.045050643, 0.02915365, 0.0060576336, -0.0056313304, -0.037982255, -0.014315543, 0.025055638, 0.018001005, 0.013201653, 0.0013794696, -0.024849363, -0.0127547225, -0.022580327, -0.029813735, 0.010031882, -0.04804852, -0.009612454, -0.0074946885, 0.008079137, 0.017547198, -0.005693213, -0.050963882, -0.0016931808, -0.057482198, 0.009846233, 0.0077422196, 0.0025457875, -0.029236162, -0.042272795, 0.0031972753, -0.015594454, 0.08482062, 0.02483561, -0.010279412, -0.013242909, 0.016474564, 0.009770598, -0.014343047, -0.028493568, 0.024065513, 0.03536943, 0.0024976565, 0.05453933, 0.032069016, -0.021040136, 0.0054250546, -0.002030098, -0.010994502, -0.042327803, 0.0095368195, 0.009193026, 0.050743856, 0.008759847, -0.030253788, -0.02603201, 0.0001669545, -0.015388178, -0.01764346, -0.0012290602, 0.034929376, 0.0027469066, -0.0053837993, 0.0063017267, -0.0025578204, 0.0069618095, 0.0057516578, 0.010753847, 0.015539446, -0.0029325548, 0.0060645095, -0.041420188, -0.031436436, -0.0015608205, 0.004159896, 0.028741099, 0.03616703, -0.0023876429, -0.024656838, -0.010960123, -0.005304727, 0.012967874, -0.029263666, -0.023432935, 0.014329295, 0.007529068, -0.0015221437, 0.012486564, 0.004366172, 0.0053803613, -0.0071508954, -0.021218907, 0.038944878, -0.004345544, -0.004462434, -0.00018414416, -0.0064220545, -0.010396302, -0.017588453, -0.010128143, -0.0050606336, -0.0012135895, 0.012245908, 0.006583637, 0.04120016, 0.015690716, 0.0031903996, 0.009612454, -0.0046308925, -0.0049231164, -0.0072059026, -0.008141019, 0.007831606, 0.0036270167, 0.04557321, 0.0048474823, 0.020297542, -0.00090761366, -0.03465434, -0.01089824, 0.010451309, -0.009131144, 0.020421308, -0.0012763317, 0.025344424, 0.009089888, 0.01137955, 0.013689839, -0.0020197842, -0.023529196, -0.02722841, 0.02882361, 0.021053886, 0.019733721, -0.0024048325, -0.012679088, 0.003166334, 0.0030580394, -0.02340543, 0.005490375, -0.018564826, -0.020077515, 0.016433308, 0.0036063893, 0.032371555, -0.040127527, -0.016887115, 0.0022621583, 0.03135393, 0.0039811237, 0.014590577, 0.006538944, 0.022016507, 0.025399432, -0.008003502, -0.0012651584, -0.006122954, -0.008972999, 0.03338918, 0.037487194, 0.01827604, -0.0053116027, -0.009523068, -0.029676218, 0.0031336737, -0.013016005, 0.028246038, -0.023983004, 0.013524819, -0.025426935, -0.021383928, 0.014618081, -0.011771474, 0.0021108894, -0.027242161, -0.0074465577, 0.016900867, -0.01229404, -0.0053666094, -0.0048612338, -0.010712592, 0.008807979, 0.005211903, 0.038119774, 0.013985502, 0.020503817, -0.025825735, -0.0033090082, -0.02770972, 0.039659966, -0.0063120406, -0.00013998823, 0.047855992, -0.0048406064, 0.0029514635, -0.00029458766, 0.0045311926, 0.007810978, 0.0077353436, -0.033884242, 0.04007252, -0.016034508, -0.029346175, 0.013511067, -0.013517943, 0.011262661, 0.03201401, 0.015539446, -0.025371928, 0.053054146, 0.020943873, -0.0020472878, -0.005490375, 0.014439308, 0.0059338682, -0.008766723, -0.010685088, -0.009172399, -0.010746971, -0.009289289, -0.014659336, 0.0011199058, -0.014920618, -0.004885299, -0.0062639094, -0.028163528, 0.024945624, -0.021755224, 0.02571572, 0.049918752, -0.016172025, -0.022924121, 0.016997129, 0.025096893, 0.009227405, -0.014301791, 0.03217903, -0.003465434, 0.020448811, 0.03553445, -0.00722653, -0.016474564, -0.0011517068, 0.014755598, 0.026375804, -0.05198151, 0.008732344, 0.024863115, -0.0024993755, -0.033196658, -0.031023886, 0.0004989297, 0.007329668, -0.01691462, 0.017258411, 0.006593951, 0.041062642, -0.039879996, 0.042767856, -0.037789732, -0.011317668, 0.029098645, 0.0148931155, 0.008409178, -0.03322416, -0.0086498335, -0.019788729, 0.008684212, -0.024450563, 0.01069884, 0.008319792, 0.006222654, -0.039632462, -0.010224406, -0.0150718875, -0.005459434, 0.026472066, 0.008216654, -0.024038011, -0.021177653, -0.003511846, 0.0074053025, 0.009694965, -0.013909867, 0.04095263, 0.0037885995, -0.0047340305, 0.015305667, -0.0006304305, 0.003668272, -0.06116766, 0.022140274, -0.022346549, -0.016282039, 0.035424437, -0.0041427063, 0.008972999, 0.009825606, 0.012685964, 0.019059887, 0.024546824, 0.038174782, 0.01644706, 0.022635335, 0.002177929, -0.0005883159, 0.014081764, 0.035974506, 0.013634833, 0.010025006, 0.023584204, 0.010306916, 0.03281161, 0.0019768102, -0.013277288, -0.009866861, 0.02682961, 0.01297475, -0.0069618095, -0.0054834993, -0.009770598, -0.008436682, 0.042272795, 0.0023463876, -0.032536574, -0.015883239, 0.00057886157, -0.023900494, -0.010279412, 0.0069721234, -0.00097121537, -0.018743597, -0.010148771, -0.0037851615, -0.033966754, 0.012080888, 0.008594827, -0.012954122, 0.008732344, -0.01923866, 0.029043637, 0.009914991, -0.0007004784, 0.016419556, 0.016020756, 0.0058444818, 0.0020885428, -0.0070546335, -0.009378674, 0.029236162, -0.02819103, 0.03586449, 0.039659966, -0.0064186165, 0.029401183, 0.0125759505, 0.001609811, -0.017340923, 0.02819103, -0.0063670473, -0.003018503, -0.017024633, 0.023487942, -0.016749598, 0.01787724, -0.0011250628, -0.0044177407, 0.017134646, 0.0203663, -0.041172657, -0.0032643152, 0.01652957, 0.019183652, -0.015869487, -0.016598329, -0.0098943645, -0.0028586392, 0.00958495, 0.035176907, -0.0050640716, 0.032921623, 0.016777102, -0.02011877, -0.008154771, 0.016419556, -0.0049196784, 0.0090073785, 0.04351045, 0.0062054647, -0.043647967, 0.011840233, -0.04477561, -0.022016507, 0.018427309, -0.020558825, 0.028521072, -0.010478812, 0.010836357, 0.04766347, -0.001830698, 0.0077009643, 0.021177653, 0.012988502, -0.0033433875, 0.0275447, 0.008099765, 0.018179776, 0.034461815, 0.019211156, -0.0048818616, -0.00021186247, -0.014425557, 0.024230534, 0.0040326924, -0.023171652, -0.011022005, -0.01835855, 0.007016816, -0.012390302, 0.0025406308, 0.0055041267, 0.0123971775, -0.03473685, -0.0033691719, -0.015704466, -0.03363671, 0.022319045, 0.022731597, -0.020943873, -0.01564946, 0.01297475, -0.03234405, -0.0020163462, 0.004703089, 0.013414805, -0.0019011756, 0.013022881, 0.0014336171, -0.021452688, -0.020050012, -0.01573197, 0.047140904, 0.010306916, -0.00032209113, -0.000012408781, -0.017134646, -0.0068071024, 0.0016957592, 0.020008756, -0.01297475, -0.002719403, 0.017258411, 0.0156082045, 0.03289412, -0.0033537012, -0.022387804, -0.015883239, 0.012617205, -0.035947002, 0.01401988, -0.00024710127, 0.026197031, 0.04056758, 0.012019006, -0.0134629365, -0.0020610394, -0.0027125273, -0.011475813, 0.020696342, -0.02075135, -0.021136397, 0.005239406, -0.0037817236, -0.006105765, -0.0021882427, 0.011400178, 0.028273541, -0.022896618, -0.030831361, -0.034516823, -0.015085639, 0.0038986134, 0.011393302, 0.053631715, -0.033499196, -0.022717845, -0.0040876996, 0.032206535, -0.00436961, 0.020091265, 0.023790479, 0.0018908618, 0.018138522, 0.011881488, -0.0051259543, -0.0034963754, 0.03902739, 0.018482314, -0.03443431, 0.031821486, 0.00025526635, 0.009234281, -0.010843233, 0.0025612584, -0.016873363, 0.0026437687, -0.0023051326, -0.01892237, -0.006133268, -0.03721216, -0.0095436955, 0.0036407686, 0.0020455688, -0.0044864994, -0.016983377, -0.020655086, 0.006012941, 0.01661208, -0.008993627, 0.0017705342, 0.0029273978, 0.023240412, 0.008285413, 0.025055638, -0.0016579421, 0.02857608, 0.027420934, -0.0187711, -0.025977004, -0.013847984, -0.0074534337, -0.0031010134, 0.05324667, -0.025138149, -0.005194713, -0.0003932133, 0.039082393, 0.0018289791, 0.011640833, 0.01141393, 0.006985875, -0.005297851, 0.014796853, -0.034791857, -0.015855735, 0.0143705495, -0.024945624, 0.009344296, -0.037954755, 0.005538506, -0.0038298548, 0.003613265, -0.0031903996, -0.0046446444, 0.014604329, -0.033994257, -0.026403306, -0.023584204, -0.005944182, -0.023185404, -0.0100800125, 0.0026901807, -0.010162523, 0.010568199, 0.0057585337, -0.011716467, 0.0036407686, -0.004747782, 0.01661208, 0.022470314, 0.032839112, 0.022607831, 0.015470687, 0.014136771, -0.004342106, 0.022497818, -0.025770728, 0.006195151, -0.0191699, 0.052119028, 0.008106641, 0.014136771, 0.024230534, -0.019761225, 0.011709591, 0.01604826, -0.011743971, 0.009990626, 0.0044830614, 0.0070065027, 0.014755598, 0.0058719856, -0.0041323924, -0.0059510577, -0.0025200031, 0.010210654, -0.038862366, -0.008436682, 0.00036592473, 0.025399432, -0.0017284196, -0.01845481, 0.01533317, -0.003905489, 0.03943994, -0.013834232, 0.011702715, -0.013889239, -0.01377235, 0.02187899, 0.02506939, 0.0004473607, 0.016832108, -0.010190026, 0.019582452, -0.02649957, 0.0059338682, 0.0034757478, -0.02570197, 0.010135019, -0.01309164, -0.02483561, 0.0053700474, 0.020806355, -0.032041512, 0.032124024, 0.026719596, -0.0583073, 0.018509818, -0.012740971, -0.015195653, 0.041172657, -0.01844106, -0.0060198163, 0.003616703, -0.0053837993, 0.028466064, -0.023735473, -0.0031955566, 0.04161271, 0.008553571, 0.0073571713, -0.015759474, -0.0143705495, 0.0032522823, -0.029043637, -0.027585955, -0.018234784, -0.026073266, -0.0033949565, -0.029511197, -0.009069261, 0.010795102, 0.013022881, -0.017450936, 0.014948122, 0.010279412, 0.009440557, -0.013841108, 0.006697089, 0.0066283303, -0.029923748, -0.004218341, -0.016804604, -0.026018258, 0.01948619, 0.001339074, -0.044610586, -0.017547198, -0.016955873, -0.0051912754, 0.004012065, 0.0021538634, 0.009282413, -0.015511943, -0.021273915, 0.022346549, -0.018221032, 0.020380052, -0.025921997, 0.05132143, -0.016722094, 0.0054113027, -0.0024185842, -0.02020128, -0.02427179, 0.027819734, -0.009275537, 0.026884617, -0.00798975, -0.007364047, -0.011008253, -0.025977004, 0.00054190384, 0.010327543, -0.013737971, -0.0043799235, -0.02171397, -0.008594827, -0.019059887, -0.020215033, 0.022401556, -0.022387804, 0.0029841238, 0.007721592, 0.010355047, 0.004909365, 0.019087391, 0.022497818, -0.008065385, -0.018537322, 0.03239906, 0.012878488, 0.01724466, -0.014425557, -0.021851487, -0.007364047, -0.01619953, 0.0044005513, 0.016832108, -0.014150523, 0.009942495, 0.0011774913, 0.0032849426, -0.0035548203, 0.0036063893, -0.025138149, -0.015058136, -0.011441433, -0.031683967, -0.039797485, 0.007982875, -0.028067265, 0.005937306, 0.018729845, 0.008436682, 0.019018631, -0.008814854, -0.0024718721, -0.0093717985, 0.025688218, 0.0383398, -0.030748852, 0.03135393, -0.00523253, 0.008209778, 0.023295417, -0.0066489577, 0.007982875, -0.014205529, -0.0027056513, -0.002671272, 0.00941993, 0.022236535, 0.009406178, -0.008869861, 0.00574822, -0.018619832, -0.033609208, 0.002179648, 0.009151772, 0.0058719856, 0.019128645, 0.004599951, -0.029978754, 0.013284164, -0.0030425685, 0.007329668, 0.010389426, 0.0071096406, 0.0219615, -0.0003828995, -0.00055264734, -0.00302366, -0.013421681, 0.0036820236, -0.03239906, -0.029098645, -0.03160146, 0.0074190544, 0.0022019946, 0.029181154, 0.044198036, -0.0035341927, 0.010148771, 0.017849736, 0.0012582826, 0.0052840994, -0.020380052, -0.027269665, 0.003967372, -0.046755854, -0.0009110516, 0.016790854, 0.020146273, -0.024051763, -0.016378302, -0.0009978594, -0.03146394, 0.022181528, -0.021810232, 0.008340419, 0.009254909, 0.01885361, 0.002726279, 0.02139768, 0.039962504, 0.018042259, -0.022882866, 0.0001039974, 0.017203405, -0.014081764, -0.010685088, 0.016405804, -0.0058376063, -0.0203663, 0.023446687, 0.030501321, -0.03745969, -0.0069480576, 0.008931744, -0.018688591, 0.0059716855, 0.007364047, -0.011365798, 0.0022415307, -0.0013717343, -0.003963934, 0.025770728, 0.0061573335, 0.02035255, -0.006978999, 0.023501694, 0.000032015727, -0.00032896697, 0.0148931155, -0.0187711, -0.0047134026, 0.042767856, -0.0335267, 0.0021607394, -0.003998313, 0.015539446, -0.027077142, 0.0017387334, -0.029648714, -0.012156523, -0.005796351, -0.01556695, -0.0032007133, -0.0007245439, 0.008141019, 0.033361677, -0.003905489, -0.009990626, 0.00069618097, 0.033719223, -0.011310792, 0.026692092, 0.0018925808, -0.013662336, -0.014604329, -0.053164158, -0.008354171, 0.005559134, -0.010320667, 0.017945997, 0.0038504822, 0.020778852, -0.0031594583, 0.004266472, 0.014109267, -0.0041461443, -0.0134629365, -0.00038053593, 0.0026609583, -0.01573197, -0.037019636, -0.0068552336, -0.026884617, 0.028301045, 0.021631459, 0.0045311926, 0.01201213, -0.00014224437, 0.023721721, -0.0075428197, 0.0035926374, -0.0028191032, 0.003668272, 0.020723846, -0.024808107, -0.00603013, 0.0335267, -0.034049265, 0.010155647, -0.019733721, -0.00019639179, -0.010609454, 0.010953247, 0.009461185, 0.026692092, 0.0015247222, 0.01684586, 0.04186024, 0.0077078403, -0.008862985, -0.0029583392, -0.00312336, -0.024601832, -0.00035496632, -0.019995004, -0.0072540333, 0.018372301, 0.031491444, 0.029263666, -0.005878861, -0.0036854616, 0.009206778, -0.015718218, -0.027558452, -0.013229157, -0.031656466, -0.009880613, 0.023432935, 0.00654582, 0.013896115, 0.02666459, 0.011881488, -0.0041461443, 0.01309164, 0.017629707, -0.0019355549, 0.008574199, -0.02099888, 0.012699716, -0.00063171977, 0.0038092271, 0.009550571, -0.0027640963, -0.014260536, 0.0010992783, -0.00524972, -0.0068449196, 0.013579826, -0.010863861, 0.04422554, -0.009199902, 0.0024907808, -0.011269537, -0.016749598, -0.013978626, -0.0030511634, -0.012115267, 0.008526068, -0.02986874, 0.01676335, -0.032069016, -0.017175902, 0.02603201, 0.0052187787, -0.026073266, 0.003998313, -0.023171652, -0.0066455198, 0.0061813993, -0.006923992, -0.029208658, -0.015360674, -0.015828233, -0.008147895, 0.011840233, -0.015044385, -0.004699651, -0.0027898806, -0.021425184, 0.0033537012, -0.01844106, -0.00546631, -0.008849233, 0.0028483255, -0.014343047, -0.0148931155, -0.02171397, -0.0049437443, 0.017973501, -0.004015503, -0.012335295, 0.004441806, -0.03569947, 0.0006897348, -0.0034121461, -0.022566576, 0.0116820885, -0.037157156, -0.0015745722, 0.041557703, 0.014081764, 0.015113143, 0.023212908, 0.022181528, -0.0007563447, 0.017038384, -0.0004314603, 0.00842293, 0.016378302, 0.03982499, -0.0056244545, 0.01229404, 0.00020638327, -0.010224406, 0.0014430714, -0.011970874, 0.018111018, -0.0024031135, 0.011640833, 0.012981626, 0.024409307, 0.029126149, 0.010348171, -0.01265846, -0.0143705495, -0.005700089, -0.0030872617, 0.004321479, 0.008367923, -0.0028981755, 0.036634587, -0.024629334, 0.0017215437, -0.00085991237, 0.010884488, -0.026733348, 0.018977378, 0.027874742, -0.0071783992, -0.037982255, 0.009715592, 0.0066592717, 0.0037473442, 0.010361923, 0.026389556, 0.0070546335, 0.019417431, -0.0024117082, -0.021232659, 0.005208465, -0.008319792, 0.0044796234, -0.032124024, -0.0007589232, -0.013016005, 0.006882737, 0.025853237, -0.0063739233, 0.013407929, -0.015264411, 0.018812357, 0.02418928, -0.009028005, 0.010169399, 0.003269472, 0.014411805, 0.037954755, 0.0018203843, -0.0271459, 0.008367923, -0.004899051, 0.0035307547, -0.000445212, 0.00830604, -0.012679088, 0.0076184543, -0.0010752128, -0.019926246, -0.0082922885, 0.005342544, 0.025688218, -0.0027950376, 0.007872861, -0.02666459, -0.005679461, -0.016997129, 0.0068621095, 0.020778852, -0.0049506202, -0.02649957, 0.027971003, 0.013799854, -0.009296164, -0.014191777, -0.008766723, -0.030473817, -0.0046240166, -0.0012771912, -0.0074190544, 0.009323668, -0.007274661, -0.0054834993, 0.027324673, 0.001558242, 0.014590577, -0.01556695, 0.020572577, 0.03248157, -0.0077628475, 0.0032385306, -0.0050846995, 0.0020919808, 0.016639585, 0.017615957, -0.0025973564, 0.009330544, -0.004221779, 0.010334419, -0.0034413685, -0.01118015, -0.01516815, -0.011847109, 0.0014525257, 0.016749598, 0.015910743, -0.006226092, -0.003271191, -0.014466812, 0.006587075, 0.011709591, 0.0010606016, -0.014521819, 0.0067624096, 0.024285542, 0.00086807745, -0.010746971, 0.0029995944, -0.009529944, -0.010877612, -0.009069261, -0.023501694, 0.0071783992, -0.016007004, -0.0033038512, -0.008897364, -0.005896051, 0.012479688, -0.045710724, 0.015910743, -0.015621956, -0.0025526634, 0.007563447, -0.0014748722, -0.0043008514, -0.008010378, -0.01041693, -0.019664964, -0.012637833, -0.011063261, -0.00941993, 0.030281292, -0.002121203, -0.013524819, -0.006480499, 0.017835984, -0.014618081, -0.013930495, -0.008897364, 0.020105017, -0.019059887, 0.008539819, -0.013627957, -0.021851487, -0.015470687, 0.0066627096, 0.009213654, 0.0018994566, 0.027077142, -0.0068449196, -0.0062604714, 0.012025882, 0.028163528, -0.015938247, 0.011537695, 0.019953748, 0.032564078, -0.013071012, 0.0070477575, 0.029043637, 0.037899747, -0.003317603, 0.002234655, -0.005882299, -0.0067417817, -0.0015659773, -0.003675148, 0.018221032, 0.011125144, 0.0071990266, -0.012266536, -0.017670963, -0.0014894834, -0.017107142, -0.0035857616, 0.0033502635, 0.0036338926, -0.011517067, 0.01337355, -0.009399302, 0.019664964, 0.01313977, 0.010685088, -0.0026575204, 0.0011345171, -0.0102587845, -0.0040567582, -0.006095451, -0.011407054, -0.0064701852, -0.0017473282, 0.00564852, -0.01669459, -0.0004159896, 0.00088011025, -0.0075084404, 0.0019957188, -0.012630957, 0.043372933, 0.0064976886, -0.0081272675, -0.008519192, 0.00042737776, 0.0020197842, -0.01938993, -0.0050846995, 0.012747847, -0.007831606, -0.009358047, -0.009873737, -0.0041736476, 0.0027589393, 0.0024529635, -0.019293666, 0.018564826, -0.020022508, -0.00040116353, -0.015319419, 0.00032767776, -0.00546631, -0.004201151, -0.021053886, -0.018743597, 0.009674337, 0.013407929, -0.0023498256, 0.0049781236, 0.0046137027, 0.012713468, -0.009440557, 0.00822353, 0.023515444, 0.03146394, 0.0148931155, -0.007082137, 0.02093012, -0.0020885428, -0.013098516, -0.012864736, 0.02498688, 0.008670461, -0.0033640151, 0.0011852266, 0.010671336, -0.025536949, -0.015415681, -0.0020541635, -0.004256158, -0.006449558, -0.008869861, 0.0035823237, -0.015085639, 0.0031061703, -0.021301419, 0.00047787235, -0.018042259, -0.003168053, 0.007927868, -0.00030490148, -0.0030047514, -0.024615582, 0.01835855, -0.005108765, 0.015553198, 0.0017705342, 0.009598702, 0.014480564, -0.020503817, 0.0093717985, -0.0020558825, 0.007687213, -0.022374053, -0.026774604, 0.01988499, 0.0060541956, 0.0059785615, 0.0067245923, -0.010410054, -0.008196026, -0.00053159, 0.013057261, 0.011077012, 0.018688591, -0.013036633, -0.006934306, 0.01317415, 0.009976874, 0.010994502, -0.0077628475, 0.0068174163, 0.004225217, -0.008326668, 0.004410865, 0.016378302, -0.033856742, 0.0010872455, -0.01900488, -0.02731092, -0.013449185, -0.016268287, -0.018784853, 0.013779226, 0.0022896617, 0.02282786, -0.017203405, 0.01938993, -0.0038401685, 0.0034551201, -0.033499196, -0.0074534337, 0.009846233, -0.014728094, -0.0074534337, 0.033251666, 0.013593578, 0.027187156, 0.010114391, -0.012053384, 0.022924121, -0.013297915, -0.020985128, -0.00798975, 0.011338295, 0.011606454, 0.012355923, 0.01780848, 0.0036545203, -0.013421681, 0.013669212, -0.0074190544, 0.011991502, 0.0057172785, 0.0076390817, 0.0038229788, -0.00015911172, 0.023267914, 0.008134143, 0.0025406308, -0.030363804, -0.018372301, -0.002133236, -0.0030442874, 0.009859985, -0.0122184055, 0.011118268, -0.010341295, 0.0038814237, -0.000029705869, 0.003675148, -0.0059304303, -0.017450936, -0.034626838, 0.011077012, 0.012019006, -0.00036313143, -0.02986874, 0.009165523, -0.00090503524, -0.0011594421, -0.0077490956, 0.0122184055, -0.0015436308, -0.00030726506, -0.01089824, -0.017849736, 0.014563074, 0.0031061703, 0.0005311603, 0.0134629365, 0.0134629365, -0.0152506605, 0.011104516, -0.0024735911, 0.01089824, -0.0024804669, 0.014329295, -0.0020163462, -0.0038298548, -0.018069763, 0.00087065593, -0.011077012, -0.008003502, 0.010719468, 0.008972999, 0.011242033, -0.0191699, 0.007776599, -0.009076137, -0.01749219, -0.008381675, -0.0131122675, -0.007467185, -0.02003626, 0.002717684, -0.03041881, 0.0017774102, 0.0013545447, 0.0010004378, 0.0070958887, 0.018069763, -0.029098645, 0.024216782, -0.012850985, -0.0076390817, -0.007584075, 0.0019132084, -0.0008362766, -0.00082553306, -0.022951625, -0.014714343, -0.006700527, -0.015346922, -0.020297542, 0.0017026351, -0.01533317, -0.0010683369, 0.0036338926, 0.010148771, -0.00019316873, -0.027022135, -0.009928743, 0.009825606, 0.020792603, 0.0040464443, 0.0030872617, -0.00922053, -0.016502067, 0.03209652, -0.034104273, -0.012520943, 0.00515002, 0.0159795, -0.0074190544, 0.01812477, 0.0027039323, 0.008202902, 0.022800356, -0.012445308, 0.01245906, -0.00930304, -0.009076137, 0.007116516, -0.0076184543, 0.0021091704, -0.0066489577, 0.013579826, 0.0070546335, 0.006133268, 0.0039879996, 0.015319419, 0.04670085, -0.0053116027, -0.005105327, -0.0004636909, -0.015181902, -0.0041495822, 0.026912121, -0.014398053, 0.017052136, -0.0050881375, 0.003125079, 0.015511943, 0.0014301791, 0.005442244, -0.032866616, -0.018083515, -0.009846233, -0.0049849995, 0.019293666, -0.014480564, 0.0070477575, -0.016557073, 0.005053758, 0.021411432, 0.017780976, 0.020737598, -0.021493942, 0.01493437, -0.0076322057, -0.020050012, 0.0058651096, -0.009997502, -0.014961874, -0.0030941374, 0.025743224, 0.0064254925, 0.011544571, 0.009392426, 0.006992751, 0.034049265, 0.004988437, 0.0062020267, -0.01892237, 0.0008216654, -0.032151528, -0.030748852, -0.004885299, 0.022236535, -0.017574701, -0.011248909, -0.011049509, 0.0065183165, -0.009399302, 0.0033622961, 0.0036270167, -0.009014254, -0.006593951, 0.0035376307, 0.007016816, -0.022786604, -0.026650839, -0.011853985, 0.0060198163, 0.02986874, -0.02731092, 0.01046506, 0.0044314926, -0.006542382, -0.008560447, 0.008546695, -0.004764972, 0.013902991, 0.013497315, 0.020325046, 0.013263537, 0.012300916, 0.016350798, 0.024821859, 0.0048715477, 0.0038986134, -0.010272536, 0.006026692, 0.00028749695, -0.0011688964, 0.036469568, 0.013964874, -0.002382486, -0.027654713, -0.0019063326, 0.011606454, 0.025041886, 0.011269537, -0.015663212, 0.0039776857, 0.018991128, -0.012465936, -0.007590951, -0.014301791, -0.022085266, -0.011159522, -0.003214465, 0.011922743, 0.01217715, 0.021755224, 0.03418678, 0.026362052, 0.029841237, -0.025495693, -0.01357295, 0.002324041, 0.017382177, 0.009433681, -0.007879737, 0.008484812, 0.00782473, -0.007975999, 0.003016784, 0.0122184055, 0.009124268, -0.0004226506, 0.005194713, 0.0029188031, -0.014047384, -0.00090675417, -0.0038367305, 0.029841237, 0.003919241, -0.017464688, 0.018991128, 0.007474061, -0.0062089027, -0.0016244221, 0.004795913, 0.028796107, 0.014301791, -0.018537322, -0.008079137, 0.0029995944, 0.007370923, 0.009124268, -0.0048681097, -0.01445306, -0.005242844, 0.024038011, -0.0068724235, 0.015113143, -0.008787351, -0.019898742, -0.008986751, 0.004366172, -0.012280288, 0.009605578, -0.0084641855, 0.03448932, 0.004902489, 0.025990754, -0.014645584, -0.022649087, 0.0048784236, -0.007927868, 0.004706527, -0.005889175, 0.000044263357, -0.018908618, 0.0003214465, -0.014205529, -0.011565198, -0.022099018, 0.021095142, -0.00961933, 0.0035238788, -0.018111018, -0.018152274, 0.0048268544, -0.00027868725, -0.018248536, 0.0037232789, -0.018221032, -0.012789101, -0.012878488, -0.018688591, -0.005246282, -0.008031006, -0.0023154463, 0.01691462, -0.006432368, -0.020875115, -0.024326798, 0.014178026, -0.014576825, 0.015786977, -0.0066558337, -0.020091265, 0.025536949, 0.006473623, 0.005538506, 0.0029153652, 0.008773599, -0.008072261, 0.000101365236, -0.022965377, 0.011840233, -0.006532068, -0.012603453, -0.013359798, 0.025811983, 0.013036633, 0.013050385, 0.018413557, 0.017657211, -0.022236535, 0.022869114, -0.0125759505, -0.00435242, -0.013614205, -0.010946371, 0.009969999, -0.0068071024, 0.030803857, -0.012802853, -0.004211465, -0.0059510577, -0.007886613, -0.0019389929, -0.0043283547, -0.029071141, -0.0009265223, 0.01309164, 0.0013519662, 0.0062020267, 0.0059476197, -0.00006129186, 0.009633081, 0.026527073, 0.03259158, -0.008924868, -0.014975626, 0.01476935, 0.015828233, -0.012596577, -0.005679461, -0.0035926374, -0.004462434, -0.004355858, 0.018001005, -0.0015006567, 0.021603955, -0.008691088, 0.0036476443, 0.014508067, -0.028411059, 0.016997129, -0.02915365, -0.0063979886, -0.02179648, -0.0005771426, 0.019761225, 0.0051878374, 0.007130268, 0.012520943, -0.012252784, 0.0002539771, 0.026362052, -0.006232968, -0.011881488, -0.010286288, -0.027819734, -0.028686093, 0.018303543, 0.00632923, -0.008807979, -0.02204401, 0.0015470687, -0.017175902, -0.014480564, -0.019348674, 0.01749219, 0.01516815, 0.011049509, 0.010994502, 0.005108765, -0.0131053915, -0.0004503689, 0.026073266, 0.013126019, -0.010135019, 0.004125517, -0.019128645, 0.018193528, -0.019802481, 0.007927868, -0.00465152, -0.014246784, -0.017574701, -0.019059887, 0.021026384, 0.013380426, 0.010141895, -0.005297851, 0.016708342, -0.007439682, -0.015113143, 0.007460309, -0.022704093, -0.00066996674, -0.011221405, 0.008037882, 0.024945624, 0.0255507, -0.009241157, 0.012390302, -0.02028379, 0.021356424, 0.02075135, -0.002573291, 0.0131122675, -0.01030004, 0.023199156, 0.006449558, -0.0057344683, -0.014879364, 0.021342672, -0.0018702343, 0.018647335, 0.01812477, 0.0017387334, 0.004163334, -0.006899927, -0.017065888, 0.000023649201, 0.014301791, 0.0055694478, -0.0000074600407, -0.009997502, 0.007838482, -0.019156149, -0.008147895, -0.0020679152, 0.012981626, 0.023914246, -0.018881114, 0.011750847, -0.013909867, -0.010918868, 0.029896244, -0.005586637, 0.0057585337, 0.011228281, 0.029786231, -0.0041942755, 0.0043730475, -0.010994502, 0.009474937, -0.027929747, 0.012967874, -0.027640961, 0.011633957, 0.0021022945, 0.0043352307, -0.0009239438, 0.03234405, 0.008966123, 0.0059785615, 0.023254162, 0.016722094, 0.011627081, -0.0057138405, -0.009963123, -0.006384237, 0.0040808236, -0.010245034, 0.006635206, -0.002231217, 0.00020530891, 0.020517569, -0.020985128, 0.011331419, 0.016790854, 0.017863488, -0.027283417, 0.0028912995, -0.013084764, 0.0057516578, -0.010210654, 0.015635708, -0.018936122, -0.0020386928, -0.013483563, 0.0023842049, -0.0009531663, 0.0036510823, 0.004813103, -0.006033568, 0.009089888, 0.020710094, -0.013050385, 0.0065251924, 0.008560447, 0.006903365, 0.0056072646, -0.0018410118, -0.008237282, -0.0026936186, -0.005095013, -0.0075565716, -0.038917374, 0.008326668, 0.009110516, 0.0004636909, -0.003068353, -0.022167776, 0.03729467, 0.003860796, -0.0043971133, -0.004008627, -0.018619832, 0.013944247, 0.013634833, -0.0108019775, 0.0042802235, 0.018303543, -0.012617205, 0.031326424, 0.007955371, -0.014136771, 0.01333917, -0.010905116, -0.002616265, 0.005806665, -0.012273412, -0.0005590935, 0.011063261, 0.010733219, 0.0003091989, 0.00069188356, -0.009241157, -0.0051809615, 0.0343243, -0.014233032, 0.01041693, -0.008216654, -0.022649087, 0.013132894, 0.007350296, -0.0007155193, 0.0097293435, 0.0311339, -0.018386053, -0.009227405, -0.011303916, 0.010856985, 0.018014757, 0.00051826803, 0.009928743, 0.011056385, 0.00845731, -0.017505942, -0.0048199785, -0.0106163295, -0.008773599, -0.010843233, -0.017148398, 0.012789101, -0.00126258, -0.01010064, -0.020187529, 0.018427309, 0.013160398, 0.023515444, 0.0069068028, 0.009495564, 0.0033313548, -0.026018258, -0.0028603582, 0.010884488, -0.0033279168, 0.02211277, 0.009144896, -0.01669459, 0.015951999, -0.0138273565, -0.031986505, 0.009798102, -0.005359734, 0.017684715, 0.019307418, 0.017822232, 0.013414805, 0.016639585, 0.016337046, -0.008175399, -0.0050606336, 0.012809729, -0.005115641, -0.009660585, 0.010148771, 0.012032757, -0.004912803, -0.01021753, 0.019431183, -0.014466812, -0.014466812, -0.0027864426, -0.02154895, -0.0035995133, 0.019692466, 0.011627081, 0.0052944133, -0.008553571, 0.0046858992, -0.0027486256, -0.0054250546, 0.01580073, 0.025138149, -0.027847238, -0.028081017, -0.013552322, -0.0025062514, 0.02689837, 0.004060196, 0.0007189572, -0.0052737854, 0.006243282, -0.021823984, -0.001980248, -0.017835984, -0.008732344, 0.024368051, -0.019829983, -0.014796853, -0.019871239, 0.010093764, 0.03721216, 0.016735846, -0.01644706, 0.023267914, 0.008966123, -0.0038264168, 0.0023601393, 0.002124641, -0.0025836048, 0.013064136, -0.002825979, 0.0058616716, 0.011482689, 0.0030889807, 0.010547571, 0.0060163783, 0.0057516578, -0.023597956, 0.0136485845, -0.0047718477, -0.005153458, -0.0051740855, 0.029566202, -0.004857796, 0.00032273572, 0.0057310304, -0.011640833, -0.007384675, -0.0143567985, -0.011936495, 0.0039914376, 0.0059476197, -0.0013614205, 0.023762977, -0.003116484, -0.01313977, 0.006920554, 0.004001751, -0.005359734, -0.014205529, -0.02187899, 0.010815729, -0.0008972999, -0.017038384, -0.009887489, 0.005882299, -0.003417303, -0.012878488, 0.00126258, 0.018991128, 0.02322666, -0.010368799, 0.0038779858, -0.008312916, 0.03250907, -0.0019957188, -0.022786604, -0.008471061, 0.037322175, -0.019376177, -0.0033880805, 0.0054869372, -0.004008627, 0.0319315, 0.009413054, 0.010348171, 0.0015230032, 0.01740968, -0.0038539201, 0.00097121537, 0.0007413038, -0.0037679719, -0.0021538634, -0.010231282, 0.015883239, 0.0077628475, -0.00012022014, 0.024863115, -0.0045861993, 0.012823481, 0.012472812, -0.0052187787, -0.0036442063, -0.01093262, 0.016804604, 0.007893489, -0.01493437, -0.0072884127, -0.0057550957, -0.017299667, 0.015511943, 0.009873737, -0.010375675, 0.017052136, -0.0088286055, -0.0048715477, -0.010382551, -0.021493942, 0.016708342, 0.0018255411, -0.0013442308, 0.012342171, 0.0026007944, 0.017505942, 0.013765474, 0.0035376307, 0.0067177163, -0.021466438, 0.00465152, 0.0042802235, -0.0029377118, -0.019142397, -0.009777474, -0.0023842049, -0.004118641, 0.0020627584, -0.010093764, 0.003864234, 0.006483937, 0.020297542, 0.012087764, 0.0036957753, -0.006538944, 0.0139992535, 0.014755598, 0.0069102403, 0.009578074, 0.009887489, -0.0033915185, -0.008711716, 0.0039536203, -0.00031328143, -0.00024581203, 0.018028509, -0.0031835237, 0.0006368766, 0.0015126894, 0.024849363, 0.0036407686, 0.0008349874, -0.0058341683, 0.0054766233, 0.016350798, 0.0014508067, -0.0062089027, 0.012734095, 0.0025543824, 0.01201213, -0.006941182, -0.008092889, -0.010238158, 0.00083799555, -0.00933742, 0.009598702, 0.0052187787, -0.014755598, 0.024450563, 0.0023498256, -0.00076708826, 0.009413054, 0.0013433713, 0.0009772318, -0.0063017267, -0.027008383, 0.01309164, -0.01357295, 0.008608578, 0.0038057892, 0.0031989943, 0.0029136462, -0.009626206, -0.0075978264, -0.0062845373, 0.011448309, -0.03209652, -0.016185777, -0.0063670473, 0.00063472794, 0.022209032, -0.01157895, -0.0029892807, -0.00006048609, 0.010746971, 0.003994875, 0.013909867, -0.016158273, -0.008278537, 0.0036923375, 0.027723473, -0.025151901, -0.00029995944, -0.0054800613, -0.004892175, -0.0063017267, -0.004988437, -0.018633584, -0.0125828255, 0.023171652, 0.004661834, -0.0017705342, -0.004555258, -0.00087409385, 0.0030253788, -0.016722094, -0.0004284521, -0.0030992944, 0.025523197, -0.02219528, 0.01940368, -0.009825606, 0.009241157, -0.018221032, 0.006133268, 0.009083013, 0.0015951998, -0.00016802886, 0.011001378, -0.003061477, 0.013002253, 0.031216409, -0.0012514067, 0.04125517, -0.010740095, -0.02244281, -0.000917068, -0.00654582, -0.020105017, -0.002427179, -0.0021865238, -0.019307418, 0.022387804, -0.0009205059, 0.007920992, -0.0028500445, 0.01229404, 0.004692775, -0.013071012, 0.016062012, 0.0049265544, -0.008718592, 0.0012359361, 0.01860608, -0.014631833, 0.007329668, 0.012679088, 0.006442682, -0.006573323, -0.0014129895, 0.015016881, 0.01360733, -0.019857487, 0.017107142, -0.009076137, -0.012252784, 0.0029927185, 0.016213281, 0.03327917, -0.011166398, -0.0053769234, 0.010863861, 0.009639957, -0.012802853, -0.013703591, 0.014741846, 0.0068208543, -0.0072127786, 0.014191777, 0.010808853, 0.001558242, 0.005995751, 0.008684212, 0.0017705342, -0.015869487, 0.024505569, -0.010575075, -0.003609827, 0.017423432, 0.006580199, -0.004606827, -0.00922053, 0.004441806, 0.0015187059, -0.02186524, -0.019252412, 0.0026042324, -0.007377799, -0.0038539201, -0.015841983, -0.008319792, 0.0015591015, -0.0032385306, 0.046783358, -0.0018152273, -0.009639957, 0.0026489254, -0.010547571, 0.009117392, -0.023611708, -0.024945624, -0.004060196, 0.008972999, -0.010162523, 0.013064136, -0.029841237, -0.004912803, -0.00465152, 0.018688591, 0.024629334, 0.02402426, -0.0068586715, 0.0044246167, -0.025275666, 0.0007047758, 0.022085266, 0.015470687, -0.012761598, 0.026458314, 0.0011594421, 0.012025882, 0.00082209514, 0.009413054, 0.003013346, -0.011255785, 0.015278163, 0.01724466, -0.017024633, 0.012252784, -0.00083240896, 0.0134766875, 0.0071783992, 0.0058341683, -0.010128143, -0.0327291, -0.0022174653, -0.015814481, 0.00017683856, -0.0057207164, -0.01691462, -0.0016218438, 0.01652957, -0.0062845373, 0.009323668, -0.008890488, -0.0055694478, 0.014824357, -0.0012539852, 0.022882866, -0.010348171, 0.005981999, 0.013359798, -0.009062385, 0.009887489, 0.029346175, 0.0084779365, -0.01297475, 0.011723343, -0.0034877807, 0.034544326, -0.0014628394, 0.008677336, -0.00012602164, 0.030886369, -0.00978435, -0.0011886645, 0.004916241, 0.017890992, -0.008842357, -0.015016881, 0.0036029513, 0.024560576, -0.017395929, -0.009378674, 0.0046412065, -0.0129403705, -0.0022930996, 0.011262661, -0.033884242, 0.00045079866, -0.012053384, -0.023694217, -0.00018274749, -0.007845358, -0.0043936754, -0.0047787237, -0.010815729, -0.011221405, 0.02259408, -0.023817983, -0.00882173, 0.009296164, -0.0010159084, -0.017767224, -0.009846233, -0.020311294, -0.0020386928, -0.0018461688, -0.0052153408, 0.0014963592, -0.0028517635, 0.0383398, -0.025564453, -0.013531695, 0.038367305, 0.013841108, -0.010877612, -0.008333543, -0.019348674, -0.0022191843, 0.00028663746, -0.011599578, -0.03943994, -0.013538571, -0.015126894, -0.0008526068, 0.017519694, 0.041310173, -0.00024710127, 0.0034121461, 0.004472748, -0.00644612, -0.0064186165, -0.0025389118, -0.0031938376, -0.008532944, 0.010141895, -0.0069068028, 0.0069996268, 0.018564826, -0.0038436064, 0.035341926, -0.0062639094, -0.028768603, 0.01932117, -0.013016005, -0.021562701, 0.0042905374, 0.017313419, -0.013428557, 0.008031006, -0.014095515, 0.022745349, -0.03248157, 0.0052806614, -0.017629707, -0.0060679475, 0.012032757, 0.0030064704, -0.01932117, 0.0033399495, 0.0015264412, -0.017395929, 0.01109764, -0.009413054, 0.0006269926, 0.025743224, 0.02083386, 0.0009162085, 0.022374053, 0.022566576, 0.018221032, 0.022002757, 0.008120392, 0.0055041267, -0.010760723, -0.01149644, 0.0069136783, -0.0009789508, 0.0021383928, 0.011915867, -0.016680839, -0.0042355303, 0.0367171, 0.019953748, 0.015264411, 0.0143705495, -0.0020851048, 0.0195687, -0.0020541635, -0.01613077, -0.009454309, 0.0028500445, 0.0102587845, 0.0006660991, 0.021411432, -0.01764346, 0.015305667, -0.031078892, -0.024079265, -0.014026756, 0.0056278924, -0.008044758, 0.002717684, 0.0033932375, 0.013923619, 0.0050812615, -0.0038882995, 0.018592328, 0.014700592, 0.0054766233, -0.021810232, 0.0037267169, 0.00091019215, -0.018001005, -0.004163334, -0.0139992535, 0.0009566042, 0.01629579, 0.0036957753, -0.006432368, 0.019252412, 0.010334419, 0.0031903996, 0.022731597, -0.011345171, -0.0065217544, 0.0195687, -0.0049574957, -0.009034881, 0.007797227, -0.026265789, -0.011173274, -0.0007864266, -0.00830604, -0.015456936, -0.0099012405, 0.010760723, 0.010602578, -0.008161647, 0.010348171, 0.016185777, -0.022305293, 0.008402302, -0.0015307385, 0.0060163783, -0.0150718875, 0.018234784, 0.015209405, 0.030501321, 0.00025913402, -0.010533819, -0.005782599, -0.0050640716, -0.0027417496, 0.030253788, -0.0048268544, -0.01380673, -0.0044280547, 0.0018874239, -0.007831606, -0.01041693, 0.014191777, 0.022016507, -0.0064633093, 0.017712219, -0.0005294413, -0.025990754, 0.0064667473, -0.0191699, -0.015291915, 0.00425272, -0.0014645584, -0.02244281, 0.00042673314, 0.016185777, 0.001168037, 0.0024323359, 0.01652957, 0.0010932619, 0.012899116, -0.014824357, -0.013036633, -0.007364047, -0.015539446, 0.0011714748, -0.00456901, 0.012768474, 0.0034551201, 0.009165523, 0.006638644, 0.019293666, -0.023047887, 0.021975253, 0.0066661476, 0.0060507576, -0.0019699342, 0.0062054647, -0.017849736, -0.010520068, 0.0071577714, 0.009935619, -0.010595702, 0.00015792994, 0.00435242, -0.010746971, 0.02148019, -0.026568327, -0.0067177163, -0.0058376063, 0.023446687, -0.0016304386, 0.0057516578, 0.028466064, -0.008484812, -0.0015144084, -0.0010924025, -0.0021177651 - ] - } - ] - } + { + "name": "springfield-illinois-sixth-largest-city", + "title": "Springfield: Illinois' Sixth-Largest City", + "content": "---\n\nSpringfield is the sixth most populous city in Illinois, with a population of 10,566 as of 2019.", + "keywords": [ + "Springfield", + "Illinois", + "population", + "sixth most populous", + "10", + "566", + "2019" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.019281438, 0.03617381, -0.00036933212, -0.017628398, 0.008584945, + 0.00047773743, 0.0048957714, 0.0038249157, -0.043123815, 0.025193768, + -0.020825882, 0.013731086, 0.0000029840915, 0.016445931, 0.002387556, + 0.018050706, -0.02086208, 0.03421912, 0.00035802028, -0.006817279, + 0.006998269, -0.014418847, -0.045802463, 0.035787698, 0.025604011, + 0.0106059965, -0.027220853, 0.008307427, -0.025628142, -0.008192801, + -0.022744372, -0.004910854, 0.057723653, -0.0066423225, 0.0073843803, + -0.006720751, 0.0035654972, -0.020138122, 0.018822929, 0.03520853, + 0.014225791, -0.0046061883, 0.015106607, 0.010467238, 0.038852457, + -0.03940749, 0.00032559293, -0.027389776, 0.02311842, 0.008759902, + 0.024337083, -0.018219631, -0.024904184, 0.042979024, -0.00092606404, + -0.02719672, 0.0076980954, 0.024216423, -0.0077825575, -0.039866, + -0.0019878703, -0.007873053, 0.012874401, -0.0053391964, + -0.0017209104, 0.006992236, 0.00855478, -0.005390477, 0.06438407, + 0.012645148, 0.010768888, 0.046550553, 0.011577308, 0.03322971, + 0.0580615, 0.0027631095, -0.017664596, -0.0023739818, -0.03272294, + 0.032843597, 0.01569784, -0.0064311675, -0.003375458, 0.00035670056, + 0.009477828, 0.029851235, 0.009785511, -0.06781081, -0.0022563383, + 0.010907646, 0.025676407, -0.0035956623, -0.0011651212, -0.036077283, + -0.043702982, -0.023033956, 0.032940127, -0.005897248, 0.009628653, + 0.020596629, 0.017326748, -0.0029772806, -0.010961943, 0.0252179, + 0.00039855443, 0.009628653, -0.018448884, 0.0061204685, -0.027462171, + 0.0050586625, 0.01037071, 0.0020406588, -0.0122711025, 0.012072014, + -0.0104008755, -0.00574944, -0.025386823, -0.007517106, 0.04194135, + 0.021260258, 0.0005467397, 0.021935953, -0.025604011, 0.018497149, + -0.0070404997, -0.010340545, 0.02104307, -0.013260513, 0.0032879796, + -0.045054372, 0.03433978, 0.0054990705, -0.020500101, 0.010636162, + 0.012150442, -0.03916617, -0.011848793, 0.016856175, 0.02780002, + 0.020934476, -0.05941289, -0.006720751, 0.019546889, 0.0050043655, + -0.0000790416, -0.0149135515, -0.012536554, 0.008802133, 0.009616586, + -0.029272068, -0.014515375, -0.005915347, -0.023709651, 0.0044372645, + 0.00055842864, 0.03151634, -0.0018596692, -0.013863811, 0.04712972, + -0.052607674, -0.012922665, -0.021935953, 0.027462171, -0.03800784, + -0.022647845, -0.010750788, -0.0077342936, -0.0003295521, + 0.0028717034, 0.0028792445, 0.022611648, -0.013694888, -0.06385317, + -0.0363186, -0.0074085123, -0.010587898, -0.033519294, -0.01605982, + -0.010931779, 0.030430403, -0.002846063, -0.00054447737, 0.030719986, + -0.003909378, -0.019281438, -0.01827996, 0.0065337284, 0.008023877, + 0.026883004, -0.014346451, -0.0033845075, 0.044619996, 0.0050526294, + -0.005483988, 0.040155582, -0.00033313417, -0.009948401, -0.027534567, + -0.004769079, 0.024035433, -0.044837184, 0.0054658893, -0.03788718, + -0.024952447, 0.0059847264, 0.0187626, -0.0014675249, -0.0018822929, + 0.02128439, 0.020319112, -0.0063587716, -0.0076317326, -0.025314428, + 0.026521025, -0.011806562, 0.022020414, -0.009972533, -0.00007545951, + -0.019040117, 0.019969197, 0.014237857, 0.03800784, -0.000021139032, + -0.028934222, -0.033374503, -0.02071729, -0.045512877, 0.034508705, + -0.021960085, -0.03762173, 0.02656929, 0.027341511, -0.013091589, + 0.054441705, -0.011378219, 0.047443435, 0.013525964, 0.004253258, + -0.014853222, 0.034750026, 0.025169635, 0.0026846807, 0.0033483095, + -0.015215201, 0.0012767316, -0.019727878, 0.02524203, -0.014274055, + 0.016204612, 0.01729055, 0.01148078, -0.008295362, 0.054683022, + -0.005987743, -0.00873577, 0.014515375, -0.013188117, 0.021272324, + 0.00019531806, -0.0066543883, -0.022261733, -0.0021658435, + -0.03385714, 0.034725893, 0.007438677, -0.012168542, 0.028789429, + 0.00074582844, 0.0054658893, -0.030768251, -0.00030504307, + -0.012838203, 0.0053210976, 0.006141584, -0.009707081, 0.030430403, + 0.045464616, 0.003161287, -0.029465124, -0.03221617, -0.028451582, + 0.04601965, 0.0029410827, -0.01383968, 0.010364678, 0.0016832043, + -0.018424753, -0.00222919, 0.011354088, -0.03026148, -0.010406909, + 0.000268091, 0.044595864, 0.025917726, -0.005118992, -0.014527441, + 0.022418592, 0.00063384103, -0.042279195, 0.03185419, 0.034388043, + 0.03680124, -0.031106098, -0.028258527, 0.0071792584, -0.03890072, + 0.016457997, -0.017543936, -0.03619794, -0.06481845, -0.01680791, + -0.050773647, -0.013562162, -0.00065118587, 0.006183815, 0.0012480748, + 0.025917726, 0.008802133, 0.029513389, -0.015082476, -0.0033603755, + 0.0071792584, 0.020017462, 0.025628142, 0.0031100065, 0.054441705, + -0.0016062837, -0.0020436754, 0.011209296, 0.0038701633, -0.011221362, + 0.022744372, -0.0028294725, -0.033181448, 0.01783352, 0.002381523, + 0.030551063, 0.011740199, -0.0066302563, 0.02299776, 0.0010384284, + 0.008566846, 0.009411464, 0.03627034, 0.02011399, -0.013743152, + -0.0007477137, -0.007070665, -0.018907392, 0.039455757, 0.0006210209, + 0.016687252, 0.058254555, -0.0071852915, 0.012452092, -0.009803609, + -0.022684043, -0.0028958353, 0.024759391, -0.05019448, -0.02356486, + 0.0064914976, -0.018485082, 0.004953085, -0.032264434, 0.02179116, + -0.01049137, 0.0061204685, 0.029996028, 0.0032186003, 0.0071370276, + 0.016771713, -0.025338558, 0.014804958, -0.0014268022, 0.012910599, + -0.018219631, -0.0017465507, -0.021079268, -0.005758489, 0.02416816, + 0.020801751, -0.010437073, -0.022503054, -0.006226046, -0.0073300833, + -0.0070646317, -0.013429436, -0.036946032, 0.01260895, 0.021489512, + -0.028982485, 0.011625572, 0.015794368, 0.032385092, 0.029682312, + -0.019474493, -0.011173098, 0.017942114, -0.0004464413, 0.016892374, + -0.022454789, -0.0005527727, 0.025579879, -0.01902805, -0.0030059374, + -0.010195754, -0.058158025, 0.008265196, -0.01148078, -0.006811246, + 0.026255574, 0.01383968, -0.010412941, -0.019933, -0.046719477, + -0.010316414, 0.03947989, 0.015553049, 0.015492719, -0.0070344666, + -0.024276752, 0.002882261, 0.012922665, -0.015335862, 0.015963292, + -0.012536554, 0.027703492, -0.022768505, -0.030333875, -0.02389064, + -0.003662025, -0.011848793, -0.015685774, -0.009115849, 0.051642396, + -0.04778128, 0.0021975166, 0.050242744, 0.03088891, -0.03064759, + 0.009152046, 0.043051418, -0.01900392, -0.0044463137, 0.02656929, + 0.017749058, 0.024325017, 0.019860605, 0.04833632, -0.013103655, + 0.01858161, 0.00990617, 0.016506262, 0.028958354, -0.0027857332, + -0.029802972, 0.010382776, -0.0075352048, 0.0117522655, 0.037959576, + -0.023830311, 0.014611903, -0.031154362, -0.037645858, 0.032071378, + 0.057627123, 0.018352356, -0.00039704618, -0.0038339654, -0.007450743, + -0.02596599, -0.049180936, 0.036487523, -0.03619794, 0.014418847, + 0.0014645084, 0.019643417, 0.01285027, -0.028210262, -0.030333875, + 0.018352356, -0.044981975, 0.0042502414, -0.0032035178, 0.056758374, + -0.018316159, -0.008814199, 0.037863046, -0.0052909325, 0.053476427, + 0.0018596692, -0.009912203, -0.0143947145, 0.014261989, -0.0038369817, + -0.031202625, -0.05130455, 0.040541694, 0.038225025, -0.0063889367, + 0.0119573865, 0.029272068, 0.02086208, -0.023178749, -0.011474747, + -0.01846095, -0.029585784, 0.003447854, 0.010527568, 0.06718338, + 0.013743152, -0.026352102, -0.0205363, 0.0145515725, -0.012777873, + 0.00533618, -0.02659342, 0.0269554, 0.0071309945, -0.013622492, + 0.022587515, 0.009556256, 0.004220077, -0.027703492, 0.036101412, + -0.0020361342, -0.003423722, 0.039142042, 0.005945512, -0.045247428, + -0.00310699, 0.00072358176, 0.025748802, 0.049156804, -0.02632797, + -0.021018937, -0.028475715, -0.0016575641, 0.03581183, -0.01149888, + -0.051883716, 0.0031703364, 0.03655992, 0.025435086, 0.013984472, + -0.011547144, -0.026255574, 0.011504913, -0.012814071, 0.047926076, + -0.0061566667, 0.0108171515, -0.006045056, -0.008741803, -0.00415673, + -0.010461205, -0.012017717, -0.03595662, -0.030575195, 0.01200565, + 0.011589374, 0.021477446, -0.0037977675, 0.0002394343, 0.01771286, + 0.007951481, 0.008934858, 0.007975613, 0.009598488, 0.025797067, + 0.024059566, 0.017133692, -0.0066362894, 0.0032246334, 0.016385602, + 0.0028415385, -0.015685774, -0.0034900848, -0.0009961976, + -0.0065216627, 0.017543936, 0.029996028, 0.027920678, -0.0046876334, + 0.014382649, -0.011679869, -0.044861317, -0.016083952, 0.051594134, + 0.050773647, 0.025145503, 0.007921317, 0.009507992, -0.00087704597, + 0.014225791, -0.00786702, 0.00509486, -0.022852967, -0.05275247, + 0.01906425, 0.0018068806, 0.010503436, -0.026183179, -0.020307045, + -0.003909378, 0.00815057, 0.0044070994, 0.022104876, -0.0112816915, + -0.0193659, -0.0007141552, 0.01154111, -0.016771713, -0.035763565, + 0.0069560376, 0.020596629, 0.027655227, 0.0013936207, -0.014479177, + 0.0031160396, -0.030454535, -0.027751755, -0.0082229655, 0.014455045, + -0.011897056, 0.013212249, -0.0100388955, -0.0003776275, -0.008422054, + -0.011299791, 0.026038386, -0.0069379387, -0.0018204548, + -0.0070344666, -0.0046273037, 0.0051069264, 0.005281883, -0.031275023, + 0.0035202499, 0.019305568, 0.06201914, 0.011510946, 0.022804704, + -0.00966485, -0.0014049326, -0.026183179, 0.009031386, -0.0019592135, + 0.01765253, 0.023806179, -0.020608695, -0.0011734166, 0.0077222274, + -0.011685902, 0.022297932, 0.02830679, -0.031081965, 0.01975201, + 0.005429691, -0.034532838, 0.004904821, 0.0026907136, 0.010774921, + 0.00756537, 0.029103145, -0.044523466, 0.023275277, 0.03738041, + 0.008753869, 0.008355691, 0.021815293, -0.007499007, 0.0141895935, + -0.015154871, 0.017954178, -0.0070827305, -0.002156794, -0.013320843, + -0.009091716, -0.008409988, -0.010225918, 0.0007164176, -0.01049137, + -0.00143736, -0.026472762, 0.045223296, 0.025555747, 0.0087961, + 0.004349786, 0.0087900665, 0.03619794, -0.005836918, 0.0034448374, + 0.036463395, -0.0014124739, -0.01242796, 0.034991343, 0.0114928465, + -0.007667931, -0.02209281, 0.022346197, -0.0071008294, -0.041893084, + 0.0061747655, 0.006244145, -0.002527823, -0.026907137, -0.053814273, + -0.0012284677, -0.0046695345, 0.012886467, 0.014322319, -0.03371235, + 0.05400733, -0.03766999, 0.0193659, -0.015601313, -0.0012608949, + 0.03655992, 0.030502798, 0.01708543, -0.045440484, -0.0039365264, + -0.031033702, 0.004325654, -0.035280928, -0.03800784, 0.017435342, + 0.009978566, -0.03137155, 0.013525964, 0.037863046, -0.005773572, + 0.0028551125, 0.002870195, 0.0003363392, -0.02891009, 0.025193768, + -0.0021703683, -0.012729609, -0.028113734, 0.038442213, 0.00691984, + 0.010099226, 0.010889548, 0.004379951, -0.0049561015, -0.05868893, + -0.0018506198, -0.031443946, -0.019788207, 0.05130455, 0.030913042, + 0.000030329917, -0.013912075, -0.0014705413, -0.0035594644, + 0.010388809, 0.049422257, 0.029947763, 0.008693539, 0.007891151, + 0.008560813, 0.015601313, -0.005016431, 0.010069061, -0.0009826233, + -0.0047177984, 0.0038611137, 0.04358232, 0.00035161022, -0.016892374, + 0.0017721909, 0.03320558, 0.0026695982, 0.00972518, -0.021356786, + 0.021537775, -0.04051756, 0.025628142, -0.0101414565, 0.012053914, + 0.011154999, 0.00024150814, -0.02116373, -0.0043286704, -0.009097749, + 0.008192801, -0.013344974, -0.015746105, -0.019643417, -0.044740655, + 0.0037917343, 0.009960467, -0.014177527, 0.0034327714, -0.02620731, + 0.05251115, 0.0059515447, 0.015975358, 0.03146808, 0.000858947, + 0.017724926, -0.04823979, 0.008253131, 0.0027284198, -0.001687729, + -0.005873116, 0.0030632508, 0.0322403, -0.03265054, 0.02780002, + -0.016252875, -0.031781793, -0.025917726, 0.015082476, -0.02434915, + -0.0042140437, -0.029489256, 0.025531614, -0.023842378, 0.02707606, + -0.018304093, 0.02311842, 0.010515502, 0.0016515311, -0.016651053, + 0.00756537, 0.024807656, -0.022020414, 0.0049561015, -0.020608695, + -0.010503436, -0.009839807, 0.023468332, 0.000548248, 0.012053914, + 0.018786732, 0.0052245697, 0.007511073, -0.012802006, 0.024421545, + -0.016289074, 0.027124325, 0.03974534, -0.0059907595, -0.039817736, + 0.024506006, -0.017761124, -0.031443946, 0.015939161, -0.022720242, + 0.017604265, -0.019196976, 0.028137866, 0.021489512, 0.009127914, + -0.017375013, 0.01569784, 0.0019441311, 0.0038068169, 0.029054882, + -0.013574228, 0.0011771872, 0.0070827305, 0.0044794953, 0.0073119844, + -0.021139598, -0.017302616, 0.0034900848, 0.002425262, -0.0007054828, + -0.018810865, -0.0010655769, 0.009616586, -0.0017797322, + -0.0013755218, -0.024095763, 0.025990123, -0.036632318, -0.008120405, + -0.01513074, -0.022129009, 0.016530393, 0.020150188, -0.014153395, + -0.028717034, 0.005957578, -0.0059243967, 0.003088891, 0.009924269, + -0.00533618, -0.016264942, 0.006226046, 0.01630114, -0.029561652, + -0.028861826, -0.0011342021, 0.0050677117, 0.017254353, -0.008409988, + 0.014455045, -0.02570054, -0.0011228903, 0.017000966, 0.026738213, + 0.008397923, -0.009507992, 0.011619539, -0.016470063, 0.022792637, + 0.004953085, -0.020825882, -0.0042985054, 0.020850014, -0.011878958, + -0.011607474, -0.0031824023, 0.018436817, 0.02419229, 0.0077161947, + -0.038973115, 0.0072637205, -0.0051491573, 0.0052034543, 0.012862335, + -0.04136218, -0.009405432, -0.0036167777, 0.019836472, 0.016554525, + 0.0008695047, -0.0045006108, -0.0031643035, -0.035980754, + -0.036897767, 0.012440026, -0.008132471, 0.027993076, -0.010087159, + 0.008741803, -0.030768251, 0.0029757724, -0.030502798, 0.03738041, + -0.027438039, 0.021079268, 0.030285612, -0.00097206555, -0.002058758, + 0.014056867, -0.009586422, 0.0022819785, 0.036366865, 0.022563383, + -0.016011557, 0.027389776, -0.01618048, 0.026110781, -0.0029214756, + 0.0193297, -0.04075888, -0.002990855, 0.008349659, -0.015215201, + -0.028861826, -0.054827817, -0.012259036, 0.012464158, 0.014684298, + -0.0039033447, 0.0071309945, -0.019920934, -0.0005101647, 0.030285612, + -0.008126438, -0.0005320343, 0.049760103, 0.0351844, 0.010425007, + 0.009514025, 0.0014094573, 0.0580615, 0.01605982, -0.012681345, + -0.013103655, -0.008699572, -0.011553177, -0.0042472254, 0.02659342, + -0.005803737, -0.010581865, -0.0000472977, 0.05733754, -0.009182211, + -0.011414418, 0.0081686685, -0.01618048, 0.023178749, 0.03125089, + 0.010799052, -0.028089603, -0.0024267703, -0.009302871, 0.032385092, + -0.041506972, -0.013996538, -0.00972518, 0.0018325208, 0.02041564, + -0.021332653, 0.033519294, -0.03730801, -0.0022563383, -0.031733528, + -0.015987424, -0.01797831, 0.005384444, -0.008174702, 0.00932097, + -0.0032065343, -0.0019049167, -0.023359738, -0.002656024, + -0.0013747676, 0.014406781, -0.010937812, 0.015251399, 0.04073475, + 0.021803226, -0.022189338, -0.005852001, -0.002790258, -0.0132001825, + -0.006401003, -0.008886594, 0.04995316, 0.021211993, 0.015106607, + 0.0038550808, -0.03482242, 0.031081965, 0.0027616012, 0.00071868, + -0.005375394, -0.0076498315, 0.0036318603, 0.03296426, 0.03113023, + -0.009339069, 0.005637829, -0.0072516543, 0.0012684362, -0.03680124, + -0.004618254, 0.027413908, 0.016337337, -0.0026786476, -0.018714337, + -0.0038701633, -0.018255828, -0.004331687, 0.006207947, 0.026038386, + -0.012826137, -0.012958863, 0.01037071, -0.013429436, -0.0038852456, + 0.0036469426, -0.0069379387, 0.034363914, -0.01377935, 0.028741166, + 0.004880689, -0.009387333, -0.024566336, -0.006268277, -0.0292962, + 0.0137190195, 0.008048009, -0.021441247, 0.017640464, -0.0021100384, + -0.0269554, 0.017749058, 0.0006549565, -0.008228999, 0.0028566208, + -0.016421799, -0.011758298, 0.02473526, 0.010213853, 0.045609407, + -0.018316159, 0.028258527, 0.04934986, -0.005794687, -0.00020870376, + -0.0090012215, 0.0007092534, 0.009477828, -0.043003153, 0.0073059513, + 0.013863811, -0.04121739, -0.019571021, -0.055165663, -0.009061552, + -0.01834029, 0.02161017, -0.032409225, -0.0021522692, 0.022925364, + 0.01920904, -0.020934476, -0.0073542153, -0.006992236, -0.0150704095, + -0.02467493, -0.012862335, -0.024904184, -0.0029018684, 0.033060785, + -0.012572751, -0.010334512, -0.001297847, 0.012717543, -0.0030723002, + -0.021006873, 0.017809387, -0.004027021, -0.043003153, 0.018714337, + 0.011523011, 0.008138504, -0.0065880255, 0.019402096, -0.007921317, + 0.005794687, -0.0006255457, 0.002277454, -0.01018972, -0.0043829675, + -0.006105386, 0.017013032, -0.0004057186, 0.0055714664, -0.016578658, + -0.028982485, 0.0005373132, 0.016651053, -0.0063829035, -0.0014712955, + -0.021803226, 0.0022518137, -0.010123358, -0.022913298, 0.017905915, + -0.015359993, -0.0038038003, -0.008035943, 0.015372059, 0.012150442, + 0.008687506, 0.0071611595, 0.0026771394, 0.0034327714, 0.027293248, + 0.00043211292, 0.0071973572, -0.023613123, -0.02524203, 0.012265069, + -0.007366281, 0.013393238, -0.017073363, 0.0060480726, 0.0077885906, + -0.0026364168, -0.004941019, -0.008054042, 0.009954434, -0.017447408, + -0.014841156, -0.017266419, -0.022044547, -0.035763565, -0.0015670692, + 0.0064794314, -0.02269611, 0.02632797, 0.02311842, -0.0026183177, + -0.018147234, -0.0055141533, 0.0041295816, 0.01771286, 0.027003665, + -0.030502798, 0.027365644, -0.016071886, -0.011776397, 0.009453696, + -0.006232079, 0.0011115784, 0.0008695047, -0.0037977675, -0.019727878, + 0.013417371, 0.006425135, 0.019788207, -0.012814071, 0.017423276, + -0.014479177, -0.017543936, 0.018919457, 0.041193258, 0.010485337, + -0.008216932, 0.001218664, -0.026496893, 0.008488418, 0.014418847, + 0.008609077, -0.03716322, 0.004977217, -0.029054882, -0.0013257496, + -0.010678393, 0.020656958, 0.000072348754, 0.009882038, 0.012958863, + -0.027244983, -0.024385346, 0.03265054, 0.01049137, 0.015601313, + 0.00018853095, -0.017930048, 0.020355308, -0.0043950332, -0.008066108, + 0.024325017, -0.035039607, -0.03347103, 0.011058471, -0.03776652, + 0.020970674, 0.011233428, 0.005987743, -0.026979532, -0.043268606, + -0.0011432517, -0.02130852, -0.0018747518, -0.02431295, -0.012156475, + 0.014961815, -0.0025082156, -0.009327003, 0.020572497, 0.00679918, + 0.017013032, -0.007897184, -0.0009509501, 0.023492465, 0.0069077737, + 0.0034931013, 0.019800274, -0.024686996, -0.0105517, 0.03349516, + 0.024325017, -0.0051310584, -0.009857906, 0.020850014, 0.010213853, + 0.014925618, -0.0045428416, 0.0035232664, 0.018629873, -0.005701176, + -0.0025534632, 0.028161999, -0.015299663, 0.020343244, 0.0045820563, + 0.026810609, 0.012180608, 0.01131789, 0.0008544222, -0.031540472, + -0.0013536522, 0.014310253, -0.038200896, -0.004235159, -0.022756439, + 0.026231442, -0.026134914, 0.008952958, -0.012156475, -0.012814071, + -0.013743152, -0.022985693, 0.01285027, 0.0025293313, 0.0031401715, + 0.011372186, 0.019836472, -0.035377454, -0.013405304, 0.0070404997, + -0.0059907595, 0.019583086, -0.0007480908, -0.002870195, 0.004410116, + -0.039672945, -0.00990617, -0.023818245, -0.0017163857, 0.008989155, + -0.00067456375, 0.010032862, 0.006714718, -0.010087159, 0.008940891, + -0.0010090176, 0.0070163677, -0.01704923, -0.004099417, -0.009393366, + -0.01605982, 0.021549841, -0.017218154, 0.05058059, 0.0013393238, + 0.018895326, 0.00984584, -0.00010746264, -0.005677044, 0.0039455756, + 0.0017344847, -0.0017269434, -0.024397412, 0.001522576, -0.020632828, + -0.0041778455, 0.01166177, -0.01975201, 0.020307045, -0.03325384, + 0.0005769047, -0.0070525655, 0.0030919076, 0.025290295, 0.009357168, + 0.021320587, 0.031926583, 0.023613123, -0.016638987, 0.005661961, + -0.0067086853, -0.010714591, -0.010708557, 0.004886722, -0.0234442, + 0.022937428, 0.020162253, 0.014117197, 0.017375013, 0.008307427, + 0.0065638935, -0.0072637205, -0.033398636, -0.0013325367, + 0.0016394651, -0.006220013, -0.0005218537, 0.0037977675, + -0.0022472888, 0.022165207, 0.016651053, 0.035642907, 0.0019501641, + -0.00475098, -0.0050103986, 0.018569544, 0.009550224, 0.018533345, + 0.0021884672, -0.01813517, 0.008301395, 0.010593931, -0.015975358, + 0.005544318, -0.0051793223, -0.009912203, -0.01723022, 0.00072094233, + -0.0073300833, 0.04811913, -0.002564021, 0.013284645, -0.018026575, + -0.007915284, 0.01981234, 0.0045911055, -0.022539252, 0.0133570405, + -0.025145503, -0.011806562, -0.0085366815, -0.010967976, 0.019776141, + 0.015058343, -0.03593249, 0.016651053, -0.014249923, -0.009019321, + 0.016844109, 0.0022005332, -0.03431565, -0.021658435, -0.016373536, + 0.0033211613, -0.0024297868, 0.008048009, 0.013948274, -0.005167256, + 0.004428215, 0.008048009, 0.008072141, 0.0028520962, -0.0077282605, + -0.024156094, 0.00281439, -0.0031039736, 0.0087900665, 0.0039274767, + 0.013067457, -0.016216679, 0.013924141, -0.024252621, -0.024880052, + -0.014237857, -0.017773189, -0.021031003, -0.001998428, 0.009954434, + -0.0093632005, 0.027775887, 0.010014764, 0.0027812086, 0.014008603, + 0.009459728, -0.0015059852, 0.018485082, -0.009514025, 0.0036016952, + 0.011565242, -0.020777619, 0.006943972, 0.0007254671, 0.015963292, + -0.008989155, 0.014455045, -0.0067086853, -0.004367885, 0.0222738, + 0.029440993, 0.0037977675, 0.029440993, 0.036390997, 0.0021975166, + -0.026472762, -0.009514025, -0.016289074, 0.01963135, 0.01353803, + 0.0069077737, 0.0065699266, 0.030454535, -0.009435597, 0.003369425, + 0.013670756, 0.031878322, -0.029730575, 0.0137190195, 0.032674674, + -0.01858161, -0.0158185, -0.0004502119, 0.026279707, 0.011251527, + 0.0069620707, 0.026810609, -0.009646751, 0.010425007, -0.030189084, + -0.009526092, -0.020355308, 0.007022401, 0.011474747, -0.034870684, + -0.012524487, -0.013984472, 0.02092241, 0.021682568, -0.004702716, + 0.025941858, -0.041603502, 0.011746232, -0.0025323476, 0.0013875877, + 0.0228409, -0.011028307, 0.01482909, 0.067376435, 0.03359169, + -0.010032862, 0.017495671, -0.0063105077, -0.00439805, 0.021875624, + -0.0028852776, -0.020282913, -0.014358517, 0.036366865, -0.01809897, + -0.0035323158, -0.0033121116, 0.0014886403, 0.0029938715, + 0.0075593367, -0.008965024, -0.008084207, -0.014201659, 0.015987424, + 0.014901486, -0.001029379, -0.0069801696, 0.020837948, 0.0027736672, + -0.014539506, -0.027993076, -0.013248446, -0.022732308, -0.002393589, + 0.013984472, 0.0021055136, 0.0034418209, 0.019148711, -0.0015504785, + 0.027896548, -0.0061265016, 0.010859382, -0.0028038323, 0.015565115, + -0.0016560558, 0.01987267, 0.00033407685, -0.007492974, 0.00035311846, + -0.017073363, 0.010871449, -0.012011684, 0.013393238, 0.005803737, + 0.0149135515, -0.0059786933, -0.02257545, -0.0058489842, -0.008355691, + -0.010841284, -0.0047720955, 0.008910727, -0.009888071, -0.0018626858, + -0.02209281, -0.022116942, 0.020608695, 0.014213725, 0.006787114, + -0.010865415, 0.010449139, 0.00043701474, -0.009538158, 0.004576023, + 0.0033060787, 0.014841156, -0.023009825, -0.015902963, 0.020777619, + -0.0029561652, -0.008216932, -0.010437073, 0.0035112004, 0.024107829, + -0.04346166, -0.0059515447, -0.00012056554, 0.00019965427, + 0.008566846, -0.008699572, -0.007517106, -0.015915029, 0.0013679805, + -0.015287598, 0.0035202499, 0.004476479, -0.012053914, 0.00024659847, + 0.0017013033, -0.015456521, -0.0023438167, 0.000011565196, + -0.0038249157, -0.013007127, -0.008440154, 0.0351844, -0.02608665, + 0.019884735, 0.0012669279, 0.0024886085, 0.007468842, 0.014744628, + 0.00960452, 0.011052438, 0.00873577, 0.006250178, -0.006762982, + -0.008108339, 0.020837948, -0.0070947967, 0.01148078, -0.008494451, + 0.045802463, -0.008530648, -0.030840646, 0.029996028, 0.009043452, + -0.009586422, 0.007511073, -0.0065035634, 0.00042004694, -0.018931523, + -0.00010991354, -0.016409734, 0.010630129, 0.02281677, 0.0014841156, + -0.02431295, 0.0024539188, -0.01587883, -0.0013913583, -0.0042140437, + 0.012874401, 0.0070766974, 0.0027254033, -0.010334512, 0.0054085758, + 0.021718765, 0.022370327, -0.010159556, -0.02332354, -0.019643417, + 0.010865415, 0.0016319238, 0.0090012215, -0.00085366814, + -0.0041265655, 0.009954434, -0.015927095, 0.0012088604, 0.010467238, + -0.0016786795, 0.0031763695, -0.005318081, 0.026255574, 0.0072335554, + 0.008621143, 0.008657341, -0.008633209, -0.0026786476, -0.006672487, + -0.024095763, 0.0022985693, -0.003094924, -0.0038520643, + -0.0056800605, -0.009115849, 0.012047881, 0.00028637852, -0.020777619, + -0.008910727, -0.027124325, -0.009019321, -0.005312048, 0.008808166, + -0.015094542, -0.010696492, -0.016264942, -0.016337337, 0.021561908, + 0.014430913, 0.0052004377, -0.0032578148, -0.014503309, 0.00089363666, + 0.0013001094, 0.016168414, 0.021634303, 0.0134777, 0.017375013, + -0.00040609567, 0.009236508, 0.0122168055, -0.012946798, -0.009869972, + 0.027172588, 0.015637511, -0.0037072725, -0.016795846, 0.002662057, + -0.005502087, -0.037356276, -0.022261733, -0.003948592, 0.00010680278, + -0.021368852, -0.014744628, -0.011589374, 0.0069801696, -0.017435342, + -0.0047479635, 0.010533601, -0.0036559922, 0.007541238, 0.02473526, + -0.007993712, -0.022116942, 0.023938905, 0.005118992, -0.011142933, + 0.005616714, -0.00038592285, 0.0016439898, 0.0011553176, 0.008289329, + 0.011577308, -0.003918427, -0.033688217, -0.031564604, 0.02239446, + 0.01012939, 0.026714081, -0.004020988, 0.0087900665, -0.0020044611, + 0.0071370276, 0.014418847, 0.011969453, 0.023251144, 0.0020889228, + 0.025555747, 0.009574356, 0.014636034, 0.034991343, -0.009097749, + 0.016542459, 0.009930302, 0.003125089, 0.0077584255, 0.011535077, + -0.030961307, 0.021151664, -0.037838914, 0.0032276497, -0.012874401, + -0.009031386, -0.026038386, 0.003716322, -0.015504785, -0.0122168055, + -0.0026514991, 0.016506262, -0.0021296456, 0.007541238, -0.0006017908, + -0.026376234, 0.012331432, -0.006726784, -0.0053452295, 0.018738467, + 0.01785765, 0.033664085, -0.017664596, -0.0018370455, -0.00025282, + -0.017314682, -0.026472762, 0.0029938715, -0.002070824, 0.009254607, + -0.02161017, 0.025869463, -0.00038422606, -0.0070103346, + -0.0072576874, -0.016856175, 0.01666312, 0.001516543, 0.008669407, + -0.015528917, -0.01945036, 0.0064613326, -0.0023136518, -0.0031160396, + -0.04058996, -0.001954689, 0.019426228, -0.016856175, -0.0036077283, + 0.0028837693, -0.0013958832, -0.031057835, 0.01963135, + -0.000018758828, -0.007830821, 0.01067236, -0.018931523, -0.012645148, + 0.0076799965, -0.010020797, 0.011221362, -0.012922665, 0.014853222, + -0.00021586793, 0.0046695345, 0.009296838, 0.019088382, 0.0027450107, + 0.012512422, 0.0109740095, -0.017773189, 0.02767936, 0.009284772, + 0.00861511, 0.0025791035, -0.0073843803, 0.0007529926, -0.0033000456, + -0.0073119844, 0.0015376584, 0.0008054042, -0.00018796536, + 0.014925618, -0.006811246, -0.023721717, -0.011794496, -0.006726784, + -0.006473399, 0.0060722046, -0.0082229655, 0.014780826, -0.03665645, + 0.008651308, -0.0034327714, -0.0017013033, -0.014093066, 0.00196223, + -0.010587898, 0.0035504147, -0.018050706, -0.008548747, -0.00944163, + 0.009218409, -0.005646879, -0.015649578, 0.013731086, -0.014925618, + 0.025024844, -0.019522756, 0.012277136, 0.026689949, 0.002894327, + -0.0053150645, -0.010286248, -0.021899754, -0.012198706, 0.011100702, + -0.021031003, -0.008385857, 0.011553177, 0.003417689, -0.014165461, + 0.000054721106, -0.0002601727, -0.0065035634, -0.025193768, + -0.013610426, 0.014865288, 0.0043890006, -0.0064070355, 0.005520186, + 0.000033912005, -0.01827996, 0.00075035315, -0.030502798, 0.012259036, + 0.0060088583, 0.011245494, -0.006376871, 0.016819976, -0.004343753, + 0.017821454, 0.021754963, -0.00393351, 0.01095591, -0.028523978, + -0.005158207, 0.016071886, -0.009043452, -0.0014034243, -0.0126330815, + 0.030213214, 0.0115954075, 0.00038837374, 0.0035021508, 0.0019516724, + 0.03245749, -0.006105386, -0.010925746, 0.025917726, -0.032915995, + 0.0152272675, 0.015746105, -0.011601441, 0.022141075, 0.011909123, + -0.011040372, -0.006853477, 0.010231951, 0.0054658893, -0.027606964, + -0.016011557, -0.0028958353, 0.0020843982, 0.0070163677, -0.018907392, + -0.005037547, -0.009224442, 0.002984822, -0.0054085758, 0.004784161, + 0.029223805, -0.017037164, 0.012946798, -0.01513074, -0.022008348, + 0.008271229, -0.0035685138, 0.012669279, 0.008066108, 0.007933382, + 0.0055141533, 0.036101412, -0.010225918, 0.010280215, 0.031926583, + 0.008228999, -0.009514025, -0.0055473344, 0.014937684, -0.009429564, + -0.009182211, -0.016457997, 0.014901486, -0.015782302, 0.00047999978, + 0.0024071631, 0.0058942316, -0.013417371, 0.0003338883, -0.0052728336, + 0.005969644, 0.0138758775, -0.0126330815, 0.009206343, 0.005444774, + -0.014455045, -0.011909123, 0.015927095, -0.010497403, -0.004033054, + -0.006084271, -0.01218664, -0.019546889, 0.0032246334, 0.010316414, + -0.011004174, -0.0030617425, 0.012379696, -0.0014886403, + -0.0047660624, 0.0026695982, 0.006171749, -0.0045277593, 0.005544318, + 0.017242286, -0.01605982, 0.0022819785, 0.0046333363, -0.004093384, + 0.039576415, -0.002710321, 0.0020150188, -0.022913298, -0.014080999, + 0.005197421, 0.027848283, -0.019522756, 0.008114372, 0.0010889547, + 0.004491561, -0.00031446962, -0.0068595097, -0.015577181, + -0.013622492, -0.0071973572, -0.000084367595, -0.0019637384, + 0.019908868, 0.010780954, 0.032023113, 0.010310381, 0.013176051, + -0.009296838, -0.031275023, 0.0026454662, 0.006051089, 0.010714591, + -0.025073107, -0.019571021, -0.0021914837, -0.0004999841, 0.013441502, + 0.026352102, -0.0022442723, 0.0077885906, 0.0041687964, -0.010449139, + -0.014334385, 0.013550096, -0.0012156476, 0.029320333, -0.0010437074, + -0.010690459, 0.03530506, 0.02209281, -0.028065471, 0.007987679, + 0.024131961, -0.0063829035, 0.024807656, -0.008397923, 0.001540675, + 0.011577308, -0.00028166524, 0.013151919, 0.020825882, -0.005610681, + 0.0155892465, 0.02056043, 0.0042381757, 0.012862335, -0.023649322, + -0.017495671, -0.023757916, 0.0043950332, -0.0076076007, 0.015939161, + -0.025024844, 0.0126330815, 0.0029169507, 0.025531614, -0.012283168, + 0.01692857, -0.0028098652, 0.0025806115, -0.0004404083, -0.009399399, + -0.0077161947, -0.00055201864, 0.033277974, -0.020331178, + -0.012572751, -0.023432134, 0.02915141, 0.014093066, -0.0044010663, + -0.020391507, -0.023673454, 0.01660279, -0.010437073, -0.011818628, + 0.016349403, -0.01043104, 0.009568322, -0.0036348766, -0.018557478, + 0.0055292356, -0.013248446, 0.009857906, 0.033808876, 0.00968295, + -0.007595535, -0.014575705, 0.00984584, -0.019221107, 0.0057856375, + -0.0070947967, -0.022599582, 0.00627431, 0.019667549, 0.020572497, + -0.0077101616, 0.011130867, -0.0012669279, -0.00072471297, + -0.013586294, 0.014780826, -0.0036318603, -0.022913298, 0.0068595097, + 0.0043105716, 0.021718765, -0.001120628, 0.022406526, 0.008771968, + -0.014817025, 0.02719672, 0.0081686685, -0.00006650429, -0.017761124, + -0.026279707, 0.0035624807, 0.0066302563, 0.023950972, 0.0099665, + -0.009507992, 0.010907646, -0.014406781, -0.024180224, 0.0065699266, + -0.000070699105, 0.0138758775, 0.005176306, -0.0056076646, + 0.019052183, 0.021429181, 0.0048113097, 0.0029169507, 0.016421799, + 0.012536554, -0.029247938, -0.012584818, 0.032771204, 0.016385602, + -0.0015851682, 0.015372059, 0.006895708, 0.005879149, -0.0065940586, + 0.017869717, 0.004506644, 0.0006440217, -0.0049259365, -0.0028158983, + 0.0041024336, -0.023697587, 0.01723022, -0.027751755, -0.018774666, + -0.01999333, 0.012572751, 0.009133947, 0.020837948, 0.012379696, + 0.019595152, -0.027100192, 0.0038912788, 0.01729055, -0.009471795, + 0.008271229, 0.002746519, -0.009248574, 0.0083255265, 0.0015444455, + -0.013815547, -0.007643799, -0.005085811, 0.018219631, -0.017628398, + 0.003173353, -0.015396191, 0.010799052, 0.01482909, 0.011607474, + 0.0140327355, 0.0019773126, -0.031419814, 0.006084271, 0.033808876, + -0.012349531, 0.002192992, -0.018798798, -0.00833156, -0.0020949559, + -0.0022125992, 0.0064311675, -0.0005143124, 0.0064613326, -0.03125089, + -0.016964769, 0.038321555, 0.0031522375, 0.0072818194, -0.018255828, + 0.0076256995, 0.009538158, 0.011746232, 0.011468714, -0.018714337, + -0.003393557, -0.018569544, 0.012072014, -0.0032005014, 0.025507484, + -0.0055051036, 0.00030051832, -0.018726401, 0.011504913, + -0.00014592295, 0.0037495035, 0.0043226373, -0.013320843, 0.021368852, + -0.006950005, -0.0005708717, 0.0039847903, 0.0066181906, 0.018955655, + -0.01785765, 0.017507738, -0.016289074, -0.0031914518, -0.020813817, + 0.014684298, 0.002114563, 0.018738467, -0.00310699, -0.0000085487, + -0.011649705, 0.015311729, -0.025290295, -0.01236763, 0.0038309488, + 0.039335098, 0.007993712, -0.02572467, 0.014225791, -0.024300884, + -0.008584945, 0.021935953, -0.0065699266, -0.010479304, 0.0026213343, + 0.019764077, 0.0053934935, -0.006702652, 0.0020723322, 0.008681473, + 0.0072033904, -0.005770555, -0.0072878525, 0.01827996, 0.012041848, + 0.011734166, -0.010111292, 0.037356276, 0.00087327533, 0.018605743, + 0.007933382, -0.009719147, 0.006147617, 0.020150188, -0.0028520962, + -0.0058339015, 0.023709651, -0.00533618, -0.005725308, 0.01125756, + 0.0008235032, 0.004265324, 0.0020120023, 0.006195881, -0.0070043015, + 0.015034212, -0.002996888, 0.02854811, 0.001194532, -0.008874529, + -0.0017827486, 0.017785255, -0.014298187, -0.0025579878, -0.009749312, + 0.0016454981, 0.014274055, -0.0052275863, 0.019196976, 0.003990823, + -0.009598488, 0.009634686, -0.0123073, 0.02011399, 0.023721717, + -0.003746487, 0.029827103, 0.0045096604, 0.0060872873, -0.011770364, + 0.015565115, -0.016337337, -0.04136218, -0.002138695, 0.0075352048, + -0.038393952, 0.012524487, -0.0042140437, 0.022309998, 0.0014833615, + 0.0036077283, 0.011758298, -0.01605982, 0.023166683, 0.0193297, + -0.004229126, 0.012349531, 0.0098337745, -0.035329193, 0.013332909, + 0.0025112322, -0.013103655, 0.01218664, -0.00533618, -0.0013755218, + 0.0036077283, -0.025410956, -0.008518582, 0.014129263, -0.001717894, + 0.0019758043, 0.015106607, -0.009399399, 0.0005795441, -0.0066785202, + 0.007975613, 0.009996665, -0.01248829, -0.02326321, 0.0003333227, + -0.02023465, -0.0065638935, 0.031057835, 0.020089857, 0.0006904003, + -0.01692857, -0.011673836, 0.007836854, -0.004204994, -0.00097206555, + -0.0028988519, 0.0012789939, 0.017990377, -0.019522756, -0.01999333, + 0.0010693475, -0.008428087, -0.024325017, -0.020331178, -0.0020421671, + 0.0022623714, 0.011897056, -0.0065638935, 0.01103434, 0.00439805, + 0.033808876, 0.010117325, 0.009242541, 0.0051461407, -0.006159683, + -0.0026334003, 0.0298995, -0.0025610044, 0.0027012713, -0.008874529, + -0.027968943, 0.023902707, -0.003381491, -0.0193297, -0.0036288437, + -0.0015836599, 0.011987552, -0.00035255286, 0.028403318, -0.011323922, + -0.018436817, 0.003918427, -0.012295234, -0.009290805, 0.0071913246, + -0.013924141, -0.0029637064, 0.0045006108, 0.031081965, 0.0075533036, + 0.012180608, 0.017061297, -0.012970929, -0.015022146, -0.008989155, + -0.00059915136, 0.011553177, 0.018967722, -0.00533618, -0.002722387, + -0.015094542, 0.028330922, -0.004186895, 0.018219631, 0.0077765244, + 0.028717034, -0.017121626, -0.023661388, -0.009544191, 0.009097749, + -0.0005497562, -0.004554908, -0.021272324, 0.010388809, + -0.00032860943, 0.009713114, 0.010014764, -0.01735088, -0.02719672, + 0.015842633, -0.0053391964, -0.0187626, 0.02828266, 0.034629364, + 0.02891009, 0.0001245247, -0.003405623, 0.015565115, 0.02257545, + 0.010805085, 0.0021899755, 0.018690204, -0.016904438, 0.021187862, + 0.0079635475, -0.0063949698, 0.0070887636, 0.014129263, 0.013381172, + 0.014080999, 0.0009856398, -0.0010037387, -0.017954178, -0.0046333363, + -0.029537521, -0.024445677, 0.035498116, -0.002102497, 0.0105517, + 0.005885182, -0.013574228, 0.010829218, -0.011468714, -0.0049168873, + -0.0075834687, -0.0078609865, -0.012802006, 0.022937428, -0.006232079, + -0.027220853, 0.015661642, 0.009918236, -0.024083696, -0.02485592, + -0.0068836417, 0.00597266, 0.0063889367, -0.0057132416, 0.009707081, + -0.01951069, 0.004705732, -0.01975201, 0.003387524, 0.017954178, + 0.014623969, -0.019619284, -0.0016077919, -0.009429564, 0.01678378, + 0.0024840836, -0.03371235, -0.00732405, 0.014792892, -0.01026815, + -0.025024844, 0.013091589, -0.006672487, 0.0152272675, -0.00873577, + -0.011028307, 0.0066664545, 0.014877354, 0.0054598562, 0.013827614, + 0.02917554, -0.0091580795, -0.0018657022, 0.006787114, -0.0055141533, + 0.0002552709, 0.012102178, 0.022358261, -0.012512422, 0.009616586, + -0.008048009, -0.026617553, -0.00085819286, -0.0033271941, + 0.010117325, 0.0030798416, -0.01142045, 0.012838203, -0.0004977217, + -0.016252875, 0.016252875, -0.0007775016, -0.022792637, 0.025507484, + 0.021236125, -0.010774921, -0.012337465, -0.026545158, 0.0013325367, + -0.0044131326, -0.004123549, 0.028813561, -0.0014599836, 0.0066181906, + -0.0033332272, 0.0014848697, 0.005541302, -0.018195499, -0.019426228, + 0.008977089, 0.01692857, -0.014901486, 0.0065638935, -0.012041848, + -0.017761124, 0.0123254, 0.0013649641, -0.0006176274, 0.009489894, + 0.038997248, 0.0039214436, 0.011891024, 0.011788463, -0.0070344666, + -0.0010203294, 0.007366281, 0.0127899395, 0.019764077, -0.027027797, + 0.005996792, 0.008584945, -0.013863811, 0.0023905723, -0.013272579, + -0.035087872, 0.0052366355, -0.013791416, 0.0018023559, 0.00092003105, + -0.01488942, -0.0053814272, 0.01095591, 0.031202625, -0.0004894263, + 0.0040360703, 0.010575832, 0.0083798235, 0.012355564, -0.004624287, + -0.013743152, 0.009314937, -0.027100192, 0.0040300377, 0.008434121, + -0.013188117, 0.0010587898, -0.0020044611, -0.007836854, 0.01999333, + 0.012621015, -0.0017375012, -0.00053165725, -0.015444455, + -0.007993712, 0.0021703683, -0.010418974, 0.002551955, 0.010847316, + -0.004367885, -0.008446187, -0.021356786, -0.011547144, 0.008524615, + -0.005915347, -0.028885957, -0.00095547485, -0.0031643035, + 0.0071973572, 0.005293949, -0.017930048, 0.01575817, -0.010111292, + 0.015625445, 0.001248829, 0.00045586782, 0.013694888, -0.00815057, + 0.021525709, 0.015347927, -0.016264942, 0.009797576, 0.014201659, + -0.012910599, -0.012222839, 0.0009622619, -0.016433865, 0.005683077, + 0.0039455756, -0.005176306, -0.009882038, -0.00020757258, 0.003022528, + 0.005128042, -0.005128042, 0.014720497, -0.001699795, 0.018955655, + -0.012090112, 0.0075050397, 0.023661388, 0.0025821198, -0.009894104, + 0.013212249, 0.0041687964, -0.02314255, 0.015685774, 0.02854811, + -0.0070827305, 0.0076739634, 0.016880307, 0.0138758775, 0.037476934, + -0.0032366994, -0.022104876, -0.019221107, -0.0049621346, + -0.018750533, 0.002826456, 0.011311857, 0.0071008294, 0.00966485, + -0.021187862, 0.004180862, -0.0030074457, -0.006256211, -0.012150442, + -0.009876005, 0.009658817, -0.014358517, -0.010913679, 0.008578912, + 0.014213725, -0.0035172333, 0.026376234, 0.004051153, 0.0025263147, + 0.000737156, 0.021923887, 0.012331432, 0.015721973, -0.011547144, + 0.016988901, -0.0050767614, -0.011305824, 0.0013551605, 0.01482909, + 0.038442213, 0.00404512, 0.017013032, 0.023830311, 0.010069061, + -0.005604648, 0.02023465, 0.016530393, -0.00492292, -0.018207565, + 0.008138504, 0.0038942953, -0.008898661, 0.007903217, 0.021598104, + 0.011378219, 0.009067585, 0.014286121, -0.010859382, -0.0030240363, + 0.018967722, -0.01149888, -0.0066061243, -0.0023423084, -0.0037645858, + 0.007885118, 0.0013287661, -0.022732308, -0.012102178, -0.013381172, + 0.0013679805, -0.026858872, -0.0052275863, 0.0046061883, -0.004763046, + 0.020934476, -0.005532252, -0.006901741, -0.0064794314, 0.0015489702, + 0.0017028115, -0.029996028, -0.025459219, -0.0029410827, 0.02707606, + -0.00014120968, -0.0029214756, -0.0081686685, 0.008464285, + 0.0053090313, 0.027003665, -0.00014733693, 0.0032155837, 0.007812723, + -0.01569784, -0.028692901, -0.006220013, 0.020777619, 0.014177527, + -0.0076377657, 0.008597011, -0.011710034, 0.019148711, 0.006793147, + 0.0016394651, 0.00007484679, -0.00060518435, 0.02005366, 0.027148455, + -0.009230475, 0.0064673657, -0.018786732, 0.0012020733, 0.017254353, + 0.0038369817, -0.012970929, -0.009230475, 0.0045187096, -0.032385092, + 0.0021764012, -0.007420578, -0.0031582704, -0.003954625, 0.003393557, + 0.005085811, 0.00809024, -0.005221553, -0.004307555, 0.009797576, + 0.0064070355, 0.027293248, -0.009495927, -0.012645148, 0.019293502, + -0.005342213, -0.0041265655, 0.01630114, 0.01414133, -0.006244145, + -0.008524615, -0.0012284677, 0.0056800605, 0.012838203, -0.0049199034, + -0.020837948, 0.010485337, 0.0032940127, 0.01248829, 0.0025459218, + 0.0077704913, 0.0064914976, -0.003076825, 0.0014222774, 0.012391762, + 0.0041386313, -0.017966244, -0.016904438, -0.023154616, -0.0035292993, + 0.014684298, -0.014358517, -0.0013385697, -0.022165207, -0.024530139, + 0.0038158663, -0.0027042879, 0.0021839426, 0.012035816, 0.0033603755, + -0.01494975, 0.004545858, -0.0066121574, -0.026062518, 0.005692126, + -0.00392446, -0.016156347, -0.018255828, -0.040638223, -0.0047600293, + -0.0011357104, -0.01414133, -0.008385857, -0.0039123944, 0.04404083, + -0.024337083, -0.008808166, 0.02719672, 0.011058471, -0.004132598, + -0.01383968, 0.006624223, -0.0005252472, 0.0009019321, -0.018171366, + -0.018328225, -0.027148455, -0.0037645858, -0.008657341, 0.005867083, + 0.015951226, -0.0010180671, 0.013694888, -0.008434121, -0.00014092689, + -0.009013288, -0.008452219, -0.012403828, 0.014503309, 0.0175922, + -0.006220013, -0.0036077283, -0.00296069, 0.0045247427, 0.028355055, + -0.01599949, -0.01632527, 0.0064191017, 0.0047720955, -0.0013310285, + -0.0064552994, 0.014961815, 0.005855017, 0.013176051, -0.013393238, + 0.0048233755, -0.00034406898, 0.0017978312, -0.022285866, + -0.011516978, 0.01142045, -0.0013740135, 0.0022125992, -0.005263784, + -0.0087900665, 0.00086120935, 0.0150704095, -0.020632828, + -0.008958991, 0.018219631, 0.026641686, 0.008898661, 0.0211034, + 0.024928316, 0.0134777, 0.0119573865, -0.0052004377, -0.00078278047, + -0.01383968, -0.0094657615, 0.0046695345, -0.0092606405, 0.022104876, + -0.00048867223, -0.013646624, 0.006925873, 0.011710034, 0.009912203, + 0.015444455, 0.0022005332, -0.004491561, 0.027172588, 0.0076136338, + -0.027413908, 0.022889165, -0.021875624, 0.0126330815, -0.018521281, + 0.0111369, 0.012379696, 0.012464158, -0.008729737, -0.01975201, + -0.020282913, 0.0059424955, -0.0023588992, -0.0041205324, + -0.0072275223, 0.009646751, -0.001200565, -0.0057222913, 0.004971184, + -0.0065156296, 0.0013008636, -0.013127787, 0.004361852, 0.016759647, + -0.019920934, -0.0049440353, -0.008259163, -0.0076799965, 0.015203135, + 0.028210262, -0.008156603, 0.003710289, 0.03460523, 0.01329671, + 0.032192037, 0.0035866126, -0.012621015, 0.03026148, 0.007873053, + -0.0032970293, 0.020367375, -0.0059123305, -0.010913679, 0.006232079, + -0.012168542, -0.010696492, 0.01061203, 0.0013611935, -0.0007526155, + -0.018485082, 0.01414133, -0.012802006, -0.015166937, -0.015299663, + -0.00021360556, -0.009296838, -0.0050345305, 0.018002443, 0.019052183, + 0.028572243, 0.00972518, 0.005827869, 0.005453823, -0.008247098, + -0.0077342936, -0.006066172, -0.017013032, 0.0017163857, -0.022973627, + 0.0045337924, 0.006298442, -0.012265069, 0.010418974, 0.019884735, + 0.0064191017, 0.007366281, -0.005257751, -0.0038399983, 0.0050586625, + -0.023540728, 0.017495671, -0.0016093001, -0.008603044, -0.00240113, + -0.012319366, 0.015420323, -0.00053995266, -0.010714591, 0.020693157, + 0.0066604214, 0.015323795, -0.013151919, -0.012681345, -0.018111037, + 0.0063105077, 0.022623714, -0.0015022146, 0.02416816, -0.0050103986, + 0.014780826, 0.008066108, 0.009737247, -0.020101923, 0.00873577, + 0.011993584, 0.023794113, -0.008253131, -0.008011811, 0.005556384, + -0.024759391, 0.026231442, 0.016108084, 0.013236381, 0.0075533036, + -0.013694888, -0.013091589, 0.004467429, -0.026641686, -0.0031884355, + -0.0022322063, 0.027872415, -0.009218409, 0.021513643, 0.017495671, + -0.021115465, 0.0032638477, -0.002527823, 0.005836918 + ] + } + ] + }, + { + "name": "springfield-illinois-sangamon-county-seat", + "title": "Springfield: Illinois' Sangamon County Seat", + "content": "Springfield serves as the county seat of Sangamon County in the U.S. state of Illinois.\n\n---", + "keywords": [ + "Springfield", + "county seat", + "Sangamon County", + "Illinois", + "U.S. state" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.0343243, 0.03184899, -0.00958495, -0.02962121, 0.009942495, + -0.025578203, 0.011001378, 0.01197775, -0.0375422, 0.0131053915, + -0.021686466, 0.010190026, -0.009413054, 0.005342544, 0.013456061, + -0.006195151, -0.012844109, 0.019197404, -0.0012668774, -0.0040464443, + 0.04590325, 0.0017739722, -0.04799351, 0.022415308, -0.0046171406, + 0.016117018, -0.04408802, 0.021383928, -0.03135393, 0.00081307057, + -0.01596575, -0.0018908618, 0.032041512, 0.03822979, -0.009639957, + -0.036139525, 0.004441806, -0.034214284, -0.002770972, 0.012163399, + -0.014728094, -0.0055694478, 0.01805601, 0.0000574779, 0.018949874, + -0.009495564, -0.0014009568, -0.020902619, 0.0048681097, 0.0109669985, + 0.011970874, -0.013531695, 0.008560447, 0.037899747, 0.03058383, + 0.0012496877, 0.021768976, 0.033251666, -0.012740971, -0.05151395, + -0.02427179, -0.011792102, -0.03377423, 0.005995751, 0.022415308, + 0.016584577, -0.004163334, 0.009406178, 0.037734725, 0.005345982, + 0.030803857, 0.035011884, 0.012947246, 0.048901126, 0.04488562, + -0.008993627, -0.03690962, -0.021466438, -0.026870865, 0.022126522, + 0.03443431, -0.033966754, -0.009323668, 0.028136024, 0.0129403705, + -0.0227591, 0.003508408, -0.070848875, 0.00008428302, 0.009426805, + 0.006989313, -0.013758598, -0.023584204, -0.01709339, -0.032949127, + -0.03872485, 0.05715216, -0.012417805, 0.0012591421, 0.02890612, + 0.005397551, -0.008752971, -0.049121153, -0.008842357, 0.01892237, + 0.019059887, 0.011475813, -0.022937873, -0.020957625, -0.009935619, + 0.012507192, 0.0020730721, 0.00014385591, -0.029566202, -0.0239555, + -0.011193902, -0.002671272, -0.020517569, 0.048433565, 0.010595702, + -0.016075764, -0.006133268, -0.016185777, -0.000042034073, + -0.016598329, 0.035561953, 0.01357295, -0.044995636, 0.031271417, + -0.011565198, 0.0131053915, -0.0014353361, 0.004462434, 0.008285413, + -0.003121641, -0.018234784, 0.029236162, 0.04903864, 0.026073266, + 0.019857487, -0.07519442, 0.0015951998, 0.020847611, -0.01701088, + 0.018344797, -0.005641644, -0.004696213, -0.022979127, -0.010348171, + -0.031491444, -0.018262288, -0.014714343, -0.03536943, 0.019995004, + 0.031188907, -0.01109764, 0.0006467607, -0.034351803, 0.034379307, + -0.04136518, -0.022415308, -0.010808853, 0.028012259, 0.011902115, + -0.01652957, -0.008010378, 0.002336074, -0.005995751, -0.007467185, + 0.007914117, 0.029428685, 0.0045346306, -0.013593578, -0.074754365, + 0.017285915, -0.03041881, 0.005242844, -0.022291541, 0.0016837266, + 0.038284793, -0.0077559715, 0.029401183, 0.021177653, -0.016240785, + -0.017780976, -0.008684212, 0.0020679152, 0.026073266, 0.042355306, + -0.016405804, -0.015745722, 0.037322175, 0.038064767, 0.0140061295, + 0.026238287, -0.014033632, -0.034709346, -0.030886369, -0.0013536852, + 0.014659336, -0.05924242, 0.022539074, 0.029896244, -0.01932117, + 0.007487813, 0.022869114, 0.0028087893, -0.028988631, 0.0026489254, + 0.040347554, 0.008003502, 0.0071646473, -0.007322792, 0.06171773, + -0.031628963, 0.031078892, -0.014425557, -0.0046412065, -0.024808107, + 0.051376436, 0.031436436, 0.00882173, -0.009976874, 0.00077783177, + -0.005397551, -0.007900365, -0.052669097, -0.03303164, 0.029676218, + 0.004658396, 0.013064136, -0.004885299, 0.01580073, 0.04408802, + -0.035781983, 0.04007252, 0.016213281, -0.007020254, -0.033251666, + 0.030308796, -0.003719841, 0.04879111, -0.013483563, -0.027049638, + -0.005552258, -0.01564946, 0.0075015645, 0.01948619, 0.0013046947, + 0.013600454, 0.00069102406, 0.004307727, 0.0343243, 0.0062707854, + -0.01177835, 0.013820481, -0.022786604, -0.0038917374, 0.010245034, + 0.019101143, -0.007838482, 0.027847238, -0.010355047, 0.017272163, + 0.007694089, -0.023460438, -0.023432935, 0.00043231976, 0.011922743, + -0.014838109, -0.023680465, -0.0066180164, 0.005706965, 0.028933624, + 0.041970257, 0.0040808236, 0.018977378, 0.031106396, -0.034269292, + -0.015841983, -0.015181902, -0.012527819, 0.0116820885, -0.04854358, + 0.012823481, 0.020215033, -0.009351172, 0.051348932, 0.0152506605, + -0.028713597, -0.0012514067, -0.02267659, 0.032921623, 0.031683967, + 0.050001264, 0.010960123, 0.02154895, 0.009096764, -0.03877986, + 0.0013313387, 0.02370797, 0.024079265, 0.010348171, 0.00001035408, + 0.01619953, -0.00825791, 0.01018315, 0.014961874, -0.03226154, + -0.055749483, 0.0025475065, -0.03825729, -0.0018135083, 0.018798605, + -0.00941993, 0.029016133, -0.003912365, -0.01229404, 0.04967122, + -0.035039388, -0.019197404, 0.017863488, 0.022030259, -0.004359296, + -0.016117018, 0.0069480576, -0.022896618, 0.004259596, 0.022662839, + 0.0145355705, -0.029648714, 0.0006360172, -0.0059269923, -0.04191525, + -0.014728094, -0.027517196, 0.017698467, 0.009234281, 0.020627584, + 0.012706592, -0.023652961, -0.020710094, -0.0003661396, 0.04576573, + 0.03160146, 0.010561323, 0.004242406, 0.0010193464, -0.027805982, + 0.0056107026, 0.00074173353, 0.009399302, 0.035176907, -0.0195687, + 0.02362546, 0.0054113027, -0.033471692, -0.03938493, -0.0070477575, + -0.010361923, -0.028603582, 0.0062742233, -0.010196902, 0.0058169784, + 0.014398053, 0.045160655, -0.017574701, 0.018908618, 0.036634587, + 0.004912803, -0.010368799, 0.013964874, 0.013951123, -0.0074053025, + 0.021892741, 0.0014714343, -0.004565572, 0.013270412, 0.005538506, + -0.009117392, 0.040980134, 0.010341295, -0.038422313, -0.017602205, + -0.00063558744, -0.029951252, -0.015910743, -0.027750976, + -0.026059514, 0.018166026, -0.042740352, -0.007920992, 0.0063601714, + -0.030171279, -0.026018258, 0.028851114, -0.05979249, 0.013751722, + 0.058967385, 0.030941375, 0.015181902, -0.030501321, 0.011984626, + 0.028273541, -0.03239906, -0.027695969, 0.0059476197, -0.060397565, + 0.008931744, -0.0060645095, -0.026967127, 0.0002116476, 0.00893862, + 0.007281537, -0.019651212, -0.05013878, -0.017698467, 0.032949127, + 0.013786102, 0.008725468, 0.0039708097, -0.042382807, -0.003661396, + -0.018908618, 0.00267471, 0.014205529, 0.02649957, 0.03734968, + -0.033471692, -0.036634587, -0.034296796, -0.009014254, -0.0058444818, + 0.035479445, -0.0030322548, 0.044995636, -0.002568134, 0.012452184, + 0.046370808, 0.014590577, -0.017272163, 0.017107142, 0.04928617, + -0.05046882, 0.00004904423, 0.004242406, 0.012555323, 0.0043352307, + 0.028521072, 0.039219912, -0.013737971, 0.019541197, 0.0049678097, + 0.012080888, -0.0011731938, 0.027737224, -0.045545705, 0.021438936, + -0.02689837, 0.045050643, 0.02915365, 0.0060576336, -0.0056313304, + -0.037982255, -0.014315543, 0.025055638, 0.018001005, 0.013201653, + 0.0013794696, -0.024849363, -0.0127547225, -0.022580327, -0.029813735, + 0.010031882, -0.04804852, -0.009612454, -0.0074946885, 0.008079137, + 0.017547198, -0.005693213, -0.050963882, -0.0016931808, -0.057482198, + 0.009846233, 0.0077422196, 0.0025457875, -0.029236162, -0.042272795, + 0.0031972753, -0.015594454, 0.08482062, 0.02483561, -0.010279412, + -0.013242909, 0.016474564, 0.009770598, -0.014343047, -0.028493568, + 0.024065513, 0.03536943, 0.0024976565, 0.05453933, 0.032069016, + -0.021040136, 0.0054250546, -0.002030098, -0.010994502, -0.042327803, + 0.0095368195, 0.009193026, 0.050743856, 0.008759847, -0.030253788, + -0.02603201, 0.0001669545, -0.015388178, -0.01764346, -0.0012290602, + 0.034929376, 0.0027469066, -0.0053837993, 0.0063017267, -0.0025578204, + 0.0069618095, 0.0057516578, 0.010753847, 0.015539446, -0.0029325548, + 0.0060645095, -0.041420188, -0.031436436, -0.0015608205, 0.004159896, + 0.028741099, 0.03616703, -0.0023876429, -0.024656838, -0.010960123, + -0.005304727, 0.012967874, -0.029263666, -0.023432935, 0.014329295, + 0.007529068, -0.0015221437, 0.012486564, 0.004366172, 0.0053803613, + -0.0071508954, -0.021218907, 0.038944878, -0.004345544, -0.004462434, + -0.00018414416, -0.0064220545, -0.010396302, -0.017588453, + -0.010128143, -0.0050606336, -0.0012135895, 0.012245908, 0.006583637, + 0.04120016, 0.015690716, 0.0031903996, 0.009612454, -0.0046308925, + -0.0049231164, -0.0072059026, -0.008141019, 0.007831606, 0.0036270167, + 0.04557321, 0.0048474823, 0.020297542, -0.00090761366, -0.03465434, + -0.01089824, 0.010451309, -0.009131144, 0.020421308, -0.0012763317, + 0.025344424, 0.009089888, 0.01137955, 0.013689839, -0.0020197842, + -0.023529196, -0.02722841, 0.02882361, 0.021053886, 0.019733721, + -0.0024048325, -0.012679088, 0.003166334, 0.0030580394, -0.02340543, + 0.005490375, -0.018564826, -0.020077515, 0.016433308, 0.0036063893, + 0.032371555, -0.040127527, -0.016887115, 0.0022621583, 0.03135393, + 0.0039811237, 0.014590577, 0.006538944, 0.022016507, 0.025399432, + -0.008003502, -0.0012651584, -0.006122954, -0.008972999, 0.03338918, + 0.037487194, 0.01827604, -0.0053116027, -0.009523068, -0.029676218, + 0.0031336737, -0.013016005, 0.028246038, -0.023983004, 0.013524819, + -0.025426935, -0.021383928, 0.014618081, -0.011771474, 0.0021108894, + -0.027242161, -0.0074465577, 0.016900867, -0.01229404, -0.0053666094, + -0.0048612338, -0.010712592, 0.008807979, 0.005211903, 0.038119774, + 0.013985502, 0.020503817, -0.025825735, -0.0033090082, -0.02770972, + 0.039659966, -0.0063120406, -0.00013998823, 0.047855992, + -0.0048406064, 0.0029514635, -0.00029458766, 0.0045311926, + 0.007810978, 0.0077353436, -0.033884242, 0.04007252, -0.016034508, + -0.029346175, 0.013511067, -0.013517943, 0.011262661, 0.03201401, + 0.015539446, -0.025371928, 0.053054146, 0.020943873, -0.0020472878, + -0.005490375, 0.014439308, 0.0059338682, -0.008766723, -0.010685088, + -0.009172399, -0.010746971, -0.009289289, -0.014659336, 0.0011199058, + -0.014920618, -0.004885299, -0.0062639094, -0.028163528, 0.024945624, + -0.021755224, 0.02571572, 0.049918752, -0.016172025, -0.022924121, + 0.016997129, 0.025096893, 0.009227405, -0.014301791, 0.03217903, + -0.003465434, 0.020448811, 0.03553445, -0.00722653, -0.016474564, + -0.0011517068, 0.014755598, 0.026375804, -0.05198151, 0.008732344, + 0.024863115, -0.0024993755, -0.033196658, -0.031023886, 0.0004989297, + 0.007329668, -0.01691462, 0.017258411, 0.006593951, 0.041062642, + -0.039879996, 0.042767856, -0.037789732, -0.011317668, 0.029098645, + 0.0148931155, 0.008409178, -0.03322416, -0.0086498335, -0.019788729, + 0.008684212, -0.024450563, 0.01069884, 0.008319792, 0.006222654, + -0.039632462, -0.010224406, -0.0150718875, -0.005459434, 0.026472066, + 0.008216654, -0.024038011, -0.021177653, -0.003511846, 0.0074053025, + 0.009694965, -0.013909867, 0.04095263, 0.0037885995, -0.0047340305, + 0.015305667, -0.0006304305, 0.003668272, -0.06116766, 0.022140274, + -0.022346549, -0.016282039, 0.035424437, -0.0041427063, 0.008972999, + 0.009825606, 0.012685964, 0.019059887, 0.024546824, 0.038174782, + 0.01644706, 0.022635335, 0.002177929, -0.0005883159, 0.014081764, + 0.035974506, 0.013634833, 0.010025006, 0.023584204, 0.010306916, + 0.03281161, 0.0019768102, -0.013277288, -0.009866861, 0.02682961, + 0.01297475, -0.0069618095, -0.0054834993, -0.009770598, -0.008436682, + 0.042272795, 0.0023463876, -0.032536574, -0.015883239, 0.00057886157, + -0.023900494, -0.010279412, 0.0069721234, -0.00097121537, + -0.018743597, -0.010148771, -0.0037851615, -0.033966754, 0.012080888, + 0.008594827, -0.012954122, 0.008732344, -0.01923866, 0.029043637, + 0.009914991, -0.0007004784, 0.016419556, 0.016020756, 0.0058444818, + 0.0020885428, -0.0070546335, -0.009378674, 0.029236162, -0.02819103, + 0.03586449, 0.039659966, -0.0064186165, 0.029401183, 0.0125759505, + 0.001609811, -0.017340923, 0.02819103, -0.0063670473, -0.003018503, + -0.017024633, 0.023487942, -0.016749598, 0.01787724, -0.0011250628, + -0.0044177407, 0.017134646, 0.0203663, -0.041172657, -0.0032643152, + 0.01652957, 0.019183652, -0.015869487, -0.016598329, -0.0098943645, + -0.0028586392, 0.00958495, 0.035176907, -0.0050640716, 0.032921623, + 0.016777102, -0.02011877, -0.008154771, 0.016419556, -0.0049196784, + 0.0090073785, 0.04351045, 0.0062054647, -0.043647967, 0.011840233, + -0.04477561, -0.022016507, 0.018427309, -0.020558825, 0.028521072, + -0.010478812, 0.010836357, 0.04766347, -0.001830698, 0.0077009643, + 0.021177653, 0.012988502, -0.0033433875, 0.0275447, 0.008099765, + 0.018179776, 0.034461815, 0.019211156, -0.0048818616, -0.00021186247, + -0.014425557, 0.024230534, 0.0040326924, -0.023171652, -0.011022005, + -0.01835855, 0.007016816, -0.012390302, 0.0025406308, 0.0055041267, + 0.0123971775, -0.03473685, -0.0033691719, -0.015704466, -0.03363671, + 0.022319045, 0.022731597, -0.020943873, -0.01564946, 0.01297475, + -0.03234405, -0.0020163462, 0.004703089, 0.013414805, -0.0019011756, + 0.013022881, 0.0014336171, -0.021452688, -0.020050012, -0.01573197, + 0.047140904, 0.010306916, -0.00032209113, -0.000012408781, + -0.017134646, -0.0068071024, 0.0016957592, 0.020008756, -0.01297475, + -0.002719403, 0.017258411, 0.0156082045, 0.03289412, -0.0033537012, + -0.022387804, -0.015883239, 0.012617205, -0.035947002, 0.01401988, + -0.00024710127, 0.026197031, 0.04056758, 0.012019006, -0.0134629365, + -0.0020610394, -0.0027125273, -0.011475813, 0.020696342, -0.02075135, + -0.021136397, 0.005239406, -0.0037817236, -0.006105765, -0.0021882427, + 0.011400178, 0.028273541, -0.022896618, -0.030831361, -0.034516823, + -0.015085639, 0.0038986134, 0.011393302, 0.053631715, -0.033499196, + -0.022717845, -0.0040876996, 0.032206535, -0.00436961, 0.020091265, + 0.023790479, 0.0018908618, 0.018138522, 0.011881488, -0.0051259543, + -0.0034963754, 0.03902739, 0.018482314, -0.03443431, 0.031821486, + 0.00025526635, 0.009234281, -0.010843233, 0.0025612584, -0.016873363, + 0.0026437687, -0.0023051326, -0.01892237, -0.006133268, -0.03721216, + -0.0095436955, 0.0036407686, 0.0020455688, -0.0044864994, + -0.016983377, -0.020655086, 0.006012941, 0.01661208, -0.008993627, + 0.0017705342, 0.0029273978, 0.023240412, 0.008285413, 0.025055638, + -0.0016579421, 0.02857608, 0.027420934, -0.0187711, -0.025977004, + -0.013847984, -0.0074534337, -0.0031010134, 0.05324667, -0.025138149, + -0.005194713, -0.0003932133, 0.039082393, 0.0018289791, 0.011640833, + 0.01141393, 0.006985875, -0.005297851, 0.014796853, -0.034791857, + -0.015855735, 0.0143705495, -0.024945624, 0.009344296, -0.037954755, + 0.005538506, -0.0038298548, 0.003613265, -0.0031903996, -0.0046446444, + 0.014604329, -0.033994257, -0.026403306, -0.023584204, -0.005944182, + -0.023185404, -0.0100800125, 0.0026901807, -0.010162523, 0.010568199, + 0.0057585337, -0.011716467, 0.0036407686, -0.004747782, 0.01661208, + 0.022470314, 0.032839112, 0.022607831, 0.015470687, 0.014136771, + -0.004342106, 0.022497818, -0.025770728, 0.006195151, -0.0191699, + 0.052119028, 0.008106641, 0.014136771, 0.024230534, -0.019761225, + 0.011709591, 0.01604826, -0.011743971, 0.009990626, 0.0044830614, + 0.0070065027, 0.014755598, 0.0058719856, -0.0041323924, -0.0059510577, + -0.0025200031, 0.010210654, -0.038862366, -0.008436682, 0.00036592473, + 0.025399432, -0.0017284196, -0.01845481, 0.01533317, -0.003905489, + 0.03943994, -0.013834232, 0.011702715, -0.013889239, -0.01377235, + 0.02187899, 0.02506939, 0.0004473607, 0.016832108, -0.010190026, + 0.019582452, -0.02649957, 0.0059338682, 0.0034757478, -0.02570197, + 0.010135019, -0.01309164, -0.02483561, 0.0053700474, 0.020806355, + -0.032041512, 0.032124024, 0.026719596, -0.0583073, 0.018509818, + -0.012740971, -0.015195653, 0.041172657, -0.01844106, -0.0060198163, + 0.003616703, -0.0053837993, 0.028466064, -0.023735473, -0.0031955566, + 0.04161271, 0.008553571, 0.0073571713, -0.015759474, -0.0143705495, + 0.0032522823, -0.029043637, -0.027585955, -0.018234784, -0.026073266, + -0.0033949565, -0.029511197, -0.009069261, 0.010795102, 0.013022881, + -0.017450936, 0.014948122, 0.010279412, 0.009440557, -0.013841108, + 0.006697089, 0.0066283303, -0.029923748, -0.004218341, -0.016804604, + -0.026018258, 0.01948619, 0.001339074, -0.044610586, -0.017547198, + -0.016955873, -0.0051912754, 0.004012065, 0.0021538634, 0.009282413, + -0.015511943, -0.021273915, 0.022346549, -0.018221032, 0.020380052, + -0.025921997, 0.05132143, -0.016722094, 0.0054113027, -0.0024185842, + -0.02020128, -0.02427179, 0.027819734, -0.009275537, 0.026884617, + -0.00798975, -0.007364047, -0.011008253, -0.025977004, 0.00054190384, + 0.010327543, -0.013737971, -0.0043799235, -0.02171397, -0.008594827, + -0.019059887, -0.020215033, 0.022401556, -0.022387804, 0.0029841238, + 0.007721592, 0.010355047, 0.004909365, 0.019087391, 0.022497818, + -0.008065385, -0.018537322, 0.03239906, 0.012878488, 0.01724466, + -0.014425557, -0.021851487, -0.007364047, -0.01619953, 0.0044005513, + 0.016832108, -0.014150523, 0.009942495, 0.0011774913, 0.0032849426, + -0.0035548203, 0.0036063893, -0.025138149, -0.015058136, -0.011441433, + -0.031683967, -0.039797485, 0.007982875, -0.028067265, 0.005937306, + 0.018729845, 0.008436682, 0.019018631, -0.008814854, -0.0024718721, + -0.0093717985, 0.025688218, 0.0383398, -0.030748852, 0.03135393, + -0.00523253, 0.008209778, 0.023295417, -0.0066489577, 0.007982875, + -0.014205529, -0.0027056513, -0.002671272, 0.00941993, 0.022236535, + 0.009406178, -0.008869861, 0.00574822, -0.018619832, -0.033609208, + 0.002179648, 0.009151772, 0.0058719856, 0.019128645, 0.004599951, + -0.029978754, 0.013284164, -0.0030425685, 0.007329668, 0.010389426, + 0.0071096406, 0.0219615, -0.0003828995, -0.00055264734, -0.00302366, + -0.013421681, 0.0036820236, -0.03239906, -0.029098645, -0.03160146, + 0.0074190544, 0.0022019946, 0.029181154, 0.044198036, -0.0035341927, + 0.010148771, 0.017849736, 0.0012582826, 0.0052840994, -0.020380052, + -0.027269665, 0.003967372, -0.046755854, -0.0009110516, 0.016790854, + 0.020146273, -0.024051763, -0.016378302, -0.0009978594, -0.03146394, + 0.022181528, -0.021810232, 0.008340419, 0.009254909, 0.01885361, + 0.002726279, 0.02139768, 0.039962504, 0.018042259, -0.022882866, + 0.0001039974, 0.017203405, -0.014081764, -0.010685088, 0.016405804, + -0.0058376063, -0.0203663, 0.023446687, 0.030501321, -0.03745969, + -0.0069480576, 0.008931744, -0.018688591, 0.0059716855, 0.007364047, + -0.011365798, 0.0022415307, -0.0013717343, -0.003963934, 0.025770728, + 0.0061573335, 0.02035255, -0.006978999, 0.023501694, 0.000032015727, + -0.00032896697, 0.0148931155, -0.0187711, -0.0047134026, 0.042767856, + -0.0335267, 0.0021607394, -0.003998313, 0.015539446, -0.027077142, + 0.0017387334, -0.029648714, -0.012156523, -0.005796351, -0.01556695, + -0.0032007133, -0.0007245439, 0.008141019, 0.033361677, -0.003905489, + -0.009990626, 0.00069618097, 0.033719223, -0.011310792, 0.026692092, + 0.0018925808, -0.013662336, -0.014604329, -0.053164158, -0.008354171, + 0.005559134, -0.010320667, 0.017945997, 0.0038504822, 0.020778852, + -0.0031594583, 0.004266472, 0.014109267, -0.0041461443, -0.0134629365, + -0.00038053593, 0.0026609583, -0.01573197, -0.037019636, + -0.0068552336, -0.026884617, 0.028301045, 0.021631459, 0.0045311926, + 0.01201213, -0.00014224437, 0.023721721, -0.0075428197, 0.0035926374, + -0.0028191032, 0.003668272, 0.020723846, -0.024808107, -0.00603013, + 0.0335267, -0.034049265, 0.010155647, -0.019733721, -0.00019639179, + -0.010609454, 0.010953247, 0.009461185, 0.026692092, 0.0015247222, + 0.01684586, 0.04186024, 0.0077078403, -0.008862985, -0.0029583392, + -0.00312336, -0.024601832, -0.00035496632, -0.019995004, + -0.0072540333, 0.018372301, 0.031491444, 0.029263666, -0.005878861, + -0.0036854616, 0.009206778, -0.015718218, -0.027558452, -0.013229157, + -0.031656466, -0.009880613, 0.023432935, 0.00654582, 0.013896115, + 0.02666459, 0.011881488, -0.0041461443, 0.01309164, 0.017629707, + -0.0019355549, 0.008574199, -0.02099888, 0.012699716, -0.00063171977, + 0.0038092271, 0.009550571, -0.0027640963, -0.014260536, 0.0010992783, + -0.00524972, -0.0068449196, 0.013579826, -0.010863861, 0.04422554, + -0.009199902, 0.0024907808, -0.011269537, -0.016749598, -0.013978626, + -0.0030511634, -0.012115267, 0.008526068, -0.02986874, 0.01676335, + -0.032069016, -0.017175902, 0.02603201, 0.0052187787, -0.026073266, + 0.003998313, -0.023171652, -0.0066455198, 0.0061813993, -0.006923992, + -0.029208658, -0.015360674, -0.015828233, -0.008147895, 0.011840233, + -0.015044385, -0.004699651, -0.0027898806, -0.021425184, 0.0033537012, + -0.01844106, -0.00546631, -0.008849233, 0.0028483255, -0.014343047, + -0.0148931155, -0.02171397, -0.0049437443, 0.017973501, -0.004015503, + -0.012335295, 0.004441806, -0.03569947, 0.0006897348, -0.0034121461, + -0.022566576, 0.0116820885, -0.037157156, -0.0015745722, 0.041557703, + 0.014081764, 0.015113143, 0.023212908, 0.022181528, -0.0007563447, + 0.017038384, -0.0004314603, 0.00842293, 0.016378302, 0.03982499, + -0.0056244545, 0.01229404, 0.00020638327, -0.010224406, 0.0014430714, + -0.011970874, 0.018111018, -0.0024031135, 0.011640833, 0.012981626, + 0.024409307, 0.029126149, 0.010348171, -0.01265846, -0.0143705495, + -0.005700089, -0.0030872617, 0.004321479, 0.008367923, -0.0028981755, + 0.036634587, -0.024629334, 0.0017215437, -0.00085991237, 0.010884488, + -0.026733348, 0.018977378, 0.027874742, -0.0071783992, -0.037982255, + 0.009715592, 0.0066592717, 0.0037473442, 0.010361923, 0.026389556, + 0.0070546335, 0.019417431, -0.0024117082, -0.021232659, 0.005208465, + -0.008319792, 0.0044796234, -0.032124024, -0.0007589232, -0.013016005, + 0.006882737, 0.025853237, -0.0063739233, 0.013407929, -0.015264411, + 0.018812357, 0.02418928, -0.009028005, 0.010169399, 0.003269472, + 0.014411805, 0.037954755, 0.0018203843, -0.0271459, 0.008367923, + -0.004899051, 0.0035307547, -0.000445212, 0.00830604, -0.012679088, + 0.0076184543, -0.0010752128, -0.019926246, -0.0082922885, 0.005342544, + 0.025688218, -0.0027950376, 0.007872861, -0.02666459, -0.005679461, + -0.016997129, 0.0068621095, 0.020778852, -0.0049506202, -0.02649957, + 0.027971003, 0.013799854, -0.009296164, -0.014191777, -0.008766723, + -0.030473817, -0.0046240166, -0.0012771912, -0.0074190544, + 0.009323668, -0.007274661, -0.0054834993, 0.027324673, 0.001558242, + 0.014590577, -0.01556695, 0.020572577, 0.03248157, -0.0077628475, + 0.0032385306, -0.0050846995, 0.0020919808, 0.016639585, 0.017615957, + -0.0025973564, 0.009330544, -0.004221779, 0.010334419, -0.0034413685, + -0.01118015, -0.01516815, -0.011847109, 0.0014525257, 0.016749598, + 0.015910743, -0.006226092, -0.003271191, -0.014466812, 0.006587075, + 0.011709591, 0.0010606016, -0.014521819, 0.0067624096, 0.024285542, + 0.00086807745, -0.010746971, 0.0029995944, -0.009529944, -0.010877612, + -0.009069261, -0.023501694, 0.0071783992, -0.016007004, -0.0033038512, + -0.008897364, -0.005896051, 0.012479688, -0.045710724, 0.015910743, + -0.015621956, -0.0025526634, 0.007563447, -0.0014748722, + -0.0043008514, -0.008010378, -0.01041693, -0.019664964, -0.012637833, + -0.011063261, -0.00941993, 0.030281292, -0.002121203, -0.013524819, + -0.006480499, 0.017835984, -0.014618081, -0.013930495, -0.008897364, + 0.020105017, -0.019059887, 0.008539819, -0.013627957, -0.021851487, + -0.015470687, 0.0066627096, 0.009213654, 0.0018994566, 0.027077142, + -0.0068449196, -0.0062604714, 0.012025882, 0.028163528, -0.015938247, + 0.011537695, 0.019953748, 0.032564078, -0.013071012, 0.0070477575, + 0.029043637, 0.037899747, -0.003317603, 0.002234655, -0.005882299, + -0.0067417817, -0.0015659773, -0.003675148, 0.018221032, 0.011125144, + 0.0071990266, -0.012266536, -0.017670963, -0.0014894834, -0.017107142, + -0.0035857616, 0.0033502635, 0.0036338926, -0.011517067, 0.01337355, + -0.009399302, 0.019664964, 0.01313977, 0.010685088, -0.0026575204, + 0.0011345171, -0.0102587845, -0.0040567582, -0.006095451, + -0.011407054, -0.0064701852, -0.0017473282, 0.00564852, -0.01669459, + -0.0004159896, 0.00088011025, -0.0075084404, 0.0019957188, + -0.012630957, 0.043372933, 0.0064976886, -0.0081272675, -0.008519192, + 0.00042737776, 0.0020197842, -0.01938993, -0.0050846995, 0.012747847, + -0.007831606, -0.009358047, -0.009873737, -0.0041736476, 0.0027589393, + 0.0024529635, -0.019293666, 0.018564826, -0.020022508, -0.00040116353, + -0.015319419, 0.00032767776, -0.00546631, -0.004201151, -0.021053886, + -0.018743597, 0.009674337, 0.013407929, -0.0023498256, 0.0049781236, + 0.0046137027, 0.012713468, -0.009440557, 0.00822353, 0.023515444, + 0.03146394, 0.0148931155, -0.007082137, 0.02093012, -0.0020885428, + -0.013098516, -0.012864736, 0.02498688, 0.008670461, -0.0033640151, + 0.0011852266, 0.010671336, -0.025536949, -0.015415681, -0.0020541635, + -0.004256158, -0.006449558, -0.008869861, 0.0035823237, -0.015085639, + 0.0031061703, -0.021301419, 0.00047787235, -0.018042259, -0.003168053, + 0.007927868, -0.00030490148, -0.0030047514, -0.024615582, 0.01835855, + -0.005108765, 0.015553198, 0.0017705342, 0.009598702, 0.014480564, + -0.020503817, 0.0093717985, -0.0020558825, 0.007687213, -0.022374053, + -0.026774604, 0.01988499, 0.0060541956, 0.0059785615, 0.0067245923, + -0.010410054, -0.008196026, -0.00053159, 0.013057261, 0.011077012, + 0.018688591, -0.013036633, -0.006934306, 0.01317415, 0.009976874, + 0.010994502, -0.0077628475, 0.0068174163, 0.004225217, -0.008326668, + 0.004410865, 0.016378302, -0.033856742, 0.0010872455, -0.01900488, + -0.02731092, -0.013449185, -0.016268287, -0.018784853, 0.013779226, + 0.0022896617, 0.02282786, -0.017203405, 0.01938993, -0.0038401685, + 0.0034551201, -0.033499196, -0.0074534337, 0.009846233, -0.014728094, + -0.0074534337, 0.033251666, 0.013593578, 0.027187156, 0.010114391, + -0.012053384, 0.022924121, -0.013297915, -0.020985128, -0.00798975, + 0.011338295, 0.011606454, 0.012355923, 0.01780848, 0.0036545203, + -0.013421681, 0.013669212, -0.0074190544, 0.011991502, 0.0057172785, + 0.0076390817, 0.0038229788, -0.00015911172, 0.023267914, 0.008134143, + 0.0025406308, -0.030363804, -0.018372301, -0.002133236, -0.0030442874, + 0.009859985, -0.0122184055, 0.011118268, -0.010341295, 0.0038814237, + -0.000029705869, 0.003675148, -0.0059304303, -0.017450936, + -0.034626838, 0.011077012, 0.012019006, -0.00036313143, -0.02986874, + 0.009165523, -0.00090503524, -0.0011594421, -0.0077490956, + 0.0122184055, -0.0015436308, -0.00030726506, -0.01089824, + -0.017849736, 0.014563074, 0.0031061703, 0.0005311603, 0.0134629365, + 0.0134629365, -0.0152506605, 0.011104516, -0.0024735911, 0.01089824, + -0.0024804669, 0.014329295, -0.0020163462, -0.0038298548, + -0.018069763, 0.00087065593, -0.011077012, -0.008003502, 0.010719468, + 0.008972999, 0.011242033, -0.0191699, 0.007776599, -0.009076137, + -0.01749219, -0.008381675, -0.0131122675, -0.007467185, -0.02003626, + 0.002717684, -0.03041881, 0.0017774102, 0.0013545447, 0.0010004378, + 0.0070958887, 0.018069763, -0.029098645, 0.024216782, -0.012850985, + -0.0076390817, -0.007584075, 0.0019132084, -0.0008362766, + -0.00082553306, -0.022951625, -0.014714343, -0.006700527, + -0.015346922, -0.020297542, 0.0017026351, -0.01533317, -0.0010683369, + 0.0036338926, 0.010148771, -0.00019316873, -0.027022135, -0.009928743, + 0.009825606, 0.020792603, 0.0040464443, 0.0030872617, -0.00922053, + -0.016502067, 0.03209652, -0.034104273, -0.012520943, 0.00515002, + 0.0159795, -0.0074190544, 0.01812477, 0.0027039323, 0.008202902, + 0.022800356, -0.012445308, 0.01245906, -0.00930304, -0.009076137, + 0.007116516, -0.0076184543, 0.0021091704, -0.0066489577, 0.013579826, + 0.0070546335, 0.006133268, 0.0039879996, 0.015319419, 0.04670085, + -0.0053116027, -0.005105327, -0.0004636909, -0.015181902, + -0.0041495822, 0.026912121, -0.014398053, 0.017052136, -0.0050881375, + 0.003125079, 0.015511943, 0.0014301791, 0.005442244, -0.032866616, + -0.018083515, -0.009846233, -0.0049849995, 0.019293666, -0.014480564, + 0.0070477575, -0.016557073, 0.005053758, 0.021411432, 0.017780976, + 0.020737598, -0.021493942, 0.01493437, -0.0076322057, -0.020050012, + 0.0058651096, -0.009997502, -0.014961874, -0.0030941374, 0.025743224, + 0.0064254925, 0.011544571, 0.009392426, 0.006992751, 0.034049265, + 0.004988437, 0.0062020267, -0.01892237, 0.0008216654, -0.032151528, + -0.030748852, -0.004885299, 0.022236535, -0.017574701, -0.011248909, + -0.011049509, 0.0065183165, -0.009399302, 0.0033622961, 0.0036270167, + -0.009014254, -0.006593951, 0.0035376307, 0.007016816, -0.022786604, + -0.026650839, -0.011853985, 0.0060198163, 0.02986874, -0.02731092, + 0.01046506, 0.0044314926, -0.006542382, -0.008560447, 0.008546695, + -0.004764972, 0.013902991, 0.013497315, 0.020325046, 0.013263537, + 0.012300916, 0.016350798, 0.024821859, 0.0048715477, 0.0038986134, + -0.010272536, 0.006026692, 0.00028749695, -0.0011688964, 0.036469568, + 0.013964874, -0.002382486, -0.027654713, -0.0019063326, 0.011606454, + 0.025041886, 0.011269537, -0.015663212, 0.0039776857, 0.018991128, + -0.012465936, -0.007590951, -0.014301791, -0.022085266, -0.011159522, + -0.003214465, 0.011922743, 0.01217715, 0.021755224, 0.03418678, + 0.026362052, 0.029841237, -0.025495693, -0.01357295, 0.002324041, + 0.017382177, 0.009433681, -0.007879737, 0.008484812, 0.00782473, + -0.007975999, 0.003016784, 0.0122184055, 0.009124268, -0.0004226506, + 0.005194713, 0.0029188031, -0.014047384, -0.00090675417, + -0.0038367305, 0.029841237, 0.003919241, -0.017464688, 0.018991128, + 0.007474061, -0.0062089027, -0.0016244221, 0.004795913, 0.028796107, + 0.014301791, -0.018537322, -0.008079137, 0.0029995944, 0.007370923, + 0.009124268, -0.0048681097, -0.01445306, -0.005242844, 0.024038011, + -0.0068724235, 0.015113143, -0.008787351, -0.019898742, -0.008986751, + 0.004366172, -0.012280288, 0.009605578, -0.0084641855, 0.03448932, + 0.004902489, 0.025990754, -0.014645584, -0.022649087, 0.0048784236, + -0.007927868, 0.004706527, -0.005889175, 0.000044263357, -0.018908618, + 0.0003214465, -0.014205529, -0.011565198, -0.022099018, 0.021095142, + -0.00961933, 0.0035238788, -0.018111018, -0.018152274, 0.0048268544, + -0.00027868725, -0.018248536, 0.0037232789, -0.018221032, + -0.012789101, -0.012878488, -0.018688591, -0.005246282, -0.008031006, + -0.0023154463, 0.01691462, -0.006432368, -0.020875115, -0.024326798, + 0.014178026, -0.014576825, 0.015786977, -0.0066558337, -0.020091265, + 0.025536949, 0.006473623, 0.005538506, 0.0029153652, 0.008773599, + -0.008072261, 0.000101365236, -0.022965377, 0.011840233, -0.006532068, + -0.012603453, -0.013359798, 0.025811983, 0.013036633, 0.013050385, + 0.018413557, 0.017657211, -0.022236535, 0.022869114, -0.0125759505, + -0.00435242, -0.013614205, -0.010946371, 0.009969999, -0.0068071024, + 0.030803857, -0.012802853, -0.004211465, -0.0059510577, -0.007886613, + -0.0019389929, -0.0043283547, -0.029071141, -0.0009265223, 0.01309164, + 0.0013519662, 0.0062020267, 0.0059476197, -0.00006129186, 0.009633081, + 0.026527073, 0.03259158, -0.008924868, -0.014975626, 0.01476935, + 0.015828233, -0.012596577, -0.005679461, -0.0035926374, -0.004462434, + -0.004355858, 0.018001005, -0.0015006567, 0.021603955, -0.008691088, + 0.0036476443, 0.014508067, -0.028411059, 0.016997129, -0.02915365, + -0.0063979886, -0.02179648, -0.0005771426, 0.019761225, 0.0051878374, + 0.007130268, 0.012520943, -0.012252784, 0.0002539771, 0.026362052, + -0.006232968, -0.011881488, -0.010286288, -0.027819734, -0.028686093, + 0.018303543, 0.00632923, -0.008807979, -0.02204401, 0.0015470687, + -0.017175902, -0.014480564, -0.019348674, 0.01749219, 0.01516815, + 0.011049509, 0.010994502, 0.005108765, -0.0131053915, -0.0004503689, + 0.026073266, 0.013126019, -0.010135019, 0.004125517, -0.019128645, + 0.018193528, -0.019802481, 0.007927868, -0.00465152, -0.014246784, + -0.017574701, -0.019059887, 0.021026384, 0.013380426, 0.010141895, + -0.005297851, 0.016708342, -0.007439682, -0.015113143, 0.007460309, + -0.022704093, -0.00066996674, -0.011221405, 0.008037882, 0.024945624, + 0.0255507, -0.009241157, 0.012390302, -0.02028379, 0.021356424, + 0.02075135, -0.002573291, 0.0131122675, -0.01030004, 0.023199156, + 0.006449558, -0.0057344683, -0.014879364, 0.021342672, -0.0018702343, + 0.018647335, 0.01812477, 0.0017387334, 0.004163334, -0.006899927, + -0.017065888, 0.000023649201, 0.014301791, 0.0055694478, + -0.0000074600407, -0.009997502, 0.007838482, -0.019156149, + -0.008147895, -0.0020679152, 0.012981626, 0.023914246, -0.018881114, + 0.011750847, -0.013909867, -0.010918868, 0.029896244, -0.005586637, + 0.0057585337, 0.011228281, 0.029786231, -0.0041942755, 0.0043730475, + -0.010994502, 0.009474937, -0.027929747, 0.012967874, -0.027640961, + 0.011633957, 0.0021022945, 0.0043352307, -0.0009239438, 0.03234405, + 0.008966123, 0.0059785615, 0.023254162, 0.016722094, 0.011627081, + -0.0057138405, -0.009963123, -0.006384237, 0.0040808236, -0.010245034, + 0.006635206, -0.002231217, 0.00020530891, 0.020517569, -0.020985128, + 0.011331419, 0.016790854, 0.017863488, -0.027283417, 0.0028912995, + -0.013084764, 0.0057516578, -0.010210654, 0.015635708, -0.018936122, + -0.0020386928, -0.013483563, 0.0023842049, -0.0009531663, + 0.0036510823, 0.004813103, -0.006033568, 0.009089888, 0.020710094, + -0.013050385, 0.0065251924, 0.008560447, 0.006903365, 0.0056072646, + -0.0018410118, -0.008237282, -0.0026936186, -0.005095013, + -0.0075565716, -0.038917374, 0.008326668, 0.009110516, 0.0004636909, + -0.003068353, -0.022167776, 0.03729467, 0.003860796, -0.0043971133, + -0.004008627, -0.018619832, 0.013944247, 0.013634833, -0.0108019775, + 0.0042802235, 0.018303543, -0.012617205, 0.031326424, 0.007955371, + -0.014136771, 0.01333917, -0.010905116, -0.002616265, 0.005806665, + -0.012273412, -0.0005590935, 0.011063261, 0.010733219, 0.0003091989, + 0.00069188356, -0.009241157, -0.0051809615, 0.0343243, -0.014233032, + 0.01041693, -0.008216654, -0.022649087, 0.013132894, 0.007350296, + -0.0007155193, 0.0097293435, 0.0311339, -0.018386053, -0.009227405, + -0.011303916, 0.010856985, 0.018014757, 0.00051826803, 0.009928743, + 0.011056385, 0.00845731, -0.017505942, -0.0048199785, -0.0106163295, + -0.008773599, -0.010843233, -0.017148398, 0.012789101, -0.00126258, + -0.01010064, -0.020187529, 0.018427309, 0.013160398, 0.023515444, + 0.0069068028, 0.009495564, 0.0033313548, -0.026018258, -0.0028603582, + 0.010884488, -0.0033279168, 0.02211277, 0.009144896, -0.01669459, + 0.015951999, -0.0138273565, -0.031986505, 0.009798102, -0.005359734, + 0.017684715, 0.019307418, 0.017822232, 0.013414805, 0.016639585, + 0.016337046, -0.008175399, -0.0050606336, 0.012809729, -0.005115641, + -0.009660585, 0.010148771, 0.012032757, -0.004912803, -0.01021753, + 0.019431183, -0.014466812, -0.014466812, -0.0027864426, -0.02154895, + -0.0035995133, 0.019692466, 0.011627081, 0.0052944133, -0.008553571, + 0.0046858992, -0.0027486256, -0.0054250546, 0.01580073, 0.025138149, + -0.027847238, -0.028081017, -0.013552322, -0.0025062514, 0.02689837, + 0.004060196, 0.0007189572, -0.0052737854, 0.006243282, -0.021823984, + -0.001980248, -0.017835984, -0.008732344, 0.024368051, -0.019829983, + -0.014796853, -0.019871239, 0.010093764, 0.03721216, 0.016735846, + -0.01644706, 0.023267914, 0.008966123, -0.0038264168, 0.0023601393, + 0.002124641, -0.0025836048, 0.013064136, -0.002825979, 0.0058616716, + 0.011482689, 0.0030889807, 0.010547571, 0.0060163783, 0.0057516578, + -0.023597956, 0.0136485845, -0.0047718477, -0.005153458, + -0.0051740855, 0.029566202, -0.004857796, 0.00032273572, 0.0057310304, + -0.011640833, -0.007384675, -0.0143567985, -0.011936495, 0.0039914376, + 0.0059476197, -0.0013614205, 0.023762977, -0.003116484, -0.01313977, + 0.006920554, 0.004001751, -0.005359734, -0.014205529, -0.02187899, + 0.010815729, -0.0008972999, -0.017038384, -0.009887489, 0.005882299, + -0.003417303, -0.012878488, 0.00126258, 0.018991128, 0.02322666, + -0.010368799, 0.0038779858, -0.008312916, 0.03250907, -0.0019957188, + -0.022786604, -0.008471061, 0.037322175, -0.019376177, -0.0033880805, + 0.0054869372, -0.004008627, 0.0319315, 0.009413054, 0.010348171, + 0.0015230032, 0.01740968, -0.0038539201, 0.00097121537, 0.0007413038, + -0.0037679719, -0.0021538634, -0.010231282, 0.015883239, 0.0077628475, + -0.00012022014, 0.024863115, -0.0045861993, 0.012823481, 0.012472812, + -0.0052187787, -0.0036442063, -0.01093262, 0.016804604, 0.007893489, + -0.01493437, -0.0072884127, -0.0057550957, -0.017299667, 0.015511943, + 0.009873737, -0.010375675, 0.017052136, -0.0088286055, -0.0048715477, + -0.010382551, -0.021493942, 0.016708342, 0.0018255411, -0.0013442308, + 0.012342171, 0.0026007944, 0.017505942, 0.013765474, 0.0035376307, + 0.0067177163, -0.021466438, 0.00465152, 0.0042802235, -0.0029377118, + -0.019142397, -0.009777474, -0.0023842049, -0.004118641, 0.0020627584, + -0.010093764, 0.003864234, 0.006483937, 0.020297542, 0.012087764, + 0.0036957753, -0.006538944, 0.0139992535, 0.014755598, 0.0069102403, + 0.009578074, 0.009887489, -0.0033915185, -0.008711716, 0.0039536203, + -0.00031328143, -0.00024581203, 0.018028509, -0.0031835237, + 0.0006368766, 0.0015126894, 0.024849363, 0.0036407686, 0.0008349874, + -0.0058341683, 0.0054766233, 0.016350798, 0.0014508067, -0.0062089027, + 0.012734095, 0.0025543824, 0.01201213, -0.006941182, -0.008092889, + -0.010238158, 0.00083799555, -0.00933742, 0.009598702, 0.0052187787, + -0.014755598, 0.024450563, 0.0023498256, -0.00076708826, 0.009413054, + 0.0013433713, 0.0009772318, -0.0063017267, -0.027008383, 0.01309164, + -0.01357295, 0.008608578, 0.0038057892, 0.0031989943, 0.0029136462, + -0.009626206, -0.0075978264, -0.0062845373, 0.011448309, -0.03209652, + -0.016185777, -0.0063670473, 0.00063472794, 0.022209032, -0.01157895, + -0.0029892807, -0.00006048609, 0.010746971, 0.003994875, 0.013909867, + -0.016158273, -0.008278537, 0.0036923375, 0.027723473, -0.025151901, + -0.00029995944, -0.0054800613, -0.004892175, -0.0063017267, + -0.004988437, -0.018633584, -0.0125828255, 0.023171652, 0.004661834, + -0.0017705342, -0.004555258, -0.00087409385, 0.0030253788, + -0.016722094, -0.0004284521, -0.0030992944, 0.025523197, -0.02219528, + 0.01940368, -0.009825606, 0.009241157, -0.018221032, 0.006133268, + 0.009083013, 0.0015951998, -0.00016802886, 0.011001378, -0.003061477, + 0.013002253, 0.031216409, -0.0012514067, 0.04125517, -0.010740095, + -0.02244281, -0.000917068, -0.00654582, -0.020105017, -0.002427179, + -0.0021865238, -0.019307418, 0.022387804, -0.0009205059, 0.007920992, + -0.0028500445, 0.01229404, 0.004692775, -0.013071012, 0.016062012, + 0.0049265544, -0.008718592, 0.0012359361, 0.01860608, -0.014631833, + 0.007329668, 0.012679088, 0.006442682, -0.006573323, -0.0014129895, + 0.015016881, 0.01360733, -0.019857487, 0.017107142, -0.009076137, + -0.012252784, 0.0029927185, 0.016213281, 0.03327917, -0.011166398, + -0.0053769234, 0.010863861, 0.009639957, -0.012802853, -0.013703591, + 0.014741846, 0.0068208543, -0.0072127786, 0.014191777, 0.010808853, + 0.001558242, 0.005995751, 0.008684212, 0.0017705342, -0.015869487, + 0.024505569, -0.010575075, -0.003609827, 0.017423432, 0.006580199, + -0.004606827, -0.00922053, 0.004441806, 0.0015187059, -0.02186524, + -0.019252412, 0.0026042324, -0.007377799, -0.0038539201, -0.015841983, + -0.008319792, 0.0015591015, -0.0032385306, 0.046783358, -0.0018152273, + -0.009639957, 0.0026489254, -0.010547571, 0.009117392, -0.023611708, + -0.024945624, -0.004060196, 0.008972999, -0.010162523, 0.013064136, + -0.029841237, -0.004912803, -0.00465152, 0.018688591, 0.024629334, + 0.02402426, -0.0068586715, 0.0044246167, -0.025275666, 0.0007047758, + 0.022085266, 0.015470687, -0.012761598, 0.026458314, 0.0011594421, + 0.012025882, 0.00082209514, 0.009413054, 0.003013346, -0.011255785, + 0.015278163, 0.01724466, -0.017024633, 0.012252784, -0.00083240896, + 0.0134766875, 0.0071783992, 0.0058341683, -0.010128143, -0.0327291, + -0.0022174653, -0.015814481, 0.00017683856, -0.0057207164, + -0.01691462, -0.0016218438, 0.01652957, -0.0062845373, 0.009323668, + -0.008890488, -0.0055694478, 0.014824357, -0.0012539852, 0.022882866, + -0.010348171, 0.005981999, 0.013359798, -0.009062385, 0.009887489, + 0.029346175, 0.0084779365, -0.01297475, 0.011723343, -0.0034877807, + 0.034544326, -0.0014628394, 0.008677336, -0.00012602164, 0.030886369, + -0.00978435, -0.0011886645, 0.004916241, 0.017890992, -0.008842357, + -0.015016881, 0.0036029513, 0.024560576, -0.017395929, -0.009378674, + 0.0046412065, -0.0129403705, -0.0022930996, 0.011262661, -0.033884242, + 0.00045079866, -0.012053384, -0.023694217, -0.00018274749, + -0.007845358, -0.0043936754, -0.0047787237, -0.010815729, + -0.011221405, 0.02259408, -0.023817983, -0.00882173, 0.009296164, + -0.0010159084, -0.017767224, -0.009846233, -0.020311294, + -0.0020386928, -0.0018461688, -0.0052153408, 0.0014963592, + -0.0028517635, 0.0383398, -0.025564453, -0.013531695, 0.038367305, + 0.013841108, -0.010877612, -0.008333543, -0.019348674, -0.0022191843, + 0.00028663746, -0.011599578, -0.03943994, -0.013538571, -0.015126894, + -0.0008526068, 0.017519694, 0.041310173, -0.00024710127, 0.0034121461, + 0.004472748, -0.00644612, -0.0064186165, -0.0025389118, -0.0031938376, + -0.008532944, 0.010141895, -0.0069068028, 0.0069996268, 0.018564826, + -0.0038436064, 0.035341926, -0.0062639094, -0.028768603, 0.01932117, + -0.013016005, -0.021562701, 0.0042905374, 0.017313419, -0.013428557, + 0.008031006, -0.014095515, 0.022745349, -0.03248157, 0.0052806614, + -0.017629707, -0.0060679475, 0.012032757, 0.0030064704, -0.01932117, + 0.0033399495, 0.0015264412, -0.017395929, 0.01109764, -0.009413054, + 0.0006269926, 0.025743224, 0.02083386, 0.0009162085, 0.022374053, + 0.022566576, 0.018221032, 0.022002757, 0.008120392, 0.0055041267, + -0.010760723, -0.01149644, 0.0069136783, -0.0009789508, 0.0021383928, + 0.011915867, -0.016680839, -0.0042355303, 0.0367171, 0.019953748, + 0.015264411, 0.0143705495, -0.0020851048, 0.0195687, -0.0020541635, + -0.01613077, -0.009454309, 0.0028500445, 0.0102587845, 0.0006660991, + 0.021411432, -0.01764346, 0.015305667, -0.031078892, -0.024079265, + -0.014026756, 0.0056278924, -0.008044758, 0.002717684, 0.0033932375, + 0.013923619, 0.0050812615, -0.0038882995, 0.018592328, 0.014700592, + 0.0054766233, -0.021810232, 0.0037267169, 0.00091019215, -0.018001005, + -0.004163334, -0.0139992535, 0.0009566042, 0.01629579, 0.0036957753, + -0.006432368, 0.019252412, 0.010334419, 0.0031903996, 0.022731597, + -0.011345171, -0.0065217544, 0.0195687, -0.0049574957, -0.009034881, + 0.007797227, -0.026265789, -0.011173274, -0.0007864266, -0.00830604, + -0.015456936, -0.0099012405, 0.010760723, 0.010602578, -0.008161647, + 0.010348171, 0.016185777, -0.022305293, 0.008402302, -0.0015307385, + 0.0060163783, -0.0150718875, 0.018234784, 0.015209405, 0.030501321, + 0.00025913402, -0.010533819, -0.005782599, -0.0050640716, + -0.0027417496, 0.030253788, -0.0048268544, -0.01380673, -0.0044280547, + 0.0018874239, -0.007831606, -0.01041693, 0.014191777, 0.022016507, + -0.0064633093, 0.017712219, -0.0005294413, -0.025990754, 0.0064667473, + -0.0191699, -0.015291915, 0.00425272, -0.0014645584, -0.02244281, + 0.00042673314, 0.016185777, 0.001168037, 0.0024323359, 0.01652957, + 0.0010932619, 0.012899116, -0.014824357, -0.013036633, -0.007364047, + -0.015539446, 0.0011714748, -0.00456901, 0.012768474, 0.0034551201, + 0.009165523, 0.006638644, 0.019293666, -0.023047887, 0.021975253, + 0.0066661476, 0.0060507576, -0.0019699342, 0.0062054647, -0.017849736, + -0.010520068, 0.0071577714, 0.009935619, -0.010595702, 0.00015792994, + 0.00435242, -0.010746971, 0.02148019, -0.026568327, -0.0067177163, + -0.0058376063, 0.023446687, -0.0016304386, 0.0057516578, 0.028466064, + -0.008484812, -0.0015144084, -0.0010924025, -0.0021177651 + ] + } + ] + } ] diff --git a/src/scrapers/document/examples/10-simple.odt.knowledge.json b/src/scrapers/document/examples/10-simple.odt.knowledge.json index f840c74a0e..6ce235f770 100644 --- a/src/scrapers/document/examples/10-simple.odt.knowledge.json +++ b/src/scrapers/document/examples/10-simple.odt.knowledge.json @@ -1,27 +1,647 @@ [ - { - "name": "springfield-illinois-sixth-largest-city", - "title": "Springfield: Illinois' Sixth Largest City", - "content": "---\n\nSpringfield is the sixth most populous city in Illinois, with a population of 10,566 as of 2019. It serves as the county seat of Sangamon County.\n\n---", - "keywords": [ - "Springfield", - "Illinois", - "sixth most populous city", - "population", - "10", - "566", - "2019", - "county seat", - "Sangamon County", - "U.S. state" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.022805648, 0.035417367, -0.0029928328, -0.024086423, 0.016179595, -0.0070115933, 0.011461635, 0.0027347174, -0.056981448, 0.017042158, -0.029248733, 0.01291231, -0.008736719, 0.018858768, 0.011193718, 0.009886804, -0.016336424, 0.033953626, 0.003763912, -0.0007968091, 0.012670531, -0.0023491778, -0.049244516, 0.03596627, 0.023171583, 0.0093379, -0.03126138, 0.008860877, -0.034188867, -0.0043977653, -0.021616356, -0.0042866776, 0.049793422, 0.0024504636, 0.015983557, -0.016349493, 0.0077042575, -0.020740723, 0.024190977, 0.02930101, 0.0013836952, -0.0013395868, 0.01671543, 0.00820742, 0.03293423, -0.036985666, -0.004815978, -0.024360877, 0.029248733, 0.016937604, 0.024870573, -0.012500632, -0.019708263, 0.037168633, 0.010141652, -0.01806155, 0.011677276, 0.022596542, -0.0067959526, -0.043232713, -0.0032231766, -0.0049597383, -0.00013263135, 0.00044802643, 0.0061326935, 0.0029536255, 0.0051002316, 0.004289945, 0.053688023, 0.011376685, 0.013317453, 0.056563236, 0.008070193, 0.043781616, 0.05980438, -0.005485771, -0.03147049, -0.002639966, -0.04252698, 0.029562393, 0.013173693, -0.013604974, -0.0145720905, 0.002955259, 0.017251264, 0.008638701, -0.00064978126, -0.07783979, -0.00006636672, 0.014206154, 0.017133642, -0.0077696033, -0.0059431912, -0.028752107, -0.043781616, -0.028124787, 0.041559864, -0.013435075, -0.00046354602, 0.01974747, 0.014781197, -0.009239881, -0.026504215, 0.01914629, 0.008187816, 0.015238617, -0.0075539625, -0.0068220906, -0.020204889, 0.0046591484, 0.018649662, -0.0037933176, -0.012964586, 0.006345067, -0.016205734, 0.00068653823, -0.014545952, -0.0107689705, 0.042370148, 0.02429553, -0.0038848016, 0.0049139964, -0.0276543, 0.011402824, -0.0046983557, -0.0016556965, 0.015277823, -0.027288362, 0.007978709, -0.0412462, 0.03463322, 0.0062960577, -0.018139966, 0.018715007, 0.00956661, -0.034136593, -0.0057275505, 0.030921584, 0.03052951, 0.03873693, -0.06984148, -0.0030941188, 0.026033726, -0.0000384927, 0.0048355814, -0.010357293, -0.01927698, 0.0032362456, 0.013604974, -0.028516863, -0.01152698, -0.010494519, -0.027419055, 0.004874789, 0.0025615513, 0.016637014, -0.0039762855, -0.013278246, 0.043337267, -0.053583473, -0.019512225, -0.009710371, 0.03450253, -0.026138278, -0.00872365, -0.013088743, -0.01488575, -0.0041233134, -0.00736446, 0.008096332, 0.022727232, -0.009488195, -0.061895445, -0.048800167, 0.0021188343, -0.018545108, -0.03688111, -0.015656829, -0.00064365513, 0.029666945, -0.0039664838, 0.01146817, 0.02883052, -0.008867411, -0.024988195, -0.019329257, 0.008436129, 0.011291737, 0.036489036, -0.010043633, -0.0066129845, 0.041978076, 0.010389966, -0.0023687815, 0.04022681, -0.003298324, -0.023511382, -0.03293423, -0.0040677697, 0.029091904, -0.05755649, 0.021603288, -0.024073355, -0.03207167, 0.0006881719, 0.023001686, -0.00842306, -0.0133566605, 0.020034991, 0.026360454, -0.006952782, -0.0030843168, -0.01944688, 0.03492074, -0.020714587, 0.023158515, -0.013944772, -0.0011966431, -0.015617621, 0.038841482, 0.020518549, 0.028647553, 0.0004202545, -0.014807335, -0.032698985, -0.025236508, -0.053034566, 0.016297217, -0.009050379, -0.037011802, 0.024413152, 0.019394603, 0.004022028, 0.06017032, -0.021433389, 0.043467958, 0.018453624, -0.00009761014, -0.014493676, 0.03267285, 0.018937184, 0.018963322, 0.0011705048, -0.030581785, 0.0012186973, -0.02080607, 0.019721331, -0.0033358978, 0.013435075, 0.021747047, 0.008717116, -0.0030124367, 0.057190552, 0.0014637436, -0.0112263905, 0.01715978, -0.015774451, 0.013173693, 0.0088478075, 0.00023606133, -0.016833052, 0.0037018338, -0.03842327, 0.0297715, 0.004678752, -0.01504258, 0.022870993, 0.00047947405, 0.00565567, -0.032123946, -0.0045578624, -0.010442242, 0.0061196247, 0.007959106, 0.0054106237, 0.019930437, 0.0479376, 0.014206154, -0.022047639, -0.03753457, -0.021015177, 0.030712478, -0.0012693402, -0.01987816, 0.007684654, 0.0034273819, -0.017695615, 0.013310919, 0.005208052, -0.045924954, -0.013853287, -0.001886857, 0.04265767, 0.025876896, 0.006122892, -0.011866778, 0.01488575, -0.0015290893, -0.04022681, 0.014062393, 0.03322175, 0.036541313, -0.023916526, -0.029327149, 0.022191398, -0.03902445, 0.018100757, -0.0052701305, -0.039677907, -0.06372512, -0.0072795106, -0.048800167, -0.010082841, 0.016022764, 0.0050316188, 0.011762225, 0.018113827, 0.00926602, 0.034084316, -0.02305396, -0.0050316188, 0.010965008, 0.024047216, 0.023615934, -0.010305016, 0.04145531, -0.008638701, 0.00018337637, 0.010461846, 0.011376685, -0.016140386, 0.01594435, -0.0068547637, -0.04116779, 0.009194139, -0.013683389, 0.024060285, 0.0058419053, -0.00070736714, 0.031784147, -0.004071037, 0.008939291, 0.0021449726, 0.041873522, 0.028412309, -0.008717116, 0.0077696033, -0.010200463, -0.023511382, 0.028046373, 0.0006722438, 0.010690556, 0.058079254, -0.0054596327, 0.016336424, -0.0018884906, -0.023459105, -0.01989123, 0.015317031, -0.038658515, -0.020126475, 0.005887647, -0.015251686, 0.011513912, -0.022910202, 0.024948986, -0.012977655, 0.0074298056, 0.038789205, -0.0012358505, -0.0005852524, 0.016780775, -0.0132129, 0.009788785, 0.000096793316, 0.012180438, -0.016323354, 0.008769392, -0.021002106, -0.0015642126, 0.026896289, 0.018244518, -0.014689713, -0.02548482, -0.010108979, -0.023733556, -0.010801644, -0.01382715, -0.028124787, 0.018610455, 0.0053714165, -0.028360032, 0.008730185, 0.0055576516, 0.009305228, 0.032123946, -0.028438447, -0.006828625, 0.030294266, 0.0009360771, 0.01714671, -0.027419055, 0.0023916524, 0.03340472, -0.020126475, -0.0104683805, -0.009520868, -0.061895445, 0.011585792, -0.011219856, -0.015852867, 0.015552276, 0.01487268, -0.007841484, -0.02535413, -0.05739966, -0.010102444, 0.04461804, 0.019852022, 0.019329257, -0.009520868, -0.030634062, 0.0014457735, 0.004613406, -0.0034731238, 0.01441526, 0.0006849046, 0.026242832, -0.026974704, -0.032881953, -0.036410622, 0.004335687, -0.011324409, -0.0025958577, -0.009527403, 0.05206745, -0.04859106, 0.005400822, 0.046892073, 0.0262559, -0.0318887, 0.01684612, 0.043311127, -0.03175801, -0.017551854, 0.008854342, 0.021747047, 0.017303541, 0.017538784, 0.055047214, -0.00926602, 0.017839376, 0.0013812446, 0.013297849, 0.017551854, 0.009625421, -0.021943085, 0.011742622, -0.018388279, 0.022099914, 0.029118042, -0.019224703, 0.011455101, -0.035286676, -0.032986507, 0.035417367, 0.049897972, 0.016088111, 0.0034437182, -0.007939503, -0.012383009, -0.026190555, -0.050943505, 0.023746626, -0.0406973, 0.0007825147, 0.0039730184, 0.021302696, 0.019512225, -0.021851601, -0.044722594, 0.010880059, -0.052041315, 0.011063026, 0.002115567, 0.04101096, -0.026504215, -0.0094555225, 0.036096964, -0.014349915, 0.069893755, 0.007606239, -0.008024451, -0.013376264, 0.014284569, -0.006959317, -0.03949494, -0.058236085, 0.037560705, 0.039129004, -0.0017643338, 0.02217833, 0.023811972, 0.003102287, -0.006518233, -0.009448987, -0.017525716, -0.03978246, 0.0035188657, 0.0002311604, 0.06267959, 0.011121837, -0.036384486, -0.024504635, 0.010625211, -0.014140809, -0.0012824093, -0.014532883, 0.028203202, 0.008429595, -0.009945615, 0.02443929, 0.0035646078, 0.0058680438, -0.023406828, 0.030503372, 0.0012366673, -0.0046166736, 0.019773608, -0.0031055543, -0.042239457, -0.0021417052, 0.0068416945, 0.028673692, 0.04521922, -0.023197722, -0.017486509, -0.025001263, -0.0058549745, 0.042840637, -0.016297217, -0.04610792, 0.0023720488, 0.0273145, 0.014781197, 0.01641484, -0.0041396497, -0.021080522, 0.009729974, -0.019107081, 0.046682965, -0.010566399, 0.015081787, -0.0052831997, -0.007018128, -0.012873102, -0.012775084, -0.010612141, -0.031548902, -0.021381112, 0.01807462, 0.010533726, 0.031627316, 0.003239513, -0.005554384, 0.020283304, 0.0073840637, -0.00076576986, 0.0024586318, 0.0029209526, 0.023171583, 0.017813237, 0.027967958, -0.009703836, 0.00398282, 0.010599072, -0.014898819, -0.01487268, -0.0037443084, 0.0025484823, -0.001069219, 0.018022344, 0.02565472, 0.021969223, 0.0053158724, 0.015748313, -0.006345067, -0.04009612, -0.021250421, 0.04328499, 0.046343166, 0.02869983, 0.00019777284, 0.005492306, -0.0043324195, 0.006730607, -0.015918212, 0.009122259, -0.031496625, -0.038240302, 0.016650084, -0.0012170636, 0.025602443, -0.022910202, -0.020518549, 0.005825569, 0.014938027, 0.009207209, 0.013918634, -0.0031643654, -0.011422427, 0.011383221, 0.00486172, -0.015369308, -0.030555647, -0.0074036676, 0.023171583, 0.034842327, 0.005208052, -0.015317031, -0.0009769181, -0.034241144, -0.011709949, -0.012644392, 0.016493253, -0.025942242, 0.009547006, -0.01701602, 0.00014753833, 0.008070193, -0.0029209526, 0.021341905, -0.014911888, -0.004110244, 0.000940978, -0.002011014, 0.0034241145, 0.012794687, -0.02596838, 0.0001425353, 0.014480607, 0.053243674, 0.0243086, 0.030634062, -0.009076517, 0.0004770236, -0.020309443, 0.019303119, -0.002390019, 0.0069266437, 0.032594435, -0.016506324, 0.0057504214, 0.0005652403, -0.0053126053, 0.018309863, 0.04119393, -0.031182967, 0.026295109, 0.003384907, -0.033012647, 0.0022397237, -0.0003771672, 0.0138140805, 0.015134064, 0.030921584, -0.043494094, 0.030059021, 0.024831364, 0.0048780562, 0.008527613, 0.019525293, -0.00090258743, 0.008227023, -0.017081365, 0.014010117, -0.009187605, -0.0069266437, -0.019956576, 0.0007616858, -0.007567032, -0.010572934, -0.009697301, -0.016048903, 0.013317453, -0.023615934, 0.035155986, 0.04116779, 0.0036070824, -0.005266863, 0.018192243, 0.03782209, -0.0033979763, -0.008429595, 0.04114165, -0.0056524025, -0.004610139, 0.042448565, 0.0076454463, -0.014049325, -0.01684612, 0.02475295, 0.0057079466, -0.04767622, 0.010017496, 0.013271711, -0.007998313, -0.03356155, -0.051387858, -0.00364629, -0.0026889753, -0.0013820615, 0.016937604, -0.02126349, 0.054263067, -0.04132462, 0.028464586, -0.023276137, -0.006482293, 0.039677907, 0.023027824, 0.0153301, -0.034084316, -0.0036299534, -0.038789205, 0.00697892, -0.029091904, -0.023249999, 0.014768127, 0.012010539, -0.037116356, 0.00516231, 0.030294266, -0.008913153, 0.01236994, 0.010455311, -0.007795742, -0.029353287, 0.021080522, 0.0021482399, -0.006459422, -0.021028245, 0.043833893, -0.0010071405, 0.008828203, 0.016532462, -0.0021433388, 0.0028033303, -0.057922427, 0.00271348, -0.033378582, -0.024426222, 0.049270656, 0.024687605, -0.002324673, -0.0021678435, -0.0014506745, 0.0034927274, 0.018989459, 0.053086843, 0.03141821, 0.0118863825, -0.00029548508, 0.013618044, 0.01640177, 0.0005403272, 0.0063483343, -0.0034829257, 0.0064626895, 0.0034633218, 0.049715005, -0.0021008642, -0.023419898, 0.0026121943, 0.039887015, 0.004338954, 0.007795742, -0.022152191, 0.008200885, -0.030059021, 0.03309106, -0.0005047955, -0.0015846331, -0.0006211925, 0.0032836213, -0.012624789, -0.0036560916, -0.0047898395, 0.011938659, -0.011383221, -0.015238617, -0.019629847, -0.038501684, 0.015016441, 0.015892074, -0.020231027, 0.004672217, -0.02885666, 0.04900927, 0.009716905, 0.0115335155, 0.023589797, -0.000035659352, 0.018518971, -0.040435918, 0.0077630687, 0.0075408933, 0.006518233, -0.014402191, 0.016950674, 0.042317875, -0.025249578, 0.029091904, -0.009827993, -0.025314923, -0.018414417, 0.02217833, -0.016271079, 0.0026791736, -0.035129845, 0.03037268, -0.01957757, 0.034580942, -0.012539839, 0.016584737, 0.010932335, 0.010494519, -0.023511382, 0.0048584524, 0.028543001, -0.01411467, 0.0027216482, -0.024086423, -0.012735876, -0.0022413575, 0.015212478, 0.0054563656, 0.003613617, 0.025615513, 0.009867201, -0.0045349915, -0.011337479, 0.020296372, -0.013591905, 0.025458684, 0.045663573, -0.0040350966, -0.04268381, 0.011350548, -0.032986507, -0.031052275, 0.009827993, -0.017656406, 0.021302696, -0.022674957, 0.01624494, 0.037247047, 0.007939503, -0.007730396, 0.009520868, 0.006893971, -0.0012276822, 0.030477233, 0.0003344883, 0.002997734, 0.006162099, 0.011559654, 0.004165788, -0.010148186, -0.018623523, 0.007338322, 0.0066554593, -0.004930333, -0.024426222, -0.009984822, 0.0022511592, -0.012428751, -0.006116357, -0.020701516, 0.024635328, -0.04085413, -0.0048029087, -0.019303119, -0.024687605, 0.0115335155, 0.014898819, -0.009448987, -0.03309106, 0.010984612, -0.0176172, 0.005701412, 0.012402614, 0.00444024, -0.015081787, 0.009958684, 0.012860033, -0.023981871, -0.03526054, -0.011579257, 0.011520446, 0.017395025, -0.008919688, 0.0028343697, -0.029091904, -0.0044239038, 0.01959064, 0.023903456, 0.0049597383, -0.019185496, 0.0074951514, -0.0088086, 0.02383811, 0.0040677697, -0.016127318, -0.01382715, 0.017970067, -0.018884907, -0.0017757693, -0.009762647, 0.026033726, 0.032516018, 0.013291314, -0.034816187, -0.0015829995, -0.0046036043, 0.0007796558, 0.022112984, -0.029536255, -0.013395868, -0.0021939818, 0.019629847, 0.010716694, 0.0026677381, 0.0032819875, 0.003979553, -0.02883052, -0.034737773, -0.006309127, -0.017120572, 0.01670236, -0.0074298056, 0.019054806, -0.026046794, -0.007913364, -0.018963322, 0.037639122, -0.01853204, 0.021211213, 0.03536509, -0.0045023183, 0.0061424957, 0.01821838, -0.008057124, 0.0063417996, 0.04294519, 0.022126053, -0.023498312, 0.030869307, -0.009168001, 0.026556492, -0.007919898, 0.011892917, -0.025236508, 0.0011582525, 0.010344224, -0.019956576, -0.017525716, -0.052903876, -0.014990303, 0.010376897, 0.012435286, -0.0019799746, -0.002046954, -0.023106238, -0.0015070351, 0.034397975, -0.01078204, 0.0030042685, 0.03008516, 0.034685496, 0.0024096225, 0.0137618035, 0.006207841, 0.051152613, 0.026504215, -0.01853204, -0.017342748, -0.009390176, -0.00805059, -0.008253162, 0.03536509, -0.008971964, -0.009893338, 0.0016017864, 0.057086002, -0.009658094, 0.003959949, 0.010089376, -0.011624999, 0.015656829, 0.028752107, -0.0030598124, -0.024021078, 0.0059791314, -0.014284569, 0.028569138, -0.0419258, -0.011794899, -0.012213111, 0.006015071, 0.011023819, -0.0220215, 0.036384486, -0.036227655, -0.0055674533, -0.036227655, -0.016140386, -0.023432966, 0.001388596, -0.0071553537, 0.0024422954, 0.0005007114, -0.0032966903, -0.0241387, -0.000017880726, -0.0011084264, 0.015918212, 0.006282989, 0.022962477, 0.034946878, 0.016833052, -0.0058909147, -0.0012742411, -0.0031153562, -0.028673692, -0.0046199406, -0.010076307, 0.05102192, 0.013787942, 0.018819561, 0.01578752, -0.03586172, 0.02279258, 0.0013902297, -0.0014735454, 0.0002234006, -0.003626686, 0.0043487563, 0.023354553, 0.02170784, -0.007710792, 0.0072795106, -0.0041167787, 0.0075735664, -0.04132462, -0.0016556965, 0.024792157, 0.027209949, -0.0047767702, -0.020021921, 0.0013036466, -0.0102135325, 0.00697892, -0.0025697195, 0.022570403, -0.018649662, -0.015303962, 0.01123946, 0.0042964797, -0.00083070714, 0.009893338, -0.009122259, 0.034345698, -0.017813237, 0.02566779, 0.002718381, -0.019094013, -0.016179595, -0.0074298056, -0.02473988, 0.0096058175, 0.009847596, -0.03248988, 0.025432546, 0.009396711, -0.042161044, 0.016061973, 0.002380217, -0.012383009, 0.01382715, -0.018153034, -0.016114248, 0.011579257, 0.006224178, 0.0473887, -0.024635328, 0.015591484, 0.052616354, -0.004979342, -0.003509064, -0.013709527, -0.0060869516, 0.003486193, -0.04540219, -0.0034731238, -0.003531935, -0.0352344, -0.017120572, -0.047702357, -0.011781829, 0.0008617464, 0.019603709, -0.023289206, 0.007998313, 0.022335159, 0.010435708, -0.021851601, 0.0035286676, -0.0033685707, -0.021158937, -0.021158937, -0.012592115, -0.028464586, 0.0031300588, 0.026817873, -0.020779932, -0.018584317, -0.005698145, 0.016427908, -0.0046591484, -0.015264755, 0.011102234, -0.011794899, -0.042788364, 0.016114248, 0.0004937684, 0.018741146, -0.020766862, 0.032280773, -0.012539839, 0.0017479974, -0.0040775714, -0.003613617, -0.018701939, 0.0013910466, -0.009305228, 0.024177907, -0.003750843, 0.0077369306, -0.017774029, -0.02794182, -0.0026121943, 0.020884484, -0.009965219, -0.0029225862, -0.022152191, -0.0022691293, -0.013709527, -0.024347806, 0.03052951, -0.021250421, -0.0020404193, -0.008377318, 0.015774451, 0.006789418, 0.017970067, 0.01593128, 0.004655881, -0.0005223572, 0.030973861, 0.002233189, 0.012102023, -0.021289628, -0.022714164, 0.011108768, -0.012860033, 0.0035842115, -0.013552697, 0.0012260487, 0.011899452, -0.004884591, -0.0032444138, -0.007939503, 0.010684022, -0.018492833, -0.016911466, -0.020322511, -0.02443929, -0.03991315, -0.000039743456, -0.001101075, -0.009429384, 0.032594435, 0.017774029, 0.004695088, -0.019538363, -0.008115935, -0.001064318, 0.015578414, 0.029536255, -0.030032882, 0.03186256, -0.0126966685, -0.0004431255, 0.0153301, -0.011330944, 0.008383852, -0.0036528246, -0.0045741987, -0.0150295105, 0.0125986505, 0.014323777, 0.016885327, -0.0132129, 0.015317031, -0.014807335, -0.0262559, 0.015879005, 0.037299324, 0.0028294686, -0.0031349598, 0.0046068714, -0.033195615, 0.011749156, 0.01236994, 0.008952361, -0.028203202, 0.011899452, -0.012820826, 0.0013926801, -0.005861509, 0.015317031, -0.013905564, 0.016075041, -0.0019309653, -0.028647553, -0.026791736, 0.022570403, 0.010108979, 0.020335581, 0.015774451, -0.012507167, 0.016192663, -0.000824581, -0.007586635, 0.020832209, -0.03384907, -0.035731025, 0.010082841, -0.039573353, 0.014362984, 0.017029088, 0.015356239, -0.031784147, -0.03826644, -0.0015241883, -0.03207167, 0.010540261, -0.026608767, -0.0019587371, 0.0070704045, -0.005469435, -0.0054171584, 0.026674114, 0.020793, 0.020675378, -0.019708263, -0.0046820194, 0.019642916, 0.002522344, 0.0028588742, 0.020217959, -0.019172428, -0.016493253, 0.032751262, 0.01989123, -0.017342748, -0.014702782, 0.019995783, 0.009599282, 0.012657462, 0.00035307097, 0.0023001686, 0.008148609, -0.010507588, -0.0015437921, 0.030294266, -0.008932756, 0.016323354, 0.0038684653, 0.030973861, 0.013232504, 0.0047277613, -0.000090360845, -0.02794182, -0.0014523081, 0.022505058, -0.038344856, -0.0000059793865, -0.016349493, 0.024099493, -0.034659356, 0.0039403457, -0.020714587, -0.011951728, -0.01032462, -0.023785833, 0.00016785676, -0.0012350336, 0.0065247677, 0.013774873, 0.012075884, -0.023746626, -0.009886804, 0.015107925, -0.012206576, 0.029954467, 0.003257483, -0.013997048, 0.003888069, -0.052747045, -0.013774873, -0.015591484, -0.0041363826, 0.016833052, 0.0040808385, 0.016963743, 0.012690134, -0.012735876, 0.012232714, -0.01123946, -0.0021874472, -0.012618254, -0.008606028, -0.011821036, -0.024112562, 0.015826728, -0.023877317, 0.0406973, 0.010409569, 0.013644181, 0.009664629, -0.00011997062, 0.003348967, 0.003590746, 0.0037998522, -0.0030679805, -0.014820404, 0.008227023, -0.01640177, -0.005966062, 0.020440133, -0.015682967, 0.023289206, -0.03141821, 0.00972344, -0.004316083, 0.0038717326, 0.020701516, 0.010710159, 0.019708263, 0.03144435, 0.031182967, -0.010474916, -0.0066521917, -0.014637436, -0.0075408933, -0.013787942, -0.0012235981, -0.028516863, 0.022674957, 0.01472892, 0.018754214, 0.024426222, 0.0030500104, -0.003522133, -0.00531914, -0.030241989, -0.011134907, -0.00093526026, -0.014650505, -0.004652614, 0.013918634, 0.011245995, 0.020596964, 0.02490978, 0.029222596, -0.0017528983, -0.0050381534, -0.0013044635, 0.011605396, 0.009514334, 0.012284991, 0.007697723, -0.011500843, -0.0014065661, 0.010056702, -0.008102867, 0.0016393602, -0.008396922, -0.0071096118, -0.016153457, 0.009971754, -0.015696036, 0.049715005, -0.0038684653, 0.0061686337, -0.013193296, -0.017682545, 0.009233347, 0.00265957, -0.018649662, 0.012958052, -0.02548482, -0.007018128, -0.017852444, -0.008161677, 0.020257166, 0.016022764, -0.021903878, 0.011840641, -0.018336002, -0.011821036, 0.011807967, 0.0041167787, -0.036018547, -0.017342748, -0.023419898, 0.0007702624, 0.007789207, -0.006893971, 0.012389544, -0.009592748, -0.007181492, 0.0071357503, -0.0060673477, 0.0009115725, -0.0056458684, -0.011017284, -0.004711425, -0.008501475, -0.0014065661, -0.00037308308, 0.020610033, -0.0086909775, 0.0047244937, -0.010527192, -0.027811129, -0.012186972, -0.0037541103, -0.02050548, 0.0025419476, -0.005322407, -0.004450042, 0.034267284, 0.009736509, 0.0013877793, 0.025772342, 0.013062605, -0.0032215428, 0.020714587, -0.007410202, 0.0073579256, 0.013801011, 0.004280143, 0.0014294371, 0.008860877, 0.00720763, -0.009926011, 0.0031300588, -0.015578414, -0.004737563, 0.013252107, 0.022988616, 0.0028670426, 0.024583051, 0.040409777, -0.002197249, -0.028020235, -0.014663574, -0.0069658514, 0.014101601, 0.007057335, 0.011317874, 0.0016434443, 0.027680438, -0.01518634, -0.002087795, 0.007560497, 0.023759695, -0.03157504, 0.015003372, 0.030895445, -0.014023187, -0.018884907, 0.0057504214, 0.024321668, 0.01031155, 0.013219435, 0.03431956, -0.0023949197, 0.018309863, -0.023328414, -0.0127489455, -0.00872365, 0.0023410097, 0.006227445, -0.03978246, -0.008782461, -0.017564923, 0.015134064, 0.016493253, -0.007978709, 0.02187774, -0.037586845, 0.009945615, 0.011219856, -0.0022233874, 0.014898819, -0.0073448564, 0.009429384, 0.062313657, 0.029065765, -0.013944772, 0.017826306, -0.011461635, 0.0075735664, 0.0114812385, -0.0022952675, -0.025942242, -0.002621996, 0.031235242, -0.019094013, 0.00035102892, -0.0028637752, 0.0044696457, 0.0005223572, 0.007874156, -0.0176172, -0.010069772, -0.011415893, 0.011579257, 0.017826306, -0.0053518126, -0.014532883, 0.021916946, 0.0015707472, -0.0110760955, -0.0220215, -0.008684443, -0.026229763, 0.0021939818, 0.007272976, 0.0013379531, 0.0059693293, 0.017669477, -0.007834949, 0.031679593, -0.0012709738, 0.007351391, -0.0076454463, 0.018244518, 0.004129848, 0.009533937, 0.0039501474, -0.007527824, 0.008939291, -0.010174325, 0.010808178, -0.013219435, 0.010246205, 0.0007812895, 0.011572723, -0.0031856026, -0.022962477, -0.012768549, -0.01564376, -0.0059693293, 0.005518444, 0.012193507, -0.015957419, -0.0006113907, -0.021694772, -0.013918634, 0.015447723, 0.011500843, 0.0039305435, -0.0032885221, 0.017303541, 0.000052582865, -0.012918844, 0.006227445, -0.0005382852, 0.013578836, -0.023759695, -0.017839376, 0.023197722, -0.009527403, -0.011004215, -0.014389123, 0.002489671, 0.021054383, -0.044461213, -0.0017349282, -0.0060183387, -0.008481871, 0.0061653666, -0.0036005478, -0.013016863, -0.016075041, -0.0029111507, -0.01671543, -0.0042703412, -0.0027918948, -0.011794899, 0.008272765, 0.0011500843, -0.009899873, -0.0022821985, 0.0053714165, -0.011337479, -0.016937604, -0.008821669, 0.03526054, -0.025576307, 0.013997048, -0.0023230396, -0.0013265177, 0.00067551114, 0.013343591, 0.009547006, 0.008802066, 0.016127318, 0.007076939, -0.0071161464, -0.008645236, 0.024556912, -0.012559443, 0.0071422844, 0.0031905037, 0.048303537, -0.0097561125, -0.020832209, 0.034058176, 0.019695193, -0.00489766, 0.004234401, -0.0080832625, -0.003750843, -0.014794266, -0.0019097279, -0.0075997044, 0.01252677, 0.017695615, -0.0004729395, -0.021786256, -0.0051459735, -0.01791779, -0.0009826359, -0.003247681, 0.016584737, 0.0023312077, 0.0070311967, -0.014467537, 0.015526137, 0.018492833, 0.019172428, -0.014598229, -0.014376054, -0.018466694, 0.010442242, -0.002690609, 0.0065541733, -0.0029585264, -0.0032182755, 0.009952149, -0.016336424, -0.0000847452, 0.005492306, -0.0039860876, 0.0048486507, -0.0054334947, 0.034554806, 0.006541104, 0.0047310283, -0.0029111507, -0.017421164, -0.0044990513, -0.0072403033, -0.015055648, 0.0036724282, -0.0029568928, -0.0056687393, -0.0094555225, -0.007076939, 0.00918107, 0.0072599067, -0.023328414, 0.0016164891, -0.02656956, -0.0066587264, -0.0055903243, 0.014741989, -0.008187816, -0.008030986, -0.018715007, -0.018989459, 0.018584317, 0.013435075, 0.0012799589, -0.004384696, -0.005995468, -0.00007565806, 0.0041363826, 0.022374367, 0.024190977, 0.020610033, 0.014820404, -0.003763912, 0.012690134, 0.012977655, -0.0122915255, -0.013323988, 0.02490978, 0.0145198135, -0.0022854658, -0.0064267493, 0.0126966685, -0.01534317, -0.029640809, -0.014349915, -0.0045970697, 0.0020502212, -0.020113405, -0.0035155984, -0.018087689, 0.006730607, -0.01714671, -0.0042180647, 0.00017571867, -0.0074755475, 0.007841484, 0.014180016, -0.008658305, -0.02170784, 0.022701096, -0.0012987457, -0.0052015176, 0.009867201, 0.0036495572, 0.0014384221, -0.00084622676, 0.010435708, 0.0028980817, -0.003362036, -0.030189712, -0.032881953, 0.023955733, 0.007253372, 0.02488364, -0.0035613405, 0.007848018, -0.0002004275, 0.0002891547, 0.0150295105, 0.012938448, 0.03113069, -0.0042017284, 0.015839797, 0.012892706, 0.017395025, 0.029693084, -0.011147976, 0.01943381, 0.009416315, 0.000097303826, 0.010305016, 0.013591905, -0.030921584, 0.010265809, -0.033195615, -0.0060510114, -0.012180438, -0.0044761803, -0.026412731, 0.009867201, -0.009285623, -0.0050806277, 0.003705101, 0.023067031, -0.0054302276, 0.009409781, -0.01823145, -0.021773186, 0.008514544, -0.013337057, -0.01152698, 0.024818296, 0.017342748, 0.0273145, -0.011977866, -0.005966062, 0.011860244, -0.021119729, -0.021668633, -0.00039922138, 0.0001421269, 0.01776096, -0.009547006, 0.01929005, 0.0009246416, -0.005573988, -0.001927698, -0.019459948, 0.017395025, 0.008991567, 0.008285834, -0.0007906829, -0.013644181, 0.013265177, 0.007730396, -0.0016230238, -0.04009612, -0.005737352, 0.014402191, -0.012539839, 0.004695088, 0.0035646078, 0.005217854, -0.026661044, 0.021446457, -0.0000255129, -0.00081804645, 0.005698145, -0.020152614, -0.017042158, 0.003362036, 0.0024047217, 0.006815556, -0.014650505, 0.014088532, 0.00421153, 0.0015054015, 0.004044899, 0.016885327, -0.00146211, 0.013487352, 0.002095963, -0.02200843, 0.025733136, 0.012330733, 0.012565978, 0.0085929595, -0.003183969, -0.0005758589, -0.00045578624, -0.0068612983, 0.00027608557, -0.005136172, 0.0015633958, 0.012559443, -0.004760434, -0.019512225, -0.009736509, -0.014689713, -0.003937078, 0.006224178, -0.0073840637, 0.014598229, -0.032280773, 0.007848018, -0.00466895, -0.009919477, -0.011860244, 0.0002234006, -0.01487268, -0.006031408, -0.012840429, -0.011932124, -0.0018525505, 0.011115303, -0.0046166736, -0.010037099, 0.017355816, -0.015068717, 0.025406407, -0.019825885, 0.008566821, 0.016140386, 0.0013510223, -0.0068547637, -0.008403457, -0.02338069, -0.014441399, 0.007919898, -0.022805648, -0.0138140805, 0.008717116, 0.004554595, -0.0057406193, 0.004002424, 0.0055347807, -0.004172323, -0.027602023, -0.012905775, 0.012186972, 0.010749367, 0.0016663152, 0.0015731977, 0.0024766019, -0.018192243, 0.013644181, -0.03445025, 0.0043716272, 0.007913364, 0.014062393, -0.005864776, 0.021812394, -0.00057871785, 0.014951095, 0.018179173, -0.0054596327, 0.011938659, -0.025981449, -0.007331787, 0.009396711, -0.009612352, -0.0020600231, -0.015408515, 0.027105395, 0.011520446, 0.008658305, 0.009710371, 0.008200885, 0.04325885, -0.010592537, -0.006080417, 0.021067452, -0.03068634, 0.0103965, 0.02822934, -0.013631112, 0.020413995, 0.0077238614, -0.009468592, 0.0020763595, 0.00531914, 0.007397133, -0.03113069, -0.016271079, -0.0047767702, 0.009331365, 0.015617621, -0.01624494, -0.004819245, -0.013435075, 0.001009591, 0.003878267, 0.015983557, 0.028752107, -0.017956998, 0.021616356, -0.017512647, -0.021851601, 0.009462057, -0.0059007164, 0.013983979, 0.007501686, 0.020100337, 0.003603815, 0.028307756, -0.007815345, 0.012873102, 0.028752107, 0.003531935, -0.0063581364, -0.006972386, 0.013108347, -0.016519392, -0.015774451, -0.011585792, 0.019028667, -0.01987816, -0.0020387857, -0.0033914417, 0.0016434443, -0.010298482, 0.0036626263, -0.0038488614, 0.0016532461, 0.012128161, -0.0138140805, 0.008285834, -0.006008537, -0.021694772, -0.011742622, 0.014689713, 0.0007224784, -0.013043001, -0.0048682545, -0.0068416945, -0.017395025, -0.003404511, 0.0069331783, -0.012409148, -0.0009246416, 0.013552697, 0.008161677, 0.008527613, 0.0002001212, 0.018244518, 0.005505375, 0.004280143, 0.022714164, -0.017564923, 0.008318507, 0.0022266547, -0.0029405563, 0.036985666, 0.012363406, 0.0006085318, -0.028072512, -0.014258431, 0.005760223, 0.03207167, -0.006224178, -0.004378162, 0.002064924, 0.00903731, -0.0043912306, -0.008762858, -0.013983979, -0.02217833, -0.011043423, -0.0029993674, 0.003018971, 0.012637857, 0.018270656, 0.0376914, 0.017695615, 0.018022344, -0.02157715, -0.02974536, 0.0050838953, 0.010474916, 0.016349493, -0.021211213, -0.014402191, 0.001932599, -0.0025305122, 0.009468592, 0.026465008, 0.0038586634, 0.009272554, 0.011500843, -0.0052243886, -0.010108979, 0.014663574, -0.0019848756, 0.02778499, 0.00039186998, -0.017538784, 0.030320404, 0.019473018, -0.02352445, 0.0036724282, 0.013016863, 0.0072337687, 0.018754214, -0.0089458255, -0.0029127845, 0.005237458, 0.005093697, 0.01336973, 0.016480185, -0.010801644, 0.012559443, 0.015656829, -0.0057275505, 0.019028667, -0.022975547, -0.018100757, -0.019773608, 0.0029928328, -0.012428751, 0.020727655, -0.02275337, 0.017290471, 0.0031267917, 0.027863406, -0.00993908, 0.00034122705, -0.008403457, -0.0018100758, -0.0052113193, -0.008246627, -0.00082907354, -0.0078088106, 0.020766862, -0.018688869, -0.008331576, -0.022844855, 0.02961467, 0.0072860452, 0.0039534145, -0.021903878, -0.028412309, 0.009716905, -0.004338954, -0.009390176, 0.0102135325, -0.01563069, 0.000013694518, -0.00706387, -0.017878583, 0.004518655, -0.01684612, 0.0062339795, 0.02734064, -0.00061915047, -0.017029088, -0.030424956, 0.010592537, -0.019054806, 0.0038161888, -0.008632166, -0.020244097, 0.013918634, 0.013539628, 0.0092006745, -0.0015650294, 0.005636066, 0.0007049167, -0.0020779932, -0.020766862, 0.006939713, -0.00074371573, -0.020884484, 0.00041351575, 0.008978499, 0.022805648, 0.0049499366, 0.017538784, 0.012794687, -0.014336846, 0.03382293, -0.00410371, -0.0027902613, -0.017120572, -0.026922427, 0.008586424, -0.0011966431, 0.026334316, 0.002011014, -0.009814924, 0.0039272765, -0.009390176, -0.019995783, 0.0028441714, -0.009377107, 0.011317874, 0.01048145, -0.0040547005, 0.023563659, 0.021930017, 0.0064169476, 0.0056033935, 0.025092747, 0.020518549, -0.024413152, -0.010331155, 0.03199325, 0.013709527, -0.008181281, 0.01442833, 0.0008707314, -0.0007543344, -0.011925589, 0.018296795, 0.0017610665, 0.00796564, -0.009272554, -0.0041396497, 0.0050414205, -0.028046373, 0.01640177, -0.035338953, -0.012820826, -0.026765598, 0.011004215, 0.013931703, 0.01504258, 0.0038488614, 0.02049241, -0.026242832, -0.0015388911, 0.019773608, -0.010069772, 0.0010496152, -0.004760434, -0.01914629, 0.0029928328, 0.007684654, -0.008285834, -0.0017888384, -0.00972344, 0.01412774, -0.02157715, -0.0029764965, -0.01261172, 0.011592327, 0.013526559, 0.013892495, 0.012252318, 0.005573988, -0.024021078, 0.0074494095, 0.034816187, -0.00903731, -0.0012383009, -0.008481871, -0.016950674, 0.0062960577, -0.006345067, 0.004649346, -0.004299747, 0.005792896, -0.029065765, -0.020374788, 0.035809442, 0.008455733, 0.008749789, -0.012160834, 0.011461635, 0.0027347174, -0.0005235824, 0.0077499994, -0.018963322, -0.005933389, -0.020596964, 0.010579469, 0.003545004, 0.031548902, -0.006959317, 0.0040547005, -0.023119308, 0.01791779, 0.007090008, -0.0012432019, 0.003247681, -0.013840219, 0.01927698, -0.0041167787, -0.00588438, -0.004946669, 0.00812247, 0.021812394, -0.008279299, 0.021629425, -0.006809022, -0.0019015598, -0.01684612, 0.0039893547, -0.0012235981, 0.019760538, 0.0006538654, -0.00000608468, -0.010474916, 0.008599494, -0.035417367, -0.007188027, -0.00041821247, 0.03330017, 0.008194351, -0.03139207, 0.005155775, -0.029065765, -0.01101075, 0.027131533, -0.006008537, -0.004652614, 0.0054563656, 0.019616779, 0.008971964, -0.00683516, 0.001351839, 0.01085392, -0.000532159, 0.0028425378, -0.019760538, 0.010644814, 0.013023397, 0.010507588, -0.0049760747, 0.038031194, -0.002535413, 0.017133642, 0.0085406825, -0.0063385325, 0.008357715, 0.009645025, -0.0048911255, -0.013029932, 0.02383811, -0.0011664206, -0.0016017864, 0.0013257008, -0.003061446, 0.015918212, -0.013252107, 0.0052733975, -0.0064430856, 0.019094013, -0.01351349, 0.02200843, -0.0044043, -0.008416526, -0.0017202254, 0.016258009, -0.010801644, -0.0034241145, -0.012017074, -0.0048780562, 0.009520868, -0.0031774344, 0.013879426, -0.005230923, -0.0068678325, 0.014807335, -0.014180016, 0.015696036, 0.023929594, -0.0012252318, 0.022988616, 0.0052505266, -0.00019164667, -0.016506324, 0.012474493, -0.021629425, -0.047101177, -0.004022028, 0.0073579256, -0.029379426, 0.010540261, -0.011455101, 0.027079256, 0.003384907, 0.0066521917, 0.0045349915, -0.019224703, 0.016963743, 0.022831786, -0.0055576516, 0.015081787, 0.0069462475, -0.034528665, 0.01989123, 0.0019358663, -0.020975968, 0.013186761, -0.0006346701, -0.012383009, 0.0020338849, -0.024726812, -0.0021907145, 0.017303541, -0.005743887, 0.006410413, 0.0069462475, -0.012200042, -0.004554595, 0.0036332207, 0.0012374842, 0.011585792, -0.010801644, -0.021054383, 0.0058909147, -0.009906407, -0.008279299, 0.028150925, 0.025903035, -0.00015203084, -0.01654553, -0.0071553537, 0.009965219, 0.006939713, 0.0008617464, -0.004672217, 0.0061424957, 0.0142192235, -0.019512225, -0.014964164, -0.006116357, -0.011860244, -0.022988616, -0.017695615, 0.0075735664, 0.0021319033, 0.012265387, -0.013487352, 0.010697091, 0.0083381105, 0.037168633, 0.009246416, 0.006354869, 0.000010152832, -0.014715851, 0.00013559232, 0.028124787, -0.003914207, 0.014075463, -0.0039501474, -0.025314923, 0.021067452, -0.0033342643, -0.02186467, -0.0024161572, -0.0054204254, 0.00805059, 0.008220488, 0.03081703, -0.002563185, -0.0062143756, 0.0037116355, -0.015199409, -0.010167791, 0.011474704, -0.00796564, -0.0033685707, 0.00683516, 0.0264258, 0.0049041947, 0.005342011, 0.016676221, -0.013029932, -0.021694772, -0.008671374, -0.006387542, 0.0061915047, 0.021067452, -0.0028490725, -0.004185392, -0.011925589, 0.015669897, -0.0031807018, 0.009305228, 0.008854342, 0.029248733, -0.02399494, -0.023067031, -0.013297849, 0.0049499366, 0.0037541103, -0.0070508006, -0.02339376, -0.0004778404, -0.00027955705, 0.0032999576, 0.0102135325, -0.018545108, -0.02064924, 0.023811972, -0.01032462, -0.020335581, 0.013036466, 0.030137435, 0.03021585, 0.005345278, -0.008344646, 0.016741568, 0.016480185, 0.0077499994, 0.0017398292, 0.011827571, -0.012265387, 0.0188457, 0.008429595, -0.0019015598, 0.011892917, 0.013239038, 0.0104683805, 0.018871836, 0.0020583894, -0.012278456, -0.007272976, -0.0041951938, -0.02791568, -0.020675378, 0.04451349, -0.0075735664, 0.013350125, 0.0033342643, -0.0073252525, 0.0034600548, -0.01367032, -0.008514544, -0.008553752, -0.0061424957, -0.011082631, 0.020178752, -0.010429173, -0.026282039, 0.0142715005, 0.015356239, -0.024726812, -0.016336424, -0.007377529, 0.006207841, 0.007919898, -0.010886593, 0.0016614143, -0.00942285, 0.0067044683, -0.021132799, 0.0060608136, 0.015120994, 0.028203202, -0.014924957, -0.0030091694, -0.009658094, 0.021930017, -0.0004239302, -0.027079256, -0.005554384, 0.021250421, -0.015970489, -0.014898819, 0.011847175, -0.0040775714, 0.020570826, -0.0068416945, -0.0041461843, 0.006015071, 0.021211213, 0.006469224, 0.01236994, 0.019838953, -0.007815345, -0.0010398134, 0.00006963401, -0.0072403033, 0.00021421137, 0.013258642, 0.02186467, -0.014846542, 0.0063810074, 0.0047473647, -0.019224703, -0.0008919688, -0.0056491354, 0.0071749575, 0.006599915, -0.014768127, 0.008416526, -0.00013600073, -0.01836214, 0.021812394, -0.0004075938, -0.016336424, 0.027131533, 0.012925379, -0.009886804, -0.011102234, -0.025942242, 0.00034796583, -0.000068000365, -0.006057546, 0.022413574, -0.0026791736, 0.01236994, 0.0006857214, 0.00012977247, 0.0073252525, -0.01623187, -0.016467115, 0.0017316609, 0.009631956, -0.019995783, 0.003394709, -0.0154346535, -0.015539207, 0.010089376, -0.0026203624, 0.0010929068, 0.007625843, 0.036985666, 0.0069462475, 0.006998524, 0.010265809, -0.005008748, 0.00095486396, 0.007351391, 0.010487984, 0.01504258, -0.021185074, 0.0005627898, 0.009187605, -0.009083052, 0.0041690553, -0.006711003, -0.030581785, 0.007377529, -0.011664207, 0.012128161, 0.0040775714, -0.011638069, -0.0043748943, 0.0062111085, 0.023981871, -0.004071037, -0.00011762225, 0.014898819, 0.01093887, 0.011389755, -0.0055119097, -0.0075408933, 0.0056491354, -0.02126349, -0.00611309, 0.0041788574, -0.012650927, -0.0038749997, 0.00294219, -0.0035352022, 0.01610118, 0.0060477443, 0.0037214374, -0.002179279, -0.009631956, -0.012128161, 0.007057335, -0.0133566605, 0.0046395445, 0.005243992, -0.0057667578, -0.0011460001, -0.020675378, -0.0077173267, 0.0054988405, 0.0010153088, -0.034737773, -0.008971964, -0.015826728, -0.0010332789, 0.011925589, -0.019303119, 0.0103965, -0.002407989, 0.01472892, 0.004093908, 0.0044075674, 0.010416104, -0.011677276, 0.0176172, 0.025066609, -0.016427908, 0.009187605, 0.0072403033, -0.01382715, -0.00835118, -0.007181492, -0.016663153, -0.0013861456, 0.012422217, -0.002489671, -0.008573355, -0.0014245362, 0.0051884484, -0.00094261166, -0.0032819875, 0.01185371, -0.006697934, 0.025131956, -0.01957757, 0.015003372, 0.0137618035, 0.009239881, -0.01594435, 0.0104683805, 0.0073840637, -0.012833895, 0.013853287, 0.01944688, -0.0058680438, 0.0092006745, 0.024543844, 0.010579469, 0.04631703, -0.0050838953, -0.015264755, -0.012762015, -0.009814924, -0.021943085, 0.005253794, 0.010788575, 0.0014833473, 0.014245362, -0.010971542, 0.007651981, -0.0029536255, 0.0066031828, -0.012200042, -0.0074298056, 0.014624367, -0.008854342, -0.006331998, 0.0060510114, 0.015016441, -0.0034600548, 0.016898397, 0.009860666, 0.0026350652, -0.0039958893, 0.01791779, 0.0114812385, 0.017669477, -0.012938448, 0.019120151, -0.00805059, -0.016480185, 0.0062470487, 0.020845277, 0.03873693, 0.0074951514, 0.017029088, 0.0230801, 0.01101075, -0.010657883, 0.011742622, 0.016937604, -0.0019064607, -0.014206154, 0.013997048, 0.008560286, 0.002759222, 0.007246838, 0.018022344, 0.009599282, 0.0023459105, 0.017839376, -0.011487774, -0.003509064, 0.016192663, -0.0022625946, -0.005952993, -0.002901349, 0.0011672375, 0.006697934, -0.0015797322, -0.017747892, -0.0025158094, -0.011579257, 0.0017316609, -0.021015177, -0.0073905983, 0.0043422217, -0.006243781, 0.025785413, -0.0029519917, -0.0029046163, -0.0031169897, -0.0019391335, 0.0002082894, -0.024465429, -0.02961467, -0.0027020446, 0.030189712, -0.007919898, -0.0016279246, -0.016623946, 0.004920531, -0.0009050379, 0.023877317, 0.0081355395, 0.011566188, 0.0051329043, -0.008070193, -0.02791568, -0.004485982, 0.02247892, 0.022439713, -0.0054400293, 0.009488195, -0.0070050587, 0.012886171, 0.0015478762, 0.0008748155, -0.0022217536, -0.0031186233, 0.020322511, 0.023315344, -0.010690556, 0.0056883427, -0.019956576, 0.0075147552, 0.0154869305, 0.0018982925, -0.01684612, -0.015120994, 0.00660645, -0.028647553, 0.0068416945, -0.0074886167, -0.0012775083, -0.0005934206, 0.001881956, 0.0038259905, 0.014493676, -0.008854342, -0.0074886167, 0.012722807, 0.0024471963, 0.026360454, -0.010167791, -0.008318507, 0.015813658, -0.00926602, 0.000111393994, 0.02200843, 0.019185496, -0.007874156, 0.002105765, 0.0020665578, 0.014376054, 0.00736446, 0.0009393444, -0.013748734, 0.020100337, 0.0005697328, 0.018937184, 0.0013142653, 0.011272132, 0.004110244, -0.003143128, -0.0012366673, 0.02004806, -0.006730607, -0.019237773, -0.014794266, -0.016127318, -0.004106977, 0.016493253, -0.013186761, -0.0022070508, -0.019198565, -0.026020657, 0.004316083, -0.0016238405, 0.00085684547, 0.014624367, -0.0033097595, -0.016976813, 0.0078088106, -0.009004637, -0.020152614, 0.0012619888, -0.009207209, -0.017434232, -0.012905775, -0.03719477, -0.0013346858, -0.00086419686, -0.0125986505, -0.009729974, 0.0037345067, 0.043102022, -0.027445193, -0.013709527, 0.036645867, 0.015774451, -0.001210529, -0.009743043, -0.0028360032, -0.002192348, -0.00040555172, -0.023602866, -0.029222596, -0.032908093, -0.01312795, -0.0040089586, 0.009814924, 0.030137435, -0.002965061, 0.015408515, -0.0023148714, -0.007887226, -0.014362984, -0.0048453836, -0.010651348, 0.004028562, 0.021315766, -0.008854342, 0.003161098, 0.005070826, -0.0014057494, 0.03565261, -0.013983979, -0.024190977, 0.014768127, 0.0010831049, -0.013539628, -0.0035646078, 0.016623946, -0.0038390597, 0.014467537, -0.013892495, 0.010461846, -0.012245784, 0.0020567558, -0.01913322, -0.009586214, 0.005861509, -0.0024635328, -0.0076781195, -0.00083519967, -0.007351391, 0.0004623208, 0.019002529, -0.015016441, -0.0052243886, 0.020100337, 0.024975125, 0.00752129, 0.022400504, 0.026504215, 0.01580059, 0.018858768, -0.0052276556, -0.00044394232, -0.012304595, -0.013500421, 0.00720763, -0.001960371, 0.023602866, -0.0027984295, -0.014376054, 0.012383009, 0.02460919, 0.014454468, 0.017813237, 0.007874156, -0.003796585, 0.031156829, 0.0019064607, -0.024569983, 0.013722597, -0.018153034, 0.01594435, -0.005400822, 0.012304595, 0.0009066715, 0.015264755, -0.018244518, -0.024151769, -0.019237773, 0.0061098225, -0.0029911993, 0.0015225548, -0.0017904721, 0.013134485, 0.001434338, -0.0050054803, 0.009703836, 0.00006279313, 0.0012897607, -0.015238617, 0.0021433388, 0.010461846, -0.016427908, -0.007913364, -0.013866357, -0.0030957523, 0.0154869305, 0.018571246, -0.013591905, 0.012735876, 0.028725969, 0.008377318, 0.024975125, 0.0006113907, -0.012337267, 0.033012647, 0.006031408, 0.0024096225, 0.013343591, -0.010951939, -0.008076728, 0.0032640176, -0.012585581, -0.018414417, 0.0014857978, 0.003404511, 0.0069658514, -0.019094013, 0.009292158, -0.004254005, -0.018610455, -0.009468592, 0.0058582416, -0.012317664, -0.008828203, 0.023302276, 0.023655143, 0.031209106, 0.007018128, -0.0002805781, 0.0025321457, -0.004799641, -0.0070246626, 0.012278456, -0.016166525, -0.004623208, -0.018897975, 0.0028245677, 0.003348967, -0.010775506, 0.02006113, 0.022505058, 0.00013201873, 0.015055648, -0.0020861614, -0.01312795, 0.003626686, -0.020897554, 0.0056066606, -0.00037267467, -0.007423271, -0.017538784, -0.012997259, 0.0174473, 0.0011639702, -0.0062731868, 0.021747047, 0.0039534145, 0.01806155, -0.019655986, -0.013539628, -0.008318507, 0.0040743044, 0.021773186, 0.0031169897, 0.017029088, 0.0006126159, 0.016297217, 0.0053616143, 0.009174536, -0.028307756, 0.012840429, 0.011082631, 0.02612521, -0.0021596754, -0.006008537, 0.0019636382, -0.023498312, 0.018950252, 0.01929005, 0.011363616, 0.0059693293, -0.006848229, -0.013408937, 0.011357082, -0.03614924, -0.008305438, 0.00051786465, 0.028490724, -0.013892495, 0.020989038, 0.023459105, -0.01564376, 0.0070115933, -0.005815767, -0.0018019076 - ] - } - ] - } + { + "name": "springfield-illinois-sixth-largest-city", + "title": "Springfield: Illinois' Sixth Largest City", + "content": "---\n\nSpringfield is the sixth most populous city in Illinois, with a population of 10,566 as of 2019. It serves as the county seat of Sangamon County.\n\n---", + "keywords": [ + "Springfield", + "Illinois", + "sixth most populous city", + "population", + "10", + "566", + "2019", + "county seat", + "Sangamon County", + "U.S. state" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.022805648, 0.035417367, -0.0029928328, -0.024086423, 0.016179595, + -0.0070115933, 0.011461635, 0.0027347174, -0.056981448, 0.017042158, + -0.029248733, 0.01291231, -0.008736719, 0.018858768, 0.011193718, + 0.009886804, -0.016336424, 0.033953626, 0.003763912, -0.0007968091, + 0.012670531, -0.0023491778, -0.049244516, 0.03596627, 0.023171583, + 0.0093379, -0.03126138, 0.008860877, -0.034188867, -0.0043977653, + -0.021616356, -0.0042866776, 0.049793422, 0.0024504636, 0.015983557, + -0.016349493, 0.0077042575, -0.020740723, 0.024190977, 0.02930101, + 0.0013836952, -0.0013395868, 0.01671543, 0.00820742, 0.03293423, + -0.036985666, -0.004815978, -0.024360877, 0.029248733, 0.016937604, + 0.024870573, -0.012500632, -0.019708263, 0.037168633, 0.010141652, + -0.01806155, 0.011677276, 0.022596542, -0.0067959526, -0.043232713, + -0.0032231766, -0.0049597383, -0.00013263135, 0.00044802643, + 0.0061326935, 0.0029536255, 0.0051002316, 0.004289945, 0.053688023, + 0.011376685, 0.013317453, 0.056563236, 0.008070193, 0.043781616, + 0.05980438, -0.005485771, -0.03147049, -0.002639966, -0.04252698, + 0.029562393, 0.013173693, -0.013604974, -0.0145720905, 0.002955259, + 0.017251264, 0.008638701, -0.00064978126, -0.07783979, -0.00006636672, + 0.014206154, 0.017133642, -0.0077696033, -0.0059431912, -0.028752107, + -0.043781616, -0.028124787, 0.041559864, -0.013435075, -0.00046354602, + 0.01974747, 0.014781197, -0.009239881, -0.026504215, 0.01914629, + 0.008187816, 0.015238617, -0.0075539625, -0.0068220906, -0.020204889, + 0.0046591484, 0.018649662, -0.0037933176, -0.012964586, 0.006345067, + -0.016205734, 0.00068653823, -0.014545952, -0.0107689705, 0.042370148, + 0.02429553, -0.0038848016, 0.0049139964, -0.0276543, 0.011402824, + -0.0046983557, -0.0016556965, 0.015277823, -0.027288362, 0.007978709, + -0.0412462, 0.03463322, 0.0062960577, -0.018139966, 0.018715007, + 0.00956661, -0.034136593, -0.0057275505, 0.030921584, 0.03052951, + 0.03873693, -0.06984148, -0.0030941188, 0.026033726, -0.0000384927, + 0.0048355814, -0.010357293, -0.01927698, 0.0032362456, 0.013604974, + -0.028516863, -0.01152698, -0.010494519, -0.027419055, 0.004874789, + 0.0025615513, 0.016637014, -0.0039762855, -0.013278246, 0.043337267, + -0.053583473, -0.019512225, -0.009710371, 0.03450253, -0.026138278, + -0.00872365, -0.013088743, -0.01488575, -0.0041233134, -0.00736446, + 0.008096332, 0.022727232, -0.009488195, -0.061895445, -0.048800167, + 0.0021188343, -0.018545108, -0.03688111, -0.015656829, -0.00064365513, + 0.029666945, -0.0039664838, 0.01146817, 0.02883052, -0.008867411, + -0.024988195, -0.019329257, 0.008436129, 0.011291737, 0.036489036, + -0.010043633, -0.0066129845, 0.041978076, 0.010389966, -0.0023687815, + 0.04022681, -0.003298324, -0.023511382, -0.03293423, -0.0040677697, + 0.029091904, -0.05755649, 0.021603288, -0.024073355, -0.03207167, + 0.0006881719, 0.023001686, -0.00842306, -0.0133566605, 0.020034991, + 0.026360454, -0.006952782, -0.0030843168, -0.01944688, 0.03492074, + -0.020714587, 0.023158515, -0.013944772, -0.0011966431, -0.015617621, + 0.038841482, 0.020518549, 0.028647553, 0.0004202545, -0.014807335, + -0.032698985, -0.025236508, -0.053034566, 0.016297217, -0.009050379, + -0.037011802, 0.024413152, 0.019394603, 0.004022028, 0.06017032, + -0.021433389, 0.043467958, 0.018453624, -0.00009761014, -0.014493676, + 0.03267285, 0.018937184, 0.018963322, 0.0011705048, -0.030581785, + 0.0012186973, -0.02080607, 0.019721331, -0.0033358978, 0.013435075, + 0.021747047, 0.008717116, -0.0030124367, 0.057190552, 0.0014637436, + -0.0112263905, 0.01715978, -0.015774451, 0.013173693, 0.0088478075, + 0.00023606133, -0.016833052, 0.0037018338, -0.03842327, 0.0297715, + 0.004678752, -0.01504258, 0.022870993, 0.00047947405, 0.00565567, + -0.032123946, -0.0045578624, -0.010442242, 0.0061196247, 0.007959106, + 0.0054106237, 0.019930437, 0.0479376, 0.014206154, -0.022047639, + -0.03753457, -0.021015177, 0.030712478, -0.0012693402, -0.01987816, + 0.007684654, 0.0034273819, -0.017695615, 0.013310919, 0.005208052, + -0.045924954, -0.013853287, -0.001886857, 0.04265767, 0.025876896, + 0.006122892, -0.011866778, 0.01488575, -0.0015290893, -0.04022681, + 0.014062393, 0.03322175, 0.036541313, -0.023916526, -0.029327149, + 0.022191398, -0.03902445, 0.018100757, -0.0052701305, -0.039677907, + -0.06372512, -0.0072795106, -0.048800167, -0.010082841, 0.016022764, + 0.0050316188, 0.011762225, 0.018113827, 0.00926602, 0.034084316, + -0.02305396, -0.0050316188, 0.010965008, 0.024047216, 0.023615934, + -0.010305016, 0.04145531, -0.008638701, 0.00018337637, 0.010461846, + 0.011376685, -0.016140386, 0.01594435, -0.0068547637, -0.04116779, + 0.009194139, -0.013683389, 0.024060285, 0.0058419053, -0.00070736714, + 0.031784147, -0.004071037, 0.008939291, 0.0021449726, 0.041873522, + 0.028412309, -0.008717116, 0.0077696033, -0.010200463, -0.023511382, + 0.028046373, 0.0006722438, 0.010690556, 0.058079254, -0.0054596327, + 0.016336424, -0.0018884906, -0.023459105, -0.01989123, 0.015317031, + -0.038658515, -0.020126475, 0.005887647, -0.015251686, 0.011513912, + -0.022910202, 0.024948986, -0.012977655, 0.0074298056, 0.038789205, + -0.0012358505, -0.0005852524, 0.016780775, -0.0132129, 0.009788785, + 0.000096793316, 0.012180438, -0.016323354, 0.008769392, -0.021002106, + -0.0015642126, 0.026896289, 0.018244518, -0.014689713, -0.02548482, + -0.010108979, -0.023733556, -0.010801644, -0.01382715, -0.028124787, + 0.018610455, 0.0053714165, -0.028360032, 0.008730185, 0.0055576516, + 0.009305228, 0.032123946, -0.028438447, -0.006828625, 0.030294266, + 0.0009360771, 0.01714671, -0.027419055, 0.0023916524, 0.03340472, + -0.020126475, -0.0104683805, -0.009520868, -0.061895445, 0.011585792, + -0.011219856, -0.015852867, 0.015552276, 0.01487268, -0.007841484, + -0.02535413, -0.05739966, -0.010102444, 0.04461804, 0.019852022, + 0.019329257, -0.009520868, -0.030634062, 0.0014457735, 0.004613406, + -0.0034731238, 0.01441526, 0.0006849046, 0.026242832, -0.026974704, + -0.032881953, -0.036410622, 0.004335687, -0.011324409, -0.0025958577, + -0.009527403, 0.05206745, -0.04859106, 0.005400822, 0.046892073, + 0.0262559, -0.0318887, 0.01684612, 0.043311127, -0.03175801, + -0.017551854, 0.008854342, 0.021747047, 0.017303541, 0.017538784, + 0.055047214, -0.00926602, 0.017839376, 0.0013812446, 0.013297849, + 0.017551854, 0.009625421, -0.021943085, 0.011742622, -0.018388279, + 0.022099914, 0.029118042, -0.019224703, 0.011455101, -0.035286676, + -0.032986507, 0.035417367, 0.049897972, 0.016088111, 0.0034437182, + -0.007939503, -0.012383009, -0.026190555, -0.050943505, 0.023746626, + -0.0406973, 0.0007825147, 0.0039730184, 0.021302696, 0.019512225, + -0.021851601, -0.044722594, 0.010880059, -0.052041315, 0.011063026, + 0.002115567, 0.04101096, -0.026504215, -0.0094555225, 0.036096964, + -0.014349915, 0.069893755, 0.007606239, -0.008024451, -0.013376264, + 0.014284569, -0.006959317, -0.03949494, -0.058236085, 0.037560705, + 0.039129004, -0.0017643338, 0.02217833, 0.023811972, 0.003102287, + -0.006518233, -0.009448987, -0.017525716, -0.03978246, 0.0035188657, + 0.0002311604, 0.06267959, 0.011121837, -0.036384486, -0.024504635, + 0.010625211, -0.014140809, -0.0012824093, -0.014532883, 0.028203202, + 0.008429595, -0.009945615, 0.02443929, 0.0035646078, 0.0058680438, + -0.023406828, 0.030503372, 0.0012366673, -0.0046166736, 0.019773608, + -0.0031055543, -0.042239457, -0.0021417052, 0.0068416945, 0.028673692, + 0.04521922, -0.023197722, -0.017486509, -0.025001263, -0.0058549745, + 0.042840637, -0.016297217, -0.04610792, 0.0023720488, 0.0273145, + 0.014781197, 0.01641484, -0.0041396497, -0.021080522, 0.009729974, + -0.019107081, 0.046682965, -0.010566399, 0.015081787, -0.0052831997, + -0.007018128, -0.012873102, -0.012775084, -0.010612141, -0.031548902, + -0.021381112, 0.01807462, 0.010533726, 0.031627316, 0.003239513, + -0.005554384, 0.020283304, 0.0073840637, -0.00076576986, 0.0024586318, + 0.0029209526, 0.023171583, 0.017813237, 0.027967958, -0.009703836, + 0.00398282, 0.010599072, -0.014898819, -0.01487268, -0.0037443084, + 0.0025484823, -0.001069219, 0.018022344, 0.02565472, 0.021969223, + 0.0053158724, 0.015748313, -0.006345067, -0.04009612, -0.021250421, + 0.04328499, 0.046343166, 0.02869983, 0.00019777284, 0.005492306, + -0.0043324195, 0.006730607, -0.015918212, 0.009122259, -0.031496625, + -0.038240302, 0.016650084, -0.0012170636, 0.025602443, -0.022910202, + -0.020518549, 0.005825569, 0.014938027, 0.009207209, 0.013918634, + -0.0031643654, -0.011422427, 0.011383221, 0.00486172, -0.015369308, + -0.030555647, -0.0074036676, 0.023171583, 0.034842327, 0.005208052, + -0.015317031, -0.0009769181, -0.034241144, -0.011709949, -0.012644392, + 0.016493253, -0.025942242, 0.009547006, -0.01701602, 0.00014753833, + 0.008070193, -0.0029209526, 0.021341905, -0.014911888, -0.004110244, + 0.000940978, -0.002011014, 0.0034241145, 0.012794687, -0.02596838, + 0.0001425353, 0.014480607, 0.053243674, 0.0243086, 0.030634062, + -0.009076517, 0.0004770236, -0.020309443, 0.019303119, -0.002390019, + 0.0069266437, 0.032594435, -0.016506324, 0.0057504214, 0.0005652403, + -0.0053126053, 0.018309863, 0.04119393, -0.031182967, 0.026295109, + 0.003384907, -0.033012647, 0.0022397237, -0.0003771672, 0.0138140805, + 0.015134064, 0.030921584, -0.043494094, 0.030059021, 0.024831364, + 0.0048780562, 0.008527613, 0.019525293, -0.00090258743, 0.008227023, + -0.017081365, 0.014010117, -0.009187605, -0.0069266437, -0.019956576, + 0.0007616858, -0.007567032, -0.010572934, -0.009697301, -0.016048903, + 0.013317453, -0.023615934, 0.035155986, 0.04116779, 0.0036070824, + -0.005266863, 0.018192243, 0.03782209, -0.0033979763, -0.008429595, + 0.04114165, -0.0056524025, -0.004610139, 0.042448565, 0.0076454463, + -0.014049325, -0.01684612, 0.02475295, 0.0057079466, -0.04767622, + 0.010017496, 0.013271711, -0.007998313, -0.03356155, -0.051387858, + -0.00364629, -0.0026889753, -0.0013820615, 0.016937604, -0.02126349, + 0.054263067, -0.04132462, 0.028464586, -0.023276137, -0.006482293, + 0.039677907, 0.023027824, 0.0153301, -0.034084316, -0.0036299534, + -0.038789205, 0.00697892, -0.029091904, -0.023249999, 0.014768127, + 0.012010539, -0.037116356, 0.00516231, 0.030294266, -0.008913153, + 0.01236994, 0.010455311, -0.007795742, -0.029353287, 0.021080522, + 0.0021482399, -0.006459422, -0.021028245, 0.043833893, -0.0010071405, + 0.008828203, 0.016532462, -0.0021433388, 0.0028033303, -0.057922427, + 0.00271348, -0.033378582, -0.024426222, 0.049270656, 0.024687605, + -0.002324673, -0.0021678435, -0.0014506745, 0.0034927274, 0.018989459, + 0.053086843, 0.03141821, 0.0118863825, -0.00029548508, 0.013618044, + 0.01640177, 0.0005403272, 0.0063483343, -0.0034829257, 0.0064626895, + 0.0034633218, 0.049715005, -0.0021008642, -0.023419898, 0.0026121943, + 0.039887015, 0.004338954, 0.007795742, -0.022152191, 0.008200885, + -0.030059021, 0.03309106, -0.0005047955, -0.0015846331, -0.0006211925, + 0.0032836213, -0.012624789, -0.0036560916, -0.0047898395, 0.011938659, + -0.011383221, -0.015238617, -0.019629847, -0.038501684, 0.015016441, + 0.015892074, -0.020231027, 0.004672217, -0.02885666, 0.04900927, + 0.009716905, 0.0115335155, 0.023589797, -0.000035659352, 0.018518971, + -0.040435918, 0.0077630687, 0.0075408933, 0.006518233, -0.014402191, + 0.016950674, 0.042317875, -0.025249578, 0.029091904, -0.009827993, + -0.025314923, -0.018414417, 0.02217833, -0.016271079, 0.0026791736, + -0.035129845, 0.03037268, -0.01957757, 0.034580942, -0.012539839, + 0.016584737, 0.010932335, 0.010494519, -0.023511382, 0.0048584524, + 0.028543001, -0.01411467, 0.0027216482, -0.024086423, -0.012735876, + -0.0022413575, 0.015212478, 0.0054563656, 0.003613617, 0.025615513, + 0.009867201, -0.0045349915, -0.011337479, 0.020296372, -0.013591905, + 0.025458684, 0.045663573, -0.0040350966, -0.04268381, 0.011350548, + -0.032986507, -0.031052275, 0.009827993, -0.017656406, 0.021302696, + -0.022674957, 0.01624494, 0.037247047, 0.007939503, -0.007730396, + 0.009520868, 0.006893971, -0.0012276822, 0.030477233, 0.0003344883, + 0.002997734, 0.006162099, 0.011559654, 0.004165788, -0.010148186, + -0.018623523, 0.007338322, 0.0066554593, -0.004930333, -0.024426222, + -0.009984822, 0.0022511592, -0.012428751, -0.006116357, -0.020701516, + 0.024635328, -0.04085413, -0.0048029087, -0.019303119, -0.024687605, + 0.0115335155, 0.014898819, -0.009448987, -0.03309106, 0.010984612, + -0.0176172, 0.005701412, 0.012402614, 0.00444024, -0.015081787, + 0.009958684, 0.012860033, -0.023981871, -0.03526054, -0.011579257, + 0.011520446, 0.017395025, -0.008919688, 0.0028343697, -0.029091904, + -0.0044239038, 0.01959064, 0.023903456, 0.0049597383, -0.019185496, + 0.0074951514, -0.0088086, 0.02383811, 0.0040677697, -0.016127318, + -0.01382715, 0.017970067, -0.018884907, -0.0017757693, -0.009762647, + 0.026033726, 0.032516018, 0.013291314, -0.034816187, -0.0015829995, + -0.0046036043, 0.0007796558, 0.022112984, -0.029536255, -0.013395868, + -0.0021939818, 0.019629847, 0.010716694, 0.0026677381, 0.0032819875, + 0.003979553, -0.02883052, -0.034737773, -0.006309127, -0.017120572, + 0.01670236, -0.0074298056, 0.019054806, -0.026046794, -0.007913364, + -0.018963322, 0.037639122, -0.01853204, 0.021211213, 0.03536509, + -0.0045023183, 0.0061424957, 0.01821838, -0.008057124, 0.0063417996, + 0.04294519, 0.022126053, -0.023498312, 0.030869307, -0.009168001, + 0.026556492, -0.007919898, 0.011892917, -0.025236508, 0.0011582525, + 0.010344224, -0.019956576, -0.017525716, -0.052903876, -0.014990303, + 0.010376897, 0.012435286, -0.0019799746, -0.002046954, -0.023106238, + -0.0015070351, 0.034397975, -0.01078204, 0.0030042685, 0.03008516, + 0.034685496, 0.0024096225, 0.0137618035, 0.006207841, 0.051152613, + 0.026504215, -0.01853204, -0.017342748, -0.009390176, -0.00805059, + -0.008253162, 0.03536509, -0.008971964, -0.009893338, 0.0016017864, + 0.057086002, -0.009658094, 0.003959949, 0.010089376, -0.011624999, + 0.015656829, 0.028752107, -0.0030598124, -0.024021078, 0.0059791314, + -0.014284569, 0.028569138, -0.0419258, -0.011794899, -0.012213111, + 0.006015071, 0.011023819, -0.0220215, 0.036384486, -0.036227655, + -0.0055674533, -0.036227655, -0.016140386, -0.023432966, 0.001388596, + -0.0071553537, 0.0024422954, 0.0005007114, -0.0032966903, -0.0241387, + -0.000017880726, -0.0011084264, 0.015918212, 0.006282989, 0.022962477, + 0.034946878, 0.016833052, -0.0058909147, -0.0012742411, -0.0031153562, + -0.028673692, -0.0046199406, -0.010076307, 0.05102192, 0.013787942, + 0.018819561, 0.01578752, -0.03586172, 0.02279258, 0.0013902297, + -0.0014735454, 0.0002234006, -0.003626686, 0.0043487563, 0.023354553, + 0.02170784, -0.007710792, 0.0072795106, -0.0041167787, 0.0075735664, + -0.04132462, -0.0016556965, 0.024792157, 0.027209949, -0.0047767702, + -0.020021921, 0.0013036466, -0.0102135325, 0.00697892, -0.0025697195, + 0.022570403, -0.018649662, -0.015303962, 0.01123946, 0.0042964797, + -0.00083070714, 0.009893338, -0.009122259, 0.034345698, -0.017813237, + 0.02566779, 0.002718381, -0.019094013, -0.016179595, -0.0074298056, + -0.02473988, 0.0096058175, 0.009847596, -0.03248988, 0.025432546, + 0.009396711, -0.042161044, 0.016061973, 0.002380217, -0.012383009, + 0.01382715, -0.018153034, -0.016114248, 0.011579257, 0.006224178, + 0.0473887, -0.024635328, 0.015591484, 0.052616354, -0.004979342, + -0.003509064, -0.013709527, -0.0060869516, 0.003486193, -0.04540219, + -0.0034731238, -0.003531935, -0.0352344, -0.017120572, -0.047702357, + -0.011781829, 0.0008617464, 0.019603709, -0.023289206, 0.007998313, + 0.022335159, 0.010435708, -0.021851601, 0.0035286676, -0.0033685707, + -0.021158937, -0.021158937, -0.012592115, -0.028464586, 0.0031300588, + 0.026817873, -0.020779932, -0.018584317, -0.005698145, 0.016427908, + -0.0046591484, -0.015264755, 0.011102234, -0.011794899, -0.042788364, + 0.016114248, 0.0004937684, 0.018741146, -0.020766862, 0.032280773, + -0.012539839, 0.0017479974, -0.0040775714, -0.003613617, -0.018701939, + 0.0013910466, -0.009305228, 0.024177907, -0.003750843, 0.0077369306, + -0.017774029, -0.02794182, -0.0026121943, 0.020884484, -0.009965219, + -0.0029225862, -0.022152191, -0.0022691293, -0.013709527, + -0.024347806, 0.03052951, -0.021250421, -0.0020404193, -0.008377318, + 0.015774451, 0.006789418, 0.017970067, 0.01593128, 0.004655881, + -0.0005223572, 0.030973861, 0.002233189, 0.012102023, -0.021289628, + -0.022714164, 0.011108768, -0.012860033, 0.0035842115, -0.013552697, + 0.0012260487, 0.011899452, -0.004884591, -0.0032444138, -0.007939503, + 0.010684022, -0.018492833, -0.016911466, -0.020322511, -0.02443929, + -0.03991315, -0.000039743456, -0.001101075, -0.009429384, 0.032594435, + 0.017774029, 0.004695088, -0.019538363, -0.008115935, -0.001064318, + 0.015578414, 0.029536255, -0.030032882, 0.03186256, -0.0126966685, + -0.0004431255, 0.0153301, -0.011330944, 0.008383852, -0.0036528246, + -0.0045741987, -0.0150295105, 0.0125986505, 0.014323777, 0.016885327, + -0.0132129, 0.015317031, -0.014807335, -0.0262559, 0.015879005, + 0.037299324, 0.0028294686, -0.0031349598, 0.0046068714, -0.033195615, + 0.011749156, 0.01236994, 0.008952361, -0.028203202, 0.011899452, + -0.012820826, 0.0013926801, -0.005861509, 0.015317031, -0.013905564, + 0.016075041, -0.0019309653, -0.028647553, -0.026791736, 0.022570403, + 0.010108979, 0.020335581, 0.015774451, -0.012507167, 0.016192663, + -0.000824581, -0.007586635, 0.020832209, -0.03384907, -0.035731025, + 0.010082841, -0.039573353, 0.014362984, 0.017029088, 0.015356239, + -0.031784147, -0.03826644, -0.0015241883, -0.03207167, 0.010540261, + -0.026608767, -0.0019587371, 0.0070704045, -0.005469435, + -0.0054171584, 0.026674114, 0.020793, 0.020675378, -0.019708263, + -0.0046820194, 0.019642916, 0.002522344, 0.0028588742, 0.020217959, + -0.019172428, -0.016493253, 0.032751262, 0.01989123, -0.017342748, + -0.014702782, 0.019995783, 0.009599282, 0.012657462, 0.00035307097, + 0.0023001686, 0.008148609, -0.010507588, -0.0015437921, 0.030294266, + -0.008932756, 0.016323354, 0.0038684653, 0.030973861, 0.013232504, + 0.0047277613, -0.000090360845, -0.02794182, -0.0014523081, + 0.022505058, -0.038344856, -0.0000059793865, -0.016349493, + 0.024099493, -0.034659356, 0.0039403457, -0.020714587, -0.011951728, + -0.01032462, -0.023785833, 0.00016785676, -0.0012350336, 0.0065247677, + 0.013774873, 0.012075884, -0.023746626, -0.009886804, 0.015107925, + -0.012206576, 0.029954467, 0.003257483, -0.013997048, 0.003888069, + -0.052747045, -0.013774873, -0.015591484, -0.0041363826, 0.016833052, + 0.0040808385, 0.016963743, 0.012690134, -0.012735876, 0.012232714, + -0.01123946, -0.0021874472, -0.012618254, -0.008606028, -0.011821036, + -0.024112562, 0.015826728, -0.023877317, 0.0406973, 0.010409569, + 0.013644181, 0.009664629, -0.00011997062, 0.003348967, 0.003590746, + 0.0037998522, -0.0030679805, -0.014820404, 0.008227023, -0.01640177, + -0.005966062, 0.020440133, -0.015682967, 0.023289206, -0.03141821, + 0.00972344, -0.004316083, 0.0038717326, 0.020701516, 0.010710159, + 0.019708263, 0.03144435, 0.031182967, -0.010474916, -0.0066521917, + -0.014637436, -0.0075408933, -0.013787942, -0.0012235981, + -0.028516863, 0.022674957, 0.01472892, 0.018754214, 0.024426222, + 0.0030500104, -0.003522133, -0.00531914, -0.030241989, -0.011134907, + -0.00093526026, -0.014650505, -0.004652614, 0.013918634, 0.011245995, + 0.020596964, 0.02490978, 0.029222596, -0.0017528983, -0.0050381534, + -0.0013044635, 0.011605396, 0.009514334, 0.012284991, 0.007697723, + -0.011500843, -0.0014065661, 0.010056702, -0.008102867, 0.0016393602, + -0.008396922, -0.0071096118, -0.016153457, 0.009971754, -0.015696036, + 0.049715005, -0.0038684653, 0.0061686337, -0.013193296, -0.017682545, + 0.009233347, 0.00265957, -0.018649662, 0.012958052, -0.02548482, + -0.007018128, -0.017852444, -0.008161677, 0.020257166, 0.016022764, + -0.021903878, 0.011840641, -0.018336002, -0.011821036, 0.011807967, + 0.0041167787, -0.036018547, -0.017342748, -0.023419898, 0.0007702624, + 0.007789207, -0.006893971, 0.012389544, -0.009592748, -0.007181492, + 0.0071357503, -0.0060673477, 0.0009115725, -0.0056458684, + -0.011017284, -0.004711425, -0.008501475, -0.0014065661, + -0.00037308308, 0.020610033, -0.0086909775, 0.0047244937, + -0.010527192, -0.027811129, -0.012186972, -0.0037541103, -0.02050548, + 0.0025419476, -0.005322407, -0.004450042, 0.034267284, 0.009736509, + 0.0013877793, 0.025772342, 0.013062605, -0.0032215428, 0.020714587, + -0.007410202, 0.0073579256, 0.013801011, 0.004280143, 0.0014294371, + 0.008860877, 0.00720763, -0.009926011, 0.0031300588, -0.015578414, + -0.004737563, 0.013252107, 0.022988616, 0.0028670426, 0.024583051, + 0.040409777, -0.002197249, -0.028020235, -0.014663574, -0.0069658514, + 0.014101601, 0.007057335, 0.011317874, 0.0016434443, 0.027680438, + -0.01518634, -0.002087795, 0.007560497, 0.023759695, -0.03157504, + 0.015003372, 0.030895445, -0.014023187, -0.018884907, 0.0057504214, + 0.024321668, 0.01031155, 0.013219435, 0.03431956, -0.0023949197, + 0.018309863, -0.023328414, -0.0127489455, -0.00872365, 0.0023410097, + 0.006227445, -0.03978246, -0.008782461, -0.017564923, 0.015134064, + 0.016493253, -0.007978709, 0.02187774, -0.037586845, 0.009945615, + 0.011219856, -0.0022233874, 0.014898819, -0.0073448564, 0.009429384, + 0.062313657, 0.029065765, -0.013944772, 0.017826306, -0.011461635, + 0.0075735664, 0.0114812385, -0.0022952675, -0.025942242, -0.002621996, + 0.031235242, -0.019094013, 0.00035102892, -0.0028637752, 0.0044696457, + 0.0005223572, 0.007874156, -0.0176172, -0.010069772, -0.011415893, + 0.011579257, 0.017826306, -0.0053518126, -0.014532883, 0.021916946, + 0.0015707472, -0.0110760955, -0.0220215, -0.008684443, -0.026229763, + 0.0021939818, 0.007272976, 0.0013379531, 0.0059693293, 0.017669477, + -0.007834949, 0.031679593, -0.0012709738, 0.007351391, -0.0076454463, + 0.018244518, 0.004129848, 0.009533937, 0.0039501474, -0.007527824, + 0.008939291, -0.010174325, 0.010808178, -0.013219435, 0.010246205, + 0.0007812895, 0.011572723, -0.0031856026, -0.022962477, -0.012768549, + -0.01564376, -0.0059693293, 0.005518444, 0.012193507, -0.015957419, + -0.0006113907, -0.021694772, -0.013918634, 0.015447723, 0.011500843, + 0.0039305435, -0.0032885221, 0.017303541, 0.000052582865, + -0.012918844, 0.006227445, -0.0005382852, 0.013578836, -0.023759695, + -0.017839376, 0.023197722, -0.009527403, -0.011004215, -0.014389123, + 0.002489671, 0.021054383, -0.044461213, -0.0017349282, -0.0060183387, + -0.008481871, 0.0061653666, -0.0036005478, -0.013016863, -0.016075041, + -0.0029111507, -0.01671543, -0.0042703412, -0.0027918948, + -0.011794899, 0.008272765, 0.0011500843, -0.009899873, -0.0022821985, + 0.0053714165, -0.011337479, -0.016937604, -0.008821669, 0.03526054, + -0.025576307, 0.013997048, -0.0023230396, -0.0013265177, + 0.00067551114, 0.013343591, 0.009547006, 0.008802066, 0.016127318, + 0.007076939, -0.0071161464, -0.008645236, 0.024556912, -0.012559443, + 0.0071422844, 0.0031905037, 0.048303537, -0.0097561125, -0.020832209, + 0.034058176, 0.019695193, -0.00489766, 0.004234401, -0.0080832625, + -0.003750843, -0.014794266, -0.0019097279, -0.0075997044, 0.01252677, + 0.017695615, -0.0004729395, -0.021786256, -0.0051459735, -0.01791779, + -0.0009826359, -0.003247681, 0.016584737, 0.0023312077, 0.0070311967, + -0.014467537, 0.015526137, 0.018492833, 0.019172428, -0.014598229, + -0.014376054, -0.018466694, 0.010442242, -0.002690609, 0.0065541733, + -0.0029585264, -0.0032182755, 0.009952149, -0.016336424, + -0.0000847452, 0.005492306, -0.0039860876, 0.0048486507, + -0.0054334947, 0.034554806, 0.006541104, 0.0047310283, -0.0029111507, + -0.017421164, -0.0044990513, -0.0072403033, -0.015055648, + 0.0036724282, -0.0029568928, -0.0056687393, -0.0094555225, + -0.007076939, 0.00918107, 0.0072599067, -0.023328414, 0.0016164891, + -0.02656956, -0.0066587264, -0.0055903243, 0.014741989, -0.008187816, + -0.008030986, -0.018715007, -0.018989459, 0.018584317, 0.013435075, + 0.0012799589, -0.004384696, -0.005995468, -0.00007565806, + 0.0041363826, 0.022374367, 0.024190977, 0.020610033, 0.014820404, + -0.003763912, 0.012690134, 0.012977655, -0.0122915255, -0.013323988, + 0.02490978, 0.0145198135, -0.0022854658, -0.0064267493, 0.0126966685, + -0.01534317, -0.029640809, -0.014349915, -0.0045970697, 0.0020502212, + -0.020113405, -0.0035155984, -0.018087689, 0.006730607, -0.01714671, + -0.0042180647, 0.00017571867, -0.0074755475, 0.007841484, 0.014180016, + -0.008658305, -0.02170784, 0.022701096, -0.0012987457, -0.0052015176, + 0.009867201, 0.0036495572, 0.0014384221, -0.00084622676, 0.010435708, + 0.0028980817, -0.003362036, -0.030189712, -0.032881953, 0.023955733, + 0.007253372, 0.02488364, -0.0035613405, 0.007848018, -0.0002004275, + 0.0002891547, 0.0150295105, 0.012938448, 0.03113069, -0.0042017284, + 0.015839797, 0.012892706, 0.017395025, 0.029693084, -0.011147976, + 0.01943381, 0.009416315, 0.000097303826, 0.010305016, 0.013591905, + -0.030921584, 0.010265809, -0.033195615, -0.0060510114, -0.012180438, + -0.0044761803, -0.026412731, 0.009867201, -0.009285623, -0.0050806277, + 0.003705101, 0.023067031, -0.0054302276, 0.009409781, -0.01823145, + -0.021773186, 0.008514544, -0.013337057, -0.01152698, 0.024818296, + 0.017342748, 0.0273145, -0.011977866, -0.005966062, 0.011860244, + -0.021119729, -0.021668633, -0.00039922138, 0.0001421269, 0.01776096, + -0.009547006, 0.01929005, 0.0009246416, -0.005573988, -0.001927698, + -0.019459948, 0.017395025, 0.008991567, 0.008285834, -0.0007906829, + -0.013644181, 0.013265177, 0.007730396, -0.0016230238, -0.04009612, + -0.005737352, 0.014402191, -0.012539839, 0.004695088, 0.0035646078, + 0.005217854, -0.026661044, 0.021446457, -0.0000255129, -0.00081804645, + 0.005698145, -0.020152614, -0.017042158, 0.003362036, 0.0024047217, + 0.006815556, -0.014650505, 0.014088532, 0.00421153, 0.0015054015, + 0.004044899, 0.016885327, -0.00146211, 0.013487352, 0.002095963, + -0.02200843, 0.025733136, 0.012330733, 0.012565978, 0.0085929595, + -0.003183969, -0.0005758589, -0.00045578624, -0.0068612983, + 0.00027608557, -0.005136172, 0.0015633958, 0.012559443, -0.004760434, + -0.019512225, -0.009736509, -0.014689713, -0.003937078, 0.006224178, + -0.0073840637, 0.014598229, -0.032280773, 0.007848018, -0.00466895, + -0.009919477, -0.011860244, 0.0002234006, -0.01487268, -0.006031408, + -0.012840429, -0.011932124, -0.0018525505, 0.011115303, -0.0046166736, + -0.010037099, 0.017355816, -0.015068717, 0.025406407, -0.019825885, + 0.008566821, 0.016140386, 0.0013510223, -0.0068547637, -0.008403457, + -0.02338069, -0.014441399, 0.007919898, -0.022805648, -0.0138140805, + 0.008717116, 0.004554595, -0.0057406193, 0.004002424, 0.0055347807, + -0.004172323, -0.027602023, -0.012905775, 0.012186972, 0.010749367, + 0.0016663152, 0.0015731977, 0.0024766019, -0.018192243, 0.013644181, + -0.03445025, 0.0043716272, 0.007913364, 0.014062393, -0.005864776, + 0.021812394, -0.00057871785, 0.014951095, 0.018179173, -0.0054596327, + 0.011938659, -0.025981449, -0.007331787, 0.009396711, -0.009612352, + -0.0020600231, -0.015408515, 0.027105395, 0.011520446, 0.008658305, + 0.009710371, 0.008200885, 0.04325885, -0.010592537, -0.006080417, + 0.021067452, -0.03068634, 0.0103965, 0.02822934, -0.013631112, + 0.020413995, 0.0077238614, -0.009468592, 0.0020763595, 0.00531914, + 0.007397133, -0.03113069, -0.016271079, -0.0047767702, 0.009331365, + 0.015617621, -0.01624494, -0.004819245, -0.013435075, 0.001009591, + 0.003878267, 0.015983557, 0.028752107, -0.017956998, 0.021616356, + -0.017512647, -0.021851601, 0.009462057, -0.0059007164, 0.013983979, + 0.007501686, 0.020100337, 0.003603815, 0.028307756, -0.007815345, + 0.012873102, 0.028752107, 0.003531935, -0.0063581364, -0.006972386, + 0.013108347, -0.016519392, -0.015774451, -0.011585792, 0.019028667, + -0.01987816, -0.0020387857, -0.0033914417, 0.0016434443, -0.010298482, + 0.0036626263, -0.0038488614, 0.0016532461, 0.012128161, -0.0138140805, + 0.008285834, -0.006008537, -0.021694772, -0.011742622, 0.014689713, + 0.0007224784, -0.013043001, -0.0048682545, -0.0068416945, + -0.017395025, -0.003404511, 0.0069331783, -0.012409148, -0.0009246416, + 0.013552697, 0.008161677, 0.008527613, 0.0002001212, 0.018244518, + 0.005505375, 0.004280143, 0.022714164, -0.017564923, 0.008318507, + 0.0022266547, -0.0029405563, 0.036985666, 0.012363406, 0.0006085318, + -0.028072512, -0.014258431, 0.005760223, 0.03207167, -0.006224178, + -0.004378162, 0.002064924, 0.00903731, -0.0043912306, -0.008762858, + -0.013983979, -0.02217833, -0.011043423, -0.0029993674, 0.003018971, + 0.012637857, 0.018270656, 0.0376914, 0.017695615, 0.018022344, + -0.02157715, -0.02974536, 0.0050838953, 0.010474916, 0.016349493, + -0.021211213, -0.014402191, 0.001932599, -0.0025305122, 0.009468592, + 0.026465008, 0.0038586634, 0.009272554, 0.011500843, -0.0052243886, + -0.010108979, 0.014663574, -0.0019848756, 0.02778499, 0.00039186998, + -0.017538784, 0.030320404, 0.019473018, -0.02352445, 0.0036724282, + 0.013016863, 0.0072337687, 0.018754214, -0.0089458255, -0.0029127845, + 0.005237458, 0.005093697, 0.01336973, 0.016480185, -0.010801644, + 0.012559443, 0.015656829, -0.0057275505, 0.019028667, -0.022975547, + -0.018100757, -0.019773608, 0.0029928328, -0.012428751, 0.020727655, + -0.02275337, 0.017290471, 0.0031267917, 0.027863406, -0.00993908, + 0.00034122705, -0.008403457, -0.0018100758, -0.0052113193, + -0.008246627, -0.00082907354, -0.0078088106, 0.020766862, + -0.018688869, -0.008331576, -0.022844855, 0.02961467, 0.0072860452, + 0.0039534145, -0.021903878, -0.028412309, 0.009716905, -0.004338954, + -0.009390176, 0.0102135325, -0.01563069, 0.000013694518, -0.00706387, + -0.017878583, 0.004518655, -0.01684612, 0.0062339795, 0.02734064, + -0.00061915047, -0.017029088, -0.030424956, 0.010592537, -0.019054806, + 0.0038161888, -0.008632166, -0.020244097, 0.013918634, 0.013539628, + 0.0092006745, -0.0015650294, 0.005636066, 0.0007049167, -0.0020779932, + -0.020766862, 0.006939713, -0.00074371573, -0.020884484, + 0.00041351575, 0.008978499, 0.022805648, 0.0049499366, 0.017538784, + 0.012794687, -0.014336846, 0.03382293, -0.00410371, -0.0027902613, + -0.017120572, -0.026922427, 0.008586424, -0.0011966431, 0.026334316, + 0.002011014, -0.009814924, 0.0039272765, -0.009390176, -0.019995783, + 0.0028441714, -0.009377107, 0.011317874, 0.01048145, -0.0040547005, + 0.023563659, 0.021930017, 0.0064169476, 0.0056033935, 0.025092747, + 0.020518549, -0.024413152, -0.010331155, 0.03199325, 0.013709527, + -0.008181281, 0.01442833, 0.0008707314, -0.0007543344, -0.011925589, + 0.018296795, 0.0017610665, 0.00796564, -0.009272554, -0.0041396497, + 0.0050414205, -0.028046373, 0.01640177, -0.035338953, -0.012820826, + -0.026765598, 0.011004215, 0.013931703, 0.01504258, 0.0038488614, + 0.02049241, -0.026242832, -0.0015388911, 0.019773608, -0.010069772, + 0.0010496152, -0.004760434, -0.01914629, 0.0029928328, 0.007684654, + -0.008285834, -0.0017888384, -0.00972344, 0.01412774, -0.02157715, + -0.0029764965, -0.01261172, 0.011592327, 0.013526559, 0.013892495, + 0.012252318, 0.005573988, -0.024021078, 0.0074494095, 0.034816187, + -0.00903731, -0.0012383009, -0.008481871, -0.016950674, 0.0062960577, + -0.006345067, 0.004649346, -0.004299747, 0.005792896, -0.029065765, + -0.020374788, 0.035809442, 0.008455733, 0.008749789, -0.012160834, + 0.011461635, 0.0027347174, -0.0005235824, 0.0077499994, -0.018963322, + -0.005933389, -0.020596964, 0.010579469, 0.003545004, 0.031548902, + -0.006959317, 0.0040547005, -0.023119308, 0.01791779, 0.007090008, + -0.0012432019, 0.003247681, -0.013840219, 0.01927698, -0.0041167787, + -0.00588438, -0.004946669, 0.00812247, 0.021812394, -0.008279299, + 0.021629425, -0.006809022, -0.0019015598, -0.01684612, 0.0039893547, + -0.0012235981, 0.019760538, 0.0006538654, -0.00000608468, + -0.010474916, 0.008599494, -0.035417367, -0.007188027, -0.00041821247, + 0.03330017, 0.008194351, -0.03139207, 0.005155775, -0.029065765, + -0.01101075, 0.027131533, -0.006008537, -0.004652614, 0.0054563656, + 0.019616779, 0.008971964, -0.00683516, 0.001351839, 0.01085392, + -0.000532159, 0.0028425378, -0.019760538, 0.010644814, 0.013023397, + 0.010507588, -0.0049760747, 0.038031194, -0.002535413, 0.017133642, + 0.0085406825, -0.0063385325, 0.008357715, 0.009645025, -0.0048911255, + -0.013029932, 0.02383811, -0.0011664206, -0.0016017864, 0.0013257008, + -0.003061446, 0.015918212, -0.013252107, 0.0052733975, -0.0064430856, + 0.019094013, -0.01351349, 0.02200843, -0.0044043, -0.008416526, + -0.0017202254, 0.016258009, -0.010801644, -0.0034241145, -0.012017074, + -0.0048780562, 0.009520868, -0.0031774344, 0.013879426, -0.005230923, + -0.0068678325, 0.014807335, -0.014180016, 0.015696036, 0.023929594, + -0.0012252318, 0.022988616, 0.0052505266, -0.00019164667, + -0.016506324, 0.012474493, -0.021629425, -0.047101177, -0.004022028, + 0.0073579256, -0.029379426, 0.010540261, -0.011455101, 0.027079256, + 0.003384907, 0.0066521917, 0.0045349915, -0.019224703, 0.016963743, + 0.022831786, -0.0055576516, 0.015081787, 0.0069462475, -0.034528665, + 0.01989123, 0.0019358663, -0.020975968, 0.013186761, -0.0006346701, + -0.012383009, 0.0020338849, -0.024726812, -0.0021907145, 0.017303541, + -0.005743887, 0.006410413, 0.0069462475, -0.012200042, -0.004554595, + 0.0036332207, 0.0012374842, 0.011585792, -0.010801644, -0.021054383, + 0.0058909147, -0.009906407, -0.008279299, 0.028150925, 0.025903035, + -0.00015203084, -0.01654553, -0.0071553537, 0.009965219, 0.006939713, + 0.0008617464, -0.004672217, 0.0061424957, 0.0142192235, -0.019512225, + -0.014964164, -0.006116357, -0.011860244, -0.022988616, -0.017695615, + 0.0075735664, 0.0021319033, 0.012265387, -0.013487352, 0.010697091, + 0.0083381105, 0.037168633, 0.009246416, 0.006354869, 0.000010152832, + -0.014715851, 0.00013559232, 0.028124787, -0.003914207, 0.014075463, + -0.0039501474, -0.025314923, 0.021067452, -0.0033342643, -0.02186467, + -0.0024161572, -0.0054204254, 0.00805059, 0.008220488, 0.03081703, + -0.002563185, -0.0062143756, 0.0037116355, -0.015199409, -0.010167791, + 0.011474704, -0.00796564, -0.0033685707, 0.00683516, 0.0264258, + 0.0049041947, 0.005342011, 0.016676221, -0.013029932, -0.021694772, + -0.008671374, -0.006387542, 0.0061915047, 0.021067452, -0.0028490725, + -0.004185392, -0.011925589, 0.015669897, -0.0031807018, 0.009305228, + 0.008854342, 0.029248733, -0.02399494, -0.023067031, -0.013297849, + 0.0049499366, 0.0037541103, -0.0070508006, -0.02339376, -0.0004778404, + -0.00027955705, 0.0032999576, 0.0102135325, -0.018545108, -0.02064924, + 0.023811972, -0.01032462, -0.020335581, 0.013036466, 0.030137435, + 0.03021585, 0.005345278, -0.008344646, 0.016741568, 0.016480185, + 0.0077499994, 0.0017398292, 0.011827571, -0.012265387, 0.0188457, + 0.008429595, -0.0019015598, 0.011892917, 0.013239038, 0.0104683805, + 0.018871836, 0.0020583894, -0.012278456, -0.007272976, -0.0041951938, + -0.02791568, -0.020675378, 0.04451349, -0.0075735664, 0.013350125, + 0.0033342643, -0.0073252525, 0.0034600548, -0.01367032, -0.008514544, + -0.008553752, -0.0061424957, -0.011082631, 0.020178752, -0.010429173, + -0.026282039, 0.0142715005, 0.015356239, -0.024726812, -0.016336424, + -0.007377529, 0.006207841, 0.007919898, -0.010886593, 0.0016614143, + -0.00942285, 0.0067044683, -0.021132799, 0.0060608136, 0.015120994, + 0.028203202, -0.014924957, -0.0030091694, -0.009658094, 0.021930017, + -0.0004239302, -0.027079256, -0.005554384, 0.021250421, -0.015970489, + -0.014898819, 0.011847175, -0.0040775714, 0.020570826, -0.0068416945, + -0.0041461843, 0.006015071, 0.021211213, 0.006469224, 0.01236994, + 0.019838953, -0.007815345, -0.0010398134, 0.00006963401, + -0.0072403033, 0.00021421137, 0.013258642, 0.02186467, -0.014846542, + 0.0063810074, 0.0047473647, -0.019224703, -0.0008919688, + -0.0056491354, 0.0071749575, 0.006599915, -0.014768127, 0.008416526, + -0.00013600073, -0.01836214, 0.021812394, -0.0004075938, -0.016336424, + 0.027131533, 0.012925379, -0.009886804, -0.011102234, -0.025942242, + 0.00034796583, -0.000068000365, -0.006057546, 0.022413574, + -0.0026791736, 0.01236994, 0.0006857214, 0.00012977247, 0.0073252525, + -0.01623187, -0.016467115, 0.0017316609, 0.009631956, -0.019995783, + 0.003394709, -0.0154346535, -0.015539207, 0.010089376, -0.0026203624, + 0.0010929068, 0.007625843, 0.036985666, 0.0069462475, 0.006998524, + 0.010265809, -0.005008748, 0.00095486396, 0.007351391, 0.010487984, + 0.01504258, -0.021185074, 0.0005627898, 0.009187605, -0.009083052, + 0.0041690553, -0.006711003, -0.030581785, 0.007377529, -0.011664207, + 0.012128161, 0.0040775714, -0.011638069, -0.0043748943, 0.0062111085, + 0.023981871, -0.004071037, -0.00011762225, 0.014898819, 0.01093887, + 0.011389755, -0.0055119097, -0.0075408933, 0.0056491354, -0.02126349, + -0.00611309, 0.0041788574, -0.012650927, -0.0038749997, 0.00294219, + -0.0035352022, 0.01610118, 0.0060477443, 0.0037214374, -0.002179279, + -0.009631956, -0.012128161, 0.007057335, -0.0133566605, 0.0046395445, + 0.005243992, -0.0057667578, -0.0011460001, -0.020675378, + -0.0077173267, 0.0054988405, 0.0010153088, -0.034737773, -0.008971964, + -0.015826728, -0.0010332789, 0.011925589, -0.019303119, 0.0103965, + -0.002407989, 0.01472892, 0.004093908, 0.0044075674, 0.010416104, + -0.011677276, 0.0176172, 0.025066609, -0.016427908, 0.009187605, + 0.0072403033, -0.01382715, -0.00835118, -0.007181492, -0.016663153, + -0.0013861456, 0.012422217, -0.002489671, -0.008573355, -0.0014245362, + 0.0051884484, -0.00094261166, -0.0032819875, 0.01185371, -0.006697934, + 0.025131956, -0.01957757, 0.015003372, 0.0137618035, 0.009239881, + -0.01594435, 0.0104683805, 0.0073840637, -0.012833895, 0.013853287, + 0.01944688, -0.0058680438, 0.0092006745, 0.024543844, 0.010579469, + 0.04631703, -0.0050838953, -0.015264755, -0.012762015, -0.009814924, + -0.021943085, 0.005253794, 0.010788575, 0.0014833473, 0.014245362, + -0.010971542, 0.007651981, -0.0029536255, 0.0066031828, -0.012200042, + -0.0074298056, 0.014624367, -0.008854342, -0.006331998, 0.0060510114, + 0.015016441, -0.0034600548, 0.016898397, 0.009860666, 0.0026350652, + -0.0039958893, 0.01791779, 0.0114812385, 0.017669477, -0.012938448, + 0.019120151, -0.00805059, -0.016480185, 0.0062470487, 0.020845277, + 0.03873693, 0.0074951514, 0.017029088, 0.0230801, 0.01101075, + -0.010657883, 0.011742622, 0.016937604, -0.0019064607, -0.014206154, + 0.013997048, 0.008560286, 0.002759222, 0.007246838, 0.018022344, + 0.009599282, 0.0023459105, 0.017839376, -0.011487774, -0.003509064, + 0.016192663, -0.0022625946, -0.005952993, -0.002901349, 0.0011672375, + 0.006697934, -0.0015797322, -0.017747892, -0.0025158094, -0.011579257, + 0.0017316609, -0.021015177, -0.0073905983, 0.0043422217, -0.006243781, + 0.025785413, -0.0029519917, -0.0029046163, -0.0031169897, + -0.0019391335, 0.0002082894, -0.024465429, -0.02961467, -0.0027020446, + 0.030189712, -0.007919898, -0.0016279246, -0.016623946, 0.004920531, + -0.0009050379, 0.023877317, 0.0081355395, 0.011566188, 0.0051329043, + -0.008070193, -0.02791568, -0.004485982, 0.02247892, 0.022439713, + -0.0054400293, 0.009488195, -0.0070050587, 0.012886171, 0.0015478762, + 0.0008748155, -0.0022217536, -0.0031186233, 0.020322511, 0.023315344, + -0.010690556, 0.0056883427, -0.019956576, 0.0075147552, 0.0154869305, + 0.0018982925, -0.01684612, -0.015120994, 0.00660645, -0.028647553, + 0.0068416945, -0.0074886167, -0.0012775083, -0.0005934206, + 0.001881956, 0.0038259905, 0.014493676, -0.008854342, -0.0074886167, + 0.012722807, 0.0024471963, 0.026360454, -0.010167791, -0.008318507, + 0.015813658, -0.00926602, 0.000111393994, 0.02200843, 0.019185496, + -0.007874156, 0.002105765, 0.0020665578, 0.014376054, 0.00736446, + 0.0009393444, -0.013748734, 0.020100337, 0.0005697328, 0.018937184, + 0.0013142653, 0.011272132, 0.004110244, -0.003143128, -0.0012366673, + 0.02004806, -0.006730607, -0.019237773, -0.014794266, -0.016127318, + -0.004106977, 0.016493253, -0.013186761, -0.0022070508, -0.019198565, + -0.026020657, 0.004316083, -0.0016238405, 0.00085684547, 0.014624367, + -0.0033097595, -0.016976813, 0.0078088106, -0.009004637, -0.020152614, + 0.0012619888, -0.009207209, -0.017434232, -0.012905775, -0.03719477, + -0.0013346858, -0.00086419686, -0.0125986505, -0.009729974, + 0.0037345067, 0.043102022, -0.027445193, -0.013709527, 0.036645867, + 0.015774451, -0.001210529, -0.009743043, -0.0028360032, -0.002192348, + -0.00040555172, -0.023602866, -0.029222596, -0.032908093, -0.01312795, + -0.0040089586, 0.009814924, 0.030137435, -0.002965061, 0.015408515, + -0.0023148714, -0.007887226, -0.014362984, -0.0048453836, + -0.010651348, 0.004028562, 0.021315766, -0.008854342, 0.003161098, + 0.005070826, -0.0014057494, 0.03565261, -0.013983979, -0.024190977, + 0.014768127, 0.0010831049, -0.013539628, -0.0035646078, 0.016623946, + -0.0038390597, 0.014467537, -0.013892495, 0.010461846, -0.012245784, + 0.0020567558, -0.01913322, -0.009586214, 0.005861509, -0.0024635328, + -0.0076781195, -0.00083519967, -0.007351391, 0.0004623208, + 0.019002529, -0.015016441, -0.0052243886, 0.020100337, 0.024975125, + 0.00752129, 0.022400504, 0.026504215, 0.01580059, 0.018858768, + -0.0052276556, -0.00044394232, -0.012304595, -0.013500421, 0.00720763, + -0.001960371, 0.023602866, -0.0027984295, -0.014376054, 0.012383009, + 0.02460919, 0.014454468, 0.017813237, 0.007874156, -0.003796585, + 0.031156829, 0.0019064607, -0.024569983, 0.013722597, -0.018153034, + 0.01594435, -0.005400822, 0.012304595, 0.0009066715, 0.015264755, + -0.018244518, -0.024151769, -0.019237773, 0.0061098225, -0.0029911993, + 0.0015225548, -0.0017904721, 0.013134485, 0.001434338, -0.0050054803, + 0.009703836, 0.00006279313, 0.0012897607, -0.015238617, 0.0021433388, + 0.010461846, -0.016427908, -0.007913364, -0.013866357, -0.0030957523, + 0.0154869305, 0.018571246, -0.013591905, 0.012735876, 0.028725969, + 0.008377318, 0.024975125, 0.0006113907, -0.012337267, 0.033012647, + 0.006031408, 0.0024096225, 0.013343591, -0.010951939, -0.008076728, + 0.0032640176, -0.012585581, -0.018414417, 0.0014857978, 0.003404511, + 0.0069658514, -0.019094013, 0.009292158, -0.004254005, -0.018610455, + -0.009468592, 0.0058582416, -0.012317664, -0.008828203, 0.023302276, + 0.023655143, 0.031209106, 0.007018128, -0.0002805781, 0.0025321457, + -0.004799641, -0.0070246626, 0.012278456, -0.016166525, -0.004623208, + -0.018897975, 0.0028245677, 0.003348967, -0.010775506, 0.02006113, + 0.022505058, 0.00013201873, 0.015055648, -0.0020861614, -0.01312795, + 0.003626686, -0.020897554, 0.0056066606, -0.00037267467, -0.007423271, + -0.017538784, -0.012997259, 0.0174473, 0.0011639702, -0.0062731868, + 0.021747047, 0.0039534145, 0.01806155, -0.019655986, -0.013539628, + -0.008318507, 0.0040743044, 0.021773186, 0.0031169897, 0.017029088, + 0.0006126159, 0.016297217, 0.0053616143, 0.009174536, -0.028307756, + 0.012840429, 0.011082631, 0.02612521, -0.0021596754, -0.006008537, + 0.0019636382, -0.023498312, 0.018950252, 0.01929005, 0.011363616, + 0.0059693293, -0.006848229, -0.013408937, 0.011357082, -0.03614924, + -0.008305438, 0.00051786465, 0.028490724, -0.013892495, 0.020989038, + 0.023459105, -0.01564376, 0.0070115933, -0.005815767, -0.0018019076 + ] + } + ] + } ] diff --git a/src/scrapers/document/playground/document-scraper-playground.ts b/src/scrapers/document/playground/document-scraper-playground.ts index 3afd9ad332..96b9df5d50 100644 --- a/src/scrapers/document/playground/document-scraper-playground.ts +++ b/src/scrapers/document/playground/document-scraper-playground.ts @@ -1,77 +1,79 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { writeFile } from 'fs/promises'; -import { join } from 'path'; -import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode'; -import { usageToHuman } from '../../../execution/utils/usageToHuman'; -import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground'; -import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson'; -import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode'; -import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler'; -import { DocumentScraper } from '../DocumentScraper'; +import { join } from "path"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { writeFile } from "fs/promises"; +import { $provideExecutablesForNode } from "../../../executables/$provideExecutablesForNode"; +import { usageToHuman } from "../../../execution/utils/usageToHuman"; +import { $provideLlmToolsForTestingAndScriptsAndPlayground } from "../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground"; +import { stringifyPipelineJson } from "../../../utils/editable/utils/stringifyPipelineJson"; +import { $provideFilesystemForNode } from "../../_common/register/$provideFilesystemForNode"; +import { makeKnowledgeSourceHandler } from "../../_common/utils/makeKnowledgeSourceHandler"; +import { DocumentScraper } from "../DocumentScraper"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 Scrape knowledge from documents (playground)`); + console.info(`🧸 Scrape knowledge from documents (playground)`); - // Do here stuff you want to test - //========================================> + // Do here stuff you want to test + //========================================> - //const example = '10-simple.docx'; - const example = '10-simple.odt'; - // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them + //const example = '10-simple.docx'; + const example = "10-simple.odt"; + // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them - const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true }); - const rootDirname = join(__dirname, '..', 'examples'); + const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ + isCacheReloaded: true, + }); + const rootDirname = join(__dirname, "..", "examples"); - const documentScraper = new DocumentScraper( - { - fs: $provideFilesystemForNode(), - llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), - executables: await $provideExecutablesForNode(), - }, - { - rootDirname, - }, - ); + const documentScraper = new DocumentScraper( + { + fs: $provideFilesystemForNode(), + llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), + executables: await $provideExecutablesForNode(), + }, + { + rootDirname, + }, + ); - const knowledge = await documentScraper.scrape( - await makeKnowledgeSourceHandler( - { knowledgeSourceContent: example }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ); + const knowledge = await documentScraper.scrape( + await makeKnowledgeSourceHandler( + { knowledgeSourceContent: example }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ); - console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); - console.info(colors.bgGreen(' Knowledge: ')); - console.info(knowledge); + console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); + console.info(colors.bgGreen(" Knowledge: ")); + console.info(knowledge); - await writeFile( - join( - __dirname, - `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, - ), - stringifyPipelineJson(knowledge), - 'utf-8', - ); - /**/ + await writeFile( + join( + __dirname, + `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, + ), + stringifyPipelineJson(knowledge), + "utf-8", + ); + /**/ - //========================================/ + //========================================/ } /** diff --git a/src/scrapers/document/playground/tsconfig.json b/src/scrapers/document/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/scrapers/document/playground/tsconfig.json +++ b/src/scrapers/document/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/scrapers/document/register-constructor.ts b/src/scrapers/document/register-constructor.ts index 4b691b367e..ab4496430d 100644 --- a/src/scrapers/document/register-constructor.ts +++ b/src/scrapers/document/register-constructor.ts @@ -1,6 +1,6 @@ -import type { Registration } from '../../utils/$Register'; -import { $scrapersRegister } from '../_common/register/$scrapersRegister'; -import { createDocumentScraper } from './createDocumentScraper'; +import type { Registration } from "../../utils/$Register"; +import { $scrapersRegister } from "../_common/register/$scrapersRegister"; +import { createDocumentScraper } from "./createDocumentScraper"; /** * Registration of known scraper @@ -11,7 +11,8 @@ import { createDocumentScraper } from './createDocumentScraper'; * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _DocumentScraperRegistration: Registration = $scrapersRegister.register(createDocumentScraper); +export const _DocumentScraperRegistration: Registration = + $scrapersRegister.register(createDocumentScraper); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/scrapers/document/register-metadata.ts b/src/scrapers/document/register-metadata.ts index 9187a3f9c9..97820d1474 100644 --- a/src/scrapers/document/register-metadata.ts +++ b/src/scrapers/document/register-metadata.ts @@ -1,8 +1,8 @@ -import type { Registration } from '../../utils/$Register'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import { $deepFreeze } from '../../utils/serialization/$deepFreeze'; -import { $scrapersMetadataRegister } from '../_common/register/$scrapersMetadataRegister'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; +import type { Registration } from "../../utils/$Register"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import { $deepFreeze } from "../../utils/serialization/$deepFreeze"; +import { $scrapersMetadataRegister } from "../_common/register/$scrapersMetadataRegister"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; keepTypeImported(); @@ -12,14 +12,16 @@ keepTypeImported(); * @private within the scraper directory */ export const documentScraperMetadata = $deepFreeze({ - title: 'Document scraper', - packageName: '@promptbook/documents', - className: 'DocumentScraper', - mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'], - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@', - isAvilableInBrowser: false, - // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server - requiredExecutables: ['Pandoc'], + title: "Document scraper", + packageName: "@promptbook/documents", + className: "DocumentScraper", + mimeTypes: [ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ], + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@", + isAvilableInBrowser: false, + // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server + requiredExecutables: ["Pandoc"], }) satisfies ScraperAndConverterMetadata; /* <- Note: [🤛] */ /** @@ -32,7 +34,7 @@ export const documentScraperMetadata = $deepFreeze({ * @public exported from `@promptbook/cli` */ export const _DocumentScraperMetadataRegistration: Registration = - $scrapersMetadataRegister.register(documentScraperMetadata); + $scrapersMetadataRegister.register(documentScraperMetadata); /** * Note: [💞] Ignore a discrepancy between file name and entity name diff --git a/src/scrapers/markdown/MarkdownScraper.test.ts b/src/scrapers/markdown/MarkdownScraper.test.ts index db8057e007..1c6852a717 100644 --- a/src/scrapers/markdown/MarkdownScraper.test.ts +++ b/src/scrapers/markdown/MarkdownScraper.test.ts @@ -1,65 +1,65 @@ -import { describe, expect, it } from '@jest/globals'; -import { join } from 'path'; -import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground'; -import { $provideFilesystemForNode } from '../_common/register/$provideFilesystemForNode'; -import { makeKnowledgeSourceHandler } from '../_common/utils/makeKnowledgeSourceHandler'; -import { MarkdownScraper } from './MarkdownScraper'; +import { join } from "path"; +import { describe, expect, it } from "@jest/globals"; +import { $provideLlmToolsForTestingAndScriptsAndPlayground } from "../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground"; +import { $provideFilesystemForNode } from "../_common/register/$provideFilesystemForNode"; +import { makeKnowledgeSourceHandler } from "../_common/utils/makeKnowledgeSourceHandler"; +import { MarkdownScraper } from "./MarkdownScraper"; -describe('how creating knowledge from markdown works', () => { - const rootDirname = join(__dirname, 'examples'); - let markdownScraper: MarkdownScraper; +describe("how creating knowledge from markdown works", () => { + const rootDirname = join(__dirname, "examples"); + let markdownScraper: MarkdownScraper; - beforeAll(async () => { - const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground(); - markdownScraper = new MarkdownScraper( - { llm: llmTools }, - { - rootDirname, - }, - ); - }); + beforeAll(async () => { + const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground(); + markdownScraper = new MarkdownScraper( + { llm: llmTools }, + { + rootDirname, + }, + ); + }); - it('should scrape simple information from a markdown', () => - expect( - Promise.resolve() - .then(() => - makeKnowledgeSourceHandler( - { - knowledgeSourceContent: '10-simple.md', - }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ) - .then((sourceHandler) => markdownScraper!.scrape(sourceHandler)) - .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) - .then((knowledge) => knowledge?.slice(0, 1)), - ).resolves.toMatchObject([ - { - content: expect.stringMatching(/Springfield (is )?.*/i), - }, - ])); + it("should scrape simple information from a markdown", () => + expect( + Promise.resolve() + .then(() => + makeKnowledgeSourceHandler( + { + knowledgeSourceContent: "10-simple.md", + }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ) + .then((sourceHandler) => markdownScraper!.scrape(sourceHandler)) + .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) + .then((knowledge) => knowledge?.slice(0, 1)), + ).resolves.toMatchObject([ + { + content: expect.stringMatching(/Springfield (is )?.*/i), + }, + ])); - it('should NOT scrape irrelevant information', () => - expect( - Promise.resolve() - .then(() => - makeKnowledgeSourceHandler( - { - knowledgeSourceContent: '10-simple.md', - }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ) - .then((sourceHandler) => markdownScraper!.scrape(sourceHandler)) - .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) - .then((knowledge) => knowledge?.slice(0, 1)), - ).resolves.toMatchObject([ - { - content: expect.not.stringMatching(/London (is )?.*/i), - }, - ])); + it("should NOT scrape irrelevant information", () => + expect( + Promise.resolve() + .then(() => + makeKnowledgeSourceHandler( + { + knowledgeSourceContent: "10-simple.md", + }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ) + .then((sourceHandler) => markdownScraper!.scrape(sourceHandler)) + .then((knowledge) => knowledge?.map(({ content }) => ({ content }))) + .then((knowledge) => knowledge?.slice(0, 1)), + ).resolves.toMatchObject([ + { + content: expect.not.stringMatching(/London (is )?.*/i), + }, + ])); }); /** diff --git a/src/scrapers/markdown/MarkdownScraper.ts b/src/scrapers/markdown/MarkdownScraper.ts index d5e361e23a..06b91bcf36 100644 --- a/src/scrapers/markdown/MarkdownScraper.ts +++ b/src/scrapers/markdown/MarkdownScraper.ts @@ -1,27 +1,27 @@ -import spaceTrim from 'spacetrim'; -import type { KnowledgePiecePreparedJson } from '../../pipeline/PipelineJson/KnowledgePieceJson'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; -import type { Scraper } from '../_common/Scraper'; -import type { ScraperSourceHandler } from '../_common/Scraper'; -// TODO: [🏳‍🌈] Finally take pick of .json vs .ts -import PipelineCollection from '../../../books/index.json'; +import spaceTrim from "spacetrim"; // import PipelineCollection from '../../../books/books'; -import type { WritableDeep } from 'type-fest'; -import { createCollectionFromJson } from '../../collection/constructors/createCollectionFromJson'; -import { DEFAULT_IS_VERBOSE } from '../../config'; -import { DEFAULT_MAX_PARALLEL_COUNT } from '../../config'; -import { MissingToolsError } from '../../errors/MissingToolsError'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import { createPipelineExecutor } from '../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import { joinLlmExecutionTools } from '../../llm-providers/multiple/joinLlmExecutionTools'; -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { arrayableToArray } from '../../utils/arrayableToArray'; -import { titleToName } from '../../utils/normalization/titleToName'; -import type { TODO_any } from '../../utils/organization/TODO_any'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; -import { markdownScraperMetadata } from './register-metadata'; +import type { WritableDeep } from "type-fest"; +// TODO: [🏳‍🌈] Finally take pick of .json vs .ts +import PipelineCollection from "../../../books/index.json"; +import { createCollectionFromJson } from "../../collection/constructors/createCollectionFromJson"; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import { DEFAULT_MAX_PARALLEL_COUNT } from "../../config"; +import { MissingToolsError } from "../../errors/MissingToolsError"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import { createPipelineExecutor } from "../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { joinLlmExecutionTools } from "../../llm-providers/multiple/joinLlmExecutionTools"; +import type { KnowledgePiecePreparedJson } from "../../pipeline/PipelineJson/KnowledgePieceJson"; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { arrayableToArray } from "../../utils/arrayableToArray"; +import { titleToName } from "../../utils/normalization/titleToName"; +import { TODO_USE } from "../../utils/organization/TODO_USE"; +import type { TODO_any } from "../../utils/organization/TODO_any"; +import type { Scraper } from "../_common/Scraper"; +import type { ScraperSourceHandler } from "../_common/Scraper"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; +import { markdownScraperMetadata } from "./register-metadata"; /** * Scraper for markdown files @@ -30,166 +30,186 @@ import { markdownScraperMetadata } from './register-metadata'; * @public exported from `@promptbook/markdown-utils` */ export class MarkdownScraper implements Scraper { - /** - * Metadata of the scraper which includes title, mime types, etc. - */ - public get metadata(): ScraperAndConverterMetadata { - return markdownScraperMetadata; - } - - public constructor( - private readonly tools: Pick, - private readonly options: PrepareAndScrapeOptions, - ) {} - - /** - * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it - */ - public async scrape( - source: ScraperSourceHandler, - ): Promise> | null> { - const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, isVerbose = DEFAULT_IS_VERBOSE } = this.options; - const { llm } = this.tools; - - if (llm === undefined) { - throw new MissingToolsError('LLM tools are required for scraping external files'); - // <- Note: This scraper is used in all other scrapers, so saying "external files" not "markdown files" - } - - // TODO: [🚐] Make arrayable LLMs -> single LLM DRY - const _llms = arrayableToArray(llm); - const llmTools = _llms.length === 1 ? _llms[0]! : joinLlmExecutionTools(..._llms); - - TODO_USE(maxParallelCount); // <- [🪂] - - // TODO: [🌼] In future use `ptbk make` and made getPipelineCollection - const collection = createCollectionFromJson(...(PipelineCollection as TODO_any as ReadonlyArray)); - - const prepareKnowledgeFromMarkdownExecutor = createPipelineExecutor({ - pipeline: await collection.getPipelineByUrl( - 'https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book', - ), - tools: { - llm: llm, - }, - }); - - const prepareTitleExecutor = createPipelineExecutor({ - pipeline: await collection.getPipelineByUrl( - 'https://promptbook.studio/promptbook/prepare-knowledge-title.book', - ), - tools: { - llm: llm, - }, - }); - - const prepareKeywordsExecutor = createPipelineExecutor({ - pipeline: await collection.getPipelineByUrl( - 'https://promptbook.studio/promptbook/prepare-knowledge-keywords.book', - ), - tools: { - llm: llm, - }, - }); - - const knowledgeContent = await source.asText(); - - const result = await prepareKnowledgeFromMarkdownExecutor({ knowledgeContent }).asPromise(); - - const { outputParameters } = result; - const { knowledgePieces: knowledgePiecesRaw } = outputParameters; - - const knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n'); - // <- TODO: [main] Smarter split and filter out empty pieces - - if (isVerbose) { - console.info('knowledgeTextPieces:', knowledgeTextPieces); - } - - // const usage = ; - - const knowledge = await Promise.all( - // TODO: [🪂] Do not send all at once but in chunks - knowledgeTextPieces.map(async (knowledgeTextPiece, i) => { - // Note: Theese are just default values, they will be overwritten by the actual values: - let name: KnowledgePiecePreparedJson['name'] = `piece-${i}`; - let title: KnowledgePiecePreparedJson['title'] = spaceTrim(knowledgeTextPiece.substring(0, 100)); - const knowledgePieceContent: KnowledgePiecePreparedJson['content'] = spaceTrim(knowledgeTextPiece); - let keywords: KnowledgePiecePreparedJson['keywords'] = []; - const index: WritableDeep = []; - - /* + /** + * Metadata of the scraper which includes title, mime types, etc. + */ + public get metadata(): ScraperAndConverterMetadata { + return markdownScraperMetadata; + } + + public constructor( + private readonly tools: Pick, + private readonly options: PrepareAndScrapeOptions, + ) {} + + /** + * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it + */ + public async scrape( + source: ScraperSourceHandler, + ): Promise + > | null> { + const { + maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, + isVerbose = DEFAULT_IS_VERBOSE, + } = this.options; + const { llm } = this.tools; + + if (llm === undefined) { + throw new MissingToolsError( + "LLM tools are required for scraping external files", + ); + // <- Note: This scraper is used in all other scrapers, so saying "external files" not "markdown files" + } + + // TODO: [🚐] Make arrayable LLMs -> single LLM DRY + const _llms = arrayableToArray(llm); + const llmTools = + _llms.length === 1 ? _llms[0]! : joinLlmExecutionTools(..._llms); + + TODO_USE(maxParallelCount); // <- [🪂] + + // TODO: [🌼] In future use `ptbk make` and made getPipelineCollection + const collection = createCollectionFromJson( + ...(PipelineCollection as TODO_any as ReadonlyArray), + ); + + const prepareKnowledgeFromMarkdownExecutor = createPipelineExecutor({ + pipeline: await collection.getPipelineByUrl( + "https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book", + ), + tools: { + llm: llm, + }, + }); + + const prepareTitleExecutor = createPipelineExecutor({ + pipeline: await collection.getPipelineByUrl( + "https://promptbook.studio/promptbook/prepare-knowledge-title.book", + ), + tools: { + llm: llm, + }, + }); + + const prepareKeywordsExecutor = createPipelineExecutor({ + pipeline: await collection.getPipelineByUrl( + "https://promptbook.studio/promptbook/prepare-knowledge-keywords.book", + ), + tools: { + llm: llm, + }, + }); + + const knowledgeContent = await source.asText(); + + const result = await prepareKnowledgeFromMarkdownExecutor({ + knowledgeContent, + }).asPromise(); + + const { outputParameters } = result; + const { knowledgePieces: knowledgePiecesRaw } = outputParameters; + + const knowledgeTextPieces = (knowledgePiecesRaw || "").split("\n---\n"); + // <- TODO: [main] Smarter split and filter out empty pieces + + if (isVerbose) { + console.info("knowledgeTextPieces:", knowledgeTextPieces); + } + + // const usage = ; + + const knowledge = await Promise.all( + // TODO: [🪂] Do not send all at once but in chunks + knowledgeTextPieces.map(async (knowledgeTextPiece, i) => { + // Note: Theese are just default values, they will be overwritten by the actual values: + let name: KnowledgePiecePreparedJson["name"] = `piece-${i}`; + let title: KnowledgePiecePreparedJson["title"] = spaceTrim( + knowledgeTextPiece.substring(0, 100), + ); + const knowledgePieceContent: KnowledgePiecePreparedJson["content"] = + spaceTrim(knowledgeTextPiece); + let keywords: KnowledgePiecePreparedJson["keywords"] = []; + const index: WritableDeep = []; + + /* TODO: [☀] Track line and column of the source const sources: KnowledgePiecePreparedJson['sources'] = [ ]; */ - try { - const titleResult = await prepareTitleExecutor({ knowledgePieceContent }).asPromise(); - const { title: titleRaw = 'Untitled' } = titleResult.outputParameters; - title = spaceTrim(titleRaw) /* <- TODO: Maybe do in pipeline */; - name = titleToName(title); - - // --- Keywords - const keywordsResult = await prepareKeywordsExecutor({ knowledgePieceContent }).asPromise(); - const { keywords: keywordsRaw = '' } = keywordsResult.outputParameters; - keywords = (keywordsRaw || '') - .split(',') - .map((keyword) => keyword.trim()) - .filter((keyword) => keyword !== ''); - if (isVerbose) { - console.info(`Keywords for "${title}":`, keywords); - } - // --- - - if (!llmTools.callEmbeddingModel) { - // TODO: [🟥] Detect browser / node and make it colorfull - console.error('No callEmbeddingModel function provided'); - } else { - // TODO: [🧠][🎛] Embedding via multiple models - - const embeddingResult = await llmTools.callEmbeddingModel({ - title: `Embedding for ${title}` /* <- Note: No impact on embedding result itself, just for logging */, - parameters: {}, - content: knowledgePieceContent, - modelRequirements: { - modelVariant: 'EMBEDDING', - }, - }); - - index.push({ - modelName: embeddingResult.modelName, - position: [...embeddingResult.content], - // <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content` - }); - } - } catch (error) { - // Note: Here is expected error: - // > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING - if (!(error instanceof PipelineExecutionError)) { - throw error; - } - - // TODO: [🟥] Detect browser / node and make it colorfull - console.error( - error, - "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings", - ); - } - - return { - name, - title, - content: knowledgePieceContent, - keywords, - index, - // <- TODO: [☀] sources, - }; - }), - ); - - return knowledge; - } + try { + const titleResult = await prepareTitleExecutor({ + knowledgePieceContent, + }).asPromise(); + const { title: titleRaw = "Untitled" } = titleResult.outputParameters; + title = spaceTrim(titleRaw) /* <- TODO: Maybe do in pipeline */; + name = titleToName(title); + + // --- Keywords + const keywordsResult = await prepareKeywordsExecutor({ + knowledgePieceContent, + }).asPromise(); + const { keywords: keywordsRaw = "" } = + keywordsResult.outputParameters; + keywords = (keywordsRaw || "") + .split(",") + .map((keyword) => keyword.trim()) + .filter((keyword) => keyword !== ""); + if (isVerbose) { + console.info(`Keywords for "${title}":`, keywords); + } + // --- + + if (!llmTools.callEmbeddingModel) { + // TODO: [🟥] Detect browser / node and make it colorfull + console.error("No callEmbeddingModel function provided"); + } else { + // TODO: [🧠][🎛] Embedding via multiple models + + const embeddingResult = await llmTools.callEmbeddingModel({ + title: `Embedding for ${title}` /* <- Note: No impact on embedding result itself, just for logging */, + parameters: {}, + content: knowledgePieceContent, + modelRequirements: { + modelVariant: "EMBEDDING", + }, + }); + + index.push({ + modelName: embeddingResult.modelName, + position: [...embeddingResult.content], + // <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content` + }); + } + } catch (error) { + // Note: Here is expected error: + // > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING + if (!(error instanceof PipelineExecutionError)) { + throw error; + } + + // TODO: [🟥] Detect browser / node and make it colorfull + console.error( + error, + "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings", + ); + } + + return { + name, + title, + content: knowledgePieceContent, + keywords, + index, + // <- TODO: [☀] sources, + }; + }), + ); + + return knowledge; + } } /** diff --git a/src/scrapers/markdown/createMarkdownScraper.ts b/src/scrapers/markdown/createMarkdownScraper.ts index 66e90483b7..51cf23fcf4 100644 --- a/src/scrapers/markdown/createMarkdownScraper.ts +++ b/src/scrapers/markdown/createMarkdownScraper.ts @@ -1,9 +1,9 @@ -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import type { ScraperConstructor } from '../_common/register/ScraperConstructor'; -import { MarkdownScraper } from './MarkdownScraper'; -import { markdownScraperMetadata } from './register-metadata'; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import type { ScraperConstructor } from "../_common/register/ScraperConstructor"; +import { MarkdownScraper } from "./MarkdownScraper"; +import { markdownScraperMetadata } from "./register-metadata"; keepTypeImported(); @@ -13,10 +13,13 @@ keepTypeImported(); * @public exported from `@promptbook/markdown-utils` */ export const createMarkdownScraper = Object.assign( - (tools: Pick, options: PrepareAndScrapeOptions): MarkdownScraper => { - return new MarkdownScraper(tools, options); - }, - markdownScraperMetadata, + ( + tools: Pick, + options: PrepareAndScrapeOptions, + ): MarkdownScraper => { + return new MarkdownScraper(tools, options); + }, + markdownScraperMetadata, ) satisfies ScraperConstructor; /* <- Note: [🤛] */ /** diff --git a/src/scrapers/markdown/examples/10-simple.md.knowledge.json b/src/scrapers/markdown/examples/10-simple.md.knowledge.json index e00b006482..f6bb65fe94 100644 --- a/src/scrapers/markdown/examples/10-simple.md.knowledge.json +++ b/src/scrapers/markdown/examples/10-simple.md.knowledge.json @@ -1,44 +1,1285 @@ [ - { - "name": "springfield-illinois-sangamon-county-seat", - "title": "Springfield: Illinois' Sangamon County Seat", - "content": "---\n\nSpringfield is the county seat of Sangamon County, located in the U.S. state of Illinois.", - "keywords": [ - "Springfield", - "county seat", - "Sangamon County", - "Illinois", - "United States" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.03425075, 0.036835715, -0.0047420897, -0.01650307, -0.016335526, -0.023121051, -0.0037727286, -0.0065581454, -0.03973183, 0.0018070801, -0.008377193, 0.021445613, -0.018022932, -0.0042693764, 0.009579918, -0.010297963, -0.013068421, 0.015940601, -0.0019432096, -0.0073539787, 0.04102431, 0.004221507, -0.05011955, 0.047079824, -0.0009072799, 0.013511215, -0.04844411, 0.016814223, -0.02558634, 0.011303226, -0.019123934, -0.009436309, 0.02197218, 0.022283332, -0.00955, -0.02637619, -0.00906532, -0.016933898, 0.0044399123, 0.019399185, -0.0073599624, -0.018058835, 0.0028512373, 0.007347995, 0.021062655, 0.0017861372, 0.01714931, -0.03578258, -0.005478086, 0.003970191, 0.020380514, -0.0067615914, 0.005510996, 0.02687882, 0.02140971, 0.012948747, 0.0008489387, 0.033173684, -0.008299405, -0.045260776, -0.021804636, -0.008269486, -0.040665288, 0.0038116227, 0.014336967, 0.01200332, 0.0005220786, -0.0015587562, 0.050598245, 0.005202835, 0.029583458, 0.02294154, -0.0066119987, 0.037194736, 0.048635587, -0.010561246, -0.031498246, -0.014073684, -0.018429825, 0.029056892, 0.039516415, -0.04023446, -0.0055798087, 0.022331202, 0.009466228, -0.02642406, 0.010016729, -0.059358396, -0.009723527, -0.0009357025, 0.015749123, -0.018250313, -0.021242168, -0.02243891, -0.024389599, -0.031019548, 0.059262656, -0.012900877, 0.010734774, 0.035591103, 0.012864974, -0.009466228, -0.053326815, -0.010052632, 0.009424342, 0.006235025, 0.009137124, -0.022965476, -0.011626347, 0.008556704, 0.025323058, 0.013810401, 0.00042259946, -0.020380514, -0.0074138157, -0.004766024, -0.009023434, -0.028027693, 0.039324936, 0.019159837, -0.0072103697, 0.00053442, -0.015282393, -0.001687406, -0.009400407, 0.039157394, 0.02011723, -0.039157394, 0.019674435, -0.016993733, 0.025610276, -0.00456557, 0.009801316, 0.015593546, 0.011404949, -0.02271416, 0.012230702, 0.049401503, 0.021768734, 0.023707455, -0.057539348, 0.011668233, 0.01571322, -0.008999499, 0.023934837, -0.015796993, -0.017568171, -0.009286717, -0.012733333, -0.031091353, -0.01775965, -0.014157456, -0.026208647, 0.010764693, 0.025251253, 0.006402569, -0.017352756, -0.043681078, 0.04978446, -0.0498802, -0.013678759, -0.008173747, 0.03949248, 0.009747462, -0.007383897, -0.012601691, -0.028578196, -0.00004270405, -0.00567854, 0.0010224663, 0.035710778, -0.0016889019, -0.015366165, -0.08645263, 0.003847525, -0.0333891, 0.012625626, -0.024820426, 0.00625896, 0.043465663, 0.0005658345, 0.02127807, 0.026184712, -0.016311591, -0.009202945, -0.022809898, -0.013678759, 0.029176567, 0.032216292, -0.019674435, -0.016754385, 0.030421177, 0.032048747, 0.02100282, 0.030277569, -0.011267324, -0.03973183, -0.034705512, 0.011805858, 0.02197218, -0.051412027, 0.011375031, 0.0112553565, -0.012122995, 0.013606954, 0.025921429, -0.0023022322, -0.038032457, 0.008742199, 0.0403302, 0.004398026, -0.003784696, -0.021397743, 0.06883659, -0.01209906, 0.02405451, -0.015066979, -0.015964536, -0.031139223, 0.039205264, 0.01464812, 0.022043984, -0.001913291, -0.000095599105, -0.011213471, 0.00563067, -0.055050123, -0.027261779, 0.015162719, -0.0077309525, 0.021158395, -0.001358302, 0.001442074, 0.050646115, -0.046648998, 0.04588308, 0.020667732, 0.0023620692, -0.04696015, 0.023348434, 0.01714931, 0.031546116, -0.005229762, -0.023156954, -0.019016229, -0.009472212, 0.014085651, 0.00059388316, -0.009442293, 0.0154978065, 0.0034047305, 0.010160338, 0.050502505, 0.015318296, -0.0053195176, 0.012027255, -0.016862093, 0.0077907895, 0.01072879, 0.029631328, -0.0001544545, 0.021577256, -0.019913783, 0.021302005, 0.013307769, -0.037912782, -0.008628509, -0.0009214912, 0.01242218, -0.03980363, -0.014097619, -0.0038714597, 0.0038535087, 0.019638533, 0.029774936, 0.011578477, 0.030469047, 0.01497124, -0.032694988, -0.020548057, -0.024078446, -0.009029417, 0.0039432645, -0.026447995, 0.025993232, 0.002610393, -0.009364505, 0.04112005, 0.0035752663, -0.027572932, 0.010549279, -0.0206797, 0.025275188, 0.02697456, 0.045763407, 0.014348934, 0.015976503, 0.0042065475, -0.034801252, 0.004110808, 0.03389173, 0.033915665, 0.008987531, -0.009603853, 0.0270703, -0.008754167, 0.019877883, 0.0066479007, -0.043585338, -0.055002254, 0.010190257, -0.03389173, 0.003353869, 0.01566535, -0.017400626, 0.016323559, 0.005119063, -0.0030202772, 0.075538345, -0.031953007, -0.022857768, 0.016215852, 0.031857267, 0.00226633, -0.01343941, 0.02558634, -0.0072702067, -0.0025475642, 0.009639756, 0.009436309, -0.02850639, 0.0036799812, -0.012218734, -0.04167055, -0.009310652, -0.026136842, 0.0265198, -0.009777381, 0.018070802, 0.0057563283, -0.009149091, -0.01460025, -0.0065521616, 0.03350877, 0.026400125, 0.014755826, -0.0067915097, -0.010525345, -0.024269924, -0.0010890351, 0.0005239485, 0.011668233, 0.021026755, -0.020224936, 0.029152632, 0.0080421055, -0.04325025, -0.039612155, 0.015952569, -0.018190475, -0.028243108, 0.007407832, -0.021589223, 0.00003340125, -0.0012326441, 0.04013872, -0.0270703, 0.016263722, 0.030205764, 0.0008302397, -0.010926253, 0.019111967, 0.013654824, -0.009514097, 0.01321203, 0.0012490993, -0.00885589, 0.012458082, 0.0017203164, -0.0021870458, 0.037242606, 0.023695488, -0.020751504, -0.014408772, -0.014815664, -0.030469047, -0.0022453868, -0.049305763, -0.027740475, 0.011135682, -0.021098558, -0.016814223, 0.009029417, -0.03384386, -0.017831454, 0.023216791, -0.07290551, 0.007982268, 0.05394912, 0.035615038, 0.016634712, -0.04432732, 0.011883646, 0.01650307, -0.03425075, -0.027166039, -0.009232863, -0.056103256, 0.010848464, -0.010830513, -0.023839097, 0.009657706, 0.0047779917, -0.01103396, -0.0050981203, -0.046194233, -0.024174185, 0.03341303, 0.023527944, 0.0058999374, 0.01492337, -0.02563421, -0.001666463, -0.024772556, -0.019662468, 0.024964035, 0.0015333255, 0.031761527, -0.04236466, -0.026807018, -0.030756265, -0.006276911, -0.006863314, 0.020871177, -0.0061632204, 0.05988496, 0.0006338992, -0.0018968357, 0.03087594, 0.027477192, -0.010806579, 0.014360902, 0.042699747, -0.03712293, -0.0033209587, 0.009807299, 0.032072682, 0.002409939, 0.020883145, 0.044758145, -0.01033985, 0.016311591, -0.0030965696, 0.015485839, 0.008030138, 0.03425075, -0.040306266, 0.01469599, -0.0222953, 0.040521678, 0.03082807, -0.008586623, -0.0016335526, -0.03370025, -0.01714931, 0.010166322, 0.02924837, 0.004652334, -0.00014314154, -0.01919574, -0.00966369, -0.03142644, -0.031211028, 0.006833396, -0.042699747, -0.021122493, 0.00063913496, 0.0053554196, 0.009490163, 0.0033778038, -0.046984084, -0.0007445978, -0.069123805, 0.009406391, 0.014253195, 0.0061572366, -0.0292005, -0.046553258, 0.01659881, -0.012338408, 0.09966466, 0.020751504, -0.009059336, -0.010812563, 0.020296741, 0.020404449, -0.02943985, -0.024437468, 0.04107218, 0.040593483, -0.0085926065, 0.054906514, 0.038367543, -0.037434086, -0.0080421055, 0.004389051, -0.023384335, -0.034801252, 0.013379574, 0.02006936, 0.050502505, 0.00033564868, -0.032240223, -0.01974624, 0.010028697, -0.024605012, -0.011686184, -0.0024952067, 0.03671604, 0.0056635807, 0.00080181705, 0.009143108, 0.003288048, -0.0008751175, -0.003557315, 0.012769235, 0.005026316, -0.006151253, -0.0009708568, -0.029559523, -0.035615038, -0.010764693, -0.000013346476, 0.031641856, 0.035638973, -0.008778102, -0.013223997, -0.003964207, -0.009113189, 0.022450877, -0.030995613, -0.029224435, 0.008520802, 0.011895614, 0.006438471, 0.007090695, 0.0028317904, 0.0012244165, 0.0028557251, -0.021373808, 0.04265188, -0.0031324718, -0.0013590499, -0.003476535, 0.0029753994, -0.0051938593, -0.010579198, -0.015402067, -0.0074197995, -0.006600031, 0.012924812, 0.011249374, 0.022690225, 0.01942312, 0.012493985, 0.0036351033, 0.00038333138, -0.005777271, -0.008072023, 0.0063606827, 0.010154354, 0.00026571407, 0.0419099, 0.0077429195, 0.025107644, 0.0050682016, -0.028985087, 0.0043740915, 0.0009573935, -0.0066778194, 0.009376472, 0.00538833, 0.021804636, 0.02132594, 0.013092356, 0.017137343, -0.010806579, -0.028769674, -0.02340827, 0.027381454, 0.017125376, 0.008251535, -0.0052776313, -0.0022154683, -0.012864974, -0.0063247806, -0.020691667, 0.005373371, -0.024581077, -0.018106705, 0.022199562, 0.005346444, 0.014839599, -0.043681078, -0.017843422, -0.002053908, 0.032671053, 0.0040360116, 0.024509273, 0.006486341, -0.0036949404, 0.017651942, -0.016814223, -0.0067974934, -0.01188963, 0.0019835995, 0.03137857, 0.034968797, 0.015952569, -0.017819487, -0.005166933, -0.017520301, -0.008209649, -0.00908327, 0.017113408, -0.01919574, 0.01501911, -0.024605012, -0.019782143, 0.009897055, -0.015844861, 0.0036111686, -0.021206265, 0.008975564, 0.013726628, -0.013798433, -0.0046014725, -0.003189317, -0.009514097, 0.0020344611, 0.017568171, 0.040402003, 0.026830953, 0.023611717, -0.02085921, -0.0015752114, -0.010764693, 0.038965914, -0.015916666, -0.0051280386, 0.057730827, -0.017699812, -0.0045236843, -0.000005852231, 0.011668233, 0.013606954, 0.0009319627, -0.025394863, 0.030205764, -0.010656986, -0.02656767, 0.00987312, -0.0004902902, 0.018501628, 0.03958822, 0.007754887, -0.036955386, 0.05849674, 0.017639974, 0.00082350796, 0.0077070175, 0.014025814, 0.002736051, -0.010070583, -0.015689285, 0.0026298403, -0.0042244988, 0.001585683, -0.010645019, 0.001563244, -0.010309931, -0.012709398, -0.0084849, -0.021948244, 0.017687844, -0.030660525, 0.024652882, 0.034561902, -0.013403509, -0.024389599, 0.0055468986, 0.02544273, 0.023875, -0.0087840855, 0.02647193, 0.0018684132, 0.005540915, 0.042077444, -0.006151253, -0.0077070175, 0.0032581296, 0.014025814, 0.02841065, -0.04710376, 0.009047369, 0.025179448, -0.01029198, -0.022773998, -0.04533258, -0.012057174, 0.003787688, -0.0026014175, 0.017376691, -0.006276911, 0.039061654, -0.04277155, 0.04277155, -0.04157481, -0.007814724, 0.039468545, 0.026041102, 0.014289098, -0.03621341, -0.019770175, -0.021146428, 0.005905921, -0.042244986, -0.01040567, 0.016574875, 0.0061572366, -0.037960652, -0.0049814377, -0.006067481, -0.016993733, 0.020344611, 0.00906532, -0.013846303, -0.02979887, 0.00038015252, 0.004628399, -0.00085866224, -0.0039402726, 0.05083759, -0.0047630323, -0.00559776, 0.021481516, -0.000905036, 0.0014839598, -0.04399223, 0.0270703, -0.01831015, -0.0023815162, 0.035997994, 0.011039943, 0.012864974, 0.012948747, 0.0156174805, 0.009442293, 0.024365664, 0.04454273, 0.011416917, 0.016407331, 0.0066359336, 0.0055498905, 0.014492543, 0.022367105, 0.006091416, 0.0074497177, 0.006025595, 0.004030028, 0.029176567, 0.0049305763, -0.0016485119, 0.0021945254, 0.021696929, 0.014324999, 0.0117101185, -0.0050682016, 0.0050083646, -0.021110525, 0.05011955, -0.003787688, -0.019506892, -0.016000438, 0.006917168, -0.018920489, -0.01956673, 0.006971021, 0.0127453, -0.0073120925, -0.010603133, -0.004338189, -0.032551378, 0.017879322, 0.001706853, -0.023635652, 0.0075813597, -0.024964035, 0.030708395, 0.013188095, 0.005534931, 0.013942042, 0.0117579885, 0.0034406327, 0.0008698817, 0.00030965696, -0.0040210527, 0.02563421, -0.035088472, 0.039564285, 0.040545613, -0.02145758, 0.03647669, -0.0019073073, 0.007491604, -0.0075813597, 0.028027693, -0.010656986, 0.005534931, -0.01279317, 0.028985087, -0.026352255, 0.030133959, -0.017376691, 0.00074160594, 0.01868114, 0.0034885025, -0.03410714, -0.004215523, 0.014217293, 0.0035722745, -0.008999499, -0.02090708, -0.013918107, 0.0014981711, 0.0016844141, 0.0292005, -0.0028063597, 0.031211028, 0.010531328, -0.013774498, -0.010555263, 0.022199562, -0.012613659, 0.012362343, 0.053278945, 0.013283835, -0.037577692, 0.00827547, -0.060650878, -0.019088032, 0.015928634, -0.026687343, 0.023432204, -0.0249401, 0.004816886, 0.043417793, 0.018046867, 0.005537923, 0.016167982, 0.0042334744, -0.008610558, 0.043537468, 0.014241228, 0.021254135, 0.01761604, 0.021349873, -0.017508334, -0.003024765, -0.009238847, 0.014624185, -0.006378634, -0.021637091, 0.00456557, -0.0077907895, -0.0030786183, -0.010758709, -0.018298183, -0.0047510653, 0.01172807, -0.040928572, -0.005140006, -0.020966917, -0.028649999, 0.020691667, 0.032479573, -0.028290978, -0.013606954, 0.016658647, -0.03583045, -0.0016200893, -0.0080421055, 0.008443014, -0.0060465382, 0.012673496, 0.0117101185, -0.021792669, -0.023623684, -0.008359241, 0.031402506, 0.017879322, -0.0043471646, 0.009633772, -0.025131578, 0.0023156954, 0.0018564458, 0.032623183, -0.008413095, -0.0011959939, 0.012015288, 0.0056276782, 0.032503508, -0.013989912, -0.026950626, -0.008586623, 0.010262061, -0.030062154, 0.007814724, 0.003907362, 0.03355664, 0.031929072, 0.009125156, -0.011219455, -0.009089254, -0.0032341948, -0.008742199, 0.026112907, -0.019554762, -0.022450877, -0.0027270755, -0.0022528665, -0.001831015, -0.009137124, 0.003844533, 0.03003822, -0.009220895, -0.04116792, -0.026232582, -0.012948747, 0.004798935, 0.0055887843, 0.031833332, -0.027716542, -0.011363064, -0.016167982, 0.031091353, -0.006869298, 0.022355137, 0.021816604, -0.00044616032, 0.018740976, 0.0040838816, -0.01793916, -0.0016425282, 0.027309649, 0.014743859, -0.030014286, 0.038822304, -0.0078266915, 0.007527506, -0.015234523, 0.0058999374, -0.00917901, -0.0032940318, 0.0031982926, -0.006911184, -0.013415476, -0.050454635, 0.0005632166, -0.0015976503, 0.012290538, -0.0069829887, -0.015210588, -0.017831454, 0.0036710056, 0.018334085, 0.0010321898, 0.010136403, 0.010016729, 0.02400664, 0.0012019776, 0.024162218, 0.0013253916, 0.026088972, 0.016311591, -0.009274749, -0.02800376, -0.017137343, -0.0019357299, -0.007587343, 0.03958822, -0.022403007, -0.016670614, 0.010639035, 0.0398515, 0.004996397, 0.010704855, 0.0010942708, 0.000308535, -0.003949248, 0.02081134, -0.029822806, -0.022187594, 0.017352756, -0.024581077, 0.01316416, -0.052560903, 0.005947807, 0.0074796365, 0.006546178, 0.002369549, -0.009173026, 0.013008583, -0.02252268, -0.013343671, -0.024533208, -0.006318797, -0.025921429, -0.0050053727, 0.0005127291, 0.0061183427, 0.015557644, -0.0028931233, -0.011644298, 0.0018878602, -0.0009199953, 0.023468107, 0.012434147, 0.032551378, 0.014313032, 0.016132079, 0.011273308, -0.0072103697, 0.021266103, -0.013475313, 0.0081617795, -0.012912844, 0.054475687, 0.007521522, 0.012649561, 0.01098609, -0.023851065, 0.023875, 0.0019611607, 0.0022543625, 0.010495426, 0.007383897, 0.007892513, 0.011692167, 0.02503584, -0.0016350485, -0.014253195, -0.0024024593, 0.005531939, -0.038128193, -0.0082276, 0.0024996945, 0.025251253, 0.001871405, -0.016527005, 0.011075846, 0.002469776, 0.030516917, -0.01077666, 0.022223495, -0.016347494, -0.0069291354, 0.028123433, 0.022474812, 0.0059507987, 0.01119552, -0.014133521, 0.032647118, -0.042627946, 0.0025774827, 0.00695307, -0.01789129, -0.001276774, -0.0031324718, -0.029368045, -0.003147431, 0.020715602, -0.02373139, 0.040761027, 0.016263722, -0.058353133, 0.017723747, -0.0021331923, -0.007431767, 0.02776441, -0.015150752, -0.0072283205, 0.0074557015, -0.0056964913, 0.03587832, -0.029918546, 0.003042716, 0.041598745, 0.0037368264, -0.0072103697, -0.02484436, -0.006498308, 0.010321898, -0.038319673, -0.014301065, -0.023360401, -0.014348934, -0.0066359336, -0.02697456, -0.007814724, -0.014707957, 0.011183552, -0.011165601, 0.011560526, 0.01710144, 0.013080388, -0.012493985, 0.005044267, 0.007832675, -0.031905137, -0.007521522, -0.030540852, -0.022127757, 0.011584461, 0.002816831, -0.038008522, -0.0117819235, -0.019602632, 0.0027839206, -0.01427713, 0.0038744516, 0.017628007, -0.009968859, -0.02568208, 0.020248871, -0.004876723, 0.016084211, -0.01673045, 0.04389649, -0.017580137, 0.0069829887, -0.007964317, -0.022319235, -0.019554762, 0.03003822, -0.003416698, 0.024820426, -0.0036889566, -0.0071086464, -0.01692193, -0.019997556, -0.0056695645, 0.016814223, -0.0018130639, -0.0038535087, -0.02687882, -0.008652443, -0.0154738715, -0.025753884, 0.012577756, -0.024796491, 0.0021750783, 0.006725689, 0.0164552, 0.0067795427, 0.017747682, 0.012829073, -0.0015976503, -0.024329761, 0.034370426, 0.011021992, 0.013104323, -0.017628007, -0.019111967, 0.015868796, -0.020392481, 0.010477475, 0.0061632204, -0.013571053, 0.011704135, 0.002697157, 0.002406947, -0.0030157894, 0.0036889566, -0.013020551, -0.013152192, -0.006175188, -0.026591603, -0.051459897, -0.002429386, -0.028817544, 0.0021077616, 0.02558634, 0.006546178, 0.017675877, -0.006516259, 0.00047159108, -0.004221507, 0.02632832, 0.03374812, -0.031091353, 0.02568208, -0.0011638314, -0.00642052, 0.027477192, -0.013678759, 0.0074197995, -0.01714931, -0.007718985, -0.0005179648, 0.002409939, 0.023300564, 0.0206797, -0.012613659, 0.0038594925, -0.024868295, -0.027429324, 0.0023755326, 0.013918107, -0.0056127193, 0.0071505327, -0.005409273, -0.033963535, 0.009250814, -0.002857221, 0.0031504228, 0.008730232, -0.003144439, 0.02392287, 0.0043262215, -0.0027091242, -0.0013650337, -0.0018848684, 0.0008930686, -0.018334085, -0.044614535, -0.029032957, 0.012248653, 0.0075155385, 0.031689722, 0.042747617, -0.019782143, 0.019387217, 0.021661026, 0.003934289, 0.013523183, -0.028578196, -0.018501628, 0.0046014725, -0.046936214, 0.011704135, 0.0047271303, 0.011787907, -0.016993733, 0.0027824247, -0.0010134907, -0.027477192, 0.022654323, -0.022546615, 0.00642052, 0.014073684, 0.024557142, 0.01177594, 0.02132594, 0.041694485, 0.00998681, -0.013942042, 0.0052118106, 0.011135682, -0.007509555, -0.007838659, 0.01969837, -0.01659881, -0.009974843, 0.016527005, 0.015090914, -0.03573471, -0.016132079, 0.022534648, -0.011692167, 0.011004041, 0.014815664, -0.008700313, 0.0031205043, 0.005059226, -0.0115904445, 0.023659587, -0.01003468, 0.011674217, -0.0034526002, 0.023240726, -0.010926253, 0.007916447, 0.0043292134, -0.024748622, -0.009502131, 0.03384386, -0.03954035, -0.0072702067, -0.007982268, 0.0120452065, -0.025227318, -0.0047839754, -0.022881703, -0.015940601, -0.0010815554, -0.019004261, 0.013391541, -0.000047822927, 0.010226159, 0.020691667, 0.004834837, -0.023755325, 0.0005018836, 0.023743358, -0.0043740915, 0.019590665, -0.014169423, -0.017998997, -0.010094517, -0.05868822, -0.009238847, -0.009573935, -0.0065102754, 0.01184176, -0.0017562186, 0.01692193, 0.00088783284, 0.007569392, 0.015102882, 0.00064698857, 0.00374281, -0.008251535, 0.0081617795, -0.017340789, -0.02350401, 0.011584461, -0.007443734, 0.03537569, 0.021038722, 0.0048946743, 0.02776441, -0.00010602385, 0.01418139, -0.0029050908, 0.0035333803, -0.0069231517, -0.0077907895, 0.015366165, -0.026735213, -0.0006679315, 0.030660525, -0.022235462, 0.013559085, -0.020512154, 0.008861873, -0.010172306, 0.014947305, 0.01400188, 0.030325439, -0.005720426, 0.02800376, 0.048994612, -0.0039253132, -0.0072702067, -0.0006009888, -0.008072023, -0.014887468, -0.008843922, -0.030564787, -0.0022379071, 0.020942982, 0.018657206, 0.022223495, 0.002610393, -0.007754887, 0.0081139095, -0.018298183, -0.018717043, -0.004018061, -0.009843201, -0.010770677, 0.018597368, 0.000556111, 0.014672055, 0.025418796, 0.018298183, 0.009262782, 0.0038146146, 0.016491102, -0.0021750783, 0.010579198, -0.013008583, 0.011608396, -0.0015796992, 0.009011466, 0.013977945, -0.0064504384, -0.014935338, -0.0028108475, -0.0085686715, 0.010315915, 0.02197218, -0.01151864, 0.036189474, -0.002368053, 0.0057234177, -0.008981547, -0.009364505, -0.01687406, -0.007952349, -0.015246491, 0.005328493, -0.031234961, 0.0053225094, -0.027884085, -0.010794612, 0.03676391, 0.008496867, -0.013307769, 0.0057922304, -0.013882205, -0.017041603, 0.0035094453, -0.004302287, -0.024724687, -0.014301065, -0.009639756, -0.010854448, 0.008526785, -0.02202005, -0.011722086, -0.00096636906, -0.024174185, 0.013523183, -0.010627068, -0.006827412, 0.0005325501, 0.007300125, -0.009250814, -0.018561466, -0.016072243, -0.0017726738, 0.010178289, -0.008586623, -0.00551698, -0.005331485, -0.034418296, -0.0066778194, -0.0036321115, -0.023958772, 0.007318076, -0.02943985, -0.0077907895, 0.032742858, 0.010788628, 0.01400188, 0.031115288, 0.016706515, -0.0058460836, 0.013846303, 0.0027629777, 0.006085432, 0.009633772, 0.02563421, -0.013690727, 0.01673045, 0.007892513, -0.009274749, 0.0020479246, -0.0077489032, 0.0222953, 0.0033209587, 0.013750563, 0.0061243265, 0.026184712, 0.031881202, 0.014408772, -0.011273308, -0.014540413, -0.00832334, 0.012290538, -0.0022708175, 0.0020434367, -0.0106151, 0.03300614, -0.022582518, 0.0008863369, -0.0000023724472, 0.016622745, -0.02934411, 0.01659881, 0.013104323, -0.006606015, -0.021720864, 0.012912844, 0.017867355, 0.007030858, 0.024401566, 0.0222953, 0.002411435, 0.0040659304, -0.0065940474, -0.029176567, 0.01469599, -0.009639756, 0.017388659, -0.034968797, -0.0026418075, -0.0030531876, 0.01997362, 0.034801252, -0.008024154, 0.02355188, -0.027357519, 0.005923872, 0.01724505, -0.011327161, 0.017352756, 0.011021992, 0.019016229, 0.0355193, 0.007240288, -0.030445112, 0.021912344, -0.0033239503, 0.005947807, 0.01024411, -0.0051729167, -0.0078087403, -0.010627068, 0.009256798, -0.01905213, 0.000030339275, -0.0057114502, 0.026759148, -0.007258239, 0.0077010337, -0.025155513, -0.00855072, -0.012876942, 0.008149812, 0.025059775, -0.00023691749, -0.020763472, 0.015509774, 0.014205325, -0.0062410086, -0.011392983, -0.0066419174, -0.030492982, -0.010752725, 0.0037577692, -0.009424342, 0.013200062, 0.0005935092, -0.010465507, 0.035136342, 0.0036710056, 0.021290038, -0.014085651, 0.011381015, 0.024138283, -0.0033897713, -0.0061243265, -0.013032518, -0.005971742, 0.013355639, 0.00094318215, -0.006091416, 0.00625896, 0.011883646, 0.012434147, -0.0081378445, -0.012446115, -0.0055468986, -0.013774498, -0.006438471, 0.011680201, 0.021672994, -0.0080899745, 0.0044309366, -0.012290538, 0.012218734, 0.0053524277, -0.0046822526, -0.01242218, 0.0030127976, 0.0254906, -0.0039402726, -0.0048707393, 0.003644079, -0.0037577692, -0.0038116227, -0.014289098, -0.02053609, 0.007521522, -0.006456422, -0.004963487, -0.012817105, -0.0067795427, 0.019901816, -0.040258396, 0.0085926065, -0.013200062, 0.00036145342, 0.011668233, 0.0060375626, 0.0002223322, -0.01771178, -0.013068421, -0.020607894, -0.0053165257, -0.008999499, -0.006252976, 0.022151692, 0.0013396029, -0.018202443, -0.004505733, 0.011716102, -0.02063183, -0.007527506, -0.020105263, 0.02489223, -0.022008082, 0.026495865, -0.010968138, -0.008371209, -0.011985369, 0.0046164314, 0.006665852, 0.002754002, 0.023743358, -0.00567854, -0.009861153, 0.009286717, 0.021768734, -0.018717043, 0.011045927, 0.012302506, 0.037840977, -0.018872619, 0.00563067, 0.027501127, 0.032671053, -0.00022121026, -0.0009768405, -0.013475313, -0.0021017778, -0.0024054511, -0.010393702, 0.015055012, 0.012254637, 0.013415476, -0.018465726, -0.01914787, 0.0056486214, -0.016024373, -0.010603133, 0.009777381, 0.0022872728, -0.011117731, 0.0077010337, -0.0061033834, 0.022223495, 0.0082814535, 0.017879322, -0.0028617089, -0.0007846139, -0.0120212715, 0.0028213188, -0.01576109, -0.008299405, -0.0076471805, 0.00226633, 0.00453266, -0.018812781, 0.0041826125, 0.009280733, -0.0076651312, 0.0021735823, -0.007425783, 0.054954384, 0.0024757597, -0.00855072, -0.005457143, -0.0020105264, -0.0010718319, -0.020512154, -0.004816886, 0.01771178, -0.0040090852, -0.004383067, -0.012996616, -0.005744361, 0.0089037595, -0.0014779762, -0.012721365, 0.011740037, -0.023863032, -0.00016576744, -0.005247713, -0.007407832, -0.006462406, -0.009137124, -0.018657206, -0.02257055, 0.010351817, 0.010603133, -0.0012692943, 0.0028033678, 0.0044578635, 0.005738377, -0.0044698305, 0.0071864347, 0.026400125, 0.03434649, 0.015940601, -0.000946174, 0.031306766, 0.0019581688, -0.017029636, -0.016048308, 0.032216292, 0.0074138157, -0.0015752114, -0.00975943, 0.013786466, -0.01682619, -0.015198621, -0.0071624997, -0.0029065867, -0.006336748, -0.009454261, -0.0019731282, -0.0036949404, -0.0024129308, -0.011626347, 0.0041796206, -0.016586842, -0.0035483395, 0.009370488, 0.015844861, 0.0046343827, -0.025897494, 0.027429324, 0.0017547227, 0.0079703005, -0.0021556313, 0.0121409455, 0.006462406, -0.013152192, 0.006827412, 0.002240899, 0.009615821, -0.03082807, -0.035112407, 0.021349873, 0.000060211074, 0.0054990286, -0.00025000685, 0.008024154, -0.0033688282, -0.00604953, 0.021026755, 0.010555263, 0.017639974, -0.017843422, 0.0004095101, 0.018884586, 0.009047369, 0.020942982, -0.012206767, 0.010256077, 0.007425783, -0.006719705, 0.008353258, 0.024269924, -0.02934411, 0.00067129737, -0.025897494, -0.023348434, -0.017352756, -0.011913565, -0.0254906, 0.008670394, 0.0019566729, 0.020476252, -0.012769235, 0.03147431, -0.009035401, 0.005825141, -0.024964035, -0.009532048, 0.010585181, -0.016467167, -0.008359241, 0.03365238, 0.012242669, 0.016024373, -0.003084602, -0.007509555, 0.015773058, -0.02234317, -0.024305826, 0.0016559915, -0.0001748365, 0.013942042, 0.009047369, 0.02127807, 0.0022992403, -0.007689066, 0.0064504384, -0.0153541975, 0.016491102, -0.000513851, 0.0071624997, -0.0029155624, -0.008814003, 0.021206265, 0.0071924184, 0.003273089, -0.034705512, -0.018118672, 0.005002381, -0.01251792, 0.010860432, -0.010788628, 0.001914787, -0.013032518, 0.017723747, -0.000023747845, 0.008443014, -0.008419079, -0.006336748, -0.02656767, 0.00966369, 0.014217293, 0.0051818923, -0.031354636, 0.010573214, -0.009639756, 0.0003949248, -0.007425783, 0.021637091, 0.0013134242, 0.0051938593, -0.007802757, -0.017340789, 0.019399185, -0.002181062, -0.0024159225, 0.016527005, 0.004834837, -0.00785661, -0.0047809836, -0.007222337, 0.012769235, -0.0070727444, 0.005513988, 0.0021167372, -0.017041603, -0.017137343, -0.0022872728, -0.010956172, 0.0006525983, 0.0080421055, -0.0033897713, 0.006809461, -0.019303447, 0.0008511826, -0.005250705, -0.019626565, -0.0060704728, -0.022187594, -0.012948747, -0.024772556, -0.003907362, -0.03157005, -0.0033957548, 0.0030501958, -0.0044339285, 0.0038744516, 0.027046366, -0.027381454, 0.024174185, -0.014121554, -0.005699483, -0.006977005, 0.00035733962, -0.0015572603, -0.0015198621, -0.026088972, -0.014552381, 0.0011765468, -0.010950188, -0.011919549, 0.008837938, -0.018250313, 0.00032143737, 0.0043681078, 0.003910354, 0.0004712171, -0.025777819, -0.009274749, 0.0077668545, 0.019279512, -0.0017726738, -0.0008758654, -0.010489442, -0.016467167, 0.028697869, -0.041550875, -0.020320676, 0.012063158, 0.011548558, 0.0024159225, 0.01400188, 0.0082335835, 0.012864974, 0.025370928, -0.013068421, 0.012721365, -0.0114229005, -0.0040928572, 0.007240288, -0.011303226, -0.0004297051, -0.012087093, 0.017221116, 0.012122995, 0.011123715, -0.0010351817, 0.020009523, 0.046385713, -0.00987312, -0.009609837, 0.0063606827, -0.015294361, -0.00043980262, 0.03425075, -0.006462406, 0.023803195, 0.009250814, 0.0036739975, 0.012948747, 0.0066718357, 0.0067795427, -0.02508371, -0.016718483, -0.004792951, -0.02415025, 0.0170057, -0.008101942, 0.0027914003, -0.014887468, 0.0072761904, 0.012087093, 0.019064097, 0.019865915, -0.013044486, 0.009149091, 0.00139944, -0.028578196, -0.007868578, -0.014217293, -0.006486341, 0.0072283205, 0.02155332, 0.01056723, 0.0067974934, 0.015916666, 0.006659868, 0.034035336, 0.015066979, 0.0082335835, -0.022091854, 0.0038116227, -0.024138283, -0.028530326, -0.004131751, 0.02711817, -0.014875501, -0.011231422, -0.0047181547, 0.015880764, -0.012326441, 0.0016679589, 0.0026223606, -0.000822012, -0.0078087403, -0.011943484, 0.004050971, -0.02382713, -0.021792669, -0.011668233, 0.007820708, 0.020895112, -0.028267043, 0.00575932, 0.002386004, -0.012176848, -0.003329934, 0.015689285, -0.012326441, 0.009585902, 0.009107205, 0.015653383, 0.0044638473, 0.01284104, 0.010208208, 0.013355639, 0.009208929, -0.0019985589, -0.014420739, 0.012936779, -0.0024518247, -0.009208929, 0.032431703, 0.014169423, -0.014217293, -0.019853948, -0.00604953, 0.0044668387, 0.024365664, 0.013786466, -0.011985369, -0.0034885025, 0.017532269, -0.013044486, -0.0066778194, -0.015449937, -0.011410933, -0.008694329, -0.0061572366, 0.00896958, 0.005370379, 0.015677318, 0.039013784, 0.017340789, 0.029966416, -0.022426942, -0.010375752, -0.00021952733, 0.017568171, 0.0058999374, -0.0072761904, 0.007689066, 0.002942489, -0.0081857145, 0.010058615, 0.01719718, 0.012302506, -0.0031564066, 0.0056964913, -0.010435589, -0.0060614976, 0.008921711, -0.009944925, 0.03518421, -0.0019701363, -0.017233083, 0.013511215, 0.0120691415, 0.0016230812, -0.00012397497, 0.0039761746, 0.033173684, 0.007802757, -0.024186153, -0.0015722195, -0.0031324718, 0.008203666, 0.011638314, 0.0032012844, -0.012601691, 0.005864035, 0.028793609, -0.010621084, 0.01821441, -0.016012406, -0.024700752, -0.00964574, 0.003144439, -0.021158395, 0.018058835, -0.015270426, 0.03597406, 0.00065110234, 0.033724185, -0.014265163, -0.017855389, 0.0065940474, -0.015294361, 0.0082276, -0.003147431, -0.0035273966, -0.008772118, 0.0034017386, -0.012697431, -0.018142607, -0.036357015, 0.016251754, -0.003329934, 0.0029873669, -0.01793916, -0.02132594, 0.015581579, -0.006540194, -0.016562907, 0.013271867, -0.016670614, -0.00424245, -0.011207487, -0.015832894, -0.003249154, -0.0153302625, 0.01177594, 0.017843422, -0.0037936715, -0.015689285, -0.028793609, 0.011207487, -0.022187594, 0.010537311, -0.013930075, -0.020308709, 0.020715602, 0.015605514, 0.0062709274, 0.010172306, 0.012613659, -0.0067316727, 0.00088708487, -0.022426942, 0.009149091, -0.008030138, -0.011392983, -0.016227819, 0.020883145, 0.013068421, 0.00966369, 0.01543797, 0.0156174805, -0.022582518, 0.021146428, -0.009962875, -0.004568562, -0.018453758, -0.019818045, 0.009968859, 0.008532769, 0.030708395, -0.015402067, -0.010788628, -0.0007315084, 0.004766024, -0.00321026, -0.0040808897, -0.02303728, 0.0044159773, 0.006169204, 0.0059926845, 0.005657597, 0.012529887, -0.003644079, 0.011698151, 0.025107644, 0.024437468, -0.016371429, -0.016395364, 0.006576096, 0.012685464, -0.015880764, 0.0044578635, 0.003931297, -0.0059507987, -0.0037787124, 0.017998997, -0.0050472585, 0.0119973365, -0.013606954, 0.0074377507, 0.010866416, -0.028075563, 0.022558583, -0.02400664, -0.006600031, -0.016706515, -0.0016948857, 0.0180349, -0.0008451989, 0.008293421, 0.016156014, -0.016000438, 0.001974624, 0.023515977, -0.01200332, -0.007916447, -0.005487061, -0.023791227, -0.004481798, 0.008395144, 0.009705576, -0.011345113, -0.010148371, 0.011931516, -0.019806078, -0.006402569, -0.021613158, 0.021565288, 0.016012406, 0.0017726738, 0.0044997493, 0.0021571273, -0.03324549, 0.0027943922, 0.020129198, 0.012697431, -0.0042663845, -0.005905921, -0.01284104, 0.010884367, -0.010393702, 0.0012072133, -0.007940382, -0.010435589, -0.008670394, -0.017747682, 0.030708395, 0.009747462, 0.007401848, 0.0031354637, 0.0075155385, 0.0029469768, -0.0081617795, 0.0115665095, -0.022630388, -0.0069231517, -0.013319736, 0.009974843, 0.019482957, 0.015832894, -0.010674937, 0.00992099, -0.0153541975, 0.017161278, 0.018369988, 0.00017315359, 0.015964536, -0.013965977, 0.027213909, 0.0016395363, -0.0010591166, -0.017484399, 0.010890351, 0.0025670112, 0.011883646, 0.026711278, -0.007204386, 0.0069052004, -0.011105765, -0.018740976, 0.0032461623, 0.015078947, 0.008454981, -0.0000072400544, -0.015485839, 0.0054152566, -0.0155217415, -0.0021750783, -0.0058939536, 0.012374311, 0.025969299, -0.021481516, 0.015485839, -0.014420739, -0.0038176065, 0.013666792, -0.012482017, 0.003539364, 0.0028647007, 0.035136342, -0.0017951127, 0.001974624, 0.0001276213, 0.015892731, -0.0180349, 0.01184176, -0.029511653, 0.013032518, 0.0036680137, -0.0035214128, -0.005107096, 0.04231679, 0.007240288, 0.0122127505, 0.019028194, 0.01576109, 0.014253195, -0.0041885963, -0.007204386, 0.0041885963, 0.008885808, -0.0042723683, 0.009956892, 0.010040664, 0.0034675593, 0.010351817, -0.01497124, 0.015318296, 0.0015946585, 0.014576316, -0.017879322, 0.00748562, -0.0057114502, 0.008586623, -0.013571053, 0.013056453, -0.011488722, 0.0048886905, -0.008006203, 0.0048049185, 0.0005475094, 0.0056486214, 0.0020045426, -0.004335197, 0.012434147, 0.0222953, -0.0038116227, 0.007718985, 0.0044608554, -0.0029125705, 0.0061003915, 0.0020598918, -0.016371429, -0.013965977, 0.0029320174, -0.004912625, -0.04250827, 0.0051340223, 0.0035303885, -0.00039679473, -0.0033658363, -0.015282393, 0.023073183, -0.002179566, 0.0014338463, 0.0018998276, -0.022654323, 0.013020551, 0.008909743, -0.014935338, 0.0040060934, -0.00012855625, -0.014911403, 0.02415025, 0.012589724, -0.009370488, 0.009023434, -0.0048378287, -0.009609837, 0.009950908, -0.012170864, -0.00025655152, 0.008580639, -0.0007341263, -0.004562578, -0.0014525454, -0.014707957, 0.00094766996, 0.018800814, -0.015402067, 0.01992575, -0.010196241, -0.026735213, 0.008766134, 0.000113970964, 0.0047031953, 0.011608396, 0.030971678, -0.018932456, -0.013188095, -0.007551441, 0.0077608707, 0.0073898807, 0.009089254, 0.011829793, 0.005280623, 0.0021750783, -0.01348728, -0.0065521616, -0.011273308, -0.021206265, -0.011506673, -0.013032518, 0.009573935, 0.0064923246, -0.008951629, -0.014684022, 0.021577256, 0.0035214128, 0.01761604, 0.006378634, 0.009867136, 0.004571554, -0.028985087, -0.004631391, 0.009932958, -0.0075155385, 0.025801754, 0.007862594, -0.021421678, 0.01209906, -0.0068752817, -0.04408797, 0.01655094, -0.0013216517, 0.013319736, 0.004792951, 0.019267544, 0.010022713, 0.006863314, 0.011644298, -0.013618922, -0.0081857145, 0.014516478, -0.0035872336, -0.0050382833, 0.010375752, 0.01418139, -0.0030142935, -0.004538643, 0.010280012, -0.017520301, -0.019064097, -0.004398026, -0.01984198, -0.0051519736, 0.03147431, 0.0034226817, 0.0053225094, -0.018992294, 0.018501628, -0.0067974934, -0.0028557251, 0.012984649, 0.030492982, -0.027501127, -0.023013346, 0.0036799812, -0.0050322996, 0.016838158, 0.008095958, 0.0020195018, -0.009358522, 0.013523183, -0.01008255, 0.0004297051, -0.023300564, -0.015138784, 0.027333584, -0.02484436, -0.01914787, -0.020763472, 0.012912844, 0.031929072, 0.0017203164, -0.0025191416, 0.015964536, 0.020296741, 0.002202005, 0.0038415412, 0.00019484453, -0.006438471, 0.018800814, -0.0012423677, 0.00065596413, 0.013235965, 0.012900877, 0.016191917, 0.0061572366, 0.0053793546, -0.01789129, 0.00534046, -0.009573935, -0.013571053, 0.0029394971, 0.02656767, -0.0021556313, -0.009095238, 0.013331704, -0.006833396, -0.0039163376, -0.0071924184, -0.007138565, 0.0030053179, 0.006288878, -0.0010194745, 0.029559523, -0.016838158, -0.021302005, 0.01316416, 0.003473543, -0.011722086, -0.015677318, -0.01816654, 0.010543295, 0.002840766, -0.011129699, -0.0011757988, -0.001729292, 0.0035752663, -0.016538972, -0.007090695, 0.025729949, 0.019076064, -0.006971021, -0.0027510102, -0.004218515, 0.034992732, -0.0014637648, -0.020033458, 0.003991134, 0.032383833, -0.021266103, -0.007982268, 0.012625626, -0.001953681, 0.033867795, -0.0007120614, 0.011548558, 0.0012827576, 0.023515977, -0.0050921366, 0.0034914943, 0.006109367, 0.0026926692, -0.0036680137, -0.0079463655, 0.01766391, 0.0019910792, 0.00055985077, 0.015689285, 0.0013425947, 0.017795552, 0.016574875, -0.010411654, -0.009035401, -0.009926974, 0.015078947, 0.012757268, -0.020260839, -0.003844533, -0.005822149, -0.019231642, 0.014887468, 0.0035303885, -0.010537311, 0.02544273, -0.002342622, -0.00032947797, -0.0073360275, -0.019435087, 0.013295801, -0.0028258066, 0.0040031015, 0.015043044, -0.0013119283, 0.0077907895, 0.015258458, 0.013128258, 0.005902929, -0.020799372, -0.00021990131, 0.0017412594, -0.0030726346, -0.015785025, -0.0060764565, -0.009143108, -0.010112468, 0.0006488585, -0.01710144, 0.00057069625, 0.0017592105, 0.020775437, 0.021684961, 0.00321026, -0.010710839, 0.010830513, 0.013618922, 0.010166322, 0.0120691415, 0.0080421055, -0.009974843, -0.019830013, 0.008317356, 0.0033987467, 0.0002795514, 0.020871177, -0.0077309525, -0.0005822897, -0.015366165, 0.013223997, -0.002674718, 0.0003392015, -0.0023605733, 0.00089232065, 0.02002149, 0.0010329378, -0.009980827, 0.015832894, 0.0015961544, 0.013451378, -0.003413706, -0.0005288103, -0.0014428219, 0.0012939771, -0.006432487, 0.023252694, 0.0060196114, -0.015689285, 0.0169698, 0.0040599466, 0.013786466, 0.012625626, -0.005780263, 0.005765304, -0.008981547, -0.02419812, 0.010285996, -0.010495426, 0.013140226, -0.006767575, -0.0080421055, 0.0015916666, -0.019219674, -0.0017622024, -0.006402569, 0.011273308, -0.022558583, -0.018058835, -0.0055439067, 0.003889411, 0.016993733, -0.018178508, 0.0041048243, 0.009454261, 0.014863534, -0.0050951284, 0.003802647, -0.014767794, -0.005965758, 0.00987312, 0.034609772, -0.02647193, -0.006893233, 0.007431767, -0.008526785, -0.0117101185, -0.005762312, -0.016467167, -0.011542575, 0.015318296, 0.009202945, -0.0050951284, -0.0029365052, -0.0040270365, -0.00033078692, -0.022965476, 0.0062230574, -0.010363785, 0.020189034, -0.02702243, 0.02234317, -0.004712171, 0.0012505952, -0.014121554, 0.016718483, 0.016215852, -0.0018803806, 0.00018886082, 0.021337908, -0.0039971177, 0.009717544, 0.026447995, -0.0003603315, 0.03994724, -0.0039582234, -0.027692607, -0.0007524514, -0.0077070175, -0.030852005, 0.004655326, -0.005510996, -0.018238345, 0.019614598, 0.003024765, 0.012248653, 0.005427224, 0.019111967, 0.005196851, -0.012278571, 0.021242168, -0.00350047, -0.011763972, 0.0024039552, 0.020655764, -0.017532269, 0.0015333255, 0.0066179824, 0.010698872, -0.0050322996, 0.0023336466, 0.02632832, 0.015031077, -0.015210588, 0.015043044, -0.013355639, -0.010022713, 0.004502741, 0.01705357, 0.039971177, -0.0048408206, 0.0039402726, 0.018968359, 0.01497124, -0.018896554, -0.0069052004, 0.011327161, -0.002406947, -0.009298684, 0.019830013, 0.009119173, -0.012553822, 0.00019119821, 0.00987312, 0.008933677, -0.007898496, 0.01719718, -0.01135708, 0.0035124372, 0.0076471805, 0.0119973365, -0.0006410048, -0.010327882, -0.01068092, -0.00025168978, -0.018417858, -0.014540413, -0.00088184915, -0.014157456, -0.010507394, -0.025107644, -0.014073684, 0.000452144, -0.007234304, 0.042580076, -0.005570833, -0.009256798, 0.0047779917, -0.0046972116, 0.0027854166, -0.027357519, -0.022678258, 0.0056964913, 0.019315412, -0.004418969, 0.019123934, -0.027453257, -0.002079339, -0.011057895, 0.016060276, 0.023144986, 0.025227318, 0.002653775, 0.0009012962, -0.02553847, 0.0023471098, 0.023049248, 0.014360902, -0.021984147, 0.024772556, -0.007288158, 0.009430326, 0.0044309366, 0.013319736, -0.008006203, -0.015162719, 0.01427713, 0.020703634, -0.02081134, 0.007491604, 0.0008459469, 0.010465507, 0.011213471, 0.003991134, -0.020775437, -0.027261779, -0.008837938, -0.028697869, 0.005911905, -0.009202945, -0.019279512, -0.0018250312, 0.010046648, 0.00078087405, 0.013786466, -0.015605514, -0.0019252584, 0.024162218, -0.0024578085, 0.01831015, -0.009125156, -0.002281289, -0.0015871789, -0.011159617, 0.003126488, 0.03389173, 0.0075035715, -0.012541855, 0.006994956, -0.0025999215, 0.031833332, 0.0067855264, 0.003228211, 0.002221452, 0.035615038, -0.005902929, -0.0059777256, 0.005675548, 0.019387217, -0.013714662, -0.009208929, -0.00119375, 0.016203884, -0.012924812, -0.013930075, -0.007916447, -0.017280951, 0.0027719534, 0.004628399, -0.025227318, 0.011644298, -0.009944925, -0.028745739, -0.0016275689, -0.0006540942, -0.0038385494, -0.0015243499, -0.008748183, -0.013750563, 0.016395364, -0.017041603, -0.0026074012, 0.006623966, -0.008634493, -0.01808277, -0.016383396, -0.020763472, -0.0034914943, -0.003599201, -0.009005482, 0.01209906, -0.009693609, 0.04171842, -0.03334123, -0.020739537, 0.025394863, 0.016048308, -0.004448888, -0.008287437, -0.006007644, -0.00010733278, 0.003416698, -0.023695488, -0.025323058, -0.022199562, -0.014432707, 0.005902929, 0.013223997, 0.03587832, 0.0011039943, 0.0017532267, 0.007611278, -0.0013051966, 0.0008040609, -0.0047450815, -0.005304558, 0.0017711779, 0.008987531, -0.017400626, 0.0064504384, 0.01984198, 0.001132417, 0.041383334, -0.012805138, -0.022331202, 0.013176127, -0.007078728, -0.022929573, 0.013295801, 0.021074623, -0.010806579, 0.0062469924, -0.017065538, 0.031689722, -0.028889349, 0.008443014, -0.020141166, -0.008209649, 0.010280012, 0.0062828944, -0.017400626, 0.007096679, 0.008766134, -0.010639035, 0.0022289318, -0.018657206, -0.0085926065, 0.023946805, 0.016814223, 0.0026821976, 0.018752944, 0.024533208, 0.02211579, 0.022139724, -0.001750235, -0.005537923, -0.01710144, -0.01348728, -0.0008324835, 0.0038983866, 0.0079463655, 0.014217293, -0.0074497177, -0.013283835, 0.041479073, 0.016407331, 0.01997362, -0.000596501, -0.00522677, 0.016957833, -0.0040689223, -0.020978885, -0.003476535, -0.005681532, 0.020189034, -0.000761053, 0.026112907, -0.015629448, 0.013355639, -0.027501127, -0.016347494, -0.009035401, 0.006994956, -0.008107926, 0.0028228147, -0.0019521851, 0.008867857, 0.014660087, -0.00913114, 0.023719423, 0.0011361567, 0.0058999374, -0.016527005, 0.0075035715, 0.011464787, -0.020452319, -0.0054691103, -0.0063247806, -0.002611889, 0.012458082, 0.0016814222, -0.010740758, 0.022127757, 0.0070847115, -0.0069231517, 0.0169698, -0.015174686, 0.00006310943, 0.024628947, 0.0040629385, -0.004709179, 0.013223997, -0.029774936, -0.01798703, -0.003189317, -0.0020733553, -0.007629229, -0.012900877, 0.0075813597, 0.009591886, -0.003991134, 0.009029417, 0.005023324, -0.025729949, 0.005140006, 0.0012588229, 0.014169423, -0.005475094, 0.0249401, 0.021876441, 0.021493483, -0.0000093729195, -0.0050472585, -0.008221616, -0.010950188, -0.00022794193, 0.022845801, -0.0050143483, -0.009316635, -0.004709179, 0.0008122885, -0.0037158835, -0.01400188, 0.008078007, 0.00901745, -0.0010262061, 0.015593546, 0.00096188125, -0.019830013, 0.0112792915, -0.032120552, -0.009621805, 0.00790448, -0.010274028, -0.012057174, 0.0026851895, 0.015126817, -0.009077286, -0.0022319236, 0.011141666, 0.0030142935, 0.009621805, -0.0121409455, -0.005101112, -0.008347275, -0.016227819, 0.013678759, 0.0028467497, 0.009352538, -0.0017861372, 0.011045927, 0.00790448, 0.022151692, -0.023049248, 0.01905213, 0.0044309366, 0.01418139, 0.0008459469, 0.014265163, -0.007347995, -0.0030816102, 0.013977945, 0.013008583, -0.011105765, -0.0007584351, 0.0010665961, -0.016191917, 0.02276203, -0.021218233, -0.0121648805, -0.00992099, 0.014049749, 0.0084250625, 0.008101942, 0.02692669, -0.0080181705, 0.0000027990202, -0.0021661026, 0.0004895422 - ] - } - ] - }, - { - "name": "springfield-illinois-sixth-largest-city", - "title": "Springfield: Illinois' Sixth Largest City", - "content": "As of 2019, Springfield had a population of 10,566, making it the sixth most populous city in Illinois.", - "keywords": [ - "Springfield", - "population", - "10", - "566", - "sixth most populous city", - "Illinois", - "2019" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.012984714, 0.04321177, -0.0020374667, -0.02062091, 0.02117165, -0.0060951896, 0.000027884435, 0.004514465, -0.03594626, 0.02429603, -0.017475348, 0.017665988, -0.011523139, 0.017676579, -0.011131267, 0.0033044296, -0.023724109, 0.031031983, -0.00076256064, -0.0025299538, 0.0068577505, -0.023999479, -0.04482162, 0.018661553, 0.023639381, 0.011427819, -0.020938644, 0.010718213, -0.032853656, -0.0030793683, -0.022813274, -0.017560078, 0.06100485, 0.0028145902, 0.0050413734, -0.005692727, 0.0053114467, -0.029485678, 0.025397506, 0.030078782, 0.021341108, 0.009547895, 0.011332499, 0.0011895152, 0.037111286, -0.046177283, -0.012232744, -0.016479783, 0.017835446, 0.0076891533, 0.0097809, -0.031243805, -0.02440194, 0.039780248, -0.009738535, -0.029379768, 0.00081882597, 0.025376324, -0.003119085, -0.027028538, 0.024232483, -0.00260012, 0.0013980279, 0.0016667775, 0.0028675457, 0.0045038736, 0.003910771, -0.012328064, 0.07684917, -0.00032584244, 0.026837898, 0.041474827, 0.02222017, 0.011501957, 0.048549697, 0.009722648, -0.011565504, 0.00009771963, -0.028151197, 0.03204873, 0.00420997, -0.02249554, -0.004498578, -0.016490374, 0.02260145, 0.017570669, 0.02486795, -0.049524076, -0.00017293314, 0.0059045497, 0.04431325, -0.0105169825, 0.011332499, -0.026647259, -0.039377786, -0.006656519, 0.041729014, -0.0040537515, 0.013376585, 0.014414514, 0.0131223975, -0.004670684, -0.004440327, 0.02844775, -0.01418151, 0.015664266, -0.012126832, 0.012052694, -0.026414255, 0.010358116, 0.0105699375, -0.002214868, -0.022262534, 0.006481766, -0.012486931, -0.0020229039, -0.039653152, -0.01570663, 0.030523608, 0.028998487, 0.009214275, 0.032832474, -0.03545907, 0.016479783, -0.019455887, -0.0037598477, 0.015071164, -0.009235457, 0.0076520843, -0.028426567, 0.02155293, 0.015484218, -0.020176085, 0.008970679, 0.013376585, -0.041940834, 0.0023154838, 0.019710075, 0.027621642, 0.020451453, -0.06909647, -0.008404054, 0.021436427, 0.00012452841, -0.003227644, -0.018417958, 0.0105169825, 0.0045806593, 0.011692597, -0.014944071, -0.020546773, -0.020059582, -0.03204873, 0.0008419941, 0.004588603, 0.0425763, 0.0026040918, -0.008298143, 0.042004384, -0.053633433, -0.005213479, -0.023851203, 0.028574843, -0.03526843, -0.029295038, -0.012783482, -0.009833856, 0.009166614, 0.0017263526, -0.006444697, 0.025291596, -0.0062805344, -0.08477133, -0.044016697, -0.004535647, -0.0024491965, -0.019837169, -0.021775343, -0.01314358, 0.048761517, -0.008928315, -0.010930036, 0.019148745, -0.0040378645, -0.019064017, -0.021002192, 0.018312046, 0.007948636, 0.028045285, -0.020440862, 0.010887671, 0.041072365, 0.003370624, -0.0066882926, 0.044948716, 0.009569077, -0.0069318884, -0.015801951, -0.0053749937, 0.027007356, -0.04431325, 0.0015396841, -0.025694057, -0.01749653, 0.0075673554, 0.015293578, -0.0048454376, 0.010707622, 0.02760046, 0.024550216, 0.008292847, -0.0031932227, -0.036263995, 0.028151197, -0.0032435306, 0.024846768, -0.0041358327, 0.0066988836, -0.03395513, 0.020917462, 0.012497522, 0.042915218, -0.0050122477, -0.026964992, -0.014954662, -0.020599728, -0.044101425, 0.036052175, -0.025100956, -0.027918193, 0.028723119, 0.02751573, -0.009097773, 0.048083685, -0.00018848885, 0.044101425, 0.016331507, 0.005830412, -0.015018209, 0.03016351, 0.017316481, -0.0015794009, -0.00527173, -0.009251343, 0.0029655136, -0.014785204, 0.037577294, 0.006164032, 0.023469923, 0.022728544, 0.0033520896, -0.009637919, 0.051006835, -0.01823791, 0.0010915473, 0.00531939, -0.0039451923, 0.017644806, -0.0040007955, -0.0030184693, -0.023003913, -0.014234466, -0.024380758, 0.039928522, -0.003611572, -0.012614024, 0.021860072, 0.014075599, 0.020281995, -0.029612772, 0.0072019617, -0.011936192, 0.020451453, 0.017062295, 0.0010419014, 0.027685188, 0.039271872, -0.0011822338, -0.020080764, -0.019053426, -0.02440194, 0.0368571, 0.0064129233, -0.021055147, 0.011205405, -0.0021076328, -0.01703052, 0.0118726455, 0.0010816181, -0.0130906245, -0.012137424, -0.0060951896, 0.04206793, 0.018079042, -0.008827698, -0.017422393, 0.0072390307, -0.0019355272, -0.039886158, 0.033489123, 0.028002921, 0.044270884, -0.028490113, -0.022580268, 0.0177719, -0.03223937, 0.02050441, -0.01938175, -0.034971878, -0.050922107, -0.013514269, -0.047109302, -0.004691866, -0.001457603, 0.013853185, -0.006836568, 0.012730527, 0.00770504, 0.028299473, -0.018693328, -0.0023260748, 0.015759587, 0.03147681, 0.013641363, -0.0015635141, 0.050879743, -0.005427949, 0.0041120024, 0.021987166, 0.007932749, -0.02429603, 0.027240362, -0.014764021, -0.03120144, 0.026922628, 0.013874368, 0.04774477, 0.013567225, -0.0068577505, 0.035776805, -0.00765738, 0.015674857, 0.017930767, 0.038424585, 0.0099185845, -0.016278552, -0.023279281, 0.01063878, -0.016183231, 0.041834924, 0.00040312452, 0.018068451, 0.051769394, 0.008928315, 0.0049195755, -0.00893361, -0.016426828, -0.0027431, 0.019392341, -0.052065946, -0.017072886, 0.018259091, -0.024550216, 0.0044244407, -0.041538373, 0.03130735, -0.00068047945, 0.01883101, 0.03715365, -0.0080280695, 0.0058092293, 0.016437419, -0.030989617, 0.018936923, 0.0041199457, 0.018206136, -0.016098503, 0.010840012, -0.0077685867, -0.01342954, 0.020663276, 0.023046277, -0.0027483958, -0.010013904, -0.007058982, -0.0043555982, -0.005830412, 0.0049513485, -0.04111473, 0.022453174, 0.02675317, -0.043847237, 0.006910706, 0.015187667, 0.02089628, 0.018386185, -0.030099964, -0.013832003, 0.018703919, 0.0011299402, 0.023088641, -0.031370897, 0.0024822939, 0.014033235, -0.0017170854, -0.00044879873, 0.0011226587, -0.047998957, 0.0054332446, -0.023385193, -0.0065400167, 0.018746283, 0.018576825, -0.005756274, -0.02296155, -0.03384922, -0.019731257, 0.039165962, -0.0015873441, 0.009050112, 0.0057350914, -0.043635417, 0.01210565, 0.00058284256, 0.0011590658, 0.004840142, -0.0118408725, 0.011321908, -0.01465811, -0.03571326, -0.0017965188, 0.009224866, -0.018936923, -0.02402066, -0.016013773, 0.054480724, -0.054057077, -0.012306882, 0.052701414, 0.020366725, -0.028786665, 0.012222153, 0.029633954, -0.029549226, -0.001841531, 0.026880262, 0.0117455525, 0.033997495, 0.018386185, 0.044397976, -0.022262534, 0.015950227, 0.027197996, 0.013683727, 0.029739866, -0.0059151407, -0.026075337, 0.012963532, -0.012677571, 0.00902893, 0.037979756, -0.023618199, 0.017697763, -0.017401211, -0.040902905, 0.04168665, 0.0425763, 0.009187797, -0.0026808772, -0.007810951, -0.012497522, -0.013418949, -0.05481964, 0.03272656, -0.03302311, 0.015823133, 0.013556634, 0.021510566, 0.01730589, -0.016331507, -0.027918193, 0.021574112, -0.057319142, 0.0028966714, -0.023512287, 0.044567432, -0.018862786, 0.0027642823, 0.0155901285, -0.022431992, 0.04511817, -0.0009101744, -0.01266698, -0.013556634, 0.012338655, -0.006873637, -0.029189127, -0.04605019, 0.03346794, 0.045245264, 0.002613359, 0.01891574, 0.021192832, 0.028829029, -0.03024824, -0.011067721, -0.024338394, -0.041326553, -0.0008764152, 0.013313038, 0.070028484, 0.020779777, -0.030375334, -0.023364011, 0.017898994, -0.0064129233, 0.009134841, -0.019402932, 0.021404654, 0.001984511, -0.017602442, 0.022855638, 0.00964851, 0.023236917, -0.029104399, 0.040627535, -0.00093863806, 0.0055285646, 0.041051183, -0.010125111, -0.03632754, 0.00016010796, 0.0067359526, 0.0201549, 0.04897334, -0.027219178, -0.016437419, -0.040225074, -0.012889394, 0.030502426, -0.0033150206, -0.06896938, 0.0029099102, 0.031879272, 0.007021913, 0.0030555383, -0.007450853, -0.024190119, 0.00204541, -0.023406375, 0.038128033, -0.011427819, 0.0070642773, -0.0017541543, -0.016977565, 0.005205536, -0.0025842334, -0.012232744, -0.032747746, -0.030460062, 0.0072866906, 0.0022334026, 0.014891115, -0.009833856, 0.0070113214, 0.025863515, 0.0030396516, -0.0048666196, 0.00902893, 0.017369438, 0.017835446, 0.026986174, 0.029909324, -0.010739396, 0.009712057, 0.0010558023, 0.0099503575, -0.019551208, 0.004006091, -0.0063758544, -0.0027047072, 0.017263526, 0.031497993, 0.022198988, -0.006391741, 0.0109724, 0.0018918389, -0.03128617, -0.0117561435, 0.059691552, 0.045711275, 0.017136432, 0.0030846638, 0.007932749, -0.0075144, 0.012084468, -0.013207127, 0.0070007304, -0.027536912, -0.062191058, 0.021129284, -0.0077262223, 0.006825977, -0.014869933, -0.010464027, -0.007069573, 0.0033070773, 0.0072284397, 0.01816377, -0.020398498, -0.014795795, 0.006608859, 0.011237179, -0.011967965, -0.042682216, 0.0118302815, 0.007837429, 0.019710075, 0.007318464, -0.008737674, 0.0049698832, -0.029337402, -0.032874838, -0.0034500575, 0.021796525, -0.0142768305, 0.0060475296, -0.0029046147, 0.003669823, -0.0074561485, -0.0023909453, 0.02817238, 0.0021883901, -0.0029549226, -0.004620376, -0.004403258, -0.0016707492, 0.011279543, -0.03537434, 0.018682737, 0.012868211, 0.0614285, -0.0007440262, 0.03262065, -0.0054914956, -0.0014655463, -0.037683208, 0.003818099, -0.01087708, 0.0050122477, 0.017009338, -0.024783222, 0.0023882976, -0.004249687, -0.011035947, 0.024656128, 0.007429671, -0.022728544, 0.01521944, 0.0002454161, -0.037344288, -0.00032336015, 0.0036142198, 0.0029866959, 0.0007923482, 0.03319257, -0.030099964, 0.028426567, 0.038657587, 0.00037466089, -0.0039425446, 0.028659571, -0.016045548, 0.024889132, -0.0072602127, 0.018746283, -0.013842594, -0.005420006, -0.018015495, -0.0025736422, -0.010649371, -0.0075302864, -0.0012325416, -0.021436427, 0.0048083686, -0.025270414, 0.047575314, 0.028214743, 0.0070007304, 0.006148145, -0.000068180336, 0.025545781, -0.012433975, -0.0006778317, 0.033912767, -0.009103068, 0.00024839488, 0.02760046, 0.006937184, 0.00049182517, -0.008525852, 0.020271404, 0.003526843, -0.037132468, 0.00382869, 0.00016068715, -0.00089759746, -0.031031983, -0.054311264, -0.0027536913, -0.008065138, 0.011364272, 0.013715501, -0.03194282, 0.0555822, -0.042131476, 0.026181249, -0.02628716, 0.0026874968, 0.026837898, 0.021256378, 0.016215006, -0.055666927, 0.00084133213, -0.018365003, 0.007911567, -0.019519435, -0.029422132, 0.018693328, 0.011163041, -0.026160067, 0.021711797, 0.029718684, -0.0035877419, 0.0021817707, 0.008970679, -0.0034156363, -0.03141326, 0.029676318, -0.0021274912, -0.012921167, -0.029083217, 0.03024824, 0.01626796, 0.0074084885, 0.007964523, -0.0012888069, -0.010707622, -0.065410756, -0.0037572, -0.023597015, -0.020218449, 0.04431325, 0.028807847, -0.005046669, -0.010109224, -0.006645928, 0.0057880473, 0.01399087, 0.040500443, 0.022855638, -0.0019447944, 0.009208979, -0.00013478856, 0.00760972, -0.001432449, 0.004180845, -0.010607007, -0.023364011, 0.006015756, 0.040013254, -0.007069573, -0.015886681, 0.00043655274, 0.030205876, -0.0010432253, 0.0031455627, -0.020292586, 0.024846768, -0.017348254, 0.027897011, -0.005017543, 0.016310325, 0.004143776, -0.0015277691, -0.015494809, -0.006116372, -0.016564513, -0.012592842, -0.021987166, -0.008118094, -0.011501957, -0.047575314, -0.0008942877, 0.011470183, -0.0055444515, -0.010840012, -0.032218188, 0.060072836, 0.010241613, 0.0063652634, 0.020186676, 0.0021129285, 0.016003182, -0.03668764, -0.0019553856, 0.008345802, 0.0010915473, -0.006031643, 0.013779047, 0.020207858, -0.035395525, 0.020970417, -0.017380029, -0.040796995, -0.01711525, 0.01928643, -0.024317212, -0.012222153, -0.03395513, 0.017231753, -0.026498983, 0.02383002, -0.0142980125, 0.015876088, -0.00043853858, 0.015198258, -0.008954792, 0.011967965, 0.02050441, -0.016130276, 0.0117773255, -0.02296155, -0.016712788, -0.004710401, 0.025969427, 0.009113659, 0.0264778, 0.019064017, 0.0044959304, 0.0020056935, -0.0118302815, 0.0155159915, -0.017507121, 0.024825586, 0.027155632, 0.0042073224, -0.03620045, 0.034357592, -0.017602442, -0.03859404, 0.008822403, -0.017337663, 0.0255246, -0.02467731, 0.040140346, 0.030523608, 0.0017223809, -0.017846037, 0.0050493167, 0.009722648, 0.0040749335, 0.021224605, -0.021341108, 0.0007334351, 0.009547895, 0.013937914, 0.004297347, -0.021860072, -0.0118514635, 0.0077685867, 0.0044853394, -0.00046170666, -0.026731987, 0.014689884, 0.015272396, -0.001229232, 0.006788908, -0.028829029, 0.025736421, -0.027346272, -0.009897402, -0.017898994, -0.023639381, 0.012656389, 0.009521417, -0.022919184, -0.03035415, -0.0048851543, 0.0067730215, 0.01201033, 0.0053882324, 0.015007617, 0.002658371, 0.0059839827, 0.007069573, -0.017898994, -0.025651693, 0.00818164, -0.0069054104, 0.019159336, -0.0031985184, 0.0010280006, -0.019392341, -0.012328064, 0.021775343, 0.028214743, 0.0068577505, -0.014816977, 0.010278682, -0.030375334, 0.019244066, 0.0024015366, -0.016564513, -0.0065082433, 0.023533469, -0.019116972, -0.008504669, 0.0032753039, 0.015399489, 0.023321647, -0.00031607875, -0.030078782, 0.0075144, 0.011682006, 0.0064023323, 0.011046538, -0.035988625, -0.0067942035, -0.00845701, 0.023597015, 0.015664266, 0.004501226, 0.0020414384, -0.007980409, -0.025842333, -0.027388636, 0.0040511033, -0.015727814, 0.03999207, -0.013662545, -0.0017290004, -0.019116972, 0.012391611, -0.030121146, 0.03668764, -0.032112278, 0.016935201, 0.037068922, -0.002010989, -0.0038472244, 0.014170919, 0.0080386605, 0.008938906, 0.032091096, 0.030693067, -0.0143191945, 0.02260145, -0.012063286, 0.02128815, 0.010384593, 0.008250482, -0.039081234, -0.0027139746, 0.008843585, -0.0118514635, -0.030693067, -0.046092555, -0.009214275, 0.010824125, 0.016437419, -0.0073978975, 0.016204413, -0.009394323, 0.0036433453, 0.03300193, -0.015071164, 0.0018243204, 0.052065946, 0.03545907, 0.026096521, 0.0031455627, 0.0006056797, 0.049693536, 0.02628716, 0.0010028468, -0.013916732, -0.009865629, 0.0003634078, 0.00026113732, 0.03772557, -0.008483487, -0.010564642, 0.00077116594, 0.0580817, -0.016956383, -0.0076414933, 0.006921297, -0.012868211, 0.029167945, 0.03725956, 0.012921167, -0.028977305, -0.0055656335, -0.00027867884, 0.02062091, -0.03753493, -0.02081155, -0.02817238, 0.0075673554, 0.01625737, -0.02675317, 0.025588147, -0.039822612, -0.0014536312, -0.027918193, -0.01995367, -0.00818164, -0.0033362028, -0.025291596, 0.019455887, 0.004143776, 0.00021629053, -0.012275108, -0.0055550425, -0.010103929, 0.016373873, -0.014192101, 0.00026113732, 0.05151521, 0.017613033, -0.019064017, -0.0070960503, 0.002394917, -0.023025095, -0.01579136, -0.0020374667, 0.045075808, 0.022559086, 0.014742839, 0.0012451186, -0.026456619, 0.032366462, -0.0043873717, 0.0023882976, -0.016151458, -0.008176344, 0.013005896, 0.03158272, 0.047575314, -0.0036433453, 0.011999739, -0.009727944, -0.0038233944, -0.028511295, -0.0015926397, 0.031180257, 0.022834456, -0.005512678, -0.01918052, -0.0016270608, -0.015113529, -0.0070748683, -0.0034871264, 0.02599061, -0.0069636614, -0.015145302, 0.016956383, -0.027875828, -0.0037598477, 0.0008234596, -0.013863777, 0.02429603, -0.014573381, 0.018587416, -0.00041139883, -0.00529556, -0.019053426, -0.010358116, -0.020101946, 0.018693328, 0.0092672305, -0.011682006, 0.015886681, -0.0022069246, -0.015876088, 0.015134711, 0.0061587365, 0.0024995045, 0.0016416237, -0.009886811, -0.013546043, 0.023236917, 0.009600851, 0.039970886, -0.012359837, 0.03130735, 0.033997495, 0.0016932554, 0.003516252, -0.0055179736, 0.008107503, 0.004323825, -0.05320979, 0.0080916155, 0.013418949, -0.047448218, -0.023766473, -0.056556582, -0.009373141, -0.014816977, 0.01266698, -0.033912767, 0.0007599129, 0.017676579, 0.023109823, -0.017485939, -0.012423384, -0.0036142198, -0.027092086, -0.021256378, -0.0080810245, -0.028892577, -0.010109224, 0.03505661, -0.0027589868, -0.012719936, -0.009770309, 0.014213284, -0.002683525, -0.020906871, 0.009293708, 0.0020665922, -0.048210777, 0.021860072, 0.01115245, 0.0062805344, -0.0041570147, 0.021118693, -0.006359968, 0.018545052, 0.0024862655, 0.002359172, -0.008901836, 0.0005368374, 0.00022075867, 0.021383472, -0.01984776, 0.008345802, -0.017464757, -0.036624093, 0.005197592, 0.011491366, -0.008615877, 0.0008916399, -0.02155293, 0.01314358, 0.00012006028, -0.02249554, 0.022728544, -0.021139875, -0.010405775, -0.009579668, 0.015166484, 0.01314358, 0.011035947, -0.0027669303, 0.011671415, 0.010035086, 0.03168863, 0.004429736, 0.0030370038, -0.02222017, -0.025757605, 0.0118302815, 0.0032938384, 0.00902893, -0.014149737, 0.00062653096, 0.004668036, -0.0117243705, 0.002201629, -0.011184223, 0.013387176, -0.014689884, -0.018280273, -0.011258361, -0.007953932, -0.030587155, -0.0024002127, 0.006836568, -0.02912558, 0.026202431, 0.025461053, -0.0118302815, -0.010501096, -0.004850733, 0.00089892134, 0.013270673, 0.020790368, -0.026435437, 0.016278552, -0.016522147, 0.0007413784, 0.0093201855, -0.0024491965, 0.0066353367, 0.0038842934, -0.0040511033, -0.025100956, 0.0054094144, 0.007344942, 0.010273387, -0.0063017164, 0.017994313, -0.0052770255, -0.00041636341, 0.025418688, 0.047448218, 0.025545781, -0.008096911, 0.0037598477, -0.028087651, 0.011989148, 0.018788647, 0.0065611987, -0.028341837, 0.007583242, -0.025164502, -0.007832133, -0.016522147, 0.015166484, -0.010326343, 0.018788647, 0.012486931, -0.020070173, -0.014922888, 0.024338394, 0.008266369, 0.0097809, 0.015039391, -0.011544321, 0.021266969, -0.0019553856, -0.0016548625, 0.028723119, -0.044906348, -0.021923618, 0.006116372, -0.03969552, 0.013376585, 0.013132989, 0.0011511224, -0.019625345, -0.0342305, -0.0009915936, -0.011565504, 0.0059310272, -0.022368446, -0.020313768, 0.017824855, 0.0015661619, -0.009404914, 0.005597407, 0.003021117, 0.015918454, 0.0032832474, 0.0017250287, 0.02088569, 0.005028134, 0.007853315, 0.0127623, -0.025969427, -0.023300465, 0.03836104, 0.023236917, -0.004620376, -0.0022347264, 0.03346794, 0.015632493, 0.0021023373, -0.01124777, 0.0008486135, 0.011046538, -0.009706762, -0.006201101, 0.018703919, -0.012169197, 0.025736421, 0.011650233, 0.027282726, 0.010199249, 0.012539886, -0.002458464, -0.026160067, -0.012709344, 0.024232483, -0.04045808, -0.008488783, -0.017613033, 0.018184954, -0.024995044, 0.003092607, -0.006455288, -0.017793082, -0.009627328, -0.015240622, 0.011523139, -0.003937249, 0.00527173, 0.023469923, 0.0142980125, -0.034421142, -0.0130694425, 0.0028860802, -0.0015079108, 0.0131118065, 0.0009843123, -0.004021978, 0.0062805344, -0.04075463, -0.008986565, -0.0143509675, -0.01210565, 0.006677701, -0.0008386843, 0.016204413, 0.014202693, -0.009664397, 0.010776465, 0.0087429695, 0.012349246, -0.012285699, -0.007667971, -0.010982991, -0.017824855, 0.014192101, -0.020101946, 0.05049846, -0.006148145, 0.007863907, 0.008806516, -0.0000075658245, -0.0042152656, -0.00024210638, -0.001709142, 0.00044515802, -0.021669433, 0.0043052905, -0.018566234, -0.0029310926, 0.011586686, -0.021203423, 0.017983722, -0.032493558, -0.002336666, -0.010283978, -0.005316742, 0.02885021, 0.007117233, 0.017380029, 0.023914749, 0.026922628, -0.016056139, -0.0009062027, -0.013948506, -0.004180845, -0.015282987, 0.0072655086, -0.020440862, 0.030502426, 0.022389628, 0.015261804, 0.018661553, -0.0012305558, 0.0073820106, -0.007810951, -0.029803412, -0.0023287225, -0.004392667, -0.014393332, -0.0051552276, -0.0030158216, 0.006995435, 0.015939636, 0.014605155, 0.041856106, 0.014583972, -0.00945787, -0.0010697031, 0.018852195, 0.0105699375, 0.006921297, 0.013207127, -0.020589137, 0.017951949, 0.0025445167, -0.012804665, 0.003910771, -0.016162049, -0.009394323, -0.023364011, -0.004480044, -0.004657445, 0.053929985, -0.0028490114, 0.020462044, -0.015611311, -0.0059628007, 0.012052694, 0.0039081234, -0.019604163, 0.01437215, -0.03243001, -0.0020599728, -0.0038339854, -0.004413849, 0.012518704, 0.0059151407, -0.014626337, 0.014065008, -0.013789639, -0.008303438, 0.014615746, 0.0017144376, -0.031137893, -0.015357125, -0.015727814, 0.0010683793, -0.0019037538, 0.0012199647, 0.008949497, -0.007969818, -0.0087111965, 0.0008942877, 0.015049982, 0.008435828, 0.0049910652, -0.011205405, 0.009484348, -0.0047818907, 0.008435828, -0.005234661, 0.02069505, -0.0117985075, 0.013969688, -0.024571398, -0.03289602, -0.02440194, -0.02081155, -0.023427557, 0.0045806593, 0.00864765, 0.00072019617, 0.03338321, 0.004265574, 0.010363411, 0.0037757345, -0.0032223484, 0.0007605748, 0.01664924, -0.0081022065, 0.0009353283, 0.006863046, -0.008944201, 0.008578807, -0.0080386605, 0.013694318, -0.018280273, 0.011936192, -0.016988156, -0.005698023, 0.01797313, 0.027261544, -0.0019328794, 0.03082016, 0.035416707, 0.006370559, -0.030099964, -0.015420671, -0.01153373, 0.02393593, 0.012974123, 0.007339646, 0.0155795375, 0.031540357, -0.010866489, -0.004668036, 0.019127563, 0.020928053, -0.015388898, 0.008478192, 0.0310108, -0.023491105, -0.008218709, -0.007524991, 0.023491105, 0.015230031, 0.0025365732, 0.024529034, -0.010787056, 0.019127563, -0.02062091, -0.0050731464, -0.02118224, 0.0029708091, 0.012709344, -0.030947253, -0.0027828168, -0.005465018, 0.02429603, 0.020599728, -0.0033547373, 0.02836302, -0.037895028, 0.0040616947, 0.004247039, -0.0005729134, 0.034844786, -0.0055656335, 0.021595294, 0.06930829, 0.02940095, -0.014838159, 0.024889132, -0.003296486, -0.0150288, 0.018110815, -0.008160458, -0.015378307, -0.008817107, 0.02268618, -0.018280273, -0.0033123728, -0.0072019617, 0.0016416237, 0.002540545, 0.007991, -0.01937116, -0.0019659768, -0.010231022, 0.0072866906, 0.02183889, -0.0036009809, -0.0028198857, 0.01797313, -0.00085523294, -0.011618459, -0.025863515, -0.010199249, -0.033531487, -0.006052825, 0.008287552, 0.008224005, -0.004429736, 0.017125841, 0.007498513, 0.028765483, -0.007943341, 0.0020612967, -0.009351959, 0.0143297855, -0.005213479, 0.014414514, -0.0019487661, 0.0019183167, 0.0032435306, -0.023215735, 0.013005896, -0.008975974, 0.013757865, 0.003841929, 0.010607007, -0.0057986383, -0.016553922, 0.0019236122, -0.008123389, 0.00031872652, -0.0048216074, 0.010326343, -0.0069318884, -0.004202027, -0.020938644, -0.026371889, 0.021023374, 0.0038710544, -0.0046600928, -0.010511687, -0.0007062953, 0.007535582, -0.007885089, 0.0049645877, 0.0026782295, 0.022855638, -0.018142588, -0.009505531, 0.015198258, -0.011660824, 0.00081882597, -0.00964851, -0.005698023, 0.028341837, -0.047787134, -0.008165753, -0.005946914, -0.010336934, 0.010331638, -0.010675849, -0.0035003652, -0.0010736748, 0.0011219968, -0.0058886628, 0.0105381645, 0.012063286, -0.004247039, 0.0071490062, -0.0010041706, -0.011904419, -0.0026663146, 0.008986565, -0.013757865, -0.010808238, -0.011639641, 0.035035424, -0.025736421, 0.012433975, -0.003068777, -0.005851594, 0.005830412, 0.013736683, 0.0058092293, 0.016585695, 0.01078176, 0.00252863, -0.0014284773, -0.007006026, 0.018365003, -0.007318464, 0.011078312, -0.011321908, 0.03166745, -0.008345802, -0.020123128, 0.03344676, 0.016871655, -0.0076150154, 0.013514269, -0.008462305, 0.0035692076, -0.011607868, 0.0058674808, -0.017877812, 0.014075599, 0.02249554, 0.0027483958, -0.019064017, 0.0003564574, -0.017009338, -0.0011564179, -0.01578077, 0.011406637, 0.005597407, -0.0027245656, -0.013959097, -0.0031879272, 0.023978297, 0.016013773, -0.018365003, -0.027452184, -0.012169197, 0.012222153, -0.00007839911, 0.014605155, -0.0056874314, -0.0038101554, 0.012614024, -0.01937116, -0.00042894037, 0.01785663, -0.011279543, 0.008234596, -0.006370559, 0.030989617, 0.005465018, 0.013662545, 0.007186075, -0.0056768404, -0.0068577505, -0.008017478, -0.019837169, 0.0039319536, -0.008202823, 0.0027245656, -0.009246048, -0.009833856, 0.0080810245, 0.0038948844, -0.024995044, -0.00066657865, -0.026604895, -0.0017475348, -0.004442975, 0.0005560338, -0.014912297, -0.007879794, -0.0255246, -0.01910638, 0.02108692, 0.00931489, 0.003119085, -0.0076520843, -0.006513539, -0.0072655086, -0.002925797, 0.016183231, 0.022559086, 0.022241352, 0.017814264, 0.00404316, 0.0035903899, 0.013260082, -0.014583972, -0.013588407, 0.025482235, 0.01446747, -0.005777456, -0.024910314, -0.0028331247, -0.0014430401, -0.037174832, -0.021711797, 0.0016402997, -0.004284108, -0.013927323, -0.015049982, -0.010887671, 0.009664397, -0.0059522097, -0.007196666, 0.016966974, -0.007630902, 0.0017740126, 0.013842594, -0.0014549551, -0.028320655, 0.030142328, 0.005708614, -0.010797647, 0.0064341053, -0.0024253665, -0.0007235059, 0.011025356, 0.007413784, 0.0057245004, -0.002527306, -0.02334283, -0.026075337, 0.026604895, 0.01816377, 0.033997495, -0.0071595972, 0.008197527, -0.0013113131, 0.010024495, 0.013217718, 0.006296421, 0.022326082, -0.0023287225, 0.03213346, 0.0048692673, 0.008472896, 0.031222623, 0.0015754291, 0.020091355, 0.00489045, 0.005597407, 0.0056768404, 0.0065611987, -0.021214014, 0.02666844, -0.033129025, 0.011035947, -0.0051446366, -0.0036274586, -0.03213346, 0.0073025776, -0.022792092, -0.013387176, -0.0004478058, 0.018470913, 0.004525056, -0.00067386, 0.01804727, -0.023575833, 0.0069530704, -0.012868211, -0.0014734896, 0.013874368, 0.013270673, 0.039547242, -0.01389555, -0.0071119373, 0.0009207655, -0.018481504, -0.026435437, 0.011194814, -0.0013993517, -0.0010895615, -0.022559086, 0.02402066, -0.0027669303, -0.008801221, 0.005560338, -0.017549487, 0.01465811, 0.009627328, 0.013821412, -0.023618199, -0.017284708, 0.016130276, -0.0006682335, -0.0038472244, -0.03770439, -0.003383863, 0.021669433, -0.013588407, -0.0037889732, 0.00041735632, -0.010215135, -0.033404395, 0.01579136, -0.000018658575, 0.0010902234, 0.008827698, -0.018682737, -0.015431263, 0.0051949443, -0.008499374, 0.006767726, -0.018131997, 0.013323629, 0.004143776, -0.008382872, 0.009765013, 0.007890385, 0.0018322638, 0.0046521495, -0.002973457, -0.012656389, 0.018862786, 0.0092566395, 0.013980279, 0.012444566, -0.014266239, -0.0036512888, -0.0015674857, -0.0064499923, 0.0074084885, 0.012232744, 0.003031708, 0.007953932, -0.012825847, -0.022198988, -0.011936192, -0.005030782, -0.009616737, 0.016575104, -0.0046865707, 0.020303177, -0.03704774, 0.0012669627, -0.016744561, 0.0023485809, -0.0077950642, 0.009907993, -0.0063652634, 0.010522278, -0.014859342, -0.0019911306, -0.0059257317, 0.0034844785, -0.001865361, -0.015251213, 0.010755283, -0.01059112, 0.025821151, -0.017665988, 0.021214014, 0.031159075, 0.007058982, -0.0130906245, -0.0073237596, -0.0143297855, -0.016098503, 0.014170919, -0.014764021, -0.013969688, 0.012084468, -0.0038022122, -0.013408358, -0.0044376794, -0.0037757345, -0.0010902234, -0.019657118, -0.01984776, 0.01220097, 0.006063416, 0.001432449, 0.0019011061, 0.0059839827, -0.022919184, 0.0025829095, -0.033595033, 0.0009982131, 0.0012987361, 0.016246779, -0.00034454238, 0.02459258, 0.0063864454, 0.015335942, 0.014541608, -0.003479183, 0.004758061, -0.027282726, 0.0012782158, 0.012370428, -0.008382872, 0.0023499047, -0.017676579, 0.03035415, 0.009643215, -0.0040828767, 0.0065188343, 0.0009472433, 0.019011062, -0.009373141, -0.019233475, 0.022940366, -0.028384201, 0.0131012155, 0.010765874, -0.004755413, 0.014255648, 0.00079631986, -0.007053686, -0.00416231, 0.0050704987, 0.005539156, -0.028426567, -0.02070564, -0.0077474043, -0.00409082, -0.0013437484, -0.01749653, -0.00267161, -0.010283978, 0.012794074, 0.001191501, 0.0005666249, 0.0264778, -0.012158606, 0.008960088, -0.017051704, -0.018365003, 0.006269943, -0.011311316, 0.009669693, 0.010246909, 0.0031323237, -0.0035718554, 0.038424585, -0.0029072624, 0.0092990035, 0.031625085, 0.003179984, -0.0070272083, -0.0023393137, 0.008472896, -0.0031932227, -0.0064129233, -0.018026087, 0.017051704, -0.009966244, -0.00047726237, 0.005075794, 0.006492357, -0.019159336, 0.005356459, -0.009293708, -0.0009081886, 0.0069636614, -0.0076256064, 0.0013887606, 0.014382741, -0.011163041, -0.015071164, 0.0061428496, -0.007890385, 0.009441984, -0.0014602507, -0.0065664947, -0.023236917, 0.0045700683, 0.017814264, -0.0066512236, -0.007021913, 0.013291856, -0.0017303242, -0.0043873717, 0.0012358513, 0.008949497, -0.012232744, 0.002095718, 0.01730589, -0.01446747, -0.006275239, 0.0010412395, 0.0007532934, 0.04255512, -0.0038075077, -0.0018521221, -0.027176814, -0.013906141, 0.012931758, 0.02402066, -0.016130276, 0.0031031983, 0.004286756, 0.0076785623, -0.0028463635, -0.0032488261, -0.017930767, -0.016183231, -0.004623024, -0.010125111, 0.0024730265, 0.01797313, 0.004877211, 0.03141326, 0.011809099, 0.0075196954, -0.010013904, -0.024931498, 0.0014681941, 0.0009929176, 0.0019911306, -0.029528042, -0.016405646, 0.009452575, -0.00024889133, 0.01153373, 0.026541347, -0.013461314, 0.012116241, 0.0080598425, -0.011586686, -0.014615746, 0.016988156, 0.00015357124, 0.029083217, 0.001552923, -0.009473757, 0.032154642, 0.0223049, -0.02959159, 0.011268952, 0.013461314, -0.002128815, 0.0155054, 0.0010789704, 0.0009942414, 0.015166484, 0.0032223484, 0.0073820106, 0.012518704, 0.00010425634, 0.015166484, 0.0200384, 0.0015515991, 0.016236188, -0.022156624, -0.013726092, -0.01823791, 0.0030475948, -0.006137554, 0.00907659, -0.031434447, 0.018470913, 0.007535582, 0.016045548, -0.016659832, 0.015049982, -0.0033653285, -0.0031084938, -0.009118955, -0.014425105, 0.00071225286, -0.0070166173, 0.037471384, -0.02127756, -0.001576753, -0.02514332, 0.028638389, 0.009060703, -0.00489045, -0.02041968, -0.024041843, 0.020080764, -0.009839151, -0.020726822, 0.016606877, -0.0027510435, 0.0013463962, -0.015177075, -0.020472635, 0.006836568, -0.021531748, 0.007874498, 0.028468931, 0.0071331193, 0.0046971617, -0.0155159915, 0.0066724056, -0.005655658, 0.00936255, -0.0018746282, -0.026032973, 0.0034738875, 0.020758595, 0.015367716, -0.011565504, 0.010580529, 0.0032779516, 0.0025312777, -0.016034955, 0.008329916, -0.0012146691, -0.016797516, 0.007911567, 0.006042234, 0.010797647, -0.008605286, 0.015494809, 0.008329916, -0.014541608, 0.03147681, 0.004935462, -0.0040246258, -0.02080096, -0.024423122, 0.01295294, 0.009892106, 0.013874368, 0.01976303, -0.0164586, 0.012137424, -0.01626796, -0.023279281, 0.0018163772, -0.0026755817, 0.012243335, 0.004585955, -0.0020387906, 0.017644806, 0.021574112, 0.006026347, 0.006344081, 0.015441854, 0.019053426, -0.020186676, -0.020673867, 0.029019669, 0.023702927, -0.0021010134, 0.017369438, 0.0071119373, 0.0059522097, -0.002155293, 0.014054417, 0.0025193626, 0.0013927323, 0.002888728, 0.00038459006, 0.000111785965, -0.023639381, 0.013260082, -0.029655136, -0.023300465, -0.020599728, 0.010644075, 0.009765013, 0.024359576, 0.013365993, 0.018216727, -0.026986174, 0.0022664997, 0.018481504, -0.0067571346, 0.013281264, -0.0010187334, -0.011025356, 0.011226587, -0.0052558435, -0.012137424, 0.0028357725, -0.0030714248, 0.008414645, -0.018587416, 0.0041199457, -0.017613033, 0.016818698, 0.009765013, 0.011205405, 0.01995367, 0.0025114194, -0.02978223, 0.0039531356, 0.026964992, -0.007344942, -0.0000709936, -0.026138885, -0.0029496269, 0.0006291787, -0.0026279218, 0.0066353367, -0.00056033646, 0.010167476, -0.03243001, -0.016352689, 0.029019669, 0.006502948, 0.0043503027, -0.024973862, 0.0067200656, 0.008631763, 0.0027642823, 0.0117667345, -0.026371889, -0.005131398, -0.018693328, 0.010982991, -0.002216192, 0.023914749, -0.0034447617, -0.005404119, -0.02666844, 0.01054346, -0.0016628059, -0.0080916155, 0.008054547, -0.013853185, 0.017210571, -0.0022122203, -0.002070564, 0.006116372, 0.0065982677, 0.014901706, -0.02155293, 0.009616737, -0.021711797, -0.0021169002, -0.01597141, 0.012402202, -0.00081882597, 0.007906271, -0.0068206815, -0.000005848905, -0.01352486, 0.019593572, -0.026096521, -0.008684719, 0.005994574, 0.04414379, 0.01040048, -0.024656128, 0.017507121, -0.02061032, -0.011353681, 0.03715365, -0.0080386605, 0.0008995833, -0.00195009, 0.023893567, 0.0019514139, -0.0038445767, 0.0023909453, 0.008822403, 0.0118302815, -0.008292847, -0.002106309, 0.014382741, 0.014117964, 0.01683988, -0.008832994, 0.036708824, -0.0055444515, 0.009489643, 0.008303438, -0.00385252, 0.014573381, 0.027473366, -0.007858612, -0.000037979094, 0.024550216, 0.002445225, -0.004302643, 0.013027078, -0.002803999, 0.0057033184, 0.0032011662, 0.0118832365, -0.0015383603, 0.01711525, 0.0016217653, 0.028468931, 0.0013199183, -0.0070960503, -0.0014218579, 0.019212293, -0.009373141, 0.00512875, -0.014403923, 0.002394917, 0.004358246, 0.000698352, 0.021510566, -0.001770041, -0.0067836125, 0.01333422, -0.018407367, 0.018947514, 0.019604163, -0.001492024, 0.031116711, -0.0025776138, 0.008562921, -0.0015568946, 0.011163041, -0.014732248, -0.03573444, -0.0044006105, 0.0036380498, -0.030078782, 0.018470913, -0.016204413, 0.021542339, -0.0011537701, -0.008261073, 0.010130406, -0.003468592, 0.019307612, 0.023279281, 0.009907993, 0.005216127, 0.019699484, -0.03412459, 0.015685448, 0.008472896, -0.014234466, 0.015674857, -0.0018759521, 0.0030608338, 0.0072125527, -0.014838159, -0.005719205, 0.018365003, 0.005179058, 0.004498578, 0.019169927, -0.011544321, -0.0039187144, -0.007127824, 0.006677701, 0.0008797249, -0.012815256, -0.019349977, -0.0042311526, -0.014149737, -0.014827568, 0.020398498, 0.016215006, -0.0029522749, -0.0066829966, -0.019868942, 0.007991, -0.015049982, -0.009484348, -0.007964523, -0.00014496596, 0.012836438, -0.02156352, -0.020165494, -0.004104059, -0.0033388506, -0.03035415, -0.025079772, -0.004585955, 0.000007720968, 0.012328064, -0.0018097577, 0.0071648927, 0.010940627, 0.040119164, 0.008372281, 0.0080386605, 0.00780036, 0.00075263146, 0.0026689623, 0.03658173, -0.00012105319, 0.004779243, -0.013609589, -0.028765483, 0.024444306, -0.009208979, -0.015388898, -0.010262796, -0.0063652634, 0.012868211, -0.008054547, 0.024105389, -0.012878803, -0.008822403, 0.006481766, -0.005075794, -0.015240622, 0.010246909, -0.008435828, -0.0017713648, 0.0055921115, 0.029845776, 0.008239891, 0.012772891, 0.021192832, -0.014403923, -0.010622893, -0.00936255, 0.00025716564, 0.019455887, 0.027812282, -0.006148145, 0.009622033, -0.0130800335, 0.029443314, -0.00921957, 0.024380758, 0.0130800335, 0.026160067, -0.010744692, -0.018936923, -0.012328064, 0.007832133, -0.0008439799, -0.0022228113, -0.021097511, 0.01030516, -0.00271927, 0.0027669303, 0.0020626206, -0.010193953, -0.021542339, 0.01692461, -0.0032726561, -0.025376324, 0.019222884, 0.04020389, 0.025778787, 0.01257166, -0.005851594, 0.012306882, 0.018407367, 0.011215996, 0.0044932826, 0.017147023, -0.012042103, 0.02081155, 0.015357125, -0.002780169, 0.0037148355, 0.010654667, 0.009817968, 0.011300725, 0.00082875515, -0.0030846638, -0.0086900145, -0.002817238, -0.028405385, -0.022792092, 0.030481244, 0.0007043095, 0.0143297855, 0.004705105, -0.014668701, 0.0071490062, -0.016966974, -0.0041464237, -0.0029602181, -0.013376585, -0.017464757, 0.03564971, -0.0016005831, -0.023639381, 0.020176085, 0.007069573, -0.022135442, -0.025418688, -0.0042073224, -0.000039944243, -0.00095651054, -0.0030264126, 0.0044482704, -0.013440131, 0.0092672305, -0.021923618, -0.00013718811, 0.013757865, 0.014043826, -0.01910638, 0.0035083087, -0.015166484, 0.018121406, 0.010093338, -0.029612772, -0.0018759521, 0.018672144, -0.012031512, -0.02959159, 0.014689884, -0.010278682, 0.024444306, -0.008155162, -0.009579668, 0.0027854645, 0.015092346, 0.01257166, 0.015441854, 0.029252674, -0.0012795397, -0.015039391, 0.0132389, -0.0070007304, 0.0035586164, 0.0045488863, 0.015897272, -0.008462305, 0.009426097, -0.003720131, -0.016903428, 0.0006271929, -0.0071384152, 0.0155689465, -0.0011478127, -0.0029814004, 0.0071437107, -0.010003313, -0.01541008, 0.01852387, -0.0011286163, -0.022982731, 0.023491105, 0.018322637, -0.011449001, -0.015007617, -0.02950686, 0.0038737021, -0.009611442, -0.019741848, 0.01948766, 0.009701466, 0.0026914685, 0.00009498911, 0.0030846638, 0.002602768, -0.01597141, -0.022792092, 0.01044814, 0.0017184092, -0.010797647, 0.0024266904, -0.0130906245, -0.008001591, 0.024317212, 0.0015118824, 0.0035453774, 0.0078056557, 0.040013254, 0.0013781695, 0.01570663, 0.009394323, -0.009913288, -0.007816247, 0.0067200656, 0.01871451, 0.019890124, -0.01778249, 0.008340507, 0.0080280695, -0.013355402, 0.0027047072, -0.010125111, -0.03793739, 0.004321177, -0.0045594773, 0.0059628007, 0.0025246583, -0.0074932175, -0.0018084338, 0.009600851, 0.031921636, 0.011131267, 0.0010392537, 0.00661945, 0.0059151407, 0.011321908, -0.0073978975, -0.013927323, 0.009150728, -0.023745291, 0.007032504, 0.005719205, -0.013842594, -0.008149867, 0.000987622, -0.002708679, 0.014096781, 0.015155893, -0.00065797335, 0.0015780769, -0.017538896, -0.018131997, 0.0066724056, -0.0130800335, -0.0038445767, 0.013514269, -0.007842724, -0.014202693, -0.020313768, -0.020387907, -0.0015423319, -0.01162905, -0.027282726, -0.00019212953, -0.0037280743, 0.011512548, 0.009367846, -0.007810951, 0.01710466, -0.008282256, 0.013514269, 0.004487987, -0.0038233944, 0.022749726, -0.013217718, 0.016797516, 0.0068206815, -0.02459258, 0.013514269, 0.011046538, -0.0078056557, -0.012317473, 0.0131118065, -0.017189387, 0.00067121227, -0.0016164697, 0.002310188, -0.014816977, 0.0001792216, 0.0067571346, -0.0059733917, -0.0013066794, 0.009701466, 0.007858612, 0.024126573, -0.0069901394, 0.007922158, 0.008912427, 0.006322899, -0.011459592, 0.017263526, 0.006269943, -0.026520165, 0.017528305, 0.026986174, 0.0039346013, 0.0077474043, 0.016617468, 0.011120676, 0.04196202, -0.002010989, -0.022664998, -0.020197267, -0.0044588614, -0.013418949, -0.003142915, 0.0063387854, 0.008176344, 0.009283117, -0.021012783, 0.008107503, 0.0025021522, -0.009807377, -0.008684719, -0.014149737, 0.014912297, -0.011396046, -0.012328064, 0.0039796135, 0.008176344, 0.00012709344, 0.018333228, 0.0059045497, -0.0034209318, 0.0063387854, 0.02467731, 0.008467601, 0.017983722, -0.019466478, 0.022834456, -0.0052320133, -0.0059204362, -0.0013861129, 0.020567955, 0.04217384, -0.0052531958, 0.013588407, 0.024613764, 0.009945062, -0.004683923, 0.018449731, 0.014361559, -0.00623817, -0.018502686, 0.0049963607, 0.005454427, -0.012529295, 0.008441123, 0.018598007, 0.01257166, -0.0012126833, 0.014255648, -0.016469192, -0.0048348466, 0.01918052, -0.009955653, -0.006370559, -0.003929306, -0.0065506077, 0.012497522, 0.0052479003, -0.01778249, -0.011131267, -0.004127889, 0.0009141461, -0.032874838, -0.006322899, -0.00066955737, 0.001984511, 0.013408358, -0.002841068, -0.011989148, -0.011014765, 0.0019342033, 0.004104059, -0.020599728, -0.023639381, -0.0048004254, 0.023131007, -0.007117233, -0.0047262874, -0.007572651, 0.011279543, 0.0037254265, 0.018206136, -0.0037836777, 0.006825977, 0.0043264725, -0.010220431, -0.025884697, -0.0039902045, 0.030671883, 0.010395184, -0.0019911306, -0.005682136, -0.0037598477, 0.020070173, 0.007080164, 0.0025100955, -0.0016839881, -0.008197527, 0.0072125527, 0.01804727, -0.009007747, 0.0018891911, -0.013906141, -0.0013291856, 0.012550478, 0.0021499975, -0.017210571, -0.010575234, 0.010151588, -0.03505661, -0.001625737, -0.00060932036, -0.00041437757, -0.004069638, 0.011025356, 0.0077579957, 0.0058727763, -0.008504669, -0.002682201, 0.01938175, 0.012518704, 0.02599061, -0.0099715395, -0.015081755, 0.017210571, -0.009584963, -0.0043635415, 0.0053882324, 0.007667971, -0.006863046, -0.014997026, 0.0044588614, 0.0017620976, 0.008769448, 0.0006341433, -0.027303908, 0.010479913, -0.0013305094, 0.008075729, -0.00013983589, 0.0038631111, 0.004755413, 0.0002477329, -0.019710075, 0.016945792, -0.0010604359, -0.022664998, -0.0131012155, -0.027070904, -0.0072655086, 0.02041968, -0.014954662, -0.009436688, -0.024147755, -0.012878803, 0.0030025826, -0.0031826317, 0.0024121276, 0.016405646, 0.011268952, -0.015198258, 0.01408619, -0.013037669, -0.027494548, 0.0015741052, 0.0028066467, -0.021192832, -0.01672338, -0.035988625, -0.0018825716, -0.0019368511, -0.014033235, -0.0040193303, -0.0118408725, 0.048337873, -0.01823791, -0.00931489, 0.0109724, 0.00053220376, -0.004358246, -0.01229629, 0.005692727, 0.004887802, 0.0070483903, -0.023872385, -0.019604163, -0.026541347, -0.0040166825, -0.010252205, 0.003958431, 0.010728804, 0.0064976523, 0.0065241302, -0.019699484, -0.0017885754, -0.012592842, 0.0059257317, -0.014764021, 0.010342229, 0.0195618, -0.007058982, -0.0099291755, -0.0049672355, 0.006042234, 0.033700943, -0.0155477645, -0.011989148, 0.009542599, 0.012158606, -0.0020864506, -0.0069265924, 0.015134711, 0.007291986, 0.022643816, -0.01475343, 0.000053245203, 0.0017501826, 0.0033494418, -0.017803673, -0.018322637, 0.014922888, 0.0012821875, 0.009097773, -0.010140997, -0.013175353, 0.000041371564, 0.02694381, -0.027197996, -0.0025723183, 0.02579997, 0.03384922, 0.01710466, 0.019466478, 0.02089628, 0.006397037, 0.01663865, -0.0032594171, -0.011078312, -0.009018339, -0.0074720355, -0.0022082485, -0.00539088, 0.010368707, -0.0093307765, -0.017443575, -0.0024836177, -0.0039187144, 0.008917723, 0.0143509675, 0.013249491, -0.004993713, 0.023554651, 0.012317473, -0.027769918, 0.031031983, -0.0118408725, 0.0073714196, -0.023194553, 0.015685448, 0.023681745, 0.01541008, -0.0051843533, -0.024889132, -0.013673136, 0.008404054, 0.0025233345, -0.007440262, -0.0035480252, 0.015081755, 0.0003961741, 0.00035579543, 0.014075599, -0.009976835, 0.010024495, -0.015748996, 0.008292847, 0.007344942, -0.022008348, -0.0010286625, -0.0064023323, -0.014065008, 0.013270673, 0.035628527, -0.01162905, 0.0017793082, 0.037217196, 0.009383732, 0.029104399, 0.008065138, -0.014456879, 0.019868942, 0.0074932175, -0.0028092947, 0.020133719, -0.0010299864, -0.007524991, 0.010426958, -0.009283117, -0.009690875, 0.009706762, -0.0013609589, -0.0013490439, -0.023088641, 0.01778249, -0.007551469, -0.017252935, -0.012137424, -0.0063493764, -0.016511556, -0.009018339, 0.013376585, 0.019932488, 0.017676579, 0.011215996, 0.0075091044, 0.010712918, -0.00510492, -0.007927453, -0.006359968, -0.0104958005, 0.0005070499, -0.026562529, 0.0052849688, 0.0025087716, -0.012063286, 0.013567225, 0.015960818, -0.0015357124, 0.0053008557, -0.008499374, -0.008192232, 0.006370559, -0.020281995, 0.015876088, 0.005586816, -0.0036486408, 0.0051552276, -0.010140997, 0.0011570799, 0.007599129, -0.013037669, 0.021923618, 0.010204544, 0.012179788, -0.0037545522, -0.012836438, -0.020176085, 0.009463166, 0.034971878, -0.006222283, 0.0155054, -0.005851594, 0.011353681, 0.009261935, 0.0076256064, -0.026223613, 0.016553922, 0.00888595, 0.015484218, -0.0073343506, -0.012444566, 0.01257166, -0.01843914, 0.024973862, 0.019508844, 0.024910314, 0.008096911, -0.015325351, -0.0043608937, 0.002156617, -0.02904085, 0.006492357, 0.00070364756, 0.027897011, -0.011449001, 0.0039822613, 0.0067942035, -0.014626337, 0.0105063915, -0.0059204362, 0.004562125 - ] - } - ] - } + { + "name": "springfield-illinois-sangamon-county-seat", + "title": "Springfield: Illinois' Sangamon County Seat", + "content": "---\n\nSpringfield is the county seat of Sangamon County, located in the U.S. state of Illinois.", + "keywords": [ + "Springfield", + "county seat", + "Sangamon County", + "Illinois", + "United States" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.03425075, 0.036835715, -0.0047420897, -0.01650307, -0.016335526, + -0.023121051, -0.0037727286, -0.0065581454, -0.03973183, 0.0018070801, + -0.008377193, 0.021445613, -0.018022932, -0.0042693764, 0.009579918, + -0.010297963, -0.013068421, 0.015940601, -0.0019432096, -0.0073539787, + 0.04102431, 0.004221507, -0.05011955, 0.047079824, -0.0009072799, + 0.013511215, -0.04844411, 0.016814223, -0.02558634, 0.011303226, + -0.019123934, -0.009436309, 0.02197218, 0.022283332, -0.00955, + -0.02637619, -0.00906532, -0.016933898, 0.0044399123, 0.019399185, + -0.0073599624, -0.018058835, 0.0028512373, 0.007347995, 0.021062655, + 0.0017861372, 0.01714931, -0.03578258, -0.005478086, 0.003970191, + 0.020380514, -0.0067615914, 0.005510996, 0.02687882, 0.02140971, + 0.012948747, 0.0008489387, 0.033173684, -0.008299405, -0.045260776, + -0.021804636, -0.008269486, -0.040665288, 0.0038116227, 0.014336967, + 0.01200332, 0.0005220786, -0.0015587562, 0.050598245, 0.005202835, + 0.029583458, 0.02294154, -0.0066119987, 0.037194736, 0.048635587, + -0.010561246, -0.031498246, -0.014073684, -0.018429825, 0.029056892, + 0.039516415, -0.04023446, -0.0055798087, 0.022331202, 0.009466228, + -0.02642406, 0.010016729, -0.059358396, -0.009723527, -0.0009357025, + 0.015749123, -0.018250313, -0.021242168, -0.02243891, -0.024389599, + -0.031019548, 0.059262656, -0.012900877, 0.010734774, 0.035591103, + 0.012864974, -0.009466228, -0.053326815, -0.010052632, 0.009424342, + 0.006235025, 0.009137124, -0.022965476, -0.011626347, 0.008556704, + 0.025323058, 0.013810401, 0.00042259946, -0.020380514, -0.0074138157, + -0.004766024, -0.009023434, -0.028027693, 0.039324936, 0.019159837, + -0.0072103697, 0.00053442, -0.015282393, -0.001687406, -0.009400407, + 0.039157394, 0.02011723, -0.039157394, 0.019674435, -0.016993733, + 0.025610276, -0.00456557, 0.009801316, 0.015593546, 0.011404949, + -0.02271416, 0.012230702, 0.049401503, 0.021768734, 0.023707455, + -0.057539348, 0.011668233, 0.01571322, -0.008999499, 0.023934837, + -0.015796993, -0.017568171, -0.009286717, -0.012733333, -0.031091353, + -0.01775965, -0.014157456, -0.026208647, 0.010764693, 0.025251253, + 0.006402569, -0.017352756, -0.043681078, 0.04978446, -0.0498802, + -0.013678759, -0.008173747, 0.03949248, 0.009747462, -0.007383897, + -0.012601691, -0.028578196, -0.00004270405, -0.00567854, 0.0010224663, + 0.035710778, -0.0016889019, -0.015366165, -0.08645263, 0.003847525, + -0.0333891, 0.012625626, -0.024820426, 0.00625896, 0.043465663, + 0.0005658345, 0.02127807, 0.026184712, -0.016311591, -0.009202945, + -0.022809898, -0.013678759, 0.029176567, 0.032216292, -0.019674435, + -0.016754385, 0.030421177, 0.032048747, 0.02100282, 0.030277569, + -0.011267324, -0.03973183, -0.034705512, 0.011805858, 0.02197218, + -0.051412027, 0.011375031, 0.0112553565, -0.012122995, 0.013606954, + 0.025921429, -0.0023022322, -0.038032457, 0.008742199, 0.0403302, + 0.004398026, -0.003784696, -0.021397743, 0.06883659, -0.01209906, + 0.02405451, -0.015066979, -0.015964536, -0.031139223, 0.039205264, + 0.01464812, 0.022043984, -0.001913291, -0.000095599105, -0.011213471, + 0.00563067, -0.055050123, -0.027261779, 0.015162719, -0.0077309525, + 0.021158395, -0.001358302, 0.001442074, 0.050646115, -0.046648998, + 0.04588308, 0.020667732, 0.0023620692, -0.04696015, 0.023348434, + 0.01714931, 0.031546116, -0.005229762, -0.023156954, -0.019016229, + -0.009472212, 0.014085651, 0.00059388316, -0.009442293, 0.0154978065, + 0.0034047305, 0.010160338, 0.050502505, 0.015318296, -0.0053195176, + 0.012027255, -0.016862093, 0.0077907895, 0.01072879, 0.029631328, + -0.0001544545, 0.021577256, -0.019913783, 0.021302005, 0.013307769, + -0.037912782, -0.008628509, -0.0009214912, 0.01242218, -0.03980363, + -0.014097619, -0.0038714597, 0.0038535087, 0.019638533, 0.029774936, + 0.011578477, 0.030469047, 0.01497124, -0.032694988, -0.020548057, + -0.024078446, -0.009029417, 0.0039432645, -0.026447995, 0.025993232, + 0.002610393, -0.009364505, 0.04112005, 0.0035752663, -0.027572932, + 0.010549279, -0.0206797, 0.025275188, 0.02697456, 0.045763407, + 0.014348934, 0.015976503, 0.0042065475, -0.034801252, 0.004110808, + 0.03389173, 0.033915665, 0.008987531, -0.009603853, 0.0270703, + -0.008754167, 0.019877883, 0.0066479007, -0.043585338, -0.055002254, + 0.010190257, -0.03389173, 0.003353869, 0.01566535, -0.017400626, + 0.016323559, 0.005119063, -0.0030202772, 0.075538345, -0.031953007, + -0.022857768, 0.016215852, 0.031857267, 0.00226633, -0.01343941, + 0.02558634, -0.0072702067, -0.0025475642, 0.009639756, 0.009436309, + -0.02850639, 0.0036799812, -0.012218734, -0.04167055, -0.009310652, + -0.026136842, 0.0265198, -0.009777381, 0.018070802, 0.0057563283, + -0.009149091, -0.01460025, -0.0065521616, 0.03350877, 0.026400125, + 0.014755826, -0.0067915097, -0.010525345, -0.024269924, -0.0010890351, + 0.0005239485, 0.011668233, 0.021026755, -0.020224936, 0.029152632, + 0.0080421055, -0.04325025, -0.039612155, 0.015952569, -0.018190475, + -0.028243108, 0.007407832, -0.021589223, 0.00003340125, -0.0012326441, + 0.04013872, -0.0270703, 0.016263722, 0.030205764, 0.0008302397, + -0.010926253, 0.019111967, 0.013654824, -0.009514097, 0.01321203, + 0.0012490993, -0.00885589, 0.012458082, 0.0017203164, -0.0021870458, + 0.037242606, 0.023695488, -0.020751504, -0.014408772, -0.014815664, + -0.030469047, -0.0022453868, -0.049305763, -0.027740475, 0.011135682, + -0.021098558, -0.016814223, 0.009029417, -0.03384386, -0.017831454, + 0.023216791, -0.07290551, 0.007982268, 0.05394912, 0.035615038, + 0.016634712, -0.04432732, 0.011883646, 0.01650307, -0.03425075, + -0.027166039, -0.009232863, -0.056103256, 0.010848464, -0.010830513, + -0.023839097, 0.009657706, 0.0047779917, -0.01103396, -0.0050981203, + -0.046194233, -0.024174185, 0.03341303, 0.023527944, 0.0058999374, + 0.01492337, -0.02563421, -0.001666463, -0.024772556, -0.019662468, + 0.024964035, 0.0015333255, 0.031761527, -0.04236466, -0.026807018, + -0.030756265, -0.006276911, -0.006863314, 0.020871177, -0.0061632204, + 0.05988496, 0.0006338992, -0.0018968357, 0.03087594, 0.027477192, + -0.010806579, 0.014360902, 0.042699747, -0.03712293, -0.0033209587, + 0.009807299, 0.032072682, 0.002409939, 0.020883145, 0.044758145, + -0.01033985, 0.016311591, -0.0030965696, 0.015485839, 0.008030138, + 0.03425075, -0.040306266, 0.01469599, -0.0222953, 0.040521678, + 0.03082807, -0.008586623, -0.0016335526, -0.03370025, -0.01714931, + 0.010166322, 0.02924837, 0.004652334, -0.00014314154, -0.01919574, + -0.00966369, -0.03142644, -0.031211028, 0.006833396, -0.042699747, + -0.021122493, 0.00063913496, 0.0053554196, 0.009490163, 0.0033778038, + -0.046984084, -0.0007445978, -0.069123805, 0.009406391, 0.014253195, + 0.0061572366, -0.0292005, -0.046553258, 0.01659881, -0.012338408, + 0.09966466, 0.020751504, -0.009059336, -0.010812563, 0.020296741, + 0.020404449, -0.02943985, -0.024437468, 0.04107218, 0.040593483, + -0.0085926065, 0.054906514, 0.038367543, -0.037434086, -0.0080421055, + 0.004389051, -0.023384335, -0.034801252, 0.013379574, 0.02006936, + 0.050502505, 0.00033564868, -0.032240223, -0.01974624, 0.010028697, + -0.024605012, -0.011686184, -0.0024952067, 0.03671604, 0.0056635807, + 0.00080181705, 0.009143108, 0.003288048, -0.0008751175, -0.003557315, + 0.012769235, 0.005026316, -0.006151253, -0.0009708568, -0.029559523, + -0.035615038, -0.010764693, -0.000013346476, 0.031641856, 0.035638973, + -0.008778102, -0.013223997, -0.003964207, -0.009113189, 0.022450877, + -0.030995613, -0.029224435, 0.008520802, 0.011895614, 0.006438471, + 0.007090695, 0.0028317904, 0.0012244165, 0.0028557251, -0.021373808, + 0.04265188, -0.0031324718, -0.0013590499, -0.003476535, 0.0029753994, + -0.0051938593, -0.010579198, -0.015402067, -0.0074197995, + -0.006600031, 0.012924812, 0.011249374, 0.022690225, 0.01942312, + 0.012493985, 0.0036351033, 0.00038333138, -0.005777271, -0.008072023, + 0.0063606827, 0.010154354, 0.00026571407, 0.0419099, 0.0077429195, + 0.025107644, 0.0050682016, -0.028985087, 0.0043740915, 0.0009573935, + -0.0066778194, 0.009376472, 0.00538833, 0.021804636, 0.02132594, + 0.013092356, 0.017137343, -0.010806579, -0.028769674, -0.02340827, + 0.027381454, 0.017125376, 0.008251535, -0.0052776313, -0.0022154683, + -0.012864974, -0.0063247806, -0.020691667, 0.005373371, -0.024581077, + -0.018106705, 0.022199562, 0.005346444, 0.014839599, -0.043681078, + -0.017843422, -0.002053908, 0.032671053, 0.0040360116, 0.024509273, + 0.006486341, -0.0036949404, 0.017651942, -0.016814223, -0.0067974934, + -0.01188963, 0.0019835995, 0.03137857, 0.034968797, 0.015952569, + -0.017819487, -0.005166933, -0.017520301, -0.008209649, -0.00908327, + 0.017113408, -0.01919574, 0.01501911, -0.024605012, -0.019782143, + 0.009897055, -0.015844861, 0.0036111686, -0.021206265, 0.008975564, + 0.013726628, -0.013798433, -0.0046014725, -0.003189317, -0.009514097, + 0.0020344611, 0.017568171, 0.040402003, 0.026830953, 0.023611717, + -0.02085921, -0.0015752114, -0.010764693, 0.038965914, -0.015916666, + -0.0051280386, 0.057730827, -0.017699812, -0.0045236843, + -0.000005852231, 0.011668233, 0.013606954, 0.0009319627, -0.025394863, + 0.030205764, -0.010656986, -0.02656767, 0.00987312, -0.0004902902, + 0.018501628, 0.03958822, 0.007754887, -0.036955386, 0.05849674, + 0.017639974, 0.00082350796, 0.0077070175, 0.014025814, 0.002736051, + -0.010070583, -0.015689285, 0.0026298403, -0.0042244988, 0.001585683, + -0.010645019, 0.001563244, -0.010309931, -0.012709398, -0.0084849, + -0.021948244, 0.017687844, -0.030660525, 0.024652882, 0.034561902, + -0.013403509, -0.024389599, 0.0055468986, 0.02544273, 0.023875, + -0.0087840855, 0.02647193, 0.0018684132, 0.005540915, 0.042077444, + -0.006151253, -0.0077070175, 0.0032581296, 0.014025814, 0.02841065, + -0.04710376, 0.009047369, 0.025179448, -0.01029198, -0.022773998, + -0.04533258, -0.012057174, 0.003787688, -0.0026014175, 0.017376691, + -0.006276911, 0.039061654, -0.04277155, 0.04277155, -0.04157481, + -0.007814724, 0.039468545, 0.026041102, 0.014289098, -0.03621341, + -0.019770175, -0.021146428, 0.005905921, -0.042244986, -0.01040567, + 0.016574875, 0.0061572366, -0.037960652, -0.0049814377, -0.006067481, + -0.016993733, 0.020344611, 0.00906532, -0.013846303, -0.02979887, + 0.00038015252, 0.004628399, -0.00085866224, -0.0039402726, 0.05083759, + -0.0047630323, -0.00559776, 0.021481516, -0.000905036, 0.0014839598, + -0.04399223, 0.0270703, -0.01831015, -0.0023815162, 0.035997994, + 0.011039943, 0.012864974, 0.012948747, 0.0156174805, 0.009442293, + 0.024365664, 0.04454273, 0.011416917, 0.016407331, 0.0066359336, + 0.0055498905, 0.014492543, 0.022367105, 0.006091416, 0.0074497177, + 0.006025595, 0.004030028, 0.029176567, 0.0049305763, -0.0016485119, + 0.0021945254, 0.021696929, 0.014324999, 0.0117101185, -0.0050682016, + 0.0050083646, -0.021110525, 0.05011955, -0.003787688, -0.019506892, + -0.016000438, 0.006917168, -0.018920489, -0.01956673, 0.006971021, + 0.0127453, -0.0073120925, -0.010603133, -0.004338189, -0.032551378, + 0.017879322, 0.001706853, -0.023635652, 0.0075813597, -0.024964035, + 0.030708395, 0.013188095, 0.005534931, 0.013942042, 0.0117579885, + 0.0034406327, 0.0008698817, 0.00030965696, -0.0040210527, 0.02563421, + -0.035088472, 0.039564285, 0.040545613, -0.02145758, 0.03647669, + -0.0019073073, 0.007491604, -0.0075813597, 0.028027693, -0.010656986, + 0.005534931, -0.01279317, 0.028985087, -0.026352255, 0.030133959, + -0.017376691, 0.00074160594, 0.01868114, 0.0034885025, -0.03410714, + -0.004215523, 0.014217293, 0.0035722745, -0.008999499, -0.02090708, + -0.013918107, 0.0014981711, 0.0016844141, 0.0292005, -0.0028063597, + 0.031211028, 0.010531328, -0.013774498, -0.010555263, 0.022199562, + -0.012613659, 0.012362343, 0.053278945, 0.013283835, -0.037577692, + 0.00827547, -0.060650878, -0.019088032, 0.015928634, -0.026687343, + 0.023432204, -0.0249401, 0.004816886, 0.043417793, 0.018046867, + 0.005537923, 0.016167982, 0.0042334744, -0.008610558, 0.043537468, + 0.014241228, 0.021254135, 0.01761604, 0.021349873, -0.017508334, + -0.003024765, -0.009238847, 0.014624185, -0.006378634, -0.021637091, + 0.00456557, -0.0077907895, -0.0030786183, -0.010758709, -0.018298183, + -0.0047510653, 0.01172807, -0.040928572, -0.005140006, -0.020966917, + -0.028649999, 0.020691667, 0.032479573, -0.028290978, -0.013606954, + 0.016658647, -0.03583045, -0.0016200893, -0.0080421055, 0.008443014, + -0.0060465382, 0.012673496, 0.0117101185, -0.021792669, -0.023623684, + -0.008359241, 0.031402506, 0.017879322, -0.0043471646, 0.009633772, + -0.025131578, 0.0023156954, 0.0018564458, 0.032623183, -0.008413095, + -0.0011959939, 0.012015288, 0.0056276782, 0.032503508, -0.013989912, + -0.026950626, -0.008586623, 0.010262061, -0.030062154, 0.007814724, + 0.003907362, 0.03355664, 0.031929072, 0.009125156, -0.011219455, + -0.009089254, -0.0032341948, -0.008742199, 0.026112907, -0.019554762, + -0.022450877, -0.0027270755, -0.0022528665, -0.001831015, + -0.009137124, 0.003844533, 0.03003822, -0.009220895, -0.04116792, + -0.026232582, -0.012948747, 0.004798935, 0.0055887843, 0.031833332, + -0.027716542, -0.011363064, -0.016167982, 0.031091353, -0.006869298, + 0.022355137, 0.021816604, -0.00044616032, 0.018740976, 0.0040838816, + -0.01793916, -0.0016425282, 0.027309649, 0.014743859, -0.030014286, + 0.038822304, -0.0078266915, 0.007527506, -0.015234523, 0.0058999374, + -0.00917901, -0.0032940318, 0.0031982926, -0.006911184, -0.013415476, + -0.050454635, 0.0005632166, -0.0015976503, 0.012290538, -0.0069829887, + -0.015210588, -0.017831454, 0.0036710056, 0.018334085, 0.0010321898, + 0.010136403, 0.010016729, 0.02400664, 0.0012019776, 0.024162218, + 0.0013253916, 0.026088972, 0.016311591, -0.009274749, -0.02800376, + -0.017137343, -0.0019357299, -0.007587343, 0.03958822, -0.022403007, + -0.016670614, 0.010639035, 0.0398515, 0.004996397, 0.010704855, + 0.0010942708, 0.000308535, -0.003949248, 0.02081134, -0.029822806, + -0.022187594, 0.017352756, -0.024581077, 0.01316416, -0.052560903, + 0.005947807, 0.0074796365, 0.006546178, 0.002369549, -0.009173026, + 0.013008583, -0.02252268, -0.013343671, -0.024533208, -0.006318797, + -0.025921429, -0.0050053727, 0.0005127291, 0.0061183427, 0.015557644, + -0.0028931233, -0.011644298, 0.0018878602, -0.0009199953, 0.023468107, + 0.012434147, 0.032551378, 0.014313032, 0.016132079, 0.011273308, + -0.0072103697, 0.021266103, -0.013475313, 0.0081617795, -0.012912844, + 0.054475687, 0.007521522, 0.012649561, 0.01098609, -0.023851065, + 0.023875, 0.0019611607, 0.0022543625, 0.010495426, 0.007383897, + 0.007892513, 0.011692167, 0.02503584, -0.0016350485, -0.014253195, + -0.0024024593, 0.005531939, -0.038128193, -0.0082276, 0.0024996945, + 0.025251253, 0.001871405, -0.016527005, 0.011075846, 0.002469776, + 0.030516917, -0.01077666, 0.022223495, -0.016347494, -0.0069291354, + 0.028123433, 0.022474812, 0.0059507987, 0.01119552, -0.014133521, + 0.032647118, -0.042627946, 0.0025774827, 0.00695307, -0.01789129, + -0.001276774, -0.0031324718, -0.029368045, -0.003147431, 0.020715602, + -0.02373139, 0.040761027, 0.016263722, -0.058353133, 0.017723747, + -0.0021331923, -0.007431767, 0.02776441, -0.015150752, -0.0072283205, + 0.0074557015, -0.0056964913, 0.03587832, -0.029918546, 0.003042716, + 0.041598745, 0.0037368264, -0.0072103697, -0.02484436, -0.006498308, + 0.010321898, -0.038319673, -0.014301065, -0.023360401, -0.014348934, + -0.0066359336, -0.02697456, -0.007814724, -0.014707957, 0.011183552, + -0.011165601, 0.011560526, 0.01710144, 0.013080388, -0.012493985, + 0.005044267, 0.007832675, -0.031905137, -0.007521522, -0.030540852, + -0.022127757, 0.011584461, 0.002816831, -0.038008522, -0.0117819235, + -0.019602632, 0.0027839206, -0.01427713, 0.0038744516, 0.017628007, + -0.009968859, -0.02568208, 0.020248871, -0.004876723, 0.016084211, + -0.01673045, 0.04389649, -0.017580137, 0.0069829887, -0.007964317, + -0.022319235, -0.019554762, 0.03003822, -0.003416698, 0.024820426, + -0.0036889566, -0.0071086464, -0.01692193, -0.019997556, + -0.0056695645, 0.016814223, -0.0018130639, -0.0038535087, -0.02687882, + -0.008652443, -0.0154738715, -0.025753884, 0.012577756, -0.024796491, + 0.0021750783, 0.006725689, 0.0164552, 0.0067795427, 0.017747682, + 0.012829073, -0.0015976503, -0.024329761, 0.034370426, 0.011021992, + 0.013104323, -0.017628007, -0.019111967, 0.015868796, -0.020392481, + 0.010477475, 0.0061632204, -0.013571053, 0.011704135, 0.002697157, + 0.002406947, -0.0030157894, 0.0036889566, -0.013020551, -0.013152192, + -0.006175188, -0.026591603, -0.051459897, -0.002429386, -0.028817544, + 0.0021077616, 0.02558634, 0.006546178, 0.017675877, -0.006516259, + 0.00047159108, -0.004221507, 0.02632832, 0.03374812, -0.031091353, + 0.02568208, -0.0011638314, -0.00642052, 0.027477192, -0.013678759, + 0.0074197995, -0.01714931, -0.007718985, -0.0005179648, 0.002409939, + 0.023300564, 0.0206797, -0.012613659, 0.0038594925, -0.024868295, + -0.027429324, 0.0023755326, 0.013918107, -0.0056127193, 0.0071505327, + -0.005409273, -0.033963535, 0.009250814, -0.002857221, 0.0031504228, + 0.008730232, -0.003144439, 0.02392287, 0.0043262215, -0.0027091242, + -0.0013650337, -0.0018848684, 0.0008930686, -0.018334085, + -0.044614535, -0.029032957, 0.012248653, 0.0075155385, 0.031689722, + 0.042747617, -0.019782143, 0.019387217, 0.021661026, 0.003934289, + 0.013523183, -0.028578196, -0.018501628, 0.0046014725, -0.046936214, + 0.011704135, 0.0047271303, 0.011787907, -0.016993733, 0.0027824247, + -0.0010134907, -0.027477192, 0.022654323, -0.022546615, 0.00642052, + 0.014073684, 0.024557142, 0.01177594, 0.02132594, 0.041694485, + 0.00998681, -0.013942042, 0.0052118106, 0.011135682, -0.007509555, + -0.007838659, 0.01969837, -0.01659881, -0.009974843, 0.016527005, + 0.015090914, -0.03573471, -0.016132079, 0.022534648, -0.011692167, + 0.011004041, 0.014815664, -0.008700313, 0.0031205043, 0.005059226, + -0.0115904445, 0.023659587, -0.01003468, 0.011674217, -0.0034526002, + 0.023240726, -0.010926253, 0.007916447, 0.0043292134, -0.024748622, + -0.009502131, 0.03384386, -0.03954035, -0.0072702067, -0.007982268, + 0.0120452065, -0.025227318, -0.0047839754, -0.022881703, -0.015940601, + -0.0010815554, -0.019004261, 0.013391541, -0.000047822927, + 0.010226159, 0.020691667, 0.004834837, -0.023755325, 0.0005018836, + 0.023743358, -0.0043740915, 0.019590665, -0.014169423, -0.017998997, + -0.010094517, -0.05868822, -0.009238847, -0.009573935, -0.0065102754, + 0.01184176, -0.0017562186, 0.01692193, 0.00088783284, 0.007569392, + 0.015102882, 0.00064698857, 0.00374281, -0.008251535, 0.0081617795, + -0.017340789, -0.02350401, 0.011584461, -0.007443734, 0.03537569, + 0.021038722, 0.0048946743, 0.02776441, -0.00010602385, 0.01418139, + -0.0029050908, 0.0035333803, -0.0069231517, -0.0077907895, + 0.015366165, -0.026735213, -0.0006679315, 0.030660525, -0.022235462, + 0.013559085, -0.020512154, 0.008861873, -0.010172306, 0.014947305, + 0.01400188, 0.030325439, -0.005720426, 0.02800376, 0.048994612, + -0.0039253132, -0.0072702067, -0.0006009888, -0.008072023, + -0.014887468, -0.008843922, -0.030564787, -0.0022379071, 0.020942982, + 0.018657206, 0.022223495, 0.002610393, -0.007754887, 0.0081139095, + -0.018298183, -0.018717043, -0.004018061, -0.009843201, -0.010770677, + 0.018597368, 0.000556111, 0.014672055, 0.025418796, 0.018298183, + 0.009262782, 0.0038146146, 0.016491102, -0.0021750783, 0.010579198, + -0.013008583, 0.011608396, -0.0015796992, 0.009011466, 0.013977945, + -0.0064504384, -0.014935338, -0.0028108475, -0.0085686715, + 0.010315915, 0.02197218, -0.01151864, 0.036189474, -0.002368053, + 0.0057234177, -0.008981547, -0.009364505, -0.01687406, -0.007952349, + -0.015246491, 0.005328493, -0.031234961, 0.0053225094, -0.027884085, + -0.010794612, 0.03676391, 0.008496867, -0.013307769, 0.0057922304, + -0.013882205, -0.017041603, 0.0035094453, -0.004302287, -0.024724687, + -0.014301065, -0.009639756, -0.010854448, 0.008526785, -0.02202005, + -0.011722086, -0.00096636906, -0.024174185, 0.013523183, -0.010627068, + -0.006827412, 0.0005325501, 0.007300125, -0.009250814, -0.018561466, + -0.016072243, -0.0017726738, 0.010178289, -0.008586623, -0.00551698, + -0.005331485, -0.034418296, -0.0066778194, -0.0036321115, + -0.023958772, 0.007318076, -0.02943985, -0.0077907895, 0.032742858, + 0.010788628, 0.01400188, 0.031115288, 0.016706515, -0.0058460836, + 0.013846303, 0.0027629777, 0.006085432, 0.009633772, 0.02563421, + -0.013690727, 0.01673045, 0.007892513, -0.009274749, 0.0020479246, + -0.0077489032, 0.0222953, 0.0033209587, 0.013750563, 0.0061243265, + 0.026184712, 0.031881202, 0.014408772, -0.011273308, -0.014540413, + -0.00832334, 0.012290538, -0.0022708175, 0.0020434367, -0.0106151, + 0.03300614, -0.022582518, 0.0008863369, -0.0000023724472, 0.016622745, + -0.02934411, 0.01659881, 0.013104323, -0.006606015, -0.021720864, + 0.012912844, 0.017867355, 0.007030858, 0.024401566, 0.0222953, + 0.002411435, 0.0040659304, -0.0065940474, -0.029176567, 0.01469599, + -0.009639756, 0.017388659, -0.034968797, -0.0026418075, -0.0030531876, + 0.01997362, 0.034801252, -0.008024154, 0.02355188, -0.027357519, + 0.005923872, 0.01724505, -0.011327161, 0.017352756, 0.011021992, + 0.019016229, 0.0355193, 0.007240288, -0.030445112, 0.021912344, + -0.0033239503, 0.005947807, 0.01024411, -0.0051729167, -0.0078087403, + -0.010627068, 0.009256798, -0.01905213, 0.000030339275, -0.0057114502, + 0.026759148, -0.007258239, 0.0077010337, -0.025155513, -0.00855072, + -0.012876942, 0.008149812, 0.025059775, -0.00023691749, -0.020763472, + 0.015509774, 0.014205325, -0.0062410086, -0.011392983, -0.0066419174, + -0.030492982, -0.010752725, 0.0037577692, -0.009424342, 0.013200062, + 0.0005935092, -0.010465507, 0.035136342, 0.0036710056, 0.021290038, + -0.014085651, 0.011381015, 0.024138283, -0.0033897713, -0.0061243265, + -0.013032518, -0.005971742, 0.013355639, 0.00094318215, -0.006091416, + 0.00625896, 0.011883646, 0.012434147, -0.0081378445, -0.012446115, + -0.0055468986, -0.013774498, -0.006438471, 0.011680201, 0.021672994, + -0.0080899745, 0.0044309366, -0.012290538, 0.012218734, 0.0053524277, + -0.0046822526, -0.01242218, 0.0030127976, 0.0254906, -0.0039402726, + -0.0048707393, 0.003644079, -0.0037577692, -0.0038116227, + -0.014289098, -0.02053609, 0.007521522, -0.006456422, -0.004963487, + -0.012817105, -0.0067795427, 0.019901816, -0.040258396, 0.0085926065, + -0.013200062, 0.00036145342, 0.011668233, 0.0060375626, 0.0002223322, + -0.01771178, -0.013068421, -0.020607894, -0.0053165257, -0.008999499, + -0.006252976, 0.022151692, 0.0013396029, -0.018202443, -0.004505733, + 0.011716102, -0.02063183, -0.007527506, -0.020105263, 0.02489223, + -0.022008082, 0.026495865, -0.010968138, -0.008371209, -0.011985369, + 0.0046164314, 0.006665852, 0.002754002, 0.023743358, -0.00567854, + -0.009861153, 0.009286717, 0.021768734, -0.018717043, 0.011045927, + 0.012302506, 0.037840977, -0.018872619, 0.00563067, 0.027501127, + 0.032671053, -0.00022121026, -0.0009768405, -0.013475313, + -0.0021017778, -0.0024054511, -0.010393702, 0.015055012, 0.012254637, + 0.013415476, -0.018465726, -0.01914787, 0.0056486214, -0.016024373, + -0.010603133, 0.009777381, 0.0022872728, -0.011117731, 0.0077010337, + -0.0061033834, 0.022223495, 0.0082814535, 0.017879322, -0.0028617089, + -0.0007846139, -0.0120212715, 0.0028213188, -0.01576109, -0.008299405, + -0.0076471805, 0.00226633, 0.00453266, -0.018812781, 0.0041826125, + 0.009280733, -0.0076651312, 0.0021735823, -0.007425783, 0.054954384, + 0.0024757597, -0.00855072, -0.005457143, -0.0020105264, -0.0010718319, + -0.020512154, -0.004816886, 0.01771178, -0.0040090852, -0.004383067, + -0.012996616, -0.005744361, 0.0089037595, -0.0014779762, -0.012721365, + 0.011740037, -0.023863032, -0.00016576744, -0.005247713, -0.007407832, + -0.006462406, -0.009137124, -0.018657206, -0.02257055, 0.010351817, + 0.010603133, -0.0012692943, 0.0028033678, 0.0044578635, 0.005738377, + -0.0044698305, 0.0071864347, 0.026400125, 0.03434649, 0.015940601, + -0.000946174, 0.031306766, 0.0019581688, -0.017029636, -0.016048308, + 0.032216292, 0.0074138157, -0.0015752114, -0.00975943, 0.013786466, + -0.01682619, -0.015198621, -0.0071624997, -0.0029065867, -0.006336748, + -0.009454261, -0.0019731282, -0.0036949404, -0.0024129308, + -0.011626347, 0.0041796206, -0.016586842, -0.0035483395, 0.009370488, + 0.015844861, 0.0046343827, -0.025897494, 0.027429324, 0.0017547227, + 0.0079703005, -0.0021556313, 0.0121409455, 0.006462406, -0.013152192, + 0.006827412, 0.002240899, 0.009615821, -0.03082807, -0.035112407, + 0.021349873, 0.000060211074, 0.0054990286, -0.00025000685, + 0.008024154, -0.0033688282, -0.00604953, 0.021026755, 0.010555263, + 0.017639974, -0.017843422, 0.0004095101, 0.018884586, 0.009047369, + 0.020942982, -0.012206767, 0.010256077, 0.007425783, -0.006719705, + 0.008353258, 0.024269924, -0.02934411, 0.00067129737, -0.025897494, + -0.023348434, -0.017352756, -0.011913565, -0.0254906, 0.008670394, + 0.0019566729, 0.020476252, -0.012769235, 0.03147431, -0.009035401, + 0.005825141, -0.024964035, -0.009532048, 0.010585181, -0.016467167, + -0.008359241, 0.03365238, 0.012242669, 0.016024373, -0.003084602, + -0.007509555, 0.015773058, -0.02234317, -0.024305826, 0.0016559915, + -0.0001748365, 0.013942042, 0.009047369, 0.02127807, 0.0022992403, + -0.007689066, 0.0064504384, -0.0153541975, 0.016491102, -0.000513851, + 0.0071624997, -0.0029155624, -0.008814003, 0.021206265, 0.0071924184, + 0.003273089, -0.034705512, -0.018118672, 0.005002381, -0.01251792, + 0.010860432, -0.010788628, 0.001914787, -0.013032518, 0.017723747, + -0.000023747845, 0.008443014, -0.008419079, -0.006336748, -0.02656767, + 0.00966369, 0.014217293, 0.0051818923, -0.031354636, 0.010573214, + -0.009639756, 0.0003949248, -0.007425783, 0.021637091, 0.0013134242, + 0.0051938593, -0.007802757, -0.017340789, 0.019399185, -0.002181062, + -0.0024159225, 0.016527005, 0.004834837, -0.00785661, -0.0047809836, + -0.007222337, 0.012769235, -0.0070727444, 0.005513988, 0.0021167372, + -0.017041603, -0.017137343, -0.0022872728, -0.010956172, 0.0006525983, + 0.0080421055, -0.0033897713, 0.006809461, -0.019303447, 0.0008511826, + -0.005250705, -0.019626565, -0.0060704728, -0.022187594, -0.012948747, + -0.024772556, -0.003907362, -0.03157005, -0.0033957548, 0.0030501958, + -0.0044339285, 0.0038744516, 0.027046366, -0.027381454, 0.024174185, + -0.014121554, -0.005699483, -0.006977005, 0.00035733962, + -0.0015572603, -0.0015198621, -0.026088972, -0.014552381, + 0.0011765468, -0.010950188, -0.011919549, 0.008837938, -0.018250313, + 0.00032143737, 0.0043681078, 0.003910354, 0.0004712171, -0.025777819, + -0.009274749, 0.0077668545, 0.019279512, -0.0017726738, -0.0008758654, + -0.010489442, -0.016467167, 0.028697869, -0.041550875, -0.020320676, + 0.012063158, 0.011548558, 0.0024159225, 0.01400188, 0.0082335835, + 0.012864974, 0.025370928, -0.013068421, 0.012721365, -0.0114229005, + -0.0040928572, 0.007240288, -0.011303226, -0.0004297051, -0.012087093, + 0.017221116, 0.012122995, 0.011123715, -0.0010351817, 0.020009523, + 0.046385713, -0.00987312, -0.009609837, 0.0063606827, -0.015294361, + -0.00043980262, 0.03425075, -0.006462406, 0.023803195, 0.009250814, + 0.0036739975, 0.012948747, 0.0066718357, 0.0067795427, -0.02508371, + -0.016718483, -0.004792951, -0.02415025, 0.0170057, -0.008101942, + 0.0027914003, -0.014887468, 0.0072761904, 0.012087093, 0.019064097, + 0.019865915, -0.013044486, 0.009149091, 0.00139944, -0.028578196, + -0.007868578, -0.014217293, -0.006486341, 0.0072283205, 0.02155332, + 0.01056723, 0.0067974934, 0.015916666, 0.006659868, 0.034035336, + 0.015066979, 0.0082335835, -0.022091854, 0.0038116227, -0.024138283, + -0.028530326, -0.004131751, 0.02711817, -0.014875501, -0.011231422, + -0.0047181547, 0.015880764, -0.012326441, 0.0016679589, 0.0026223606, + -0.000822012, -0.0078087403, -0.011943484, 0.004050971, -0.02382713, + -0.021792669, -0.011668233, 0.007820708, 0.020895112, -0.028267043, + 0.00575932, 0.002386004, -0.012176848, -0.003329934, 0.015689285, + -0.012326441, 0.009585902, 0.009107205, 0.015653383, 0.0044638473, + 0.01284104, 0.010208208, 0.013355639, 0.009208929, -0.0019985589, + -0.014420739, 0.012936779, -0.0024518247, -0.009208929, 0.032431703, + 0.014169423, -0.014217293, -0.019853948, -0.00604953, 0.0044668387, + 0.024365664, 0.013786466, -0.011985369, -0.0034885025, 0.017532269, + -0.013044486, -0.0066778194, -0.015449937, -0.011410933, -0.008694329, + -0.0061572366, 0.00896958, 0.005370379, 0.015677318, 0.039013784, + 0.017340789, 0.029966416, -0.022426942, -0.010375752, -0.00021952733, + 0.017568171, 0.0058999374, -0.0072761904, 0.007689066, 0.002942489, + -0.0081857145, 0.010058615, 0.01719718, 0.012302506, -0.0031564066, + 0.0056964913, -0.010435589, -0.0060614976, 0.008921711, -0.009944925, + 0.03518421, -0.0019701363, -0.017233083, 0.013511215, 0.0120691415, + 0.0016230812, -0.00012397497, 0.0039761746, 0.033173684, 0.007802757, + -0.024186153, -0.0015722195, -0.0031324718, 0.008203666, 0.011638314, + 0.0032012844, -0.012601691, 0.005864035, 0.028793609, -0.010621084, + 0.01821441, -0.016012406, -0.024700752, -0.00964574, 0.003144439, + -0.021158395, 0.018058835, -0.015270426, 0.03597406, 0.00065110234, + 0.033724185, -0.014265163, -0.017855389, 0.0065940474, -0.015294361, + 0.0082276, -0.003147431, -0.0035273966, -0.008772118, 0.0034017386, + -0.012697431, -0.018142607, -0.036357015, 0.016251754, -0.003329934, + 0.0029873669, -0.01793916, -0.02132594, 0.015581579, -0.006540194, + -0.016562907, 0.013271867, -0.016670614, -0.00424245, -0.011207487, + -0.015832894, -0.003249154, -0.0153302625, 0.01177594, 0.017843422, + -0.0037936715, -0.015689285, -0.028793609, 0.011207487, -0.022187594, + 0.010537311, -0.013930075, -0.020308709, 0.020715602, 0.015605514, + 0.0062709274, 0.010172306, 0.012613659, -0.0067316727, 0.00088708487, + -0.022426942, 0.009149091, -0.008030138, -0.011392983, -0.016227819, + 0.020883145, 0.013068421, 0.00966369, 0.01543797, 0.0156174805, + -0.022582518, 0.021146428, -0.009962875, -0.004568562, -0.018453758, + -0.019818045, 0.009968859, 0.008532769, 0.030708395, -0.015402067, + -0.010788628, -0.0007315084, 0.004766024, -0.00321026, -0.0040808897, + -0.02303728, 0.0044159773, 0.006169204, 0.0059926845, 0.005657597, + 0.012529887, -0.003644079, 0.011698151, 0.025107644, 0.024437468, + -0.016371429, -0.016395364, 0.006576096, 0.012685464, -0.015880764, + 0.0044578635, 0.003931297, -0.0059507987, -0.0037787124, 0.017998997, + -0.0050472585, 0.0119973365, -0.013606954, 0.0074377507, 0.010866416, + -0.028075563, 0.022558583, -0.02400664, -0.006600031, -0.016706515, + -0.0016948857, 0.0180349, -0.0008451989, 0.008293421, 0.016156014, + -0.016000438, 0.001974624, 0.023515977, -0.01200332, -0.007916447, + -0.005487061, -0.023791227, -0.004481798, 0.008395144, 0.009705576, + -0.011345113, -0.010148371, 0.011931516, -0.019806078, -0.006402569, + -0.021613158, 0.021565288, 0.016012406, 0.0017726738, 0.0044997493, + 0.0021571273, -0.03324549, 0.0027943922, 0.020129198, 0.012697431, + -0.0042663845, -0.005905921, -0.01284104, 0.010884367, -0.010393702, + 0.0012072133, -0.007940382, -0.010435589, -0.008670394, -0.017747682, + 0.030708395, 0.009747462, 0.007401848, 0.0031354637, 0.0075155385, + 0.0029469768, -0.0081617795, 0.0115665095, -0.022630388, + -0.0069231517, -0.013319736, 0.009974843, 0.019482957, 0.015832894, + -0.010674937, 0.00992099, -0.0153541975, 0.017161278, 0.018369988, + 0.00017315359, 0.015964536, -0.013965977, 0.027213909, 0.0016395363, + -0.0010591166, -0.017484399, 0.010890351, 0.0025670112, 0.011883646, + 0.026711278, -0.007204386, 0.0069052004, -0.011105765, -0.018740976, + 0.0032461623, 0.015078947, 0.008454981, -0.0000072400544, + -0.015485839, 0.0054152566, -0.0155217415, -0.0021750783, + -0.0058939536, 0.012374311, 0.025969299, -0.021481516, 0.015485839, + -0.014420739, -0.0038176065, 0.013666792, -0.012482017, 0.003539364, + 0.0028647007, 0.035136342, -0.0017951127, 0.001974624, 0.0001276213, + 0.015892731, -0.0180349, 0.01184176, -0.029511653, 0.013032518, + 0.0036680137, -0.0035214128, -0.005107096, 0.04231679, 0.007240288, + 0.0122127505, 0.019028194, 0.01576109, 0.014253195, -0.0041885963, + -0.007204386, 0.0041885963, 0.008885808, -0.0042723683, 0.009956892, + 0.010040664, 0.0034675593, 0.010351817, -0.01497124, 0.015318296, + 0.0015946585, 0.014576316, -0.017879322, 0.00748562, -0.0057114502, + 0.008586623, -0.013571053, 0.013056453, -0.011488722, 0.0048886905, + -0.008006203, 0.0048049185, 0.0005475094, 0.0056486214, 0.0020045426, + -0.004335197, 0.012434147, 0.0222953, -0.0038116227, 0.007718985, + 0.0044608554, -0.0029125705, 0.0061003915, 0.0020598918, -0.016371429, + -0.013965977, 0.0029320174, -0.004912625, -0.04250827, 0.0051340223, + 0.0035303885, -0.00039679473, -0.0033658363, -0.015282393, + 0.023073183, -0.002179566, 0.0014338463, 0.0018998276, -0.022654323, + 0.013020551, 0.008909743, -0.014935338, 0.0040060934, -0.00012855625, + -0.014911403, 0.02415025, 0.012589724, -0.009370488, 0.009023434, + -0.0048378287, -0.009609837, 0.009950908, -0.012170864, + -0.00025655152, 0.008580639, -0.0007341263, -0.004562578, + -0.0014525454, -0.014707957, 0.00094766996, 0.018800814, -0.015402067, + 0.01992575, -0.010196241, -0.026735213, 0.008766134, 0.000113970964, + 0.0047031953, 0.011608396, 0.030971678, -0.018932456, -0.013188095, + -0.007551441, 0.0077608707, 0.0073898807, 0.009089254, 0.011829793, + 0.005280623, 0.0021750783, -0.01348728, -0.0065521616, -0.011273308, + -0.021206265, -0.011506673, -0.013032518, 0.009573935, 0.0064923246, + -0.008951629, -0.014684022, 0.021577256, 0.0035214128, 0.01761604, + 0.006378634, 0.009867136, 0.004571554, -0.028985087, -0.004631391, + 0.009932958, -0.0075155385, 0.025801754, 0.007862594, -0.021421678, + 0.01209906, -0.0068752817, -0.04408797, 0.01655094, -0.0013216517, + 0.013319736, 0.004792951, 0.019267544, 0.010022713, 0.006863314, + 0.011644298, -0.013618922, -0.0081857145, 0.014516478, -0.0035872336, + -0.0050382833, 0.010375752, 0.01418139, -0.0030142935, -0.004538643, + 0.010280012, -0.017520301, -0.019064097, -0.004398026, -0.01984198, + -0.0051519736, 0.03147431, 0.0034226817, 0.0053225094, -0.018992294, + 0.018501628, -0.0067974934, -0.0028557251, 0.012984649, 0.030492982, + -0.027501127, -0.023013346, 0.0036799812, -0.0050322996, 0.016838158, + 0.008095958, 0.0020195018, -0.009358522, 0.013523183, -0.01008255, + 0.0004297051, -0.023300564, -0.015138784, 0.027333584, -0.02484436, + -0.01914787, -0.020763472, 0.012912844, 0.031929072, 0.0017203164, + -0.0025191416, 0.015964536, 0.020296741, 0.002202005, 0.0038415412, + 0.00019484453, -0.006438471, 0.018800814, -0.0012423677, + 0.00065596413, 0.013235965, 0.012900877, 0.016191917, 0.0061572366, + 0.0053793546, -0.01789129, 0.00534046, -0.009573935, -0.013571053, + 0.0029394971, 0.02656767, -0.0021556313, -0.009095238, 0.013331704, + -0.006833396, -0.0039163376, -0.0071924184, -0.007138565, + 0.0030053179, 0.006288878, -0.0010194745, 0.029559523, -0.016838158, + -0.021302005, 0.01316416, 0.003473543, -0.011722086, -0.015677318, + -0.01816654, 0.010543295, 0.002840766, -0.011129699, -0.0011757988, + -0.001729292, 0.0035752663, -0.016538972, -0.007090695, 0.025729949, + 0.019076064, -0.006971021, -0.0027510102, -0.004218515, 0.034992732, + -0.0014637648, -0.020033458, 0.003991134, 0.032383833, -0.021266103, + -0.007982268, 0.012625626, -0.001953681, 0.033867795, -0.0007120614, + 0.011548558, 0.0012827576, 0.023515977, -0.0050921366, 0.0034914943, + 0.006109367, 0.0026926692, -0.0036680137, -0.0079463655, 0.01766391, + 0.0019910792, 0.00055985077, 0.015689285, 0.0013425947, 0.017795552, + 0.016574875, -0.010411654, -0.009035401, -0.009926974, 0.015078947, + 0.012757268, -0.020260839, -0.003844533, -0.005822149, -0.019231642, + 0.014887468, 0.0035303885, -0.010537311, 0.02544273, -0.002342622, + -0.00032947797, -0.0073360275, -0.019435087, 0.013295801, + -0.0028258066, 0.0040031015, 0.015043044, -0.0013119283, 0.0077907895, + 0.015258458, 0.013128258, 0.005902929, -0.020799372, -0.00021990131, + 0.0017412594, -0.0030726346, -0.015785025, -0.0060764565, + -0.009143108, -0.010112468, 0.0006488585, -0.01710144, 0.00057069625, + 0.0017592105, 0.020775437, 0.021684961, 0.00321026, -0.010710839, + 0.010830513, 0.013618922, 0.010166322, 0.0120691415, 0.0080421055, + -0.009974843, -0.019830013, 0.008317356, 0.0033987467, 0.0002795514, + 0.020871177, -0.0077309525, -0.0005822897, -0.015366165, 0.013223997, + -0.002674718, 0.0003392015, -0.0023605733, 0.00089232065, 0.02002149, + 0.0010329378, -0.009980827, 0.015832894, 0.0015961544, 0.013451378, + -0.003413706, -0.0005288103, -0.0014428219, 0.0012939771, + -0.006432487, 0.023252694, 0.0060196114, -0.015689285, 0.0169698, + 0.0040599466, 0.013786466, 0.012625626, -0.005780263, 0.005765304, + -0.008981547, -0.02419812, 0.010285996, -0.010495426, 0.013140226, + -0.006767575, -0.0080421055, 0.0015916666, -0.019219674, + -0.0017622024, -0.006402569, 0.011273308, -0.022558583, -0.018058835, + -0.0055439067, 0.003889411, 0.016993733, -0.018178508, 0.0041048243, + 0.009454261, 0.014863534, -0.0050951284, 0.003802647, -0.014767794, + -0.005965758, 0.00987312, 0.034609772, -0.02647193, -0.006893233, + 0.007431767, -0.008526785, -0.0117101185, -0.005762312, -0.016467167, + -0.011542575, 0.015318296, 0.009202945, -0.0050951284, -0.0029365052, + -0.0040270365, -0.00033078692, -0.022965476, 0.0062230574, + -0.010363785, 0.020189034, -0.02702243, 0.02234317, -0.004712171, + 0.0012505952, -0.014121554, 0.016718483, 0.016215852, -0.0018803806, + 0.00018886082, 0.021337908, -0.0039971177, 0.009717544, 0.026447995, + -0.0003603315, 0.03994724, -0.0039582234, -0.027692607, -0.0007524514, + -0.0077070175, -0.030852005, 0.004655326, -0.005510996, -0.018238345, + 0.019614598, 0.003024765, 0.012248653, 0.005427224, 0.019111967, + 0.005196851, -0.012278571, 0.021242168, -0.00350047, -0.011763972, + 0.0024039552, 0.020655764, -0.017532269, 0.0015333255, 0.0066179824, + 0.010698872, -0.0050322996, 0.0023336466, 0.02632832, 0.015031077, + -0.015210588, 0.015043044, -0.013355639, -0.010022713, 0.004502741, + 0.01705357, 0.039971177, -0.0048408206, 0.0039402726, 0.018968359, + 0.01497124, -0.018896554, -0.0069052004, 0.011327161, -0.002406947, + -0.009298684, 0.019830013, 0.009119173, -0.012553822, 0.00019119821, + 0.00987312, 0.008933677, -0.007898496, 0.01719718, -0.01135708, + 0.0035124372, 0.0076471805, 0.0119973365, -0.0006410048, -0.010327882, + -0.01068092, -0.00025168978, -0.018417858, -0.014540413, + -0.00088184915, -0.014157456, -0.010507394, -0.025107644, + -0.014073684, 0.000452144, -0.007234304, 0.042580076, -0.005570833, + -0.009256798, 0.0047779917, -0.0046972116, 0.0027854166, -0.027357519, + -0.022678258, 0.0056964913, 0.019315412, -0.004418969, 0.019123934, + -0.027453257, -0.002079339, -0.011057895, 0.016060276, 0.023144986, + 0.025227318, 0.002653775, 0.0009012962, -0.02553847, 0.0023471098, + 0.023049248, 0.014360902, -0.021984147, 0.024772556, -0.007288158, + 0.009430326, 0.0044309366, 0.013319736, -0.008006203, -0.015162719, + 0.01427713, 0.020703634, -0.02081134, 0.007491604, 0.0008459469, + 0.010465507, 0.011213471, 0.003991134, -0.020775437, -0.027261779, + -0.008837938, -0.028697869, 0.005911905, -0.009202945, -0.019279512, + -0.0018250312, 0.010046648, 0.00078087405, 0.013786466, -0.015605514, + -0.0019252584, 0.024162218, -0.0024578085, 0.01831015, -0.009125156, + -0.002281289, -0.0015871789, -0.011159617, 0.003126488, 0.03389173, + 0.0075035715, -0.012541855, 0.006994956, -0.0025999215, 0.031833332, + 0.0067855264, 0.003228211, 0.002221452, 0.035615038, -0.005902929, + -0.0059777256, 0.005675548, 0.019387217, -0.013714662, -0.009208929, + -0.00119375, 0.016203884, -0.012924812, -0.013930075, -0.007916447, + -0.017280951, 0.0027719534, 0.004628399, -0.025227318, 0.011644298, + -0.009944925, -0.028745739, -0.0016275689, -0.0006540942, + -0.0038385494, -0.0015243499, -0.008748183, -0.013750563, 0.016395364, + -0.017041603, -0.0026074012, 0.006623966, -0.008634493, -0.01808277, + -0.016383396, -0.020763472, -0.0034914943, -0.003599201, -0.009005482, + 0.01209906, -0.009693609, 0.04171842, -0.03334123, -0.020739537, + 0.025394863, 0.016048308, -0.004448888, -0.008287437, -0.006007644, + -0.00010733278, 0.003416698, -0.023695488, -0.025323058, -0.022199562, + -0.014432707, 0.005902929, 0.013223997, 0.03587832, 0.0011039943, + 0.0017532267, 0.007611278, -0.0013051966, 0.0008040609, -0.0047450815, + -0.005304558, 0.0017711779, 0.008987531, -0.017400626, 0.0064504384, + 0.01984198, 0.001132417, 0.041383334, -0.012805138, -0.022331202, + 0.013176127, -0.007078728, -0.022929573, 0.013295801, 0.021074623, + -0.010806579, 0.0062469924, -0.017065538, 0.031689722, -0.028889349, + 0.008443014, -0.020141166, -0.008209649, 0.010280012, 0.0062828944, + -0.017400626, 0.007096679, 0.008766134, -0.010639035, 0.0022289318, + -0.018657206, -0.0085926065, 0.023946805, 0.016814223, 0.0026821976, + 0.018752944, 0.024533208, 0.02211579, 0.022139724, -0.001750235, + -0.005537923, -0.01710144, -0.01348728, -0.0008324835, 0.0038983866, + 0.0079463655, 0.014217293, -0.0074497177, -0.013283835, 0.041479073, + 0.016407331, 0.01997362, -0.000596501, -0.00522677, 0.016957833, + -0.0040689223, -0.020978885, -0.003476535, -0.005681532, 0.020189034, + -0.000761053, 0.026112907, -0.015629448, 0.013355639, -0.027501127, + -0.016347494, -0.009035401, 0.006994956, -0.008107926, 0.0028228147, + -0.0019521851, 0.008867857, 0.014660087, -0.00913114, 0.023719423, + 0.0011361567, 0.0058999374, -0.016527005, 0.0075035715, 0.011464787, + -0.020452319, -0.0054691103, -0.0063247806, -0.002611889, 0.012458082, + 0.0016814222, -0.010740758, 0.022127757, 0.0070847115, -0.0069231517, + 0.0169698, -0.015174686, 0.00006310943, 0.024628947, 0.0040629385, + -0.004709179, 0.013223997, -0.029774936, -0.01798703, -0.003189317, + -0.0020733553, -0.007629229, -0.012900877, 0.0075813597, 0.009591886, + -0.003991134, 0.009029417, 0.005023324, -0.025729949, 0.005140006, + 0.0012588229, 0.014169423, -0.005475094, 0.0249401, 0.021876441, + 0.021493483, -0.0000093729195, -0.0050472585, -0.008221616, + -0.010950188, -0.00022794193, 0.022845801, -0.0050143483, + -0.009316635, -0.004709179, 0.0008122885, -0.0037158835, -0.01400188, + 0.008078007, 0.00901745, -0.0010262061, 0.015593546, 0.00096188125, + -0.019830013, 0.0112792915, -0.032120552, -0.009621805, 0.00790448, + -0.010274028, -0.012057174, 0.0026851895, 0.015126817, -0.009077286, + -0.0022319236, 0.011141666, 0.0030142935, 0.009621805, -0.0121409455, + -0.005101112, -0.008347275, -0.016227819, 0.013678759, 0.0028467497, + 0.009352538, -0.0017861372, 0.011045927, 0.00790448, 0.022151692, + -0.023049248, 0.01905213, 0.0044309366, 0.01418139, 0.0008459469, + 0.014265163, -0.007347995, -0.0030816102, 0.013977945, 0.013008583, + -0.011105765, -0.0007584351, 0.0010665961, -0.016191917, 0.02276203, + -0.021218233, -0.0121648805, -0.00992099, 0.014049749, 0.0084250625, + 0.008101942, 0.02692669, -0.0080181705, 0.0000027990202, + -0.0021661026, 0.0004895422 + ] + } + ] + }, + { + "name": "springfield-illinois-sixth-largest-city", + "title": "Springfield: Illinois' Sixth Largest City", + "content": "As of 2019, Springfield had a population of 10,566, making it the sixth most populous city in Illinois.", + "keywords": [ + "Springfield", + "population", + "10", + "566", + "sixth most populous city", + "Illinois", + "2019" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.012984714, 0.04321177, -0.0020374667, -0.02062091, 0.02117165, + -0.0060951896, 0.000027884435, 0.004514465, -0.03594626, 0.02429603, + -0.017475348, 0.017665988, -0.011523139, 0.017676579, -0.011131267, + 0.0033044296, -0.023724109, 0.031031983, -0.00076256064, + -0.0025299538, 0.0068577505, -0.023999479, -0.04482162, 0.018661553, + 0.023639381, 0.011427819, -0.020938644, 0.010718213, -0.032853656, + -0.0030793683, -0.022813274, -0.017560078, 0.06100485, 0.0028145902, + 0.0050413734, -0.005692727, 0.0053114467, -0.029485678, 0.025397506, + 0.030078782, 0.021341108, 0.009547895, 0.011332499, 0.0011895152, + 0.037111286, -0.046177283, -0.012232744, -0.016479783, 0.017835446, + 0.0076891533, 0.0097809, -0.031243805, -0.02440194, 0.039780248, + -0.009738535, -0.029379768, 0.00081882597, 0.025376324, -0.003119085, + -0.027028538, 0.024232483, -0.00260012, 0.0013980279, 0.0016667775, + 0.0028675457, 0.0045038736, 0.003910771, -0.012328064, 0.07684917, + -0.00032584244, 0.026837898, 0.041474827, 0.02222017, 0.011501957, + 0.048549697, 0.009722648, -0.011565504, 0.00009771963, -0.028151197, + 0.03204873, 0.00420997, -0.02249554, -0.004498578, -0.016490374, + 0.02260145, 0.017570669, 0.02486795, -0.049524076, -0.00017293314, + 0.0059045497, 0.04431325, -0.0105169825, 0.011332499, -0.026647259, + -0.039377786, -0.006656519, 0.041729014, -0.0040537515, 0.013376585, + 0.014414514, 0.0131223975, -0.004670684, -0.004440327, 0.02844775, + -0.01418151, 0.015664266, -0.012126832, 0.012052694, -0.026414255, + 0.010358116, 0.0105699375, -0.002214868, -0.022262534, 0.006481766, + -0.012486931, -0.0020229039, -0.039653152, -0.01570663, 0.030523608, + 0.028998487, 0.009214275, 0.032832474, -0.03545907, 0.016479783, + -0.019455887, -0.0037598477, 0.015071164, -0.009235457, 0.0076520843, + -0.028426567, 0.02155293, 0.015484218, -0.020176085, 0.008970679, + 0.013376585, -0.041940834, 0.0023154838, 0.019710075, 0.027621642, + 0.020451453, -0.06909647, -0.008404054, 0.021436427, 0.00012452841, + -0.003227644, -0.018417958, 0.0105169825, 0.0045806593, 0.011692597, + -0.014944071, -0.020546773, -0.020059582, -0.03204873, 0.0008419941, + 0.004588603, 0.0425763, 0.0026040918, -0.008298143, 0.042004384, + -0.053633433, -0.005213479, -0.023851203, 0.028574843, -0.03526843, + -0.029295038, -0.012783482, -0.009833856, 0.009166614, 0.0017263526, + -0.006444697, 0.025291596, -0.0062805344, -0.08477133, -0.044016697, + -0.004535647, -0.0024491965, -0.019837169, -0.021775343, -0.01314358, + 0.048761517, -0.008928315, -0.010930036, 0.019148745, -0.0040378645, + -0.019064017, -0.021002192, 0.018312046, 0.007948636, 0.028045285, + -0.020440862, 0.010887671, 0.041072365, 0.003370624, -0.0066882926, + 0.044948716, 0.009569077, -0.0069318884, -0.015801951, -0.0053749937, + 0.027007356, -0.04431325, 0.0015396841, -0.025694057, -0.01749653, + 0.0075673554, 0.015293578, -0.0048454376, 0.010707622, 0.02760046, + 0.024550216, 0.008292847, -0.0031932227, -0.036263995, 0.028151197, + -0.0032435306, 0.024846768, -0.0041358327, 0.0066988836, -0.03395513, + 0.020917462, 0.012497522, 0.042915218, -0.0050122477, -0.026964992, + -0.014954662, -0.020599728, -0.044101425, 0.036052175, -0.025100956, + -0.027918193, 0.028723119, 0.02751573, -0.009097773, 0.048083685, + -0.00018848885, 0.044101425, 0.016331507, 0.005830412, -0.015018209, + 0.03016351, 0.017316481, -0.0015794009, -0.00527173, -0.009251343, + 0.0029655136, -0.014785204, 0.037577294, 0.006164032, 0.023469923, + 0.022728544, 0.0033520896, -0.009637919, 0.051006835, -0.01823791, + 0.0010915473, 0.00531939, -0.0039451923, 0.017644806, -0.0040007955, + -0.0030184693, -0.023003913, -0.014234466, -0.024380758, 0.039928522, + -0.003611572, -0.012614024, 0.021860072, 0.014075599, 0.020281995, + -0.029612772, 0.0072019617, -0.011936192, 0.020451453, 0.017062295, + 0.0010419014, 0.027685188, 0.039271872, -0.0011822338, -0.020080764, + -0.019053426, -0.02440194, 0.0368571, 0.0064129233, -0.021055147, + 0.011205405, -0.0021076328, -0.01703052, 0.0118726455, 0.0010816181, + -0.0130906245, -0.012137424, -0.0060951896, 0.04206793, 0.018079042, + -0.008827698, -0.017422393, 0.0072390307, -0.0019355272, -0.039886158, + 0.033489123, 0.028002921, 0.044270884, -0.028490113, -0.022580268, + 0.0177719, -0.03223937, 0.02050441, -0.01938175, -0.034971878, + -0.050922107, -0.013514269, -0.047109302, -0.004691866, -0.001457603, + 0.013853185, -0.006836568, 0.012730527, 0.00770504, 0.028299473, + -0.018693328, -0.0023260748, 0.015759587, 0.03147681, 0.013641363, + -0.0015635141, 0.050879743, -0.005427949, 0.0041120024, 0.021987166, + 0.007932749, -0.02429603, 0.027240362, -0.014764021, -0.03120144, + 0.026922628, 0.013874368, 0.04774477, 0.013567225, -0.0068577505, + 0.035776805, -0.00765738, 0.015674857, 0.017930767, 0.038424585, + 0.0099185845, -0.016278552, -0.023279281, 0.01063878, -0.016183231, + 0.041834924, 0.00040312452, 0.018068451, 0.051769394, 0.008928315, + 0.0049195755, -0.00893361, -0.016426828, -0.0027431, 0.019392341, + -0.052065946, -0.017072886, 0.018259091, -0.024550216, 0.0044244407, + -0.041538373, 0.03130735, -0.00068047945, 0.01883101, 0.03715365, + -0.0080280695, 0.0058092293, 0.016437419, -0.030989617, 0.018936923, + 0.0041199457, 0.018206136, -0.016098503, 0.010840012, -0.0077685867, + -0.01342954, 0.020663276, 0.023046277, -0.0027483958, -0.010013904, + -0.007058982, -0.0043555982, -0.005830412, 0.0049513485, -0.04111473, + 0.022453174, 0.02675317, -0.043847237, 0.006910706, 0.015187667, + 0.02089628, 0.018386185, -0.030099964, -0.013832003, 0.018703919, + 0.0011299402, 0.023088641, -0.031370897, 0.0024822939, 0.014033235, + -0.0017170854, -0.00044879873, 0.0011226587, -0.047998957, + 0.0054332446, -0.023385193, -0.0065400167, 0.018746283, 0.018576825, + -0.005756274, -0.02296155, -0.03384922, -0.019731257, 0.039165962, + -0.0015873441, 0.009050112, 0.0057350914, -0.043635417, 0.01210565, + 0.00058284256, 0.0011590658, 0.004840142, -0.0118408725, 0.011321908, + -0.01465811, -0.03571326, -0.0017965188, 0.009224866, -0.018936923, + -0.02402066, -0.016013773, 0.054480724, -0.054057077, -0.012306882, + 0.052701414, 0.020366725, -0.028786665, 0.012222153, 0.029633954, + -0.029549226, -0.001841531, 0.026880262, 0.0117455525, 0.033997495, + 0.018386185, 0.044397976, -0.022262534, 0.015950227, 0.027197996, + 0.013683727, 0.029739866, -0.0059151407, -0.026075337, 0.012963532, + -0.012677571, 0.00902893, 0.037979756, -0.023618199, 0.017697763, + -0.017401211, -0.040902905, 0.04168665, 0.0425763, 0.009187797, + -0.0026808772, -0.007810951, -0.012497522, -0.013418949, -0.05481964, + 0.03272656, -0.03302311, 0.015823133, 0.013556634, 0.021510566, + 0.01730589, -0.016331507, -0.027918193, 0.021574112, -0.057319142, + 0.0028966714, -0.023512287, 0.044567432, -0.018862786, 0.0027642823, + 0.0155901285, -0.022431992, 0.04511817, -0.0009101744, -0.01266698, + -0.013556634, 0.012338655, -0.006873637, -0.029189127, -0.04605019, + 0.03346794, 0.045245264, 0.002613359, 0.01891574, 0.021192832, + 0.028829029, -0.03024824, -0.011067721, -0.024338394, -0.041326553, + -0.0008764152, 0.013313038, 0.070028484, 0.020779777, -0.030375334, + -0.023364011, 0.017898994, -0.0064129233, 0.009134841, -0.019402932, + 0.021404654, 0.001984511, -0.017602442, 0.022855638, 0.00964851, + 0.023236917, -0.029104399, 0.040627535, -0.00093863806, 0.0055285646, + 0.041051183, -0.010125111, -0.03632754, 0.00016010796, 0.0067359526, + 0.0201549, 0.04897334, -0.027219178, -0.016437419, -0.040225074, + -0.012889394, 0.030502426, -0.0033150206, -0.06896938, 0.0029099102, + 0.031879272, 0.007021913, 0.0030555383, -0.007450853, -0.024190119, + 0.00204541, -0.023406375, 0.038128033, -0.011427819, 0.0070642773, + -0.0017541543, -0.016977565, 0.005205536, -0.0025842334, -0.012232744, + -0.032747746, -0.030460062, 0.0072866906, 0.0022334026, 0.014891115, + -0.009833856, 0.0070113214, 0.025863515, 0.0030396516, -0.0048666196, + 0.00902893, 0.017369438, 0.017835446, 0.026986174, 0.029909324, + -0.010739396, 0.009712057, 0.0010558023, 0.0099503575, -0.019551208, + 0.004006091, -0.0063758544, -0.0027047072, 0.017263526, 0.031497993, + 0.022198988, -0.006391741, 0.0109724, 0.0018918389, -0.03128617, + -0.0117561435, 0.059691552, 0.045711275, 0.017136432, 0.0030846638, + 0.007932749, -0.0075144, 0.012084468, -0.013207127, 0.0070007304, + -0.027536912, -0.062191058, 0.021129284, -0.0077262223, 0.006825977, + -0.014869933, -0.010464027, -0.007069573, 0.0033070773, 0.0072284397, + 0.01816377, -0.020398498, -0.014795795, 0.006608859, 0.011237179, + -0.011967965, -0.042682216, 0.0118302815, 0.007837429, 0.019710075, + 0.007318464, -0.008737674, 0.0049698832, -0.029337402, -0.032874838, + -0.0034500575, 0.021796525, -0.0142768305, 0.0060475296, + -0.0029046147, 0.003669823, -0.0074561485, -0.0023909453, 0.02817238, + 0.0021883901, -0.0029549226, -0.004620376, -0.004403258, + -0.0016707492, 0.011279543, -0.03537434, 0.018682737, 0.012868211, + 0.0614285, -0.0007440262, 0.03262065, -0.0054914956, -0.0014655463, + -0.037683208, 0.003818099, -0.01087708, 0.0050122477, 0.017009338, + -0.024783222, 0.0023882976, -0.004249687, -0.011035947, 0.024656128, + 0.007429671, -0.022728544, 0.01521944, 0.0002454161, -0.037344288, + -0.00032336015, 0.0036142198, 0.0029866959, 0.0007923482, 0.03319257, + -0.030099964, 0.028426567, 0.038657587, 0.00037466089, -0.0039425446, + 0.028659571, -0.016045548, 0.024889132, -0.0072602127, 0.018746283, + -0.013842594, -0.005420006, -0.018015495, -0.0025736422, -0.010649371, + -0.0075302864, -0.0012325416, -0.021436427, 0.0048083686, + -0.025270414, 0.047575314, 0.028214743, 0.0070007304, 0.006148145, + -0.000068180336, 0.025545781, -0.012433975, -0.0006778317, + 0.033912767, -0.009103068, 0.00024839488, 0.02760046, 0.006937184, + 0.00049182517, -0.008525852, 0.020271404, 0.003526843, -0.037132468, + 0.00382869, 0.00016068715, -0.00089759746, -0.031031983, -0.054311264, + -0.0027536913, -0.008065138, 0.011364272, 0.013715501, -0.03194282, + 0.0555822, -0.042131476, 0.026181249, -0.02628716, 0.0026874968, + 0.026837898, 0.021256378, 0.016215006, -0.055666927, 0.00084133213, + -0.018365003, 0.007911567, -0.019519435, -0.029422132, 0.018693328, + 0.011163041, -0.026160067, 0.021711797, 0.029718684, -0.0035877419, + 0.0021817707, 0.008970679, -0.0034156363, -0.03141326, 0.029676318, + -0.0021274912, -0.012921167, -0.029083217, 0.03024824, 0.01626796, + 0.0074084885, 0.007964523, -0.0012888069, -0.010707622, -0.065410756, + -0.0037572, -0.023597015, -0.020218449, 0.04431325, 0.028807847, + -0.005046669, -0.010109224, -0.006645928, 0.0057880473, 0.01399087, + 0.040500443, 0.022855638, -0.0019447944, 0.009208979, -0.00013478856, + 0.00760972, -0.001432449, 0.004180845, -0.010607007, -0.023364011, + 0.006015756, 0.040013254, -0.007069573, -0.015886681, 0.00043655274, + 0.030205876, -0.0010432253, 0.0031455627, -0.020292586, 0.024846768, + -0.017348254, 0.027897011, -0.005017543, 0.016310325, 0.004143776, + -0.0015277691, -0.015494809, -0.006116372, -0.016564513, -0.012592842, + -0.021987166, -0.008118094, -0.011501957, -0.047575314, -0.0008942877, + 0.011470183, -0.0055444515, -0.010840012, -0.032218188, 0.060072836, + 0.010241613, 0.0063652634, 0.020186676, 0.0021129285, 0.016003182, + -0.03668764, -0.0019553856, 0.008345802, 0.0010915473, -0.006031643, + 0.013779047, 0.020207858, -0.035395525, 0.020970417, -0.017380029, + -0.040796995, -0.01711525, 0.01928643, -0.024317212, -0.012222153, + -0.03395513, 0.017231753, -0.026498983, 0.02383002, -0.0142980125, + 0.015876088, -0.00043853858, 0.015198258, -0.008954792, 0.011967965, + 0.02050441, -0.016130276, 0.0117773255, -0.02296155, -0.016712788, + -0.004710401, 0.025969427, 0.009113659, 0.0264778, 0.019064017, + 0.0044959304, 0.0020056935, -0.0118302815, 0.0155159915, -0.017507121, + 0.024825586, 0.027155632, 0.0042073224, -0.03620045, 0.034357592, + -0.017602442, -0.03859404, 0.008822403, -0.017337663, 0.0255246, + -0.02467731, 0.040140346, 0.030523608, 0.0017223809, -0.017846037, + 0.0050493167, 0.009722648, 0.0040749335, 0.021224605, -0.021341108, + 0.0007334351, 0.009547895, 0.013937914, 0.004297347, -0.021860072, + -0.0118514635, 0.0077685867, 0.0044853394, -0.00046170666, + -0.026731987, 0.014689884, 0.015272396, -0.001229232, 0.006788908, + -0.028829029, 0.025736421, -0.027346272, -0.009897402, -0.017898994, + -0.023639381, 0.012656389, 0.009521417, -0.022919184, -0.03035415, + -0.0048851543, 0.0067730215, 0.01201033, 0.0053882324, 0.015007617, + 0.002658371, 0.0059839827, 0.007069573, -0.017898994, -0.025651693, + 0.00818164, -0.0069054104, 0.019159336, -0.0031985184, 0.0010280006, + -0.019392341, -0.012328064, 0.021775343, 0.028214743, 0.0068577505, + -0.014816977, 0.010278682, -0.030375334, 0.019244066, 0.0024015366, + -0.016564513, -0.0065082433, 0.023533469, -0.019116972, -0.008504669, + 0.0032753039, 0.015399489, 0.023321647, -0.00031607875, -0.030078782, + 0.0075144, 0.011682006, 0.0064023323, 0.011046538, -0.035988625, + -0.0067942035, -0.00845701, 0.023597015, 0.015664266, 0.004501226, + 0.0020414384, -0.007980409, -0.025842333, -0.027388636, 0.0040511033, + -0.015727814, 0.03999207, -0.013662545, -0.0017290004, -0.019116972, + 0.012391611, -0.030121146, 0.03668764, -0.032112278, 0.016935201, + 0.037068922, -0.002010989, -0.0038472244, 0.014170919, 0.0080386605, + 0.008938906, 0.032091096, 0.030693067, -0.0143191945, 0.02260145, + -0.012063286, 0.02128815, 0.010384593, 0.008250482, -0.039081234, + -0.0027139746, 0.008843585, -0.0118514635, -0.030693067, -0.046092555, + -0.009214275, 0.010824125, 0.016437419, -0.0073978975, 0.016204413, + -0.009394323, 0.0036433453, 0.03300193, -0.015071164, 0.0018243204, + 0.052065946, 0.03545907, 0.026096521, 0.0031455627, 0.0006056797, + 0.049693536, 0.02628716, 0.0010028468, -0.013916732, -0.009865629, + 0.0003634078, 0.00026113732, 0.03772557, -0.008483487, -0.010564642, + 0.00077116594, 0.0580817, -0.016956383, -0.0076414933, 0.006921297, + -0.012868211, 0.029167945, 0.03725956, 0.012921167, -0.028977305, + -0.0055656335, -0.00027867884, 0.02062091, -0.03753493, -0.02081155, + -0.02817238, 0.0075673554, 0.01625737, -0.02675317, 0.025588147, + -0.039822612, -0.0014536312, -0.027918193, -0.01995367, -0.00818164, + -0.0033362028, -0.025291596, 0.019455887, 0.004143776, 0.00021629053, + -0.012275108, -0.0055550425, -0.010103929, 0.016373873, -0.014192101, + 0.00026113732, 0.05151521, 0.017613033, -0.019064017, -0.0070960503, + 0.002394917, -0.023025095, -0.01579136, -0.0020374667, 0.045075808, + 0.022559086, 0.014742839, 0.0012451186, -0.026456619, 0.032366462, + -0.0043873717, 0.0023882976, -0.016151458, -0.008176344, 0.013005896, + 0.03158272, 0.047575314, -0.0036433453, 0.011999739, -0.009727944, + -0.0038233944, -0.028511295, -0.0015926397, 0.031180257, 0.022834456, + -0.005512678, -0.01918052, -0.0016270608, -0.015113529, -0.0070748683, + -0.0034871264, 0.02599061, -0.0069636614, -0.015145302, 0.016956383, + -0.027875828, -0.0037598477, 0.0008234596, -0.013863777, 0.02429603, + -0.014573381, 0.018587416, -0.00041139883, -0.00529556, -0.019053426, + -0.010358116, -0.020101946, 0.018693328, 0.0092672305, -0.011682006, + 0.015886681, -0.0022069246, -0.015876088, 0.015134711, 0.0061587365, + 0.0024995045, 0.0016416237, -0.009886811, -0.013546043, 0.023236917, + 0.009600851, 0.039970886, -0.012359837, 0.03130735, 0.033997495, + 0.0016932554, 0.003516252, -0.0055179736, 0.008107503, 0.004323825, + -0.05320979, 0.0080916155, 0.013418949, -0.047448218, -0.023766473, + -0.056556582, -0.009373141, -0.014816977, 0.01266698, -0.033912767, + 0.0007599129, 0.017676579, 0.023109823, -0.017485939, -0.012423384, + -0.0036142198, -0.027092086, -0.021256378, -0.0080810245, + -0.028892577, -0.010109224, 0.03505661, -0.0027589868, -0.012719936, + -0.009770309, 0.014213284, -0.002683525, -0.020906871, 0.009293708, + 0.0020665922, -0.048210777, 0.021860072, 0.01115245, 0.0062805344, + -0.0041570147, 0.021118693, -0.006359968, 0.018545052, 0.0024862655, + 0.002359172, -0.008901836, 0.0005368374, 0.00022075867, 0.021383472, + -0.01984776, 0.008345802, -0.017464757, -0.036624093, 0.005197592, + 0.011491366, -0.008615877, 0.0008916399, -0.02155293, 0.01314358, + 0.00012006028, -0.02249554, 0.022728544, -0.021139875, -0.010405775, + -0.009579668, 0.015166484, 0.01314358, 0.011035947, -0.0027669303, + 0.011671415, 0.010035086, 0.03168863, 0.004429736, 0.0030370038, + -0.02222017, -0.025757605, 0.0118302815, 0.0032938384, 0.00902893, + -0.014149737, 0.00062653096, 0.004668036, -0.0117243705, 0.002201629, + -0.011184223, 0.013387176, -0.014689884, -0.018280273, -0.011258361, + -0.007953932, -0.030587155, -0.0024002127, 0.006836568, -0.02912558, + 0.026202431, 0.025461053, -0.0118302815, -0.010501096, -0.004850733, + 0.00089892134, 0.013270673, 0.020790368, -0.026435437, 0.016278552, + -0.016522147, 0.0007413784, 0.0093201855, -0.0024491965, 0.0066353367, + 0.0038842934, -0.0040511033, -0.025100956, 0.0054094144, 0.007344942, + 0.010273387, -0.0063017164, 0.017994313, -0.0052770255, + -0.00041636341, 0.025418688, 0.047448218, 0.025545781, -0.008096911, + 0.0037598477, -0.028087651, 0.011989148, 0.018788647, 0.0065611987, + -0.028341837, 0.007583242, -0.025164502, -0.007832133, -0.016522147, + 0.015166484, -0.010326343, 0.018788647, 0.012486931, -0.020070173, + -0.014922888, 0.024338394, 0.008266369, 0.0097809, 0.015039391, + -0.011544321, 0.021266969, -0.0019553856, -0.0016548625, 0.028723119, + -0.044906348, -0.021923618, 0.006116372, -0.03969552, 0.013376585, + 0.013132989, 0.0011511224, -0.019625345, -0.0342305, -0.0009915936, + -0.011565504, 0.0059310272, -0.022368446, -0.020313768, 0.017824855, + 0.0015661619, -0.009404914, 0.005597407, 0.003021117, 0.015918454, + 0.0032832474, 0.0017250287, 0.02088569, 0.005028134, 0.007853315, + 0.0127623, -0.025969427, -0.023300465, 0.03836104, 0.023236917, + -0.004620376, -0.0022347264, 0.03346794, 0.015632493, 0.0021023373, + -0.01124777, 0.0008486135, 0.011046538, -0.009706762, -0.006201101, + 0.018703919, -0.012169197, 0.025736421, 0.011650233, 0.027282726, + 0.010199249, 0.012539886, -0.002458464, -0.026160067, -0.012709344, + 0.024232483, -0.04045808, -0.008488783, -0.017613033, 0.018184954, + -0.024995044, 0.003092607, -0.006455288, -0.017793082, -0.009627328, + -0.015240622, 0.011523139, -0.003937249, 0.00527173, 0.023469923, + 0.0142980125, -0.034421142, -0.0130694425, 0.0028860802, + -0.0015079108, 0.0131118065, 0.0009843123, -0.004021978, 0.0062805344, + -0.04075463, -0.008986565, -0.0143509675, -0.01210565, 0.006677701, + -0.0008386843, 0.016204413, 0.014202693, -0.009664397, 0.010776465, + 0.0087429695, 0.012349246, -0.012285699, -0.007667971, -0.010982991, + -0.017824855, 0.014192101, -0.020101946, 0.05049846, -0.006148145, + 0.007863907, 0.008806516, -0.0000075658245, -0.0042152656, + -0.00024210638, -0.001709142, 0.00044515802, -0.021669433, + 0.0043052905, -0.018566234, -0.0029310926, 0.011586686, -0.021203423, + 0.017983722, -0.032493558, -0.002336666, -0.010283978, -0.005316742, + 0.02885021, 0.007117233, 0.017380029, 0.023914749, 0.026922628, + -0.016056139, -0.0009062027, -0.013948506, -0.004180845, -0.015282987, + 0.0072655086, -0.020440862, 0.030502426, 0.022389628, 0.015261804, + 0.018661553, -0.0012305558, 0.0073820106, -0.007810951, -0.029803412, + -0.0023287225, -0.004392667, -0.014393332, -0.0051552276, + -0.0030158216, 0.006995435, 0.015939636, 0.014605155, 0.041856106, + 0.014583972, -0.00945787, -0.0010697031, 0.018852195, 0.0105699375, + 0.006921297, 0.013207127, -0.020589137, 0.017951949, 0.0025445167, + -0.012804665, 0.003910771, -0.016162049, -0.009394323, -0.023364011, + -0.004480044, -0.004657445, 0.053929985, -0.0028490114, 0.020462044, + -0.015611311, -0.0059628007, 0.012052694, 0.0039081234, -0.019604163, + 0.01437215, -0.03243001, -0.0020599728, -0.0038339854, -0.004413849, + 0.012518704, 0.0059151407, -0.014626337, 0.014065008, -0.013789639, + -0.008303438, 0.014615746, 0.0017144376, -0.031137893, -0.015357125, + -0.015727814, 0.0010683793, -0.0019037538, 0.0012199647, 0.008949497, + -0.007969818, -0.0087111965, 0.0008942877, 0.015049982, 0.008435828, + 0.0049910652, -0.011205405, 0.009484348, -0.0047818907, 0.008435828, + -0.005234661, 0.02069505, -0.0117985075, 0.013969688, -0.024571398, + -0.03289602, -0.02440194, -0.02081155, -0.023427557, 0.0045806593, + 0.00864765, 0.00072019617, 0.03338321, 0.004265574, 0.010363411, + 0.0037757345, -0.0032223484, 0.0007605748, 0.01664924, -0.0081022065, + 0.0009353283, 0.006863046, -0.008944201, 0.008578807, -0.0080386605, + 0.013694318, -0.018280273, 0.011936192, -0.016988156, -0.005698023, + 0.01797313, 0.027261544, -0.0019328794, 0.03082016, 0.035416707, + 0.006370559, -0.030099964, -0.015420671, -0.01153373, 0.02393593, + 0.012974123, 0.007339646, 0.0155795375, 0.031540357, -0.010866489, + -0.004668036, 0.019127563, 0.020928053, -0.015388898, 0.008478192, + 0.0310108, -0.023491105, -0.008218709, -0.007524991, 0.023491105, + 0.015230031, 0.0025365732, 0.024529034, -0.010787056, 0.019127563, + -0.02062091, -0.0050731464, -0.02118224, 0.0029708091, 0.012709344, + -0.030947253, -0.0027828168, -0.005465018, 0.02429603, 0.020599728, + -0.0033547373, 0.02836302, -0.037895028, 0.0040616947, 0.004247039, + -0.0005729134, 0.034844786, -0.0055656335, 0.021595294, 0.06930829, + 0.02940095, -0.014838159, 0.024889132, -0.003296486, -0.0150288, + 0.018110815, -0.008160458, -0.015378307, -0.008817107, 0.02268618, + -0.018280273, -0.0033123728, -0.0072019617, 0.0016416237, 0.002540545, + 0.007991, -0.01937116, -0.0019659768, -0.010231022, 0.0072866906, + 0.02183889, -0.0036009809, -0.0028198857, 0.01797313, -0.00085523294, + -0.011618459, -0.025863515, -0.010199249, -0.033531487, -0.006052825, + 0.008287552, 0.008224005, -0.004429736, 0.017125841, 0.007498513, + 0.028765483, -0.007943341, 0.0020612967, -0.009351959, 0.0143297855, + -0.005213479, 0.014414514, -0.0019487661, 0.0019183167, 0.0032435306, + -0.023215735, 0.013005896, -0.008975974, 0.013757865, 0.003841929, + 0.010607007, -0.0057986383, -0.016553922, 0.0019236122, -0.008123389, + 0.00031872652, -0.0048216074, 0.010326343, -0.0069318884, + -0.004202027, -0.020938644, -0.026371889, 0.021023374, 0.0038710544, + -0.0046600928, -0.010511687, -0.0007062953, 0.007535582, -0.007885089, + 0.0049645877, 0.0026782295, 0.022855638, -0.018142588, -0.009505531, + 0.015198258, -0.011660824, 0.00081882597, -0.00964851, -0.005698023, + 0.028341837, -0.047787134, -0.008165753, -0.005946914, -0.010336934, + 0.010331638, -0.010675849, -0.0035003652, -0.0010736748, 0.0011219968, + -0.0058886628, 0.0105381645, 0.012063286, -0.004247039, 0.0071490062, + -0.0010041706, -0.011904419, -0.0026663146, 0.008986565, -0.013757865, + -0.010808238, -0.011639641, 0.035035424, -0.025736421, 0.012433975, + -0.003068777, -0.005851594, 0.005830412, 0.013736683, 0.0058092293, + 0.016585695, 0.01078176, 0.00252863, -0.0014284773, -0.007006026, + 0.018365003, -0.007318464, 0.011078312, -0.011321908, 0.03166745, + -0.008345802, -0.020123128, 0.03344676, 0.016871655, -0.0076150154, + 0.013514269, -0.008462305, 0.0035692076, -0.011607868, 0.0058674808, + -0.017877812, 0.014075599, 0.02249554, 0.0027483958, -0.019064017, + 0.0003564574, -0.017009338, -0.0011564179, -0.01578077, 0.011406637, + 0.005597407, -0.0027245656, -0.013959097, -0.0031879272, 0.023978297, + 0.016013773, -0.018365003, -0.027452184, -0.012169197, 0.012222153, + -0.00007839911, 0.014605155, -0.0056874314, -0.0038101554, + 0.012614024, -0.01937116, -0.00042894037, 0.01785663, -0.011279543, + 0.008234596, -0.006370559, 0.030989617, 0.005465018, 0.013662545, + 0.007186075, -0.0056768404, -0.0068577505, -0.008017478, -0.019837169, + 0.0039319536, -0.008202823, 0.0027245656, -0.009246048, -0.009833856, + 0.0080810245, 0.0038948844, -0.024995044, -0.00066657865, + -0.026604895, -0.0017475348, -0.004442975, 0.0005560338, -0.014912297, + -0.007879794, -0.0255246, -0.01910638, 0.02108692, 0.00931489, + 0.003119085, -0.0076520843, -0.006513539, -0.0072655086, -0.002925797, + 0.016183231, 0.022559086, 0.022241352, 0.017814264, 0.00404316, + 0.0035903899, 0.013260082, -0.014583972, -0.013588407, 0.025482235, + 0.01446747, -0.005777456, -0.024910314, -0.0028331247, -0.0014430401, + -0.037174832, -0.021711797, 0.0016402997, -0.004284108, -0.013927323, + -0.015049982, -0.010887671, 0.009664397, -0.0059522097, -0.007196666, + 0.016966974, -0.007630902, 0.0017740126, 0.013842594, -0.0014549551, + -0.028320655, 0.030142328, 0.005708614, -0.010797647, 0.0064341053, + -0.0024253665, -0.0007235059, 0.011025356, 0.007413784, 0.0057245004, + -0.002527306, -0.02334283, -0.026075337, 0.026604895, 0.01816377, + 0.033997495, -0.0071595972, 0.008197527, -0.0013113131, 0.010024495, + 0.013217718, 0.006296421, 0.022326082, -0.0023287225, 0.03213346, + 0.0048692673, 0.008472896, 0.031222623, 0.0015754291, 0.020091355, + 0.00489045, 0.005597407, 0.0056768404, 0.0065611987, -0.021214014, + 0.02666844, -0.033129025, 0.011035947, -0.0051446366, -0.0036274586, + -0.03213346, 0.0073025776, -0.022792092, -0.013387176, -0.0004478058, + 0.018470913, 0.004525056, -0.00067386, 0.01804727, -0.023575833, + 0.0069530704, -0.012868211, -0.0014734896, 0.013874368, 0.013270673, + 0.039547242, -0.01389555, -0.0071119373, 0.0009207655, -0.018481504, + -0.026435437, 0.011194814, -0.0013993517, -0.0010895615, -0.022559086, + 0.02402066, -0.0027669303, -0.008801221, 0.005560338, -0.017549487, + 0.01465811, 0.009627328, 0.013821412, -0.023618199, -0.017284708, + 0.016130276, -0.0006682335, -0.0038472244, -0.03770439, -0.003383863, + 0.021669433, -0.013588407, -0.0037889732, 0.00041735632, -0.010215135, + -0.033404395, 0.01579136, -0.000018658575, 0.0010902234, 0.008827698, + -0.018682737, -0.015431263, 0.0051949443, -0.008499374, 0.006767726, + -0.018131997, 0.013323629, 0.004143776, -0.008382872, 0.009765013, + 0.007890385, 0.0018322638, 0.0046521495, -0.002973457, -0.012656389, + 0.018862786, 0.0092566395, 0.013980279, 0.012444566, -0.014266239, + -0.0036512888, -0.0015674857, -0.0064499923, 0.0074084885, + 0.012232744, 0.003031708, 0.007953932, -0.012825847, -0.022198988, + -0.011936192, -0.005030782, -0.009616737, 0.016575104, -0.0046865707, + 0.020303177, -0.03704774, 0.0012669627, -0.016744561, 0.0023485809, + -0.0077950642, 0.009907993, -0.0063652634, 0.010522278, -0.014859342, + -0.0019911306, -0.0059257317, 0.0034844785, -0.001865361, + -0.015251213, 0.010755283, -0.01059112, 0.025821151, -0.017665988, + 0.021214014, 0.031159075, 0.007058982, -0.0130906245, -0.0073237596, + -0.0143297855, -0.016098503, 0.014170919, -0.014764021, -0.013969688, + 0.012084468, -0.0038022122, -0.013408358, -0.0044376794, + -0.0037757345, -0.0010902234, -0.019657118, -0.01984776, 0.01220097, + 0.006063416, 0.001432449, 0.0019011061, 0.0059839827, -0.022919184, + 0.0025829095, -0.033595033, 0.0009982131, 0.0012987361, 0.016246779, + -0.00034454238, 0.02459258, 0.0063864454, 0.015335942, 0.014541608, + -0.003479183, 0.004758061, -0.027282726, 0.0012782158, 0.012370428, + -0.008382872, 0.0023499047, -0.017676579, 0.03035415, 0.009643215, + -0.0040828767, 0.0065188343, 0.0009472433, 0.019011062, -0.009373141, + -0.019233475, 0.022940366, -0.028384201, 0.0131012155, 0.010765874, + -0.004755413, 0.014255648, 0.00079631986, -0.007053686, -0.00416231, + 0.0050704987, 0.005539156, -0.028426567, -0.02070564, -0.0077474043, + -0.00409082, -0.0013437484, -0.01749653, -0.00267161, -0.010283978, + 0.012794074, 0.001191501, 0.0005666249, 0.0264778, -0.012158606, + 0.008960088, -0.017051704, -0.018365003, 0.006269943, -0.011311316, + 0.009669693, 0.010246909, 0.0031323237, -0.0035718554, 0.038424585, + -0.0029072624, 0.0092990035, 0.031625085, 0.003179984, -0.0070272083, + -0.0023393137, 0.008472896, -0.0031932227, -0.0064129233, + -0.018026087, 0.017051704, -0.009966244, -0.00047726237, 0.005075794, + 0.006492357, -0.019159336, 0.005356459, -0.009293708, -0.0009081886, + 0.0069636614, -0.0076256064, 0.0013887606, 0.014382741, -0.011163041, + -0.015071164, 0.0061428496, -0.007890385, 0.009441984, -0.0014602507, + -0.0065664947, -0.023236917, 0.0045700683, 0.017814264, -0.0066512236, + -0.007021913, 0.013291856, -0.0017303242, -0.0043873717, 0.0012358513, + 0.008949497, -0.012232744, 0.002095718, 0.01730589, -0.01446747, + -0.006275239, 0.0010412395, 0.0007532934, 0.04255512, -0.0038075077, + -0.0018521221, -0.027176814, -0.013906141, 0.012931758, 0.02402066, + -0.016130276, 0.0031031983, 0.004286756, 0.0076785623, -0.0028463635, + -0.0032488261, -0.017930767, -0.016183231, -0.004623024, -0.010125111, + 0.0024730265, 0.01797313, 0.004877211, 0.03141326, 0.011809099, + 0.0075196954, -0.010013904, -0.024931498, 0.0014681941, 0.0009929176, + 0.0019911306, -0.029528042, -0.016405646, 0.009452575, -0.00024889133, + 0.01153373, 0.026541347, -0.013461314, 0.012116241, 0.0080598425, + -0.011586686, -0.014615746, 0.016988156, 0.00015357124, 0.029083217, + 0.001552923, -0.009473757, 0.032154642, 0.0223049, -0.02959159, + 0.011268952, 0.013461314, -0.002128815, 0.0155054, 0.0010789704, + 0.0009942414, 0.015166484, 0.0032223484, 0.0073820106, 0.012518704, + 0.00010425634, 0.015166484, 0.0200384, 0.0015515991, 0.016236188, + -0.022156624, -0.013726092, -0.01823791, 0.0030475948, -0.006137554, + 0.00907659, -0.031434447, 0.018470913, 0.007535582, 0.016045548, + -0.016659832, 0.015049982, -0.0033653285, -0.0031084938, -0.009118955, + -0.014425105, 0.00071225286, -0.0070166173, 0.037471384, -0.02127756, + -0.001576753, -0.02514332, 0.028638389, 0.009060703, -0.00489045, + -0.02041968, -0.024041843, 0.020080764, -0.009839151, -0.020726822, + 0.016606877, -0.0027510435, 0.0013463962, -0.015177075, -0.020472635, + 0.006836568, -0.021531748, 0.007874498, 0.028468931, 0.0071331193, + 0.0046971617, -0.0155159915, 0.0066724056, -0.005655658, 0.00936255, + -0.0018746282, -0.026032973, 0.0034738875, 0.020758595, 0.015367716, + -0.011565504, 0.010580529, 0.0032779516, 0.0025312777, -0.016034955, + 0.008329916, -0.0012146691, -0.016797516, 0.007911567, 0.006042234, + 0.010797647, -0.008605286, 0.015494809, 0.008329916, -0.014541608, + 0.03147681, 0.004935462, -0.0040246258, -0.02080096, -0.024423122, + 0.01295294, 0.009892106, 0.013874368, 0.01976303, -0.0164586, + 0.012137424, -0.01626796, -0.023279281, 0.0018163772, -0.0026755817, + 0.012243335, 0.004585955, -0.0020387906, 0.017644806, 0.021574112, + 0.006026347, 0.006344081, 0.015441854, 0.019053426, -0.020186676, + -0.020673867, 0.029019669, 0.023702927, -0.0021010134, 0.017369438, + 0.0071119373, 0.0059522097, -0.002155293, 0.014054417, 0.0025193626, + 0.0013927323, 0.002888728, 0.00038459006, 0.000111785965, + -0.023639381, 0.013260082, -0.029655136, -0.023300465, -0.020599728, + 0.010644075, 0.009765013, 0.024359576, 0.013365993, 0.018216727, + -0.026986174, 0.0022664997, 0.018481504, -0.0067571346, 0.013281264, + -0.0010187334, -0.011025356, 0.011226587, -0.0052558435, -0.012137424, + 0.0028357725, -0.0030714248, 0.008414645, -0.018587416, 0.0041199457, + -0.017613033, 0.016818698, 0.009765013, 0.011205405, 0.01995367, + 0.0025114194, -0.02978223, 0.0039531356, 0.026964992, -0.007344942, + -0.0000709936, -0.026138885, -0.0029496269, 0.0006291787, + -0.0026279218, 0.0066353367, -0.00056033646, 0.010167476, -0.03243001, + -0.016352689, 0.029019669, 0.006502948, 0.0043503027, -0.024973862, + 0.0067200656, 0.008631763, 0.0027642823, 0.0117667345, -0.026371889, + -0.005131398, -0.018693328, 0.010982991, -0.002216192, 0.023914749, + -0.0034447617, -0.005404119, -0.02666844, 0.01054346, -0.0016628059, + -0.0080916155, 0.008054547, -0.013853185, 0.017210571, -0.0022122203, + -0.002070564, 0.006116372, 0.0065982677, 0.014901706, -0.02155293, + 0.009616737, -0.021711797, -0.0021169002, -0.01597141, 0.012402202, + -0.00081882597, 0.007906271, -0.0068206815, -0.000005848905, + -0.01352486, 0.019593572, -0.026096521, -0.008684719, 0.005994574, + 0.04414379, 0.01040048, -0.024656128, 0.017507121, -0.02061032, + -0.011353681, 0.03715365, -0.0080386605, 0.0008995833, -0.00195009, + 0.023893567, 0.0019514139, -0.0038445767, 0.0023909453, 0.008822403, + 0.0118302815, -0.008292847, -0.002106309, 0.014382741, 0.014117964, + 0.01683988, -0.008832994, 0.036708824, -0.0055444515, 0.009489643, + 0.008303438, -0.00385252, 0.014573381, 0.027473366, -0.007858612, + -0.000037979094, 0.024550216, 0.002445225, -0.004302643, 0.013027078, + -0.002803999, 0.0057033184, 0.0032011662, 0.0118832365, -0.0015383603, + 0.01711525, 0.0016217653, 0.028468931, 0.0013199183, -0.0070960503, + -0.0014218579, 0.019212293, -0.009373141, 0.00512875, -0.014403923, + 0.002394917, 0.004358246, 0.000698352, 0.021510566, -0.001770041, + -0.0067836125, 0.01333422, -0.018407367, 0.018947514, 0.019604163, + -0.001492024, 0.031116711, -0.0025776138, 0.008562921, -0.0015568946, + 0.011163041, -0.014732248, -0.03573444, -0.0044006105, 0.0036380498, + -0.030078782, 0.018470913, -0.016204413, 0.021542339, -0.0011537701, + -0.008261073, 0.010130406, -0.003468592, 0.019307612, 0.023279281, + 0.009907993, 0.005216127, 0.019699484, -0.03412459, 0.015685448, + 0.008472896, -0.014234466, 0.015674857, -0.0018759521, 0.0030608338, + 0.0072125527, -0.014838159, -0.005719205, 0.018365003, 0.005179058, + 0.004498578, 0.019169927, -0.011544321, -0.0039187144, -0.007127824, + 0.006677701, 0.0008797249, -0.012815256, -0.019349977, -0.0042311526, + -0.014149737, -0.014827568, 0.020398498, 0.016215006, -0.0029522749, + -0.0066829966, -0.019868942, 0.007991, -0.015049982, -0.009484348, + -0.007964523, -0.00014496596, 0.012836438, -0.02156352, -0.020165494, + -0.004104059, -0.0033388506, -0.03035415, -0.025079772, -0.004585955, + 0.000007720968, 0.012328064, -0.0018097577, 0.0071648927, 0.010940627, + 0.040119164, 0.008372281, 0.0080386605, 0.00780036, 0.00075263146, + 0.0026689623, 0.03658173, -0.00012105319, 0.004779243, -0.013609589, + -0.028765483, 0.024444306, -0.009208979, -0.015388898, -0.010262796, + -0.0063652634, 0.012868211, -0.008054547, 0.024105389, -0.012878803, + -0.008822403, 0.006481766, -0.005075794, -0.015240622, 0.010246909, + -0.008435828, -0.0017713648, 0.0055921115, 0.029845776, 0.008239891, + 0.012772891, 0.021192832, -0.014403923, -0.010622893, -0.00936255, + 0.00025716564, 0.019455887, 0.027812282, -0.006148145, 0.009622033, + -0.0130800335, 0.029443314, -0.00921957, 0.024380758, 0.0130800335, + 0.026160067, -0.010744692, -0.018936923, -0.012328064, 0.007832133, + -0.0008439799, -0.0022228113, -0.021097511, 0.01030516, -0.00271927, + 0.0027669303, 0.0020626206, -0.010193953, -0.021542339, 0.01692461, + -0.0032726561, -0.025376324, 0.019222884, 0.04020389, 0.025778787, + 0.01257166, -0.005851594, 0.012306882, 0.018407367, 0.011215996, + 0.0044932826, 0.017147023, -0.012042103, 0.02081155, 0.015357125, + -0.002780169, 0.0037148355, 0.010654667, 0.009817968, 0.011300725, + 0.00082875515, -0.0030846638, -0.0086900145, -0.002817238, + -0.028405385, -0.022792092, 0.030481244, 0.0007043095, 0.0143297855, + 0.004705105, -0.014668701, 0.0071490062, -0.016966974, -0.0041464237, + -0.0029602181, -0.013376585, -0.017464757, 0.03564971, -0.0016005831, + -0.023639381, 0.020176085, 0.007069573, -0.022135442, -0.025418688, + -0.0042073224, -0.000039944243, -0.00095651054, -0.0030264126, + 0.0044482704, -0.013440131, 0.0092672305, -0.021923618, + -0.00013718811, 0.013757865, 0.014043826, -0.01910638, 0.0035083087, + -0.015166484, 0.018121406, 0.010093338, -0.029612772, -0.0018759521, + 0.018672144, -0.012031512, -0.02959159, 0.014689884, -0.010278682, + 0.024444306, -0.008155162, -0.009579668, 0.0027854645, 0.015092346, + 0.01257166, 0.015441854, 0.029252674, -0.0012795397, -0.015039391, + 0.0132389, -0.0070007304, 0.0035586164, 0.0045488863, 0.015897272, + -0.008462305, 0.009426097, -0.003720131, -0.016903428, 0.0006271929, + -0.0071384152, 0.0155689465, -0.0011478127, -0.0029814004, + 0.0071437107, -0.010003313, -0.01541008, 0.01852387, -0.0011286163, + -0.022982731, 0.023491105, 0.018322637, -0.011449001, -0.015007617, + -0.02950686, 0.0038737021, -0.009611442, -0.019741848, 0.01948766, + 0.009701466, 0.0026914685, 0.00009498911, 0.0030846638, 0.002602768, + -0.01597141, -0.022792092, 0.01044814, 0.0017184092, -0.010797647, + 0.0024266904, -0.0130906245, -0.008001591, 0.024317212, 0.0015118824, + 0.0035453774, 0.0078056557, 0.040013254, 0.0013781695, 0.01570663, + 0.009394323, -0.009913288, -0.007816247, 0.0067200656, 0.01871451, + 0.019890124, -0.01778249, 0.008340507, 0.0080280695, -0.013355402, + 0.0027047072, -0.010125111, -0.03793739, 0.004321177, -0.0045594773, + 0.0059628007, 0.0025246583, -0.0074932175, -0.0018084338, 0.009600851, + 0.031921636, 0.011131267, 0.0010392537, 0.00661945, 0.0059151407, + 0.011321908, -0.0073978975, -0.013927323, 0.009150728, -0.023745291, + 0.007032504, 0.005719205, -0.013842594, -0.008149867, 0.000987622, + -0.002708679, 0.014096781, 0.015155893, -0.00065797335, 0.0015780769, + -0.017538896, -0.018131997, 0.0066724056, -0.0130800335, + -0.0038445767, 0.013514269, -0.007842724, -0.014202693, -0.020313768, + -0.020387907, -0.0015423319, -0.01162905, -0.027282726, + -0.00019212953, -0.0037280743, 0.011512548, 0.009367846, -0.007810951, + 0.01710466, -0.008282256, 0.013514269, 0.004487987, -0.0038233944, + 0.022749726, -0.013217718, 0.016797516, 0.0068206815, -0.02459258, + 0.013514269, 0.011046538, -0.0078056557, -0.012317473, 0.0131118065, + -0.017189387, 0.00067121227, -0.0016164697, 0.002310188, -0.014816977, + 0.0001792216, 0.0067571346, -0.0059733917, -0.0013066794, 0.009701466, + 0.007858612, 0.024126573, -0.0069901394, 0.007922158, 0.008912427, + 0.006322899, -0.011459592, 0.017263526, 0.006269943, -0.026520165, + 0.017528305, 0.026986174, 0.0039346013, 0.0077474043, 0.016617468, + 0.011120676, 0.04196202, -0.002010989, -0.022664998, -0.020197267, + -0.0044588614, -0.013418949, -0.003142915, 0.0063387854, 0.008176344, + 0.009283117, -0.021012783, 0.008107503, 0.0025021522, -0.009807377, + -0.008684719, -0.014149737, 0.014912297, -0.011396046, -0.012328064, + 0.0039796135, 0.008176344, 0.00012709344, 0.018333228, 0.0059045497, + -0.0034209318, 0.0063387854, 0.02467731, 0.008467601, 0.017983722, + -0.019466478, 0.022834456, -0.0052320133, -0.0059204362, + -0.0013861129, 0.020567955, 0.04217384, -0.0052531958, 0.013588407, + 0.024613764, 0.009945062, -0.004683923, 0.018449731, 0.014361559, + -0.00623817, -0.018502686, 0.0049963607, 0.005454427, -0.012529295, + 0.008441123, 0.018598007, 0.01257166, -0.0012126833, 0.014255648, + -0.016469192, -0.0048348466, 0.01918052, -0.009955653, -0.006370559, + -0.003929306, -0.0065506077, 0.012497522, 0.0052479003, -0.01778249, + -0.011131267, -0.004127889, 0.0009141461, -0.032874838, -0.006322899, + -0.00066955737, 0.001984511, 0.013408358, -0.002841068, -0.011989148, + -0.011014765, 0.0019342033, 0.004104059, -0.020599728, -0.023639381, + -0.0048004254, 0.023131007, -0.007117233, -0.0047262874, -0.007572651, + 0.011279543, 0.0037254265, 0.018206136, -0.0037836777, 0.006825977, + 0.0043264725, -0.010220431, -0.025884697, -0.0039902045, 0.030671883, + 0.010395184, -0.0019911306, -0.005682136, -0.0037598477, 0.020070173, + 0.007080164, 0.0025100955, -0.0016839881, -0.008197527, 0.0072125527, + 0.01804727, -0.009007747, 0.0018891911, -0.013906141, -0.0013291856, + 0.012550478, 0.0021499975, -0.017210571, -0.010575234, 0.010151588, + -0.03505661, -0.001625737, -0.00060932036, -0.00041437757, + -0.004069638, 0.011025356, 0.0077579957, 0.0058727763, -0.008504669, + -0.002682201, 0.01938175, 0.012518704, 0.02599061, -0.0099715395, + -0.015081755, 0.017210571, -0.009584963, -0.0043635415, 0.0053882324, + 0.007667971, -0.006863046, -0.014997026, 0.0044588614, 0.0017620976, + 0.008769448, 0.0006341433, -0.027303908, 0.010479913, -0.0013305094, + 0.008075729, -0.00013983589, 0.0038631111, 0.004755413, 0.0002477329, + -0.019710075, 0.016945792, -0.0010604359, -0.022664998, -0.0131012155, + -0.027070904, -0.0072655086, 0.02041968, -0.014954662, -0.009436688, + -0.024147755, -0.012878803, 0.0030025826, -0.0031826317, 0.0024121276, + 0.016405646, 0.011268952, -0.015198258, 0.01408619, -0.013037669, + -0.027494548, 0.0015741052, 0.0028066467, -0.021192832, -0.01672338, + -0.035988625, -0.0018825716, -0.0019368511, -0.014033235, + -0.0040193303, -0.0118408725, 0.048337873, -0.01823791, -0.00931489, + 0.0109724, 0.00053220376, -0.004358246, -0.01229629, 0.005692727, + 0.004887802, 0.0070483903, -0.023872385, -0.019604163, -0.026541347, + -0.0040166825, -0.010252205, 0.003958431, 0.010728804, 0.0064976523, + 0.0065241302, -0.019699484, -0.0017885754, -0.012592842, 0.0059257317, + -0.014764021, 0.010342229, 0.0195618, -0.007058982, -0.0099291755, + -0.0049672355, 0.006042234, 0.033700943, -0.0155477645, -0.011989148, + 0.009542599, 0.012158606, -0.0020864506, -0.0069265924, 0.015134711, + 0.007291986, 0.022643816, -0.01475343, 0.000053245203, 0.0017501826, + 0.0033494418, -0.017803673, -0.018322637, 0.014922888, 0.0012821875, + 0.009097773, -0.010140997, -0.013175353, 0.000041371564, 0.02694381, + -0.027197996, -0.0025723183, 0.02579997, 0.03384922, 0.01710466, + 0.019466478, 0.02089628, 0.006397037, 0.01663865, -0.0032594171, + -0.011078312, -0.009018339, -0.0074720355, -0.0022082485, -0.00539088, + 0.010368707, -0.0093307765, -0.017443575, -0.0024836177, + -0.0039187144, 0.008917723, 0.0143509675, 0.013249491, -0.004993713, + 0.023554651, 0.012317473, -0.027769918, 0.031031983, -0.0118408725, + 0.0073714196, -0.023194553, 0.015685448, 0.023681745, 0.01541008, + -0.0051843533, -0.024889132, -0.013673136, 0.008404054, 0.0025233345, + -0.007440262, -0.0035480252, 0.015081755, 0.0003961741, 0.00035579543, + 0.014075599, -0.009976835, 0.010024495, -0.015748996, 0.008292847, + 0.007344942, -0.022008348, -0.0010286625, -0.0064023323, -0.014065008, + 0.013270673, 0.035628527, -0.01162905, 0.0017793082, 0.037217196, + 0.009383732, 0.029104399, 0.008065138, -0.014456879, 0.019868942, + 0.0074932175, -0.0028092947, 0.020133719, -0.0010299864, -0.007524991, + 0.010426958, -0.009283117, -0.009690875, 0.009706762, -0.0013609589, + -0.0013490439, -0.023088641, 0.01778249, -0.007551469, -0.017252935, + -0.012137424, -0.0063493764, -0.016511556, -0.009018339, 0.013376585, + 0.019932488, 0.017676579, 0.011215996, 0.0075091044, 0.010712918, + -0.00510492, -0.007927453, -0.006359968, -0.0104958005, 0.0005070499, + -0.026562529, 0.0052849688, 0.0025087716, -0.012063286, 0.013567225, + 0.015960818, -0.0015357124, 0.0053008557, -0.008499374, -0.008192232, + 0.006370559, -0.020281995, 0.015876088, 0.005586816, -0.0036486408, + 0.0051552276, -0.010140997, 0.0011570799, 0.007599129, -0.013037669, + 0.021923618, 0.010204544, 0.012179788, -0.0037545522, -0.012836438, + -0.020176085, 0.009463166, 0.034971878, -0.006222283, 0.0155054, + -0.005851594, 0.011353681, 0.009261935, 0.0076256064, -0.026223613, + 0.016553922, 0.00888595, 0.015484218, -0.0073343506, -0.012444566, + 0.01257166, -0.01843914, 0.024973862, 0.019508844, 0.024910314, + 0.008096911, -0.015325351, -0.0043608937, 0.002156617, -0.02904085, + 0.006492357, 0.00070364756, 0.027897011, -0.011449001, 0.0039822613, + 0.0067942035, -0.014626337, 0.0105063915, -0.0059204362, 0.004562125 + ] + } + ] + } ] diff --git a/src/scrapers/markdown/playground/markdown-scraper-playground.ts b/src/scrapers/markdown/playground/markdown-scraper-playground.ts index 7da3282d28..08de86533a 100644 --- a/src/scrapers/markdown/playground/markdown-scraper-playground.ts +++ b/src/scrapers/markdown/playground/markdown-scraper-playground.ts @@ -1,71 +1,73 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { writeFile } from 'fs/promises'; -import { join } from 'path'; -import { usageToHuman } from '../../../execution/utils/usageToHuman'; -import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground'; -import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson'; -import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode'; -import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler'; -import { MarkdownScraper } from '../MarkdownScraper'; +import { join } from "path"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { writeFile } from "fs/promises"; +import { usageToHuman } from "../../../execution/utils/usageToHuman"; +import { $provideLlmToolsForTestingAndScriptsAndPlayground } from "../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground"; +import { stringifyPipelineJson } from "../../../utils/editable/utils/stringifyPipelineJson"; +import { $provideFilesystemForNode } from "../../_common/register/$provideFilesystemForNode"; +import { makeKnowledgeSourceHandler } from "../../_common/utils/makeKnowledgeSourceHandler"; +import { MarkdownScraper } from "../MarkdownScraper"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 Scrape knowledge from Markdown (playground)`); + console.info(`🧸 Scrape knowledge from Markdown (playground)`); - // Do here stuff you want to test - //========================================> + // Do here stuff you want to test + //========================================> - const example = '10-simple.md'; - // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them + const example = "10-simple.md"; + // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them - const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true }); - const rootDirname = join(__dirname, '..', 'examples'); + const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ + isCacheReloaded: true, + }); + const rootDirname = join(__dirname, "..", "examples"); - const markdownScraper = new MarkdownScraper( - { llm: await $provideLlmToolsForTestingAndScriptsAndPlayground() }, - { - rootDirname, - }, - ); + const markdownScraper = new MarkdownScraper( + { llm: await $provideLlmToolsForTestingAndScriptsAndPlayground() }, + { + rootDirname, + }, + ); - const knowledge = await markdownScraper.scrape( - await makeKnowledgeSourceHandler( - { knowledgeSourceContent: example }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ); + const knowledge = await markdownScraper.scrape( + await makeKnowledgeSourceHandler( + { knowledgeSourceContent: example }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ); - console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); - console.info(colors.bgGreen(' Knowledge: ')); - console.info(knowledge); + console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); + console.info(colors.bgGreen(" Knowledge: ")); + console.info(knowledge); - await writeFile( - join( - __dirname, - `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, - ), - stringifyPipelineJson(knowledge), - 'utf-8', - ); - /**/ + await writeFile( + join( + __dirname, + `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, + ), + stringifyPipelineJson(knowledge), + "utf-8", + ); + /**/ - //========================================/ + //========================================/ } /** diff --git a/src/scrapers/markdown/playground/tsconfig.json b/src/scrapers/markdown/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/scrapers/markdown/playground/tsconfig.json +++ b/src/scrapers/markdown/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/scrapers/markdown/register-constructor.ts b/src/scrapers/markdown/register-constructor.ts index 960e62dc7a..03d7e98a35 100644 --- a/src/scrapers/markdown/register-constructor.ts +++ b/src/scrapers/markdown/register-constructor.ts @@ -1,6 +1,6 @@ -import type { Registration } from '../../utils/$Register'; -import { $scrapersRegister } from '../_common/register/$scrapersRegister'; -import { createMarkdownScraper } from './createMarkdownScraper'; +import type { Registration } from "../../utils/$Register"; +import { $scrapersRegister } from "../_common/register/$scrapersRegister"; +import { createMarkdownScraper } from "./createMarkdownScraper"; /** * Registration of known scraper @@ -11,7 +11,8 @@ import { createMarkdownScraper } from './createMarkdownScraper'; * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _MarkdownScraperRegistration: Registration = $scrapersRegister.register(createMarkdownScraper); +export const _MarkdownScraperRegistration: Registration = + $scrapersRegister.register(createMarkdownScraper); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/scrapers/markdown/register-metadata.ts b/src/scrapers/markdown/register-metadata.ts index fa18214b1d..edb46b7a41 100644 --- a/src/scrapers/markdown/register-metadata.ts +++ b/src/scrapers/markdown/register-metadata.ts @@ -1,8 +1,8 @@ -import type { Registration } from '../../utils/$Register'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import { $deepFreeze } from '../../utils/serialization/$deepFreeze'; -import { $scrapersMetadataRegister } from '../_common/register/$scrapersMetadataRegister'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; +import type { Registration } from "../../utils/$Register"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import { $deepFreeze } from "../../utils/serialization/$deepFreeze"; +import { $scrapersMetadataRegister } from "../_common/register/$scrapersMetadataRegister"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; keepTypeImported(); @@ -12,15 +12,15 @@ keepTypeImported(); * @private within the scraper directory */ export const markdownScraperMetadata = $deepFreeze({ - title: 'Markdown scraper', - packageName: '@promptbook/markdown-utils', - className: 'MarkdownScraper', - mimeTypes: ['text/markdown', 'text/plain'], - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@', + title: "Markdown scraper", + packageName: "@promptbook/markdown-utils", + className: "MarkdownScraper", + mimeTypes: ["text/markdown", "text/plain"], + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@", - isAvilableInBrowser: true, - // <- Note: [🌏] This is the only scraper which makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server - requiredExecutables: [], + isAvilableInBrowser: true, + // <- Note: [🌏] This is the only scraper which makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server + requiredExecutables: [], }) satisfies ScraperAndConverterMetadata; /* <- Note: [🤛] */ /** @@ -33,7 +33,7 @@ export const markdownScraperMetadata = $deepFreeze({ * @public exported from `@promptbook/cli` */ export const _MarkdownScraperMetadataRegistration: Registration = - $scrapersMetadataRegister.register(markdownScraperMetadata); + $scrapersMetadataRegister.register(markdownScraperMetadata); /** * Note: [💞] Ignore a discrepancy between file name and entity name diff --git a/src/scrapers/markitdown/MarkitdownScraper.ts b/src/scrapers/markitdown/MarkitdownScraper.ts index c29f7c274b..085dd876a6 100644 --- a/src/scrapers/markitdown/MarkitdownScraper.ts +++ b/src/scrapers/markitdown/MarkitdownScraper.ts @@ -1,24 +1,24 @@ -import { readFile } from 'fs/promises'; -import type { MarkItDown } from 'markitdown-ts'; // <- TODO: [🍀] Use Markitdown directly not through this package -import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from '../../config'; -import { DEFAULT_IS_VERBOSE } from '../../config'; -import { DEFAULT_SCRAPE_CACHE_DIRNAME } from '../../config'; -import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError'; -import { KnowledgeScrapeError } from '../../errors/KnowledgeScrapeError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { KnowledgePiecePreparedJson } from '../../pipeline/PipelineJson/KnowledgePieceJson'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { $isRunningInNode } from '../../utils/environment/$isRunningInNode'; -import { isFileExisting } from '../../utils/files/isFileExisting'; -import type { Converter } from '../_common/Converter'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; -import type { Scraper } from '../_common/Scraper'; -import type { ScraperSourceHandler } from '../_common/Scraper'; -import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource'; -import { getScraperIntermediateSource } from '../_common/utils/getScraperIntermediateSource'; -import { MarkdownScraper } from '../markdown/MarkdownScraper'; -import { markitdownScraperMetadata } from './register-metadata'; +import { readFile } from "fs/promises"; +import type { MarkItDown } from "markitdown-ts"; // <- TODO: [🍀] Use Markitdown directly not through this package +import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from "../../config"; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import { DEFAULT_SCRAPE_CACHE_DIRNAME } from "../../config"; +import { EnvironmentMismatchError } from "../../errors/EnvironmentMismatchError"; +import { KnowledgeScrapeError } from "../../errors/KnowledgeScrapeError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { KnowledgePiecePreparedJson } from "../../pipeline/PipelineJson/KnowledgePieceJson"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { $isRunningInNode } from "../../utils/environment/$isRunningInNode"; +import { isFileExisting } from "../../utils/files/isFileExisting"; +import type { Converter } from "../_common/Converter"; +import type { Scraper } from "../_common/Scraper"; +import type { ScraperSourceHandler } from "../_common/Scraper"; +import type { ScraperIntermediateSource } from "../_common/ScraperIntermediateSource"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; +import { getScraperIntermediateSource } from "../_common/utils/getScraperIntermediateSource"; +import { MarkdownScraper } from "../markdown/MarkdownScraper"; +import { markitdownScraperMetadata } from "./register-metadata"; /** * Integration of Markitdown by Microsoft into Promptbook @@ -29,121 +29,132 @@ import { markitdownScraperMetadata } from './register-metadata'; * @public exported from `@promptbook/pdf` */ export class MarkitdownScraper implements Converter, Scraper { - /** - * Metadata of the scraper which includes title, mime types, etc. - */ - public get metadata(): ScraperAndConverterMetadata { - return markitdownScraperMetadata; - } - - /** - * Markdown scraper is used internally - */ - private readonly markdownScraper: MarkdownScraper; - - /** - * Markdown scraper is used internally - */ - private readonly markitdown: MarkItDown; - - public constructor( - private readonly tools: Pick, - private readonly options: PrepareAndScrapeOptions, - ) { - this.markdownScraper = new MarkdownScraper(tools, options); - - // Note: Module `markitdown-ts` has no types available, so it is imported using `require` - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { MarkItDown } = require('markitdown-ts'); - // <- TODO: [🍀] Use Markitdown directly not through this package - - this.markitdown = new MarkItDown(); - } - - /** - * Convert the documents to `.md` file and returns intermediate source - * - * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object - */ - public async $convert(source: ScraperSourceHandler): Promise { - const { - rootDirname = process.cwd(), - cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, - intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, - isVerbose = DEFAULT_IS_VERBOSE, - } = this.options; - - if (!$isRunningInNode()) { - throw new KnowledgeScrapeError('MarkitdownScraper is only supported in Node environment'); - } - - if (this.tools.fs === undefined) { - throw new EnvironmentMismatchError('Can not scrape boilerplates without filesystem tools'); - // <- TODO: [🧠] What is the best error type here` - } - - const cacheFilehandler = await getScraperIntermediateSource(source, { - rootDirname, - cacheDirname, - intermediateFilesStrategy, - extension: 'md', - isVerbose, - }); - - // TODO: @@@ Preserve, delete or modify - // Note: Running Pandoc ONLY if the file in the cache does not exist - if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { - const src = source.filename || source.url || null; - - // console.log('!!', { src, source, cacheFilehandler }); - - if (src === null) { - throw new UnexpectedError('Source has no filename or url'); - } - - const result = await this.markitdown.convert(src, { - // TODO: Pass when sacraping Youtube - // enableYoutubeTranscript: true, - // youtubeTranscriptLanguage: 'en', - }); - - if (result === null || result === undefined) { - throw new Error(`Markitdown could not convert the "${source.source}"`); - // <- TODO: [🍀] Make MarkitdownError - } - - // console.log('!!', { result, cacheFilehandler }); - - await this.tools.fs.writeFile(cacheFilehandler.filename, result.text_content); - } - - return cacheFilehandler; - } - - /** - * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it - */ - public async scrape( - source: ScraperSourceHandler, - ): Promise> | null> { - const cacheFilehandler = await this.$convert(source); - - // TODO: @@@ Preserve, delete or modify - const markdownSource = { - source: source.source, - filename: cacheFilehandler.filename, - url: null, - mimeType: 'text/markdown', - async asText() { - // Note: [0] In $convert we check that the file exists - return await readFile(cacheFilehandler.filename, 'utf-8'); - }, - asJson() { - throw new UnexpectedError( - 'Did not expect that `markdownScraper` would need to get the content `asJson`', - ); - }, - /* + /** + * Metadata of the scraper which includes title, mime types, etc. + */ + public get metadata(): ScraperAndConverterMetadata { + return markitdownScraperMetadata; + } + + /** + * Markdown scraper is used internally + */ + private readonly markdownScraper: MarkdownScraper; + + /** + * Markdown scraper is used internally + */ + private readonly markitdown: MarkItDown; + + public constructor( + private readonly tools: Pick, + private readonly options: PrepareAndScrapeOptions, + ) { + this.markdownScraper = new MarkdownScraper(tools, options); + + // Note: Module `markitdown-ts` has no types available, so it is imported using `require` + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { MarkItDown } = require("markitdown-ts"); + // <- TODO: [🍀] Use Markitdown directly not through this package + + this.markitdown = new MarkItDown(); + } + + /** + * Convert the documents to `.md` file and returns intermediate source + * + * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object + */ + public async $convert( + source: ScraperSourceHandler, + ): Promise { + const { + rootDirname = process.cwd(), + cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, + intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, + isVerbose = DEFAULT_IS_VERBOSE, + } = this.options; + + if (!$isRunningInNode()) { + throw new KnowledgeScrapeError( + "MarkitdownScraper is only supported in Node environment", + ); + } + + if (this.tools.fs === undefined) { + throw new EnvironmentMismatchError( + "Can not scrape boilerplates without filesystem tools", + ); + // <- TODO: [🧠] What is the best error type here` + } + + const cacheFilehandler = await getScraperIntermediateSource(source, { + rootDirname, + cacheDirname, + intermediateFilesStrategy, + extension: "md", + isVerbose, + }); + + // TODO: @@@ Preserve, delete or modify + // Note: Running Pandoc ONLY if the file in the cache does not exist + if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) { + const src = source.filename || source.url || null; + + // console.log('!!', { src, source, cacheFilehandler }); + + if (src === null) { + throw new UnexpectedError("Source has no filename or url"); + } + + const result = await this.markitdown.convert(src, { + // TODO: Pass when sacraping Youtube + // enableYoutubeTranscript: true, + // youtubeTranscriptLanguage: 'en', + }); + + if (result === null || result === undefined) { + throw new Error(`Markitdown could not convert the "${source.source}"`); + // <- TODO: [🍀] Make MarkitdownError + } + + // console.log('!!', { result, cacheFilehandler }); + + await this.tools.fs.writeFile( + cacheFilehandler.filename, + result.text_content, + ); + } + + return cacheFilehandler; + } + + /** + * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it + */ + public async scrape( + source: ScraperSourceHandler, + ): Promise + > | null> { + const cacheFilehandler = await this.$convert(source); + + // TODO: @@@ Preserve, delete or modify + const markdownSource = { + source: source.source, + filename: cacheFilehandler.filename, + url: null, + mimeType: "text/markdown", + async asText() { + // Note: [0] In $convert we check that the file exists + return await readFile(cacheFilehandler.filename, "utf-8"); + }, + asJson() { + throw new UnexpectedError( + "Did not expect that `markdownScraper` would need to get the content `asJson`", + ); + }, + /* TODO: [🥽] > asBlob() { > throw new UnexpectedError( @@ -151,14 +162,14 @@ export class MarkitdownScraper implements Converter, Scraper { > ); > }, */ - } satisfies ScraperSourceHandler; + } satisfies ScraperSourceHandler; - const knowledge = this.markdownScraper.scrape(markdownSource); + const knowledge = this.markdownScraper.scrape(markdownSource); - await cacheFilehandler.destroy(); + await cacheFilehandler.destroy(); - return knowledge; - } + return knowledge; + } } /** diff --git a/src/scrapers/markitdown/createMarkitdownScraper.ts b/src/scrapers/markitdown/createMarkitdownScraper.ts index fe20c36300..5bb05b74b6 100644 --- a/src/scrapers/markitdown/createMarkitdownScraper.ts +++ b/src/scrapers/markitdown/createMarkitdownScraper.ts @@ -1,9 +1,9 @@ -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import type { ScraperConstructor } from '../_common/register/ScraperConstructor'; -import { MarkitdownScraper } from './MarkitdownScraper'; -import { markitdownScraperMetadata } from './register-metadata'; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import type { ScraperConstructor } from "../_common/register/ScraperConstructor"; +import { MarkitdownScraper } from "./MarkitdownScraper"; +import { markitdownScraperMetadata } from "./register-metadata"; keepTypeImported(); @@ -14,10 +14,13 @@ keepTypeImported(); * @public exported from `@promptbook/pdf` */ export const createMarkitdownScraper = Object.assign( - (tools: Pick, options: PrepareAndScrapeOptions): MarkitdownScraper => { - return new MarkitdownScraper(tools, options); - }, - markitdownScraperMetadata, + ( + tools: Pick, + options: PrepareAndScrapeOptions, + ): MarkitdownScraper => { + return new MarkitdownScraper(tools, options); + }, + markitdownScraperMetadata, ) satisfies ScraperConstructor; /* <- Note: [🤛] */ /** diff --git a/src/scrapers/markitdown/examples/10-simple.pdf.knowledge.json b/src/scrapers/markitdown/examples/10-simple.pdf.knowledge.json index 4e4185016f..02ddbf26c2 100644 --- a/src/scrapers/markitdown/examples/10-simple.pdf.knowledge.json +++ b/src/scrapers/markitdown/examples/10-simple.pdf.knowledge.json @@ -1,18 +1,636 @@ [ - { - "name": "springfield-illinois-historic-capital-city", - "title": "Springfield: Illinois' Historic Capital City", - "content": "Springfield is the county seat of Sangamon County in Illinois, USA. As of 2019, it had a population of 10,566, making it the sixth most populous city in the state.", - "keywords": [ - "Springfield: Illinois' Historic Capital City" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.022805648, 0.035417367, -0.0029928328, -0.024086423, 0.016179595, -0.0070115933, 0.011461635, 0.0027347174, -0.056981448, 0.017042158, -0.029248733, 0.01291231, -0.008736719, 0.018858768, 0.011193718, 0.009886804, -0.016336424, 0.033953626, 0.003763912, -0.0007968091, 0.012670531, -0.0023491778, -0.049244516, 0.03596627, 0.023171583, 0.0093379, -0.03126138, 0.008860877, -0.034188867, -0.0043977653, -0.021616356, -0.0042866776, 0.049793422, 0.0024504636, 0.015983557, -0.016349493, 0.0077042575, -0.020740723, 0.024190977, 0.02930101, 0.0013836952, -0.0013395868, 0.01671543, 0.00820742, 0.03293423, -0.036985666, -0.004815978, -0.024360877, 0.029248733, 0.016937604, 0.024870573, -0.012500632, -0.019708263, 0.037168633, 0.010141652, -0.01806155, 0.011677276, 0.022596542, -0.0067959526, -0.043232713, -0.0032231766, -0.0049597383, -0.00013263135, 0.00044802643, 0.0061326935, 0.0029536255, 0.0051002316, 0.004289945, 0.053688023, 0.011376685, 0.013317453, 0.056563236, 0.008070193, 0.043781616, 0.05980438, -0.005485771, -0.03147049, -0.002639966, -0.04252698, 0.029562393, 0.013173693, -0.013604974, -0.0145720905, 0.002955259, 0.017251264, 0.008638701, -0.00064978126, -0.07783979, -0.00006636672, 0.014206154, 0.017133642, -0.0077696033, -0.0059431912, -0.028752107, -0.043781616, -0.028124787, 0.041559864, -0.013435075, -0.00046354602, 0.01974747, 0.014781197, -0.009239881, -0.026504215, 0.01914629, 0.008187816, 0.015238617, -0.0075539625, -0.0068220906, -0.020204889, 0.0046591484, 0.018649662, -0.0037933176, -0.012964586, 0.006345067, -0.016205734, 0.00068653823, -0.014545952, -0.0107689705, 0.042370148, 0.02429553, -0.0038848016, 0.0049139964, -0.0276543, 0.011402824, -0.0046983557, -0.0016556965, 0.015277823, -0.027288362, 0.007978709, -0.0412462, 0.03463322, 0.0062960577, -0.018139966, 0.018715007, 0.00956661, -0.034136593, -0.0057275505, 0.030921584, 0.03052951, 0.03873693, -0.06984148, -0.0030941188, 0.026033726, -0.0000384927, 0.0048355814, -0.010357293, -0.01927698, 0.0032362456, 0.013604974, -0.028516863, -0.01152698, -0.010494519, -0.027419055, 0.004874789, 0.0025615513, 0.016637014, -0.0039762855, -0.013278246, 0.043337267, -0.053583473, -0.019512225, -0.009710371, 0.03450253, -0.026138278, -0.00872365, -0.013088743, -0.01488575, -0.0041233134, -0.00736446, 0.008096332, 0.022727232, -0.009488195, -0.061895445, -0.048800167, 0.0021188343, -0.018545108, -0.03688111, -0.015656829, -0.00064365513, 0.029666945, -0.0039664838, 0.01146817, 0.02883052, -0.008867411, -0.024988195, -0.019329257, 0.008436129, 0.011291737, 0.036489036, -0.010043633, -0.0066129845, 0.041978076, 0.010389966, -0.0023687815, 0.04022681, -0.003298324, -0.023511382, -0.03293423, -0.0040677697, 0.029091904, -0.05755649, 0.021603288, -0.024073355, -0.03207167, 0.0006881719, 0.023001686, -0.00842306, -0.0133566605, 0.020034991, 0.026360454, -0.006952782, -0.0030843168, -0.01944688, 0.03492074, -0.020714587, 0.023158515, -0.013944772, -0.0011966431, -0.015617621, 0.038841482, 0.020518549, 0.028647553, 0.0004202545, -0.014807335, -0.032698985, -0.025236508, -0.053034566, 0.016297217, -0.009050379, -0.037011802, 0.024413152, 0.019394603, 0.004022028, 0.06017032, -0.021433389, 0.043467958, 0.018453624, -0.00009761014, -0.014493676, 0.03267285, 0.018937184, 0.018963322, 0.0011705048, -0.030581785, 0.0012186973, -0.02080607, 0.019721331, -0.0033358978, 0.013435075, 0.021747047, 0.008717116, -0.0030124367, 0.057190552, 0.0014637436, -0.0112263905, 0.01715978, -0.015774451, 0.013173693, 0.0088478075, 0.00023606133, -0.016833052, 0.0037018338, -0.03842327, 0.0297715, 0.004678752, -0.01504258, 0.022870993, 0.00047947405, 0.00565567, -0.032123946, -0.0045578624, -0.010442242, 0.0061196247, 0.007959106, 0.0054106237, 0.019930437, 0.0479376, 0.014206154, -0.022047639, -0.03753457, -0.021015177, 0.030712478, -0.0012693402, -0.01987816, 0.007684654, 0.0034273819, -0.017695615, 0.013310919, 0.005208052, -0.045924954, -0.013853287, -0.001886857, 0.04265767, 0.025876896, 0.006122892, -0.011866778, 0.01488575, -0.0015290893, -0.04022681, 0.014062393, 0.03322175, 0.036541313, -0.023916526, -0.029327149, 0.022191398, -0.03902445, 0.018100757, -0.0052701305, -0.039677907, -0.06372512, -0.0072795106, -0.048800167, -0.010082841, 0.016022764, 0.0050316188, 0.011762225, 0.018113827, 0.00926602, 0.034084316, -0.02305396, -0.0050316188, 0.010965008, 0.024047216, 0.023615934, -0.010305016, 0.04145531, -0.008638701, 0.00018337637, 0.010461846, 0.011376685, -0.016140386, 0.01594435, -0.0068547637, -0.04116779, 0.009194139, -0.013683389, 0.024060285, 0.0058419053, -0.00070736714, 0.031784147, -0.004071037, 0.008939291, 0.0021449726, 0.041873522, 0.028412309, -0.008717116, 0.0077696033, -0.010200463, -0.023511382, 0.028046373, 0.0006722438, 0.010690556, 0.058079254, -0.0054596327, 0.016336424, -0.0018884906, -0.023459105, -0.01989123, 0.015317031, -0.038658515, -0.020126475, 0.005887647, -0.015251686, 0.011513912, -0.022910202, 0.024948986, -0.012977655, 0.0074298056, 0.038789205, -0.0012358505, -0.0005852524, 0.016780775, -0.0132129, 0.009788785, 0.000096793316, 0.012180438, -0.016323354, 0.008769392, -0.021002106, -0.0015642126, 0.026896289, 0.018244518, -0.014689713, -0.02548482, -0.010108979, -0.023733556, -0.010801644, -0.01382715, -0.028124787, 0.018610455, 0.0053714165, -0.028360032, 0.008730185, 0.0055576516, 0.009305228, 0.032123946, -0.028438447, -0.006828625, 0.030294266, 0.0009360771, 0.01714671, -0.027419055, 0.0023916524, 0.03340472, -0.020126475, -0.0104683805, -0.009520868, -0.061895445, 0.011585792, -0.011219856, -0.015852867, 0.015552276, 0.01487268, -0.007841484, -0.02535413, -0.05739966, -0.010102444, 0.04461804, 0.019852022, 0.019329257, -0.009520868, -0.030634062, 0.0014457735, 0.004613406, -0.0034731238, 0.01441526, 0.0006849046, 0.026242832, -0.026974704, -0.032881953, -0.036410622, 0.004335687, -0.011324409, -0.0025958577, -0.009527403, 0.05206745, -0.04859106, 0.005400822, 0.046892073, 0.0262559, -0.0318887, 0.01684612, 0.043311127, -0.03175801, -0.017551854, 0.008854342, 0.021747047, 0.017303541, 0.017538784, 0.055047214, -0.00926602, 0.017839376, 0.0013812446, 0.013297849, 0.017551854, 0.009625421, -0.021943085, 0.011742622, -0.018388279, 0.022099914, 0.029118042, -0.019224703, 0.011455101, -0.035286676, -0.032986507, 0.035417367, 0.049897972, 0.016088111, 0.0034437182, -0.007939503, -0.012383009, -0.026190555, -0.050943505, 0.023746626, -0.0406973, 0.0007825147, 0.0039730184, 0.021302696, 0.019512225, -0.021851601, -0.044722594, 0.010880059, -0.052041315, 0.011063026, 0.002115567, 0.04101096, -0.026504215, -0.0094555225, 0.036096964, -0.014349915, 0.069893755, 0.007606239, -0.008024451, -0.013376264, 0.014284569, -0.006959317, -0.03949494, -0.058236085, 0.037560705, 0.039129004, -0.0017643338, 0.02217833, 0.023811972, 0.003102287, -0.006518233, -0.009448987, -0.017525716, -0.03978246, 0.0035188657, 0.0002311604, 0.06267959, 0.011121837, -0.036384486, -0.024504635, 0.010625211, -0.014140809, -0.0012824093, -0.014532883, 0.028203202, 0.008429595, -0.009945615, 0.02443929, 0.0035646078, 0.0058680438, -0.023406828, 0.030503372, 0.0012366673, -0.0046166736, 0.019773608, -0.0031055543, -0.042239457, -0.0021417052, 0.0068416945, 0.028673692, 0.04521922, -0.023197722, -0.017486509, -0.025001263, -0.0058549745, 0.042840637, -0.016297217, -0.04610792, 0.0023720488, 0.0273145, 0.014781197, 0.01641484, -0.0041396497, -0.021080522, 0.009729974, -0.019107081, 0.046682965, -0.010566399, 0.015081787, -0.0052831997, -0.007018128, -0.012873102, -0.012775084, -0.010612141, -0.031548902, -0.021381112, 0.01807462, 0.010533726, 0.031627316, 0.003239513, -0.005554384, 0.020283304, 0.0073840637, -0.00076576986, 0.0024586318, 0.0029209526, 0.023171583, 0.017813237, 0.027967958, -0.009703836, 0.00398282, 0.010599072, -0.014898819, -0.01487268, -0.0037443084, 0.0025484823, -0.001069219, 0.018022344, 0.02565472, 0.021969223, 0.0053158724, 0.015748313, -0.006345067, -0.04009612, -0.021250421, 0.04328499, 0.046343166, 0.02869983, 0.00019777284, 0.005492306, -0.0043324195, 0.006730607, -0.015918212, 0.009122259, -0.031496625, -0.038240302, 0.016650084, -0.0012170636, 0.025602443, -0.022910202, -0.020518549, 0.005825569, 0.014938027, 0.009207209, 0.013918634, -0.0031643654, -0.011422427, 0.011383221, 0.00486172, -0.015369308, -0.030555647, -0.0074036676, 0.023171583, 0.034842327, 0.005208052, -0.015317031, -0.0009769181, -0.034241144, -0.011709949, -0.012644392, 0.016493253, -0.025942242, 0.009547006, -0.01701602, 0.00014753833, 0.008070193, -0.0029209526, 0.021341905, -0.014911888, -0.004110244, 0.000940978, -0.002011014, 0.0034241145, 0.012794687, -0.02596838, 0.0001425353, 0.014480607, 0.053243674, 0.0243086, 0.030634062, -0.009076517, 0.0004770236, -0.020309443, 0.019303119, -0.002390019, 0.0069266437, 0.032594435, -0.016506324, 0.0057504214, 0.0005652403, -0.0053126053, 0.018309863, 0.04119393, -0.031182967, 0.026295109, 0.003384907, -0.033012647, 0.0022397237, -0.0003771672, 0.0138140805, 0.015134064, 0.030921584, -0.043494094, 0.030059021, 0.024831364, 0.0048780562, 0.008527613, 0.019525293, -0.00090258743, 0.008227023, -0.017081365, 0.014010117, -0.009187605, -0.0069266437, -0.019956576, 0.0007616858, -0.007567032, -0.010572934, -0.009697301, -0.016048903, 0.013317453, -0.023615934, 0.035155986, 0.04116779, 0.0036070824, -0.005266863, 0.018192243, 0.03782209, -0.0033979763, -0.008429595, 0.04114165, -0.0056524025, -0.004610139, 0.042448565, 0.0076454463, -0.014049325, -0.01684612, 0.02475295, 0.0057079466, -0.04767622, 0.010017496, 0.013271711, -0.007998313, -0.03356155, -0.051387858, -0.00364629, -0.0026889753, -0.0013820615, 0.016937604, -0.02126349, 0.054263067, -0.04132462, 0.028464586, -0.023276137, -0.006482293, 0.039677907, 0.023027824, 0.0153301, -0.034084316, -0.0036299534, -0.038789205, 0.00697892, -0.029091904, -0.023249999, 0.014768127, 0.012010539, -0.037116356, 0.00516231, 0.030294266, -0.008913153, 0.01236994, 0.010455311, -0.007795742, -0.029353287, 0.021080522, 0.0021482399, -0.006459422, -0.021028245, 0.043833893, -0.0010071405, 0.008828203, 0.016532462, -0.0021433388, 0.0028033303, -0.057922427, 0.00271348, -0.033378582, -0.024426222, 0.049270656, 0.024687605, -0.002324673, -0.0021678435, -0.0014506745, 0.0034927274, 0.018989459, 0.053086843, 0.03141821, 0.0118863825, -0.00029548508, 0.013618044, 0.01640177, 0.0005403272, 0.0063483343, -0.0034829257, 0.0064626895, 0.0034633218, 0.049715005, -0.0021008642, -0.023419898, 0.0026121943, 0.039887015, 0.004338954, 0.007795742, -0.022152191, 0.008200885, -0.030059021, 0.03309106, -0.0005047955, -0.0015846331, -0.0006211925, 0.0032836213, -0.012624789, -0.0036560916, -0.0047898395, 0.011938659, -0.011383221, -0.015238617, -0.019629847, -0.038501684, 0.015016441, 0.015892074, -0.020231027, 0.004672217, -0.02885666, 0.04900927, 0.009716905, 0.0115335155, 0.023589797, -0.000035659352, 0.018518971, -0.040435918, 0.0077630687, 0.0075408933, 0.006518233, -0.014402191, 0.016950674, 0.042317875, -0.025249578, 0.029091904, -0.009827993, -0.025314923, -0.018414417, 0.02217833, -0.016271079, 0.0026791736, -0.035129845, 0.03037268, -0.01957757, 0.034580942, -0.012539839, 0.016584737, 0.010932335, 0.010494519, -0.023511382, 0.0048584524, 0.028543001, -0.01411467, 0.0027216482, -0.024086423, -0.012735876, -0.0022413575, 0.015212478, 0.0054563656, 0.003613617, 0.025615513, 0.009867201, -0.0045349915, -0.011337479, 0.020296372, -0.013591905, 0.025458684, 0.045663573, -0.0040350966, -0.04268381, 0.011350548, -0.032986507, -0.031052275, 0.009827993, -0.017656406, 0.021302696, -0.022674957, 0.01624494, 0.037247047, 0.007939503, -0.007730396, 0.009520868, 0.006893971, -0.0012276822, 0.030477233, 0.0003344883, 0.002997734, 0.006162099, 0.011559654, 0.004165788, -0.010148186, -0.018623523, 0.007338322, 0.0066554593, -0.004930333, -0.024426222, -0.009984822, 0.0022511592, -0.012428751, -0.006116357, -0.020701516, 0.024635328, -0.04085413, -0.0048029087, -0.019303119, -0.024687605, 0.0115335155, 0.014898819, -0.009448987, -0.03309106, 0.010984612, -0.0176172, 0.005701412, 0.012402614, 0.00444024, -0.015081787, 0.009958684, 0.012860033, -0.023981871, -0.03526054, -0.011579257, 0.011520446, 0.017395025, -0.008919688, 0.0028343697, -0.029091904, -0.0044239038, 0.01959064, 0.023903456, 0.0049597383, -0.019185496, 0.0074951514, -0.0088086, 0.02383811, 0.0040677697, -0.016127318, -0.01382715, 0.017970067, -0.018884907, -0.0017757693, -0.009762647, 0.026033726, 0.032516018, 0.013291314, -0.034816187, -0.0015829995, -0.0046036043, 0.0007796558, 0.022112984, -0.029536255, -0.013395868, -0.0021939818, 0.019629847, 0.010716694, 0.0026677381, 0.0032819875, 0.003979553, -0.02883052, -0.034737773, -0.006309127, -0.017120572, 0.01670236, -0.0074298056, 0.019054806, -0.026046794, -0.007913364, -0.018963322, 0.037639122, -0.01853204, 0.021211213, 0.03536509, -0.0045023183, 0.0061424957, 0.01821838, -0.008057124, 0.0063417996, 0.04294519, 0.022126053, -0.023498312, 0.030869307, -0.009168001, 0.026556492, -0.007919898, 0.011892917, -0.025236508, 0.0011582525, 0.010344224, -0.019956576, -0.017525716, -0.052903876, -0.014990303, 0.010376897, 0.012435286, -0.0019799746, -0.002046954, -0.023106238, -0.0015070351, 0.034397975, -0.01078204, 0.0030042685, 0.03008516, 0.034685496, 0.0024096225, 0.0137618035, 0.006207841, 0.051152613, 0.026504215, -0.01853204, -0.017342748, -0.009390176, -0.00805059, -0.008253162, 0.03536509, -0.008971964, -0.009893338, 0.0016017864, 0.057086002, -0.009658094, 0.003959949, 0.010089376, -0.011624999, 0.015656829, 0.028752107, -0.0030598124, -0.024021078, 0.0059791314, -0.014284569, 0.028569138, -0.0419258, -0.011794899, -0.012213111, 0.006015071, 0.011023819, -0.0220215, 0.036384486, -0.036227655, -0.0055674533, -0.036227655, -0.016140386, -0.023432966, 0.001388596, -0.0071553537, 0.0024422954, 0.0005007114, -0.0032966903, -0.0241387, -0.000017880726, -0.0011084264, 0.015918212, 0.006282989, 0.022962477, 0.034946878, 0.016833052, -0.0058909147, -0.0012742411, -0.0031153562, -0.028673692, -0.0046199406, -0.010076307, 0.05102192, 0.013787942, 0.018819561, 0.01578752, -0.03586172, 0.02279258, 0.0013902297, -0.0014735454, 0.0002234006, -0.003626686, 0.0043487563, 0.023354553, 0.02170784, -0.007710792, 0.0072795106, -0.0041167787, 0.0075735664, -0.04132462, -0.0016556965, 0.024792157, 0.027209949, -0.0047767702, -0.020021921, 0.0013036466, -0.0102135325, 0.00697892, -0.0025697195, 0.022570403, -0.018649662, -0.015303962, 0.01123946, 0.0042964797, -0.00083070714, 0.009893338, -0.009122259, 0.034345698, -0.017813237, 0.02566779, 0.002718381, -0.019094013, -0.016179595, -0.0074298056, -0.02473988, 0.0096058175, 0.009847596, -0.03248988, 0.025432546, 0.009396711, -0.042161044, 0.016061973, 0.002380217, -0.012383009, 0.01382715, -0.018153034, -0.016114248, 0.011579257, 0.006224178, 0.0473887, -0.024635328, 0.015591484, 0.052616354, -0.004979342, -0.003509064, -0.013709527, -0.0060869516, 0.003486193, -0.04540219, -0.0034731238, -0.003531935, -0.0352344, -0.017120572, -0.047702357, -0.011781829, 0.0008617464, 0.019603709, -0.023289206, 0.007998313, 0.022335159, 0.010435708, -0.021851601, 0.0035286676, -0.0033685707, -0.021158937, -0.021158937, -0.012592115, -0.028464586, 0.0031300588, 0.026817873, -0.020779932, -0.018584317, -0.005698145, 0.016427908, -0.0046591484, -0.015264755, 0.011102234, -0.011794899, -0.042788364, 0.016114248, 0.0004937684, 0.018741146, -0.020766862, 0.032280773, -0.012539839, 0.0017479974, -0.0040775714, -0.003613617, -0.018701939, 0.0013910466, -0.009305228, 0.024177907, -0.003750843, 0.0077369306, -0.017774029, -0.02794182, -0.0026121943, 0.020884484, -0.009965219, -0.0029225862, -0.022152191, -0.0022691293, -0.013709527, -0.024347806, 0.03052951, -0.021250421, -0.0020404193, -0.008377318, 0.015774451, 0.006789418, 0.017970067, 0.01593128, 0.004655881, -0.0005223572, 0.030973861, 0.002233189, 0.012102023, -0.021289628, -0.022714164, 0.011108768, -0.012860033, 0.0035842115, -0.013552697, 0.0012260487, 0.011899452, -0.004884591, -0.0032444138, -0.007939503, 0.010684022, -0.018492833, -0.016911466, -0.020322511, -0.02443929, -0.03991315, -0.000039743456, -0.001101075, -0.009429384, 0.032594435, 0.017774029, 0.004695088, -0.019538363, -0.008115935, -0.001064318, 0.015578414, 0.029536255, -0.030032882, 0.03186256, -0.0126966685, -0.0004431255, 0.0153301, -0.011330944, 0.008383852, -0.0036528246, -0.0045741987, -0.0150295105, 0.0125986505, 0.014323777, 0.016885327, -0.0132129, 0.015317031, -0.014807335, -0.0262559, 0.015879005, 0.037299324, 0.0028294686, -0.0031349598, 0.0046068714, -0.033195615, 0.011749156, 0.01236994, 0.008952361, -0.028203202, 0.011899452, -0.012820826, 0.0013926801, -0.005861509, 0.015317031, -0.013905564, 0.016075041, -0.0019309653, -0.028647553, -0.026791736, 0.022570403, 0.010108979, 0.020335581, 0.015774451, -0.012507167, 0.016192663, -0.000824581, -0.007586635, 0.020832209, -0.03384907, -0.035731025, 0.010082841, -0.039573353, 0.014362984, 0.017029088, 0.015356239, -0.031784147, -0.03826644, -0.0015241883, -0.03207167, 0.010540261, -0.026608767, -0.0019587371, 0.0070704045, -0.005469435, -0.0054171584, 0.026674114, 0.020793, 0.020675378, -0.019708263, -0.0046820194, 0.019642916, 0.002522344, 0.0028588742, 0.020217959, -0.019172428, -0.016493253, 0.032751262, 0.01989123, -0.017342748, -0.014702782, 0.019995783, 0.009599282, 0.012657462, 0.00035307097, 0.0023001686, 0.008148609, -0.010507588, -0.0015437921, 0.030294266, -0.008932756, 0.016323354, 0.0038684653, 0.030973861, 0.013232504, 0.0047277613, -0.000090360845, -0.02794182, -0.0014523081, 0.022505058, -0.038344856, -0.0000059793865, -0.016349493, 0.024099493, -0.034659356, 0.0039403457, -0.020714587, -0.011951728, -0.01032462, -0.023785833, 0.00016785676, -0.0012350336, 0.0065247677, 0.013774873, 0.012075884, -0.023746626, -0.009886804, 0.015107925, -0.012206576, 0.029954467, 0.003257483, -0.013997048, 0.003888069, -0.052747045, -0.013774873, -0.015591484, -0.0041363826, 0.016833052, 0.0040808385, 0.016963743, 0.012690134, -0.012735876, 0.012232714, -0.01123946, -0.0021874472, -0.012618254, -0.008606028, -0.011821036, -0.024112562, 0.015826728, -0.023877317, 0.0406973, 0.010409569, 0.013644181, 0.009664629, -0.00011997062, 0.003348967, 0.003590746, 0.0037998522, -0.0030679805, -0.014820404, 0.008227023, -0.01640177, -0.005966062, 0.020440133, -0.015682967, 0.023289206, -0.03141821, 0.00972344, -0.004316083, 0.0038717326, 0.020701516, 0.010710159, 0.019708263, 0.03144435, 0.031182967, -0.010474916, -0.0066521917, -0.014637436, -0.0075408933, -0.013787942, -0.0012235981, -0.028516863, 0.022674957, 0.01472892, 0.018754214, 0.024426222, 0.0030500104, -0.003522133, -0.00531914, -0.030241989, -0.011134907, -0.00093526026, -0.014650505, -0.004652614, 0.013918634, 0.011245995, 0.020596964, 0.02490978, 0.029222596, -0.0017528983, -0.0050381534, -0.0013044635, 0.011605396, 0.009514334, 0.012284991, 0.007697723, -0.011500843, -0.0014065661, 0.010056702, -0.008102867, 0.0016393602, -0.008396922, -0.0071096118, -0.016153457, 0.009971754, -0.015696036, 0.049715005, -0.0038684653, 0.0061686337, -0.013193296, -0.017682545, 0.009233347, 0.00265957, -0.018649662, 0.012958052, -0.02548482, -0.007018128, -0.017852444, -0.008161677, 0.020257166, 0.016022764, -0.021903878, 0.011840641, -0.018336002, -0.011821036, 0.011807967, 0.0041167787, -0.036018547, -0.017342748, -0.023419898, 0.0007702624, 0.007789207, -0.006893971, 0.012389544, -0.009592748, -0.007181492, 0.0071357503, -0.0060673477, 0.0009115725, -0.0056458684, -0.011017284, -0.004711425, -0.008501475, -0.0014065661, -0.00037308308, 0.020610033, -0.0086909775, 0.0047244937, -0.010527192, -0.027811129, -0.012186972, -0.0037541103, -0.02050548, 0.0025419476, -0.005322407, -0.004450042, 0.034267284, 0.009736509, 0.0013877793, 0.025772342, 0.013062605, -0.0032215428, 0.020714587, -0.007410202, 0.0073579256, 0.013801011, 0.004280143, 0.0014294371, 0.008860877, 0.00720763, -0.009926011, 0.0031300588, -0.015578414, -0.004737563, 0.013252107, 0.022988616, 0.0028670426, 0.024583051, 0.040409777, -0.002197249, -0.028020235, -0.014663574, -0.0069658514, 0.014101601, 0.007057335, 0.011317874, 0.0016434443, 0.027680438, -0.01518634, -0.002087795, 0.007560497, 0.023759695, -0.03157504, 0.015003372, 0.030895445, -0.014023187, -0.018884907, 0.0057504214, 0.024321668, 0.01031155, 0.013219435, 0.03431956, -0.0023949197, 0.018309863, -0.023328414, -0.0127489455, -0.00872365, 0.0023410097, 0.006227445, -0.03978246, -0.008782461, -0.017564923, 0.015134064, 0.016493253, -0.007978709, 0.02187774, -0.037586845, 0.009945615, 0.011219856, -0.0022233874, 0.014898819, -0.0073448564, 0.009429384, 0.062313657, 0.029065765, -0.013944772, 0.017826306, -0.011461635, 0.0075735664, 0.0114812385, -0.0022952675, -0.025942242, -0.002621996, 0.031235242, -0.019094013, 0.00035102892, -0.0028637752, 0.0044696457, 0.0005223572, 0.007874156, -0.0176172, -0.010069772, -0.011415893, 0.011579257, 0.017826306, -0.0053518126, -0.014532883, 0.021916946, 0.0015707472, -0.0110760955, -0.0220215, -0.008684443, -0.026229763, 0.0021939818, 0.007272976, 0.0013379531, 0.0059693293, 0.017669477, -0.007834949, 0.031679593, -0.0012709738, 0.007351391, -0.0076454463, 0.018244518, 0.004129848, 0.009533937, 0.0039501474, -0.007527824, 0.008939291, -0.010174325, 0.010808178, -0.013219435, 0.010246205, 0.0007812895, 0.011572723, -0.0031856026, -0.022962477, -0.012768549, -0.01564376, -0.0059693293, 0.005518444, 0.012193507, -0.015957419, -0.0006113907, -0.021694772, -0.013918634, 0.015447723, 0.011500843, 0.0039305435, -0.0032885221, 0.017303541, 0.000052582865, -0.012918844, 0.006227445, -0.0005382852, 0.013578836, -0.023759695, -0.017839376, 0.023197722, -0.009527403, -0.011004215, -0.014389123, 0.002489671, 0.021054383, -0.044461213, -0.0017349282, -0.0060183387, -0.008481871, 0.0061653666, -0.0036005478, -0.013016863, -0.016075041, -0.0029111507, -0.01671543, -0.0042703412, -0.0027918948, -0.011794899, 0.008272765, 0.0011500843, -0.009899873, -0.0022821985, 0.0053714165, -0.011337479, -0.016937604, -0.008821669, 0.03526054, -0.025576307, 0.013997048, -0.0023230396, -0.0013265177, 0.00067551114, 0.013343591, 0.009547006, 0.008802066, 0.016127318, 0.007076939, -0.0071161464, -0.008645236, 0.024556912, -0.012559443, 0.0071422844, 0.0031905037, 0.048303537, -0.0097561125, -0.020832209, 0.034058176, 0.019695193, -0.00489766, 0.004234401, -0.0080832625, -0.003750843, -0.014794266, -0.0019097279, -0.0075997044, 0.01252677, 0.017695615, -0.0004729395, -0.021786256, -0.0051459735, -0.01791779, -0.0009826359, -0.003247681, 0.016584737, 0.0023312077, 0.0070311967, -0.014467537, 0.015526137, 0.018492833, 0.019172428, -0.014598229, -0.014376054, -0.018466694, 0.010442242, -0.002690609, 0.0065541733, -0.0029585264, -0.0032182755, 0.009952149, -0.016336424, -0.0000847452, 0.005492306, -0.0039860876, 0.0048486507, -0.0054334947, 0.034554806, 0.006541104, 0.0047310283, -0.0029111507, -0.017421164, -0.0044990513, -0.0072403033, -0.015055648, 0.0036724282, -0.0029568928, -0.0056687393, -0.0094555225, -0.007076939, 0.00918107, 0.0072599067, -0.023328414, 0.0016164891, -0.02656956, -0.0066587264, -0.0055903243, 0.014741989, -0.008187816, -0.008030986, -0.018715007, -0.018989459, 0.018584317, 0.013435075, 0.0012799589, -0.004384696, -0.005995468, -0.00007565806, 0.0041363826, 0.022374367, 0.024190977, 0.020610033, 0.014820404, -0.003763912, 0.012690134, 0.012977655, -0.0122915255, -0.013323988, 0.02490978, 0.0145198135, -0.0022854658, -0.0064267493, 0.0126966685, -0.01534317, -0.029640809, -0.014349915, -0.0045970697, 0.0020502212, -0.020113405, -0.0035155984, -0.018087689, 0.006730607, -0.01714671, -0.0042180647, 0.00017571867, -0.0074755475, 0.007841484, 0.014180016, -0.008658305, -0.02170784, 0.022701096, -0.0012987457, -0.0052015176, 0.009867201, 0.0036495572, 0.0014384221, -0.00084622676, 0.010435708, 0.0028980817, -0.003362036, -0.030189712, -0.032881953, 0.023955733, 0.007253372, 0.02488364, -0.0035613405, 0.007848018, -0.0002004275, 0.0002891547, 0.0150295105, 0.012938448, 0.03113069, -0.0042017284, 0.015839797, 0.012892706, 0.017395025, 0.029693084, -0.011147976, 0.01943381, 0.009416315, 0.000097303826, 0.010305016, 0.013591905, -0.030921584, 0.010265809, -0.033195615, -0.0060510114, -0.012180438, -0.0044761803, -0.026412731, 0.009867201, -0.009285623, -0.0050806277, 0.003705101, 0.023067031, -0.0054302276, 0.009409781, -0.01823145, -0.021773186, 0.008514544, -0.013337057, -0.01152698, 0.024818296, 0.017342748, 0.0273145, -0.011977866, -0.005966062, 0.011860244, -0.021119729, -0.021668633, -0.00039922138, 0.0001421269, 0.01776096, -0.009547006, 0.01929005, 0.0009246416, -0.005573988, -0.001927698, -0.019459948, 0.017395025, 0.008991567, 0.008285834, -0.0007906829, -0.013644181, 0.013265177, 0.007730396, -0.0016230238, -0.04009612, -0.005737352, 0.014402191, -0.012539839, 0.004695088, 0.0035646078, 0.005217854, -0.026661044, 0.021446457, -0.0000255129, -0.00081804645, 0.005698145, -0.020152614, -0.017042158, 0.003362036, 0.0024047217, 0.006815556, -0.014650505, 0.014088532, 0.00421153, 0.0015054015, 0.004044899, 0.016885327, -0.00146211, 0.013487352, 0.002095963, -0.02200843, 0.025733136, 0.012330733, 0.012565978, 0.0085929595, -0.003183969, -0.0005758589, -0.00045578624, -0.0068612983, 0.00027608557, -0.005136172, 0.0015633958, 0.012559443, -0.004760434, -0.019512225, -0.009736509, -0.014689713, -0.003937078, 0.006224178, -0.0073840637, 0.014598229, -0.032280773, 0.007848018, -0.00466895, -0.009919477, -0.011860244, 0.0002234006, -0.01487268, -0.006031408, -0.012840429, -0.011932124, -0.0018525505, 0.011115303, -0.0046166736, -0.010037099, 0.017355816, -0.015068717, 0.025406407, -0.019825885, 0.008566821, 0.016140386, 0.0013510223, -0.0068547637, -0.008403457, -0.02338069, -0.014441399, 0.007919898, -0.022805648, -0.0138140805, 0.008717116, 0.004554595, -0.0057406193, 0.004002424, 0.0055347807, -0.004172323, -0.027602023, -0.012905775, 0.012186972, 0.010749367, 0.0016663152, 0.0015731977, 0.0024766019, -0.018192243, 0.013644181, -0.03445025, 0.0043716272, 0.007913364, 0.014062393, -0.005864776, 0.021812394, -0.00057871785, 0.014951095, 0.018179173, -0.0054596327, 0.011938659, -0.025981449, -0.007331787, 0.009396711, -0.009612352, -0.0020600231, -0.015408515, 0.027105395, 0.011520446, 0.008658305, 0.009710371, 0.008200885, 0.04325885, -0.010592537, -0.006080417, 0.021067452, -0.03068634, 0.0103965, 0.02822934, -0.013631112, 0.020413995, 0.0077238614, -0.009468592, 0.0020763595, 0.00531914, 0.007397133, -0.03113069, -0.016271079, -0.0047767702, 0.009331365, 0.015617621, -0.01624494, -0.004819245, -0.013435075, 0.001009591, 0.003878267, 0.015983557, 0.028752107, -0.017956998, 0.021616356, -0.017512647, -0.021851601, 0.009462057, -0.0059007164, 0.013983979, 0.007501686, 0.020100337, 0.003603815, 0.028307756, -0.007815345, 0.012873102, 0.028752107, 0.003531935, -0.0063581364, -0.006972386, 0.013108347, -0.016519392, -0.015774451, -0.011585792, 0.019028667, -0.01987816, -0.0020387857, -0.0033914417, 0.0016434443, -0.010298482, 0.0036626263, -0.0038488614, 0.0016532461, 0.012128161, -0.0138140805, 0.008285834, -0.006008537, -0.021694772, -0.011742622, 0.014689713, 0.0007224784, -0.013043001, -0.0048682545, -0.0068416945, -0.017395025, -0.003404511, 0.0069331783, -0.012409148, -0.0009246416, 0.013552697, 0.008161677, 0.008527613, 0.0002001212, 0.018244518, 0.005505375, 0.004280143, 0.022714164, -0.017564923, 0.008318507, 0.0022266547, -0.0029405563, 0.036985666, 0.012363406, 0.0006085318, -0.028072512, -0.014258431, 0.005760223, 0.03207167, -0.006224178, -0.004378162, 0.002064924, 0.00903731, -0.0043912306, -0.008762858, -0.013983979, -0.02217833, -0.011043423, -0.0029993674, 0.003018971, 0.012637857, 0.018270656, 0.0376914, 0.017695615, 0.018022344, -0.02157715, -0.02974536, 0.0050838953, 0.010474916, 0.016349493, -0.021211213, -0.014402191, 0.001932599, -0.0025305122, 0.009468592, 0.026465008, 0.0038586634, 0.009272554, 0.011500843, -0.0052243886, -0.010108979, 0.014663574, -0.0019848756, 0.02778499, 0.00039186998, -0.017538784, 0.030320404, 0.019473018, -0.02352445, 0.0036724282, 0.013016863, 0.0072337687, 0.018754214, -0.0089458255, -0.0029127845, 0.005237458, 0.005093697, 0.01336973, 0.016480185, -0.010801644, 0.012559443, 0.015656829, -0.0057275505, 0.019028667, -0.022975547, -0.018100757, -0.019773608, 0.0029928328, -0.012428751, 0.020727655, -0.02275337, 0.017290471, 0.0031267917, 0.027863406, -0.00993908, 0.00034122705, -0.008403457, -0.0018100758, -0.0052113193, -0.008246627, -0.00082907354, -0.0078088106, 0.020766862, -0.018688869, -0.008331576, -0.022844855, 0.02961467, 0.0072860452, 0.0039534145, -0.021903878, -0.028412309, 0.009716905, -0.004338954, -0.009390176, 0.0102135325, -0.01563069, 0.000013694518, -0.00706387, -0.017878583, 0.004518655, -0.01684612, 0.0062339795, 0.02734064, -0.00061915047, -0.017029088, -0.030424956, 0.010592537, -0.019054806, 0.0038161888, -0.008632166, -0.020244097, 0.013918634, 0.013539628, 0.0092006745, -0.0015650294, 0.005636066, 0.0007049167, -0.0020779932, -0.020766862, 0.006939713, -0.00074371573, -0.020884484, 0.00041351575, 0.008978499, 0.022805648, 0.0049499366, 0.017538784, 0.012794687, -0.014336846, 0.03382293, -0.00410371, -0.0027902613, -0.017120572, -0.026922427, 0.008586424, -0.0011966431, 0.026334316, 0.002011014, -0.009814924, 0.0039272765, -0.009390176, -0.019995783, 0.0028441714, -0.009377107, 0.011317874, 0.01048145, -0.0040547005, 0.023563659, 0.021930017, 0.0064169476, 0.0056033935, 0.025092747, 0.020518549, -0.024413152, -0.010331155, 0.03199325, 0.013709527, -0.008181281, 0.01442833, 0.0008707314, -0.0007543344, -0.011925589, 0.018296795, 0.0017610665, 0.00796564, -0.009272554, -0.0041396497, 0.0050414205, -0.028046373, 0.01640177, -0.035338953, -0.012820826, -0.026765598, 0.011004215, 0.013931703, 0.01504258, 0.0038488614, 0.02049241, -0.026242832, -0.0015388911, 0.019773608, -0.010069772, 0.0010496152, -0.004760434, -0.01914629, 0.0029928328, 0.007684654, -0.008285834, -0.0017888384, -0.00972344, 0.01412774, -0.02157715, -0.0029764965, -0.01261172, 0.011592327, 0.013526559, 0.013892495, 0.012252318, 0.005573988, -0.024021078, 0.0074494095, 0.034816187, -0.00903731, -0.0012383009, -0.008481871, -0.016950674, 0.0062960577, -0.006345067, 0.004649346, -0.004299747, 0.005792896, -0.029065765, -0.020374788, 0.035809442, 0.008455733, 0.008749789, -0.012160834, 0.011461635, 0.0027347174, -0.0005235824, 0.0077499994, -0.018963322, -0.005933389, -0.020596964, 0.010579469, 0.003545004, 0.031548902, -0.006959317, 0.0040547005, -0.023119308, 0.01791779, 0.007090008, -0.0012432019, 0.003247681, -0.013840219, 0.01927698, -0.0041167787, -0.00588438, -0.004946669, 0.00812247, 0.021812394, -0.008279299, 0.021629425, -0.006809022, -0.0019015598, -0.01684612, 0.0039893547, -0.0012235981, 0.019760538, 0.0006538654, -0.00000608468, -0.010474916, 0.008599494, -0.035417367, -0.007188027, -0.00041821247, 0.03330017, 0.008194351, -0.03139207, 0.005155775, -0.029065765, -0.01101075, 0.027131533, -0.006008537, -0.004652614, 0.0054563656, 0.019616779, 0.008971964, -0.00683516, 0.001351839, 0.01085392, -0.000532159, 0.0028425378, -0.019760538, 0.010644814, 0.013023397, 0.010507588, -0.0049760747, 0.038031194, -0.002535413, 0.017133642, 0.0085406825, -0.0063385325, 0.008357715, 0.009645025, -0.0048911255, -0.013029932, 0.02383811, -0.0011664206, -0.0016017864, 0.0013257008, -0.003061446, 0.015918212, -0.013252107, 0.0052733975, -0.0064430856, 0.019094013, -0.01351349, 0.02200843, -0.0044043, -0.008416526, -0.0017202254, 0.016258009, -0.010801644, -0.0034241145, -0.012017074, -0.0048780562, 0.009520868, -0.0031774344, 0.013879426, -0.005230923, -0.0068678325, 0.014807335, -0.014180016, 0.015696036, 0.023929594, -0.0012252318, 0.022988616, 0.0052505266, -0.00019164667, -0.016506324, 0.012474493, -0.021629425, -0.047101177, -0.004022028, 0.0073579256, -0.029379426, 0.010540261, -0.011455101, 0.027079256, 0.003384907, 0.0066521917, 0.0045349915, -0.019224703, 0.016963743, 0.022831786, -0.0055576516, 0.015081787, 0.0069462475, -0.034528665, 0.01989123, 0.0019358663, -0.020975968, 0.013186761, -0.0006346701, -0.012383009, 0.0020338849, -0.024726812, -0.0021907145, 0.017303541, -0.005743887, 0.006410413, 0.0069462475, -0.012200042, -0.004554595, 0.0036332207, 0.0012374842, 0.011585792, -0.010801644, -0.021054383, 0.0058909147, -0.009906407, -0.008279299, 0.028150925, 0.025903035, -0.00015203084, -0.01654553, -0.0071553537, 0.009965219, 0.006939713, 0.0008617464, -0.004672217, 0.0061424957, 0.0142192235, -0.019512225, -0.014964164, -0.006116357, -0.011860244, -0.022988616, -0.017695615, 0.0075735664, 0.0021319033, 0.012265387, -0.013487352, 0.010697091, 0.0083381105, 0.037168633, 0.009246416, 0.006354869, 0.000010152832, -0.014715851, 0.00013559232, 0.028124787, -0.003914207, 0.014075463, -0.0039501474, -0.025314923, 0.021067452, -0.0033342643, -0.02186467, -0.0024161572, -0.0054204254, 0.00805059, 0.008220488, 0.03081703, -0.002563185, -0.0062143756, 0.0037116355, -0.015199409, -0.010167791, 0.011474704, -0.00796564, -0.0033685707, 0.00683516, 0.0264258, 0.0049041947, 0.005342011, 0.016676221, -0.013029932, -0.021694772, -0.008671374, -0.006387542, 0.0061915047, 0.021067452, -0.0028490725, -0.004185392, -0.011925589, 0.015669897, -0.0031807018, 0.009305228, 0.008854342, 0.029248733, -0.02399494, -0.023067031, -0.013297849, 0.0049499366, 0.0037541103, -0.0070508006, -0.02339376, -0.0004778404, -0.00027955705, 0.0032999576, 0.0102135325, -0.018545108, -0.02064924, 0.023811972, -0.01032462, -0.020335581, 0.013036466, 0.030137435, 0.03021585, 0.005345278, -0.008344646, 0.016741568, 0.016480185, 0.0077499994, 0.0017398292, 0.011827571, -0.012265387, 0.0188457, 0.008429595, -0.0019015598, 0.011892917, 0.013239038, 0.0104683805, 0.018871836, 0.0020583894, -0.012278456, -0.007272976, -0.0041951938, -0.02791568, -0.020675378, 0.04451349, -0.0075735664, 0.013350125, 0.0033342643, -0.0073252525, 0.0034600548, -0.01367032, -0.008514544, -0.008553752, -0.0061424957, -0.011082631, 0.020178752, -0.010429173, -0.026282039, 0.0142715005, 0.015356239, -0.024726812, -0.016336424, -0.007377529, 0.006207841, 0.007919898, -0.010886593, 0.0016614143, -0.00942285, 0.0067044683, -0.021132799, 0.0060608136, 0.015120994, 0.028203202, -0.014924957, -0.0030091694, -0.009658094, 0.021930017, -0.0004239302, -0.027079256, -0.005554384, 0.021250421, -0.015970489, -0.014898819, 0.011847175, -0.0040775714, 0.020570826, -0.0068416945, -0.0041461843, 0.006015071, 0.021211213, 0.006469224, 0.01236994, 0.019838953, -0.007815345, -0.0010398134, 0.00006963401, -0.0072403033, 0.00021421137, 0.013258642, 0.02186467, -0.014846542, 0.0063810074, 0.0047473647, -0.019224703, -0.0008919688, -0.0056491354, 0.0071749575, 0.006599915, -0.014768127, 0.008416526, -0.00013600073, -0.01836214, 0.021812394, -0.0004075938, -0.016336424, 0.027131533, 0.012925379, -0.009886804, -0.011102234, -0.025942242, 0.00034796583, -0.000068000365, -0.006057546, 0.022413574, -0.0026791736, 0.01236994, 0.0006857214, 0.00012977247, 0.0073252525, -0.01623187, -0.016467115, 0.0017316609, 0.009631956, -0.019995783, 0.003394709, -0.0154346535, -0.015539207, 0.010089376, -0.0026203624, 0.0010929068, 0.007625843, 0.036985666, 0.0069462475, 0.006998524, 0.010265809, -0.005008748, 0.00095486396, 0.007351391, 0.010487984, 0.01504258, -0.021185074, 0.0005627898, 0.009187605, -0.009083052, 0.0041690553, -0.006711003, -0.030581785, 0.007377529, -0.011664207, 0.012128161, 0.0040775714, -0.011638069, -0.0043748943, 0.0062111085, 0.023981871, -0.004071037, -0.00011762225, 0.014898819, 0.01093887, 0.011389755, -0.0055119097, -0.0075408933, 0.0056491354, -0.02126349, -0.00611309, 0.0041788574, -0.012650927, -0.0038749997, 0.00294219, -0.0035352022, 0.01610118, 0.0060477443, 0.0037214374, -0.002179279, -0.009631956, -0.012128161, 0.007057335, -0.0133566605, 0.0046395445, 0.005243992, -0.0057667578, -0.0011460001, -0.020675378, -0.0077173267, 0.0054988405, 0.0010153088, -0.034737773, -0.008971964, -0.015826728, -0.0010332789, 0.011925589, -0.019303119, 0.0103965, -0.002407989, 0.01472892, 0.004093908, 0.0044075674, 0.010416104, -0.011677276, 0.0176172, 0.025066609, -0.016427908, 0.009187605, 0.0072403033, -0.01382715, -0.00835118, -0.007181492, -0.016663153, -0.0013861456, 0.012422217, -0.002489671, -0.008573355, -0.0014245362, 0.0051884484, -0.00094261166, -0.0032819875, 0.01185371, -0.006697934, 0.025131956, -0.01957757, 0.015003372, 0.0137618035, 0.009239881, -0.01594435, 0.0104683805, 0.0073840637, -0.012833895, 0.013853287, 0.01944688, -0.0058680438, 0.0092006745, 0.024543844, 0.010579469, 0.04631703, -0.0050838953, -0.015264755, -0.012762015, -0.009814924, -0.021943085, 0.005253794, 0.010788575, 0.0014833473, 0.014245362, -0.010971542, 0.007651981, -0.0029536255, 0.0066031828, -0.012200042, -0.0074298056, 0.014624367, -0.008854342, -0.006331998, 0.0060510114, 0.015016441, -0.0034600548, 0.016898397, 0.009860666, 0.0026350652, -0.0039958893, 0.01791779, 0.0114812385, 0.017669477, -0.012938448, 0.019120151, -0.00805059, -0.016480185, 0.0062470487, 0.020845277, 0.03873693, 0.0074951514, 0.017029088, 0.0230801, 0.01101075, -0.010657883, 0.011742622, 0.016937604, -0.0019064607, -0.014206154, 0.013997048, 0.008560286, 0.002759222, 0.007246838, 0.018022344, 0.009599282, 0.0023459105, 0.017839376, -0.011487774, -0.003509064, 0.016192663, -0.0022625946, -0.005952993, -0.002901349, 0.0011672375, 0.006697934, -0.0015797322, -0.017747892, -0.0025158094, -0.011579257, 0.0017316609, -0.021015177, -0.0073905983, 0.0043422217, -0.006243781, 0.025785413, -0.0029519917, -0.0029046163, -0.0031169897, -0.0019391335, 0.0002082894, -0.024465429, -0.02961467, -0.0027020446, 0.030189712, -0.007919898, -0.0016279246, -0.016623946, 0.004920531, -0.0009050379, 0.023877317, 0.0081355395, 0.011566188, 0.0051329043, -0.008070193, -0.02791568, -0.004485982, 0.02247892, 0.022439713, -0.0054400293, 0.009488195, -0.0070050587, 0.012886171, 0.0015478762, 0.0008748155, -0.0022217536, -0.0031186233, 0.020322511, 0.023315344, -0.010690556, 0.0056883427, -0.019956576, 0.0075147552, 0.0154869305, 0.0018982925, -0.01684612, -0.015120994, 0.00660645, -0.028647553, 0.0068416945, -0.0074886167, -0.0012775083, -0.0005934206, 0.001881956, 0.0038259905, 0.014493676, -0.008854342, -0.0074886167, 0.012722807, 0.0024471963, 0.026360454, -0.010167791, -0.008318507, 0.015813658, -0.00926602, 0.000111393994, 0.02200843, 0.019185496, -0.007874156, 0.002105765, 0.0020665578, 0.014376054, 0.00736446, 0.0009393444, -0.013748734, 0.020100337, 0.0005697328, 0.018937184, 0.0013142653, 0.011272132, 0.004110244, -0.003143128, -0.0012366673, 0.02004806, -0.006730607, -0.019237773, -0.014794266, -0.016127318, -0.004106977, 0.016493253, -0.013186761, -0.0022070508, -0.019198565, -0.026020657, 0.004316083, -0.0016238405, 0.00085684547, 0.014624367, -0.0033097595, -0.016976813, 0.0078088106, -0.009004637, -0.020152614, 0.0012619888, -0.009207209, -0.017434232, -0.012905775, -0.03719477, -0.0013346858, -0.00086419686, -0.0125986505, -0.009729974, 0.0037345067, 0.043102022, -0.027445193, -0.013709527, 0.036645867, 0.015774451, -0.001210529, -0.009743043, -0.0028360032, -0.002192348, -0.00040555172, -0.023602866, -0.029222596, -0.032908093, -0.01312795, -0.0040089586, 0.009814924, 0.030137435, -0.002965061, 0.015408515, -0.0023148714, -0.007887226, -0.014362984, -0.0048453836, -0.010651348, 0.004028562, 0.021315766, -0.008854342, 0.003161098, 0.005070826, -0.0014057494, 0.03565261, -0.013983979, -0.024190977, 0.014768127, 0.0010831049, -0.013539628, -0.0035646078, 0.016623946, -0.0038390597, 0.014467537, -0.013892495, 0.010461846, -0.012245784, 0.0020567558, -0.01913322, -0.009586214, 0.005861509, -0.0024635328, -0.0076781195, -0.00083519967, -0.007351391, 0.0004623208, 0.019002529, -0.015016441, -0.0052243886, 0.020100337, 0.024975125, 0.00752129, 0.022400504, 0.026504215, 0.01580059, 0.018858768, -0.0052276556, -0.00044394232, -0.012304595, -0.013500421, 0.00720763, -0.001960371, 0.023602866, -0.0027984295, -0.014376054, 0.012383009, 0.02460919, 0.014454468, 0.017813237, 0.007874156, -0.003796585, 0.031156829, 0.0019064607, -0.024569983, 0.013722597, -0.018153034, 0.01594435, -0.005400822, 0.012304595, 0.0009066715, 0.015264755, -0.018244518, -0.024151769, -0.019237773, 0.0061098225, -0.0029911993, 0.0015225548, -0.0017904721, 0.013134485, 0.001434338, -0.0050054803, 0.009703836, 0.00006279313, 0.0012897607, -0.015238617, 0.0021433388, 0.010461846, -0.016427908, -0.007913364, -0.013866357, -0.0030957523, 0.0154869305, 0.018571246, -0.013591905, 0.012735876, 0.028725969, 0.008377318, 0.024975125, 0.0006113907, -0.012337267, 0.033012647, 0.006031408, 0.0024096225, 0.013343591, -0.010951939, -0.008076728, 0.0032640176, -0.012585581, -0.018414417, 0.0014857978, 0.003404511, 0.0069658514, -0.019094013, 0.009292158, -0.004254005, -0.018610455, -0.009468592, 0.0058582416, -0.012317664, -0.008828203, 0.023302276, 0.023655143, 0.031209106, 0.007018128, -0.0002805781, 0.0025321457, -0.004799641, -0.0070246626, 0.012278456, -0.016166525, -0.004623208, -0.018897975, 0.0028245677, 0.003348967, -0.010775506, 0.02006113, 0.022505058, 0.00013201873, 0.015055648, -0.0020861614, -0.01312795, 0.003626686, -0.020897554, 0.0056066606, -0.00037267467, -0.007423271, -0.017538784, -0.012997259, 0.0174473, 0.0011639702, -0.0062731868, 0.021747047, 0.0039534145, 0.01806155, -0.019655986, -0.013539628, -0.008318507, 0.0040743044, 0.021773186, 0.0031169897, 0.017029088, 0.0006126159, 0.016297217, 0.0053616143, 0.009174536, -0.028307756, 0.012840429, 0.011082631, 0.02612521, -0.0021596754, -0.006008537, 0.0019636382, -0.023498312, 0.018950252, 0.01929005, 0.011363616, 0.0059693293, -0.006848229, -0.013408937, 0.011357082, -0.03614924, -0.008305438, 0.00051786465, 0.028490724, -0.013892495, 0.020989038, 0.023459105, -0.01564376, 0.0070115933, -0.005815767, -0.0018019076 - ] - } - ] - } + { + "name": "springfield-illinois-historic-capital-city", + "title": "Springfield: Illinois' Historic Capital City", + "content": "Springfield is the county seat of Sangamon County in Illinois, USA. As of 2019, it had a population of 10,566, making it the sixth most populous city in the state.", + "keywords": ["Springfield: Illinois' Historic Capital City"], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.022805648, 0.035417367, -0.0029928328, -0.024086423, 0.016179595, + -0.0070115933, 0.011461635, 0.0027347174, -0.056981448, 0.017042158, + -0.029248733, 0.01291231, -0.008736719, 0.018858768, 0.011193718, + 0.009886804, -0.016336424, 0.033953626, 0.003763912, -0.0007968091, + 0.012670531, -0.0023491778, -0.049244516, 0.03596627, 0.023171583, + 0.0093379, -0.03126138, 0.008860877, -0.034188867, -0.0043977653, + -0.021616356, -0.0042866776, 0.049793422, 0.0024504636, 0.015983557, + -0.016349493, 0.0077042575, -0.020740723, 0.024190977, 0.02930101, + 0.0013836952, -0.0013395868, 0.01671543, 0.00820742, 0.03293423, + -0.036985666, -0.004815978, -0.024360877, 0.029248733, 0.016937604, + 0.024870573, -0.012500632, -0.019708263, 0.037168633, 0.010141652, + -0.01806155, 0.011677276, 0.022596542, -0.0067959526, -0.043232713, + -0.0032231766, -0.0049597383, -0.00013263135, 0.00044802643, + 0.0061326935, 0.0029536255, 0.0051002316, 0.004289945, 0.053688023, + 0.011376685, 0.013317453, 0.056563236, 0.008070193, 0.043781616, + 0.05980438, -0.005485771, -0.03147049, -0.002639966, -0.04252698, + 0.029562393, 0.013173693, -0.013604974, -0.0145720905, 0.002955259, + 0.017251264, 0.008638701, -0.00064978126, -0.07783979, -0.00006636672, + 0.014206154, 0.017133642, -0.0077696033, -0.0059431912, -0.028752107, + -0.043781616, -0.028124787, 0.041559864, -0.013435075, -0.00046354602, + 0.01974747, 0.014781197, -0.009239881, -0.026504215, 0.01914629, + 0.008187816, 0.015238617, -0.0075539625, -0.0068220906, -0.020204889, + 0.0046591484, 0.018649662, -0.0037933176, -0.012964586, 0.006345067, + -0.016205734, 0.00068653823, -0.014545952, -0.0107689705, 0.042370148, + 0.02429553, -0.0038848016, 0.0049139964, -0.0276543, 0.011402824, + -0.0046983557, -0.0016556965, 0.015277823, -0.027288362, 0.007978709, + -0.0412462, 0.03463322, 0.0062960577, -0.018139966, 0.018715007, + 0.00956661, -0.034136593, -0.0057275505, 0.030921584, 0.03052951, + 0.03873693, -0.06984148, -0.0030941188, 0.026033726, -0.0000384927, + 0.0048355814, -0.010357293, -0.01927698, 0.0032362456, 0.013604974, + -0.028516863, -0.01152698, -0.010494519, -0.027419055, 0.004874789, + 0.0025615513, 0.016637014, -0.0039762855, -0.013278246, 0.043337267, + -0.053583473, -0.019512225, -0.009710371, 0.03450253, -0.026138278, + -0.00872365, -0.013088743, -0.01488575, -0.0041233134, -0.00736446, + 0.008096332, 0.022727232, -0.009488195, -0.061895445, -0.048800167, + 0.0021188343, -0.018545108, -0.03688111, -0.015656829, -0.00064365513, + 0.029666945, -0.0039664838, 0.01146817, 0.02883052, -0.008867411, + -0.024988195, -0.019329257, 0.008436129, 0.011291737, 0.036489036, + -0.010043633, -0.0066129845, 0.041978076, 0.010389966, -0.0023687815, + 0.04022681, -0.003298324, -0.023511382, -0.03293423, -0.0040677697, + 0.029091904, -0.05755649, 0.021603288, -0.024073355, -0.03207167, + 0.0006881719, 0.023001686, -0.00842306, -0.0133566605, 0.020034991, + 0.026360454, -0.006952782, -0.0030843168, -0.01944688, 0.03492074, + -0.020714587, 0.023158515, -0.013944772, -0.0011966431, -0.015617621, + 0.038841482, 0.020518549, 0.028647553, 0.0004202545, -0.014807335, + -0.032698985, -0.025236508, -0.053034566, 0.016297217, -0.009050379, + -0.037011802, 0.024413152, 0.019394603, 0.004022028, 0.06017032, + -0.021433389, 0.043467958, 0.018453624, -0.00009761014, -0.014493676, + 0.03267285, 0.018937184, 0.018963322, 0.0011705048, -0.030581785, + 0.0012186973, -0.02080607, 0.019721331, -0.0033358978, 0.013435075, + 0.021747047, 0.008717116, -0.0030124367, 0.057190552, 0.0014637436, + -0.0112263905, 0.01715978, -0.015774451, 0.013173693, 0.0088478075, + 0.00023606133, -0.016833052, 0.0037018338, -0.03842327, 0.0297715, + 0.004678752, -0.01504258, 0.022870993, 0.00047947405, 0.00565567, + -0.032123946, -0.0045578624, -0.010442242, 0.0061196247, 0.007959106, + 0.0054106237, 0.019930437, 0.0479376, 0.014206154, -0.022047639, + -0.03753457, -0.021015177, 0.030712478, -0.0012693402, -0.01987816, + 0.007684654, 0.0034273819, -0.017695615, 0.013310919, 0.005208052, + -0.045924954, -0.013853287, -0.001886857, 0.04265767, 0.025876896, + 0.006122892, -0.011866778, 0.01488575, -0.0015290893, -0.04022681, + 0.014062393, 0.03322175, 0.036541313, -0.023916526, -0.029327149, + 0.022191398, -0.03902445, 0.018100757, -0.0052701305, -0.039677907, + -0.06372512, -0.0072795106, -0.048800167, -0.010082841, 0.016022764, + 0.0050316188, 0.011762225, 0.018113827, 0.00926602, 0.034084316, + -0.02305396, -0.0050316188, 0.010965008, 0.024047216, 0.023615934, + -0.010305016, 0.04145531, -0.008638701, 0.00018337637, 0.010461846, + 0.011376685, -0.016140386, 0.01594435, -0.0068547637, -0.04116779, + 0.009194139, -0.013683389, 0.024060285, 0.0058419053, -0.00070736714, + 0.031784147, -0.004071037, 0.008939291, 0.0021449726, 0.041873522, + 0.028412309, -0.008717116, 0.0077696033, -0.010200463, -0.023511382, + 0.028046373, 0.0006722438, 0.010690556, 0.058079254, -0.0054596327, + 0.016336424, -0.0018884906, -0.023459105, -0.01989123, 0.015317031, + -0.038658515, -0.020126475, 0.005887647, -0.015251686, 0.011513912, + -0.022910202, 0.024948986, -0.012977655, 0.0074298056, 0.038789205, + -0.0012358505, -0.0005852524, 0.016780775, -0.0132129, 0.009788785, + 0.000096793316, 0.012180438, -0.016323354, 0.008769392, -0.021002106, + -0.0015642126, 0.026896289, 0.018244518, -0.014689713, -0.02548482, + -0.010108979, -0.023733556, -0.010801644, -0.01382715, -0.028124787, + 0.018610455, 0.0053714165, -0.028360032, 0.008730185, 0.0055576516, + 0.009305228, 0.032123946, -0.028438447, -0.006828625, 0.030294266, + 0.0009360771, 0.01714671, -0.027419055, 0.0023916524, 0.03340472, + -0.020126475, -0.0104683805, -0.009520868, -0.061895445, 0.011585792, + -0.011219856, -0.015852867, 0.015552276, 0.01487268, -0.007841484, + -0.02535413, -0.05739966, -0.010102444, 0.04461804, 0.019852022, + 0.019329257, -0.009520868, -0.030634062, 0.0014457735, 0.004613406, + -0.0034731238, 0.01441526, 0.0006849046, 0.026242832, -0.026974704, + -0.032881953, -0.036410622, 0.004335687, -0.011324409, -0.0025958577, + -0.009527403, 0.05206745, -0.04859106, 0.005400822, 0.046892073, + 0.0262559, -0.0318887, 0.01684612, 0.043311127, -0.03175801, + -0.017551854, 0.008854342, 0.021747047, 0.017303541, 0.017538784, + 0.055047214, -0.00926602, 0.017839376, 0.0013812446, 0.013297849, + 0.017551854, 0.009625421, -0.021943085, 0.011742622, -0.018388279, + 0.022099914, 0.029118042, -0.019224703, 0.011455101, -0.035286676, + -0.032986507, 0.035417367, 0.049897972, 0.016088111, 0.0034437182, + -0.007939503, -0.012383009, -0.026190555, -0.050943505, 0.023746626, + -0.0406973, 0.0007825147, 0.0039730184, 0.021302696, 0.019512225, + -0.021851601, -0.044722594, 0.010880059, -0.052041315, 0.011063026, + 0.002115567, 0.04101096, -0.026504215, -0.0094555225, 0.036096964, + -0.014349915, 0.069893755, 0.007606239, -0.008024451, -0.013376264, + 0.014284569, -0.006959317, -0.03949494, -0.058236085, 0.037560705, + 0.039129004, -0.0017643338, 0.02217833, 0.023811972, 0.003102287, + -0.006518233, -0.009448987, -0.017525716, -0.03978246, 0.0035188657, + 0.0002311604, 0.06267959, 0.011121837, -0.036384486, -0.024504635, + 0.010625211, -0.014140809, -0.0012824093, -0.014532883, 0.028203202, + 0.008429595, -0.009945615, 0.02443929, 0.0035646078, 0.0058680438, + -0.023406828, 0.030503372, 0.0012366673, -0.0046166736, 0.019773608, + -0.0031055543, -0.042239457, -0.0021417052, 0.0068416945, 0.028673692, + 0.04521922, -0.023197722, -0.017486509, -0.025001263, -0.0058549745, + 0.042840637, -0.016297217, -0.04610792, 0.0023720488, 0.0273145, + 0.014781197, 0.01641484, -0.0041396497, -0.021080522, 0.009729974, + -0.019107081, 0.046682965, -0.010566399, 0.015081787, -0.0052831997, + -0.007018128, -0.012873102, -0.012775084, -0.010612141, -0.031548902, + -0.021381112, 0.01807462, 0.010533726, 0.031627316, 0.003239513, + -0.005554384, 0.020283304, 0.0073840637, -0.00076576986, 0.0024586318, + 0.0029209526, 0.023171583, 0.017813237, 0.027967958, -0.009703836, + 0.00398282, 0.010599072, -0.014898819, -0.01487268, -0.0037443084, + 0.0025484823, -0.001069219, 0.018022344, 0.02565472, 0.021969223, + 0.0053158724, 0.015748313, -0.006345067, -0.04009612, -0.021250421, + 0.04328499, 0.046343166, 0.02869983, 0.00019777284, 0.005492306, + -0.0043324195, 0.006730607, -0.015918212, 0.009122259, -0.031496625, + -0.038240302, 0.016650084, -0.0012170636, 0.025602443, -0.022910202, + -0.020518549, 0.005825569, 0.014938027, 0.009207209, 0.013918634, + -0.0031643654, -0.011422427, 0.011383221, 0.00486172, -0.015369308, + -0.030555647, -0.0074036676, 0.023171583, 0.034842327, 0.005208052, + -0.015317031, -0.0009769181, -0.034241144, -0.011709949, -0.012644392, + 0.016493253, -0.025942242, 0.009547006, -0.01701602, 0.00014753833, + 0.008070193, -0.0029209526, 0.021341905, -0.014911888, -0.004110244, + 0.000940978, -0.002011014, 0.0034241145, 0.012794687, -0.02596838, + 0.0001425353, 0.014480607, 0.053243674, 0.0243086, 0.030634062, + -0.009076517, 0.0004770236, -0.020309443, 0.019303119, -0.002390019, + 0.0069266437, 0.032594435, -0.016506324, 0.0057504214, 0.0005652403, + -0.0053126053, 0.018309863, 0.04119393, -0.031182967, 0.026295109, + 0.003384907, -0.033012647, 0.0022397237, -0.0003771672, 0.0138140805, + 0.015134064, 0.030921584, -0.043494094, 0.030059021, 0.024831364, + 0.0048780562, 0.008527613, 0.019525293, -0.00090258743, 0.008227023, + -0.017081365, 0.014010117, -0.009187605, -0.0069266437, -0.019956576, + 0.0007616858, -0.007567032, -0.010572934, -0.009697301, -0.016048903, + 0.013317453, -0.023615934, 0.035155986, 0.04116779, 0.0036070824, + -0.005266863, 0.018192243, 0.03782209, -0.0033979763, -0.008429595, + 0.04114165, -0.0056524025, -0.004610139, 0.042448565, 0.0076454463, + -0.014049325, -0.01684612, 0.02475295, 0.0057079466, -0.04767622, + 0.010017496, 0.013271711, -0.007998313, -0.03356155, -0.051387858, + -0.00364629, -0.0026889753, -0.0013820615, 0.016937604, -0.02126349, + 0.054263067, -0.04132462, 0.028464586, -0.023276137, -0.006482293, + 0.039677907, 0.023027824, 0.0153301, -0.034084316, -0.0036299534, + -0.038789205, 0.00697892, -0.029091904, -0.023249999, 0.014768127, + 0.012010539, -0.037116356, 0.00516231, 0.030294266, -0.008913153, + 0.01236994, 0.010455311, -0.007795742, -0.029353287, 0.021080522, + 0.0021482399, -0.006459422, -0.021028245, 0.043833893, -0.0010071405, + 0.008828203, 0.016532462, -0.0021433388, 0.0028033303, -0.057922427, + 0.00271348, -0.033378582, -0.024426222, 0.049270656, 0.024687605, + -0.002324673, -0.0021678435, -0.0014506745, 0.0034927274, 0.018989459, + 0.053086843, 0.03141821, 0.0118863825, -0.00029548508, 0.013618044, + 0.01640177, 0.0005403272, 0.0063483343, -0.0034829257, 0.0064626895, + 0.0034633218, 0.049715005, -0.0021008642, -0.023419898, 0.0026121943, + 0.039887015, 0.004338954, 0.007795742, -0.022152191, 0.008200885, + -0.030059021, 0.03309106, -0.0005047955, -0.0015846331, -0.0006211925, + 0.0032836213, -0.012624789, -0.0036560916, -0.0047898395, 0.011938659, + -0.011383221, -0.015238617, -0.019629847, -0.038501684, 0.015016441, + 0.015892074, -0.020231027, 0.004672217, -0.02885666, 0.04900927, + 0.009716905, 0.0115335155, 0.023589797, -0.000035659352, 0.018518971, + -0.040435918, 0.0077630687, 0.0075408933, 0.006518233, -0.014402191, + 0.016950674, 0.042317875, -0.025249578, 0.029091904, -0.009827993, + -0.025314923, -0.018414417, 0.02217833, -0.016271079, 0.0026791736, + -0.035129845, 0.03037268, -0.01957757, 0.034580942, -0.012539839, + 0.016584737, 0.010932335, 0.010494519, -0.023511382, 0.0048584524, + 0.028543001, -0.01411467, 0.0027216482, -0.024086423, -0.012735876, + -0.0022413575, 0.015212478, 0.0054563656, 0.003613617, 0.025615513, + 0.009867201, -0.0045349915, -0.011337479, 0.020296372, -0.013591905, + 0.025458684, 0.045663573, -0.0040350966, -0.04268381, 0.011350548, + -0.032986507, -0.031052275, 0.009827993, -0.017656406, 0.021302696, + -0.022674957, 0.01624494, 0.037247047, 0.007939503, -0.007730396, + 0.009520868, 0.006893971, -0.0012276822, 0.030477233, 0.0003344883, + 0.002997734, 0.006162099, 0.011559654, 0.004165788, -0.010148186, + -0.018623523, 0.007338322, 0.0066554593, -0.004930333, -0.024426222, + -0.009984822, 0.0022511592, -0.012428751, -0.006116357, -0.020701516, + 0.024635328, -0.04085413, -0.0048029087, -0.019303119, -0.024687605, + 0.0115335155, 0.014898819, -0.009448987, -0.03309106, 0.010984612, + -0.0176172, 0.005701412, 0.012402614, 0.00444024, -0.015081787, + 0.009958684, 0.012860033, -0.023981871, -0.03526054, -0.011579257, + 0.011520446, 0.017395025, -0.008919688, 0.0028343697, -0.029091904, + -0.0044239038, 0.01959064, 0.023903456, 0.0049597383, -0.019185496, + 0.0074951514, -0.0088086, 0.02383811, 0.0040677697, -0.016127318, + -0.01382715, 0.017970067, -0.018884907, -0.0017757693, -0.009762647, + 0.026033726, 0.032516018, 0.013291314, -0.034816187, -0.0015829995, + -0.0046036043, 0.0007796558, 0.022112984, -0.029536255, -0.013395868, + -0.0021939818, 0.019629847, 0.010716694, 0.0026677381, 0.0032819875, + 0.003979553, -0.02883052, -0.034737773, -0.006309127, -0.017120572, + 0.01670236, -0.0074298056, 0.019054806, -0.026046794, -0.007913364, + -0.018963322, 0.037639122, -0.01853204, 0.021211213, 0.03536509, + -0.0045023183, 0.0061424957, 0.01821838, -0.008057124, 0.0063417996, + 0.04294519, 0.022126053, -0.023498312, 0.030869307, -0.009168001, + 0.026556492, -0.007919898, 0.011892917, -0.025236508, 0.0011582525, + 0.010344224, -0.019956576, -0.017525716, -0.052903876, -0.014990303, + 0.010376897, 0.012435286, -0.0019799746, -0.002046954, -0.023106238, + -0.0015070351, 0.034397975, -0.01078204, 0.0030042685, 0.03008516, + 0.034685496, 0.0024096225, 0.0137618035, 0.006207841, 0.051152613, + 0.026504215, -0.01853204, -0.017342748, -0.009390176, -0.00805059, + -0.008253162, 0.03536509, -0.008971964, -0.009893338, 0.0016017864, + 0.057086002, -0.009658094, 0.003959949, 0.010089376, -0.011624999, + 0.015656829, 0.028752107, -0.0030598124, -0.024021078, 0.0059791314, + -0.014284569, 0.028569138, -0.0419258, -0.011794899, -0.012213111, + 0.006015071, 0.011023819, -0.0220215, 0.036384486, -0.036227655, + -0.0055674533, -0.036227655, -0.016140386, -0.023432966, 0.001388596, + -0.0071553537, 0.0024422954, 0.0005007114, -0.0032966903, -0.0241387, + -0.000017880726, -0.0011084264, 0.015918212, 0.006282989, 0.022962477, + 0.034946878, 0.016833052, -0.0058909147, -0.0012742411, -0.0031153562, + -0.028673692, -0.0046199406, -0.010076307, 0.05102192, 0.013787942, + 0.018819561, 0.01578752, -0.03586172, 0.02279258, 0.0013902297, + -0.0014735454, 0.0002234006, -0.003626686, 0.0043487563, 0.023354553, + 0.02170784, -0.007710792, 0.0072795106, -0.0041167787, 0.0075735664, + -0.04132462, -0.0016556965, 0.024792157, 0.027209949, -0.0047767702, + -0.020021921, 0.0013036466, -0.0102135325, 0.00697892, -0.0025697195, + 0.022570403, -0.018649662, -0.015303962, 0.01123946, 0.0042964797, + -0.00083070714, 0.009893338, -0.009122259, 0.034345698, -0.017813237, + 0.02566779, 0.002718381, -0.019094013, -0.016179595, -0.0074298056, + -0.02473988, 0.0096058175, 0.009847596, -0.03248988, 0.025432546, + 0.009396711, -0.042161044, 0.016061973, 0.002380217, -0.012383009, + 0.01382715, -0.018153034, -0.016114248, 0.011579257, 0.006224178, + 0.0473887, -0.024635328, 0.015591484, 0.052616354, -0.004979342, + -0.003509064, -0.013709527, -0.0060869516, 0.003486193, -0.04540219, + -0.0034731238, -0.003531935, -0.0352344, -0.017120572, -0.047702357, + -0.011781829, 0.0008617464, 0.019603709, -0.023289206, 0.007998313, + 0.022335159, 0.010435708, -0.021851601, 0.0035286676, -0.0033685707, + -0.021158937, -0.021158937, -0.012592115, -0.028464586, 0.0031300588, + 0.026817873, -0.020779932, -0.018584317, -0.005698145, 0.016427908, + -0.0046591484, -0.015264755, 0.011102234, -0.011794899, -0.042788364, + 0.016114248, 0.0004937684, 0.018741146, -0.020766862, 0.032280773, + -0.012539839, 0.0017479974, -0.0040775714, -0.003613617, -0.018701939, + 0.0013910466, -0.009305228, 0.024177907, -0.003750843, 0.0077369306, + -0.017774029, -0.02794182, -0.0026121943, 0.020884484, -0.009965219, + -0.0029225862, -0.022152191, -0.0022691293, -0.013709527, + -0.024347806, 0.03052951, -0.021250421, -0.0020404193, -0.008377318, + 0.015774451, 0.006789418, 0.017970067, 0.01593128, 0.004655881, + -0.0005223572, 0.030973861, 0.002233189, 0.012102023, -0.021289628, + -0.022714164, 0.011108768, -0.012860033, 0.0035842115, -0.013552697, + 0.0012260487, 0.011899452, -0.004884591, -0.0032444138, -0.007939503, + 0.010684022, -0.018492833, -0.016911466, -0.020322511, -0.02443929, + -0.03991315, -0.000039743456, -0.001101075, -0.009429384, 0.032594435, + 0.017774029, 0.004695088, -0.019538363, -0.008115935, -0.001064318, + 0.015578414, 0.029536255, -0.030032882, 0.03186256, -0.0126966685, + -0.0004431255, 0.0153301, -0.011330944, 0.008383852, -0.0036528246, + -0.0045741987, -0.0150295105, 0.0125986505, 0.014323777, 0.016885327, + -0.0132129, 0.015317031, -0.014807335, -0.0262559, 0.015879005, + 0.037299324, 0.0028294686, -0.0031349598, 0.0046068714, -0.033195615, + 0.011749156, 0.01236994, 0.008952361, -0.028203202, 0.011899452, + -0.012820826, 0.0013926801, -0.005861509, 0.015317031, -0.013905564, + 0.016075041, -0.0019309653, -0.028647553, -0.026791736, 0.022570403, + 0.010108979, 0.020335581, 0.015774451, -0.012507167, 0.016192663, + -0.000824581, -0.007586635, 0.020832209, -0.03384907, -0.035731025, + 0.010082841, -0.039573353, 0.014362984, 0.017029088, 0.015356239, + -0.031784147, -0.03826644, -0.0015241883, -0.03207167, 0.010540261, + -0.026608767, -0.0019587371, 0.0070704045, -0.005469435, + -0.0054171584, 0.026674114, 0.020793, 0.020675378, -0.019708263, + -0.0046820194, 0.019642916, 0.002522344, 0.0028588742, 0.020217959, + -0.019172428, -0.016493253, 0.032751262, 0.01989123, -0.017342748, + -0.014702782, 0.019995783, 0.009599282, 0.012657462, 0.00035307097, + 0.0023001686, 0.008148609, -0.010507588, -0.0015437921, 0.030294266, + -0.008932756, 0.016323354, 0.0038684653, 0.030973861, 0.013232504, + 0.0047277613, -0.000090360845, -0.02794182, -0.0014523081, + 0.022505058, -0.038344856, -0.0000059793865, -0.016349493, + 0.024099493, -0.034659356, 0.0039403457, -0.020714587, -0.011951728, + -0.01032462, -0.023785833, 0.00016785676, -0.0012350336, 0.0065247677, + 0.013774873, 0.012075884, -0.023746626, -0.009886804, 0.015107925, + -0.012206576, 0.029954467, 0.003257483, -0.013997048, 0.003888069, + -0.052747045, -0.013774873, -0.015591484, -0.0041363826, 0.016833052, + 0.0040808385, 0.016963743, 0.012690134, -0.012735876, 0.012232714, + -0.01123946, -0.0021874472, -0.012618254, -0.008606028, -0.011821036, + -0.024112562, 0.015826728, -0.023877317, 0.0406973, 0.010409569, + 0.013644181, 0.009664629, -0.00011997062, 0.003348967, 0.003590746, + 0.0037998522, -0.0030679805, -0.014820404, 0.008227023, -0.01640177, + -0.005966062, 0.020440133, -0.015682967, 0.023289206, -0.03141821, + 0.00972344, -0.004316083, 0.0038717326, 0.020701516, 0.010710159, + 0.019708263, 0.03144435, 0.031182967, -0.010474916, -0.0066521917, + -0.014637436, -0.0075408933, -0.013787942, -0.0012235981, + -0.028516863, 0.022674957, 0.01472892, 0.018754214, 0.024426222, + 0.0030500104, -0.003522133, -0.00531914, -0.030241989, -0.011134907, + -0.00093526026, -0.014650505, -0.004652614, 0.013918634, 0.011245995, + 0.020596964, 0.02490978, 0.029222596, -0.0017528983, -0.0050381534, + -0.0013044635, 0.011605396, 0.009514334, 0.012284991, 0.007697723, + -0.011500843, -0.0014065661, 0.010056702, -0.008102867, 0.0016393602, + -0.008396922, -0.0071096118, -0.016153457, 0.009971754, -0.015696036, + 0.049715005, -0.0038684653, 0.0061686337, -0.013193296, -0.017682545, + 0.009233347, 0.00265957, -0.018649662, 0.012958052, -0.02548482, + -0.007018128, -0.017852444, -0.008161677, 0.020257166, 0.016022764, + -0.021903878, 0.011840641, -0.018336002, -0.011821036, 0.011807967, + 0.0041167787, -0.036018547, -0.017342748, -0.023419898, 0.0007702624, + 0.007789207, -0.006893971, 0.012389544, -0.009592748, -0.007181492, + 0.0071357503, -0.0060673477, 0.0009115725, -0.0056458684, + -0.011017284, -0.004711425, -0.008501475, -0.0014065661, + -0.00037308308, 0.020610033, -0.0086909775, 0.0047244937, + -0.010527192, -0.027811129, -0.012186972, -0.0037541103, -0.02050548, + 0.0025419476, -0.005322407, -0.004450042, 0.034267284, 0.009736509, + 0.0013877793, 0.025772342, 0.013062605, -0.0032215428, 0.020714587, + -0.007410202, 0.0073579256, 0.013801011, 0.004280143, 0.0014294371, + 0.008860877, 0.00720763, -0.009926011, 0.0031300588, -0.015578414, + -0.004737563, 0.013252107, 0.022988616, 0.0028670426, 0.024583051, + 0.040409777, -0.002197249, -0.028020235, -0.014663574, -0.0069658514, + 0.014101601, 0.007057335, 0.011317874, 0.0016434443, 0.027680438, + -0.01518634, -0.002087795, 0.007560497, 0.023759695, -0.03157504, + 0.015003372, 0.030895445, -0.014023187, -0.018884907, 0.0057504214, + 0.024321668, 0.01031155, 0.013219435, 0.03431956, -0.0023949197, + 0.018309863, -0.023328414, -0.0127489455, -0.00872365, 0.0023410097, + 0.006227445, -0.03978246, -0.008782461, -0.017564923, 0.015134064, + 0.016493253, -0.007978709, 0.02187774, -0.037586845, 0.009945615, + 0.011219856, -0.0022233874, 0.014898819, -0.0073448564, 0.009429384, + 0.062313657, 0.029065765, -0.013944772, 0.017826306, -0.011461635, + 0.0075735664, 0.0114812385, -0.0022952675, -0.025942242, -0.002621996, + 0.031235242, -0.019094013, 0.00035102892, -0.0028637752, 0.0044696457, + 0.0005223572, 0.007874156, -0.0176172, -0.010069772, -0.011415893, + 0.011579257, 0.017826306, -0.0053518126, -0.014532883, 0.021916946, + 0.0015707472, -0.0110760955, -0.0220215, -0.008684443, -0.026229763, + 0.0021939818, 0.007272976, 0.0013379531, 0.0059693293, 0.017669477, + -0.007834949, 0.031679593, -0.0012709738, 0.007351391, -0.0076454463, + 0.018244518, 0.004129848, 0.009533937, 0.0039501474, -0.007527824, + 0.008939291, -0.010174325, 0.010808178, -0.013219435, 0.010246205, + 0.0007812895, 0.011572723, -0.0031856026, -0.022962477, -0.012768549, + -0.01564376, -0.0059693293, 0.005518444, 0.012193507, -0.015957419, + -0.0006113907, -0.021694772, -0.013918634, 0.015447723, 0.011500843, + 0.0039305435, -0.0032885221, 0.017303541, 0.000052582865, + -0.012918844, 0.006227445, -0.0005382852, 0.013578836, -0.023759695, + -0.017839376, 0.023197722, -0.009527403, -0.011004215, -0.014389123, + 0.002489671, 0.021054383, -0.044461213, -0.0017349282, -0.0060183387, + -0.008481871, 0.0061653666, -0.0036005478, -0.013016863, -0.016075041, + -0.0029111507, -0.01671543, -0.0042703412, -0.0027918948, + -0.011794899, 0.008272765, 0.0011500843, -0.009899873, -0.0022821985, + 0.0053714165, -0.011337479, -0.016937604, -0.008821669, 0.03526054, + -0.025576307, 0.013997048, -0.0023230396, -0.0013265177, + 0.00067551114, 0.013343591, 0.009547006, 0.008802066, 0.016127318, + 0.007076939, -0.0071161464, -0.008645236, 0.024556912, -0.012559443, + 0.0071422844, 0.0031905037, 0.048303537, -0.0097561125, -0.020832209, + 0.034058176, 0.019695193, -0.00489766, 0.004234401, -0.0080832625, + -0.003750843, -0.014794266, -0.0019097279, -0.0075997044, 0.01252677, + 0.017695615, -0.0004729395, -0.021786256, -0.0051459735, -0.01791779, + -0.0009826359, -0.003247681, 0.016584737, 0.0023312077, 0.0070311967, + -0.014467537, 0.015526137, 0.018492833, 0.019172428, -0.014598229, + -0.014376054, -0.018466694, 0.010442242, -0.002690609, 0.0065541733, + -0.0029585264, -0.0032182755, 0.009952149, -0.016336424, + -0.0000847452, 0.005492306, -0.0039860876, 0.0048486507, + -0.0054334947, 0.034554806, 0.006541104, 0.0047310283, -0.0029111507, + -0.017421164, -0.0044990513, -0.0072403033, -0.015055648, + 0.0036724282, -0.0029568928, -0.0056687393, -0.0094555225, + -0.007076939, 0.00918107, 0.0072599067, -0.023328414, 0.0016164891, + -0.02656956, -0.0066587264, -0.0055903243, 0.014741989, -0.008187816, + -0.008030986, -0.018715007, -0.018989459, 0.018584317, 0.013435075, + 0.0012799589, -0.004384696, -0.005995468, -0.00007565806, + 0.0041363826, 0.022374367, 0.024190977, 0.020610033, 0.014820404, + -0.003763912, 0.012690134, 0.012977655, -0.0122915255, -0.013323988, + 0.02490978, 0.0145198135, -0.0022854658, -0.0064267493, 0.0126966685, + -0.01534317, -0.029640809, -0.014349915, -0.0045970697, 0.0020502212, + -0.020113405, -0.0035155984, -0.018087689, 0.006730607, -0.01714671, + -0.0042180647, 0.00017571867, -0.0074755475, 0.007841484, 0.014180016, + -0.008658305, -0.02170784, 0.022701096, -0.0012987457, -0.0052015176, + 0.009867201, 0.0036495572, 0.0014384221, -0.00084622676, 0.010435708, + 0.0028980817, -0.003362036, -0.030189712, -0.032881953, 0.023955733, + 0.007253372, 0.02488364, -0.0035613405, 0.007848018, -0.0002004275, + 0.0002891547, 0.0150295105, 0.012938448, 0.03113069, -0.0042017284, + 0.015839797, 0.012892706, 0.017395025, 0.029693084, -0.011147976, + 0.01943381, 0.009416315, 0.000097303826, 0.010305016, 0.013591905, + -0.030921584, 0.010265809, -0.033195615, -0.0060510114, -0.012180438, + -0.0044761803, -0.026412731, 0.009867201, -0.009285623, -0.0050806277, + 0.003705101, 0.023067031, -0.0054302276, 0.009409781, -0.01823145, + -0.021773186, 0.008514544, -0.013337057, -0.01152698, 0.024818296, + 0.017342748, 0.0273145, -0.011977866, -0.005966062, 0.011860244, + -0.021119729, -0.021668633, -0.00039922138, 0.0001421269, 0.01776096, + -0.009547006, 0.01929005, 0.0009246416, -0.005573988, -0.001927698, + -0.019459948, 0.017395025, 0.008991567, 0.008285834, -0.0007906829, + -0.013644181, 0.013265177, 0.007730396, -0.0016230238, -0.04009612, + -0.005737352, 0.014402191, -0.012539839, 0.004695088, 0.0035646078, + 0.005217854, -0.026661044, 0.021446457, -0.0000255129, -0.00081804645, + 0.005698145, -0.020152614, -0.017042158, 0.003362036, 0.0024047217, + 0.006815556, -0.014650505, 0.014088532, 0.00421153, 0.0015054015, + 0.004044899, 0.016885327, -0.00146211, 0.013487352, 0.002095963, + -0.02200843, 0.025733136, 0.012330733, 0.012565978, 0.0085929595, + -0.003183969, -0.0005758589, -0.00045578624, -0.0068612983, + 0.00027608557, -0.005136172, 0.0015633958, 0.012559443, -0.004760434, + -0.019512225, -0.009736509, -0.014689713, -0.003937078, 0.006224178, + -0.0073840637, 0.014598229, -0.032280773, 0.007848018, -0.00466895, + -0.009919477, -0.011860244, 0.0002234006, -0.01487268, -0.006031408, + -0.012840429, -0.011932124, -0.0018525505, 0.011115303, -0.0046166736, + -0.010037099, 0.017355816, -0.015068717, 0.025406407, -0.019825885, + 0.008566821, 0.016140386, 0.0013510223, -0.0068547637, -0.008403457, + -0.02338069, -0.014441399, 0.007919898, -0.022805648, -0.0138140805, + 0.008717116, 0.004554595, -0.0057406193, 0.004002424, 0.0055347807, + -0.004172323, -0.027602023, -0.012905775, 0.012186972, 0.010749367, + 0.0016663152, 0.0015731977, 0.0024766019, -0.018192243, 0.013644181, + -0.03445025, 0.0043716272, 0.007913364, 0.014062393, -0.005864776, + 0.021812394, -0.00057871785, 0.014951095, 0.018179173, -0.0054596327, + 0.011938659, -0.025981449, -0.007331787, 0.009396711, -0.009612352, + -0.0020600231, -0.015408515, 0.027105395, 0.011520446, 0.008658305, + 0.009710371, 0.008200885, 0.04325885, -0.010592537, -0.006080417, + 0.021067452, -0.03068634, 0.0103965, 0.02822934, -0.013631112, + 0.020413995, 0.0077238614, -0.009468592, 0.0020763595, 0.00531914, + 0.007397133, -0.03113069, -0.016271079, -0.0047767702, 0.009331365, + 0.015617621, -0.01624494, -0.004819245, -0.013435075, 0.001009591, + 0.003878267, 0.015983557, 0.028752107, -0.017956998, 0.021616356, + -0.017512647, -0.021851601, 0.009462057, -0.0059007164, 0.013983979, + 0.007501686, 0.020100337, 0.003603815, 0.028307756, -0.007815345, + 0.012873102, 0.028752107, 0.003531935, -0.0063581364, -0.006972386, + 0.013108347, -0.016519392, -0.015774451, -0.011585792, 0.019028667, + -0.01987816, -0.0020387857, -0.0033914417, 0.0016434443, -0.010298482, + 0.0036626263, -0.0038488614, 0.0016532461, 0.012128161, -0.0138140805, + 0.008285834, -0.006008537, -0.021694772, -0.011742622, 0.014689713, + 0.0007224784, -0.013043001, -0.0048682545, -0.0068416945, + -0.017395025, -0.003404511, 0.0069331783, -0.012409148, -0.0009246416, + 0.013552697, 0.008161677, 0.008527613, 0.0002001212, 0.018244518, + 0.005505375, 0.004280143, 0.022714164, -0.017564923, 0.008318507, + 0.0022266547, -0.0029405563, 0.036985666, 0.012363406, 0.0006085318, + -0.028072512, -0.014258431, 0.005760223, 0.03207167, -0.006224178, + -0.004378162, 0.002064924, 0.00903731, -0.0043912306, -0.008762858, + -0.013983979, -0.02217833, -0.011043423, -0.0029993674, 0.003018971, + 0.012637857, 0.018270656, 0.0376914, 0.017695615, 0.018022344, + -0.02157715, -0.02974536, 0.0050838953, 0.010474916, 0.016349493, + -0.021211213, -0.014402191, 0.001932599, -0.0025305122, 0.009468592, + 0.026465008, 0.0038586634, 0.009272554, 0.011500843, -0.0052243886, + -0.010108979, 0.014663574, -0.0019848756, 0.02778499, 0.00039186998, + -0.017538784, 0.030320404, 0.019473018, -0.02352445, 0.0036724282, + 0.013016863, 0.0072337687, 0.018754214, -0.0089458255, -0.0029127845, + 0.005237458, 0.005093697, 0.01336973, 0.016480185, -0.010801644, + 0.012559443, 0.015656829, -0.0057275505, 0.019028667, -0.022975547, + -0.018100757, -0.019773608, 0.0029928328, -0.012428751, 0.020727655, + -0.02275337, 0.017290471, 0.0031267917, 0.027863406, -0.00993908, + 0.00034122705, -0.008403457, -0.0018100758, -0.0052113193, + -0.008246627, -0.00082907354, -0.0078088106, 0.020766862, + -0.018688869, -0.008331576, -0.022844855, 0.02961467, 0.0072860452, + 0.0039534145, -0.021903878, -0.028412309, 0.009716905, -0.004338954, + -0.009390176, 0.0102135325, -0.01563069, 0.000013694518, -0.00706387, + -0.017878583, 0.004518655, -0.01684612, 0.0062339795, 0.02734064, + -0.00061915047, -0.017029088, -0.030424956, 0.010592537, -0.019054806, + 0.0038161888, -0.008632166, -0.020244097, 0.013918634, 0.013539628, + 0.0092006745, -0.0015650294, 0.005636066, 0.0007049167, -0.0020779932, + -0.020766862, 0.006939713, -0.00074371573, -0.020884484, + 0.00041351575, 0.008978499, 0.022805648, 0.0049499366, 0.017538784, + 0.012794687, -0.014336846, 0.03382293, -0.00410371, -0.0027902613, + -0.017120572, -0.026922427, 0.008586424, -0.0011966431, 0.026334316, + 0.002011014, -0.009814924, 0.0039272765, -0.009390176, -0.019995783, + 0.0028441714, -0.009377107, 0.011317874, 0.01048145, -0.0040547005, + 0.023563659, 0.021930017, 0.0064169476, 0.0056033935, 0.025092747, + 0.020518549, -0.024413152, -0.010331155, 0.03199325, 0.013709527, + -0.008181281, 0.01442833, 0.0008707314, -0.0007543344, -0.011925589, + 0.018296795, 0.0017610665, 0.00796564, -0.009272554, -0.0041396497, + 0.0050414205, -0.028046373, 0.01640177, -0.035338953, -0.012820826, + -0.026765598, 0.011004215, 0.013931703, 0.01504258, 0.0038488614, + 0.02049241, -0.026242832, -0.0015388911, 0.019773608, -0.010069772, + 0.0010496152, -0.004760434, -0.01914629, 0.0029928328, 0.007684654, + -0.008285834, -0.0017888384, -0.00972344, 0.01412774, -0.02157715, + -0.0029764965, -0.01261172, 0.011592327, 0.013526559, 0.013892495, + 0.012252318, 0.005573988, -0.024021078, 0.0074494095, 0.034816187, + -0.00903731, -0.0012383009, -0.008481871, -0.016950674, 0.0062960577, + -0.006345067, 0.004649346, -0.004299747, 0.005792896, -0.029065765, + -0.020374788, 0.035809442, 0.008455733, 0.008749789, -0.012160834, + 0.011461635, 0.0027347174, -0.0005235824, 0.0077499994, -0.018963322, + -0.005933389, -0.020596964, 0.010579469, 0.003545004, 0.031548902, + -0.006959317, 0.0040547005, -0.023119308, 0.01791779, 0.007090008, + -0.0012432019, 0.003247681, -0.013840219, 0.01927698, -0.0041167787, + -0.00588438, -0.004946669, 0.00812247, 0.021812394, -0.008279299, + 0.021629425, -0.006809022, -0.0019015598, -0.01684612, 0.0039893547, + -0.0012235981, 0.019760538, 0.0006538654, -0.00000608468, + -0.010474916, 0.008599494, -0.035417367, -0.007188027, -0.00041821247, + 0.03330017, 0.008194351, -0.03139207, 0.005155775, -0.029065765, + -0.01101075, 0.027131533, -0.006008537, -0.004652614, 0.0054563656, + 0.019616779, 0.008971964, -0.00683516, 0.001351839, 0.01085392, + -0.000532159, 0.0028425378, -0.019760538, 0.010644814, 0.013023397, + 0.010507588, -0.0049760747, 0.038031194, -0.002535413, 0.017133642, + 0.0085406825, -0.0063385325, 0.008357715, 0.009645025, -0.0048911255, + -0.013029932, 0.02383811, -0.0011664206, -0.0016017864, 0.0013257008, + -0.003061446, 0.015918212, -0.013252107, 0.0052733975, -0.0064430856, + 0.019094013, -0.01351349, 0.02200843, -0.0044043, -0.008416526, + -0.0017202254, 0.016258009, -0.010801644, -0.0034241145, -0.012017074, + -0.0048780562, 0.009520868, -0.0031774344, 0.013879426, -0.005230923, + -0.0068678325, 0.014807335, -0.014180016, 0.015696036, 0.023929594, + -0.0012252318, 0.022988616, 0.0052505266, -0.00019164667, + -0.016506324, 0.012474493, -0.021629425, -0.047101177, -0.004022028, + 0.0073579256, -0.029379426, 0.010540261, -0.011455101, 0.027079256, + 0.003384907, 0.0066521917, 0.0045349915, -0.019224703, 0.016963743, + 0.022831786, -0.0055576516, 0.015081787, 0.0069462475, -0.034528665, + 0.01989123, 0.0019358663, -0.020975968, 0.013186761, -0.0006346701, + -0.012383009, 0.0020338849, -0.024726812, -0.0021907145, 0.017303541, + -0.005743887, 0.006410413, 0.0069462475, -0.012200042, -0.004554595, + 0.0036332207, 0.0012374842, 0.011585792, -0.010801644, -0.021054383, + 0.0058909147, -0.009906407, -0.008279299, 0.028150925, 0.025903035, + -0.00015203084, -0.01654553, -0.0071553537, 0.009965219, 0.006939713, + 0.0008617464, -0.004672217, 0.0061424957, 0.0142192235, -0.019512225, + -0.014964164, -0.006116357, -0.011860244, -0.022988616, -0.017695615, + 0.0075735664, 0.0021319033, 0.012265387, -0.013487352, 0.010697091, + 0.0083381105, 0.037168633, 0.009246416, 0.006354869, 0.000010152832, + -0.014715851, 0.00013559232, 0.028124787, -0.003914207, 0.014075463, + -0.0039501474, -0.025314923, 0.021067452, -0.0033342643, -0.02186467, + -0.0024161572, -0.0054204254, 0.00805059, 0.008220488, 0.03081703, + -0.002563185, -0.0062143756, 0.0037116355, -0.015199409, -0.010167791, + 0.011474704, -0.00796564, -0.0033685707, 0.00683516, 0.0264258, + 0.0049041947, 0.005342011, 0.016676221, -0.013029932, -0.021694772, + -0.008671374, -0.006387542, 0.0061915047, 0.021067452, -0.0028490725, + -0.004185392, -0.011925589, 0.015669897, -0.0031807018, 0.009305228, + 0.008854342, 0.029248733, -0.02399494, -0.023067031, -0.013297849, + 0.0049499366, 0.0037541103, -0.0070508006, -0.02339376, -0.0004778404, + -0.00027955705, 0.0032999576, 0.0102135325, -0.018545108, -0.02064924, + 0.023811972, -0.01032462, -0.020335581, 0.013036466, 0.030137435, + 0.03021585, 0.005345278, -0.008344646, 0.016741568, 0.016480185, + 0.0077499994, 0.0017398292, 0.011827571, -0.012265387, 0.0188457, + 0.008429595, -0.0019015598, 0.011892917, 0.013239038, 0.0104683805, + 0.018871836, 0.0020583894, -0.012278456, -0.007272976, -0.0041951938, + -0.02791568, -0.020675378, 0.04451349, -0.0075735664, 0.013350125, + 0.0033342643, -0.0073252525, 0.0034600548, -0.01367032, -0.008514544, + -0.008553752, -0.0061424957, -0.011082631, 0.020178752, -0.010429173, + -0.026282039, 0.0142715005, 0.015356239, -0.024726812, -0.016336424, + -0.007377529, 0.006207841, 0.007919898, -0.010886593, 0.0016614143, + -0.00942285, 0.0067044683, -0.021132799, 0.0060608136, 0.015120994, + 0.028203202, -0.014924957, -0.0030091694, -0.009658094, 0.021930017, + -0.0004239302, -0.027079256, -0.005554384, 0.021250421, -0.015970489, + -0.014898819, 0.011847175, -0.0040775714, 0.020570826, -0.0068416945, + -0.0041461843, 0.006015071, 0.021211213, 0.006469224, 0.01236994, + 0.019838953, -0.007815345, -0.0010398134, 0.00006963401, + -0.0072403033, 0.00021421137, 0.013258642, 0.02186467, -0.014846542, + 0.0063810074, 0.0047473647, -0.019224703, -0.0008919688, + -0.0056491354, 0.0071749575, 0.006599915, -0.014768127, 0.008416526, + -0.00013600073, -0.01836214, 0.021812394, -0.0004075938, -0.016336424, + 0.027131533, 0.012925379, -0.009886804, -0.011102234, -0.025942242, + 0.00034796583, -0.000068000365, -0.006057546, 0.022413574, + -0.0026791736, 0.01236994, 0.0006857214, 0.00012977247, 0.0073252525, + -0.01623187, -0.016467115, 0.0017316609, 0.009631956, -0.019995783, + 0.003394709, -0.0154346535, -0.015539207, 0.010089376, -0.0026203624, + 0.0010929068, 0.007625843, 0.036985666, 0.0069462475, 0.006998524, + 0.010265809, -0.005008748, 0.00095486396, 0.007351391, 0.010487984, + 0.01504258, -0.021185074, 0.0005627898, 0.009187605, -0.009083052, + 0.0041690553, -0.006711003, -0.030581785, 0.007377529, -0.011664207, + 0.012128161, 0.0040775714, -0.011638069, -0.0043748943, 0.0062111085, + 0.023981871, -0.004071037, -0.00011762225, 0.014898819, 0.01093887, + 0.011389755, -0.0055119097, -0.0075408933, 0.0056491354, -0.02126349, + -0.00611309, 0.0041788574, -0.012650927, -0.0038749997, 0.00294219, + -0.0035352022, 0.01610118, 0.0060477443, 0.0037214374, -0.002179279, + -0.009631956, -0.012128161, 0.007057335, -0.0133566605, 0.0046395445, + 0.005243992, -0.0057667578, -0.0011460001, -0.020675378, + -0.0077173267, 0.0054988405, 0.0010153088, -0.034737773, -0.008971964, + -0.015826728, -0.0010332789, 0.011925589, -0.019303119, 0.0103965, + -0.002407989, 0.01472892, 0.004093908, 0.0044075674, 0.010416104, + -0.011677276, 0.0176172, 0.025066609, -0.016427908, 0.009187605, + 0.0072403033, -0.01382715, -0.00835118, -0.007181492, -0.016663153, + -0.0013861456, 0.012422217, -0.002489671, -0.008573355, -0.0014245362, + 0.0051884484, -0.00094261166, -0.0032819875, 0.01185371, -0.006697934, + 0.025131956, -0.01957757, 0.015003372, 0.0137618035, 0.009239881, + -0.01594435, 0.0104683805, 0.0073840637, -0.012833895, 0.013853287, + 0.01944688, -0.0058680438, 0.0092006745, 0.024543844, 0.010579469, + 0.04631703, -0.0050838953, -0.015264755, -0.012762015, -0.009814924, + -0.021943085, 0.005253794, 0.010788575, 0.0014833473, 0.014245362, + -0.010971542, 0.007651981, -0.0029536255, 0.0066031828, -0.012200042, + -0.0074298056, 0.014624367, -0.008854342, -0.006331998, 0.0060510114, + 0.015016441, -0.0034600548, 0.016898397, 0.009860666, 0.0026350652, + -0.0039958893, 0.01791779, 0.0114812385, 0.017669477, -0.012938448, + 0.019120151, -0.00805059, -0.016480185, 0.0062470487, 0.020845277, + 0.03873693, 0.0074951514, 0.017029088, 0.0230801, 0.01101075, + -0.010657883, 0.011742622, 0.016937604, -0.0019064607, -0.014206154, + 0.013997048, 0.008560286, 0.002759222, 0.007246838, 0.018022344, + 0.009599282, 0.0023459105, 0.017839376, -0.011487774, -0.003509064, + 0.016192663, -0.0022625946, -0.005952993, -0.002901349, 0.0011672375, + 0.006697934, -0.0015797322, -0.017747892, -0.0025158094, -0.011579257, + 0.0017316609, -0.021015177, -0.0073905983, 0.0043422217, -0.006243781, + 0.025785413, -0.0029519917, -0.0029046163, -0.0031169897, + -0.0019391335, 0.0002082894, -0.024465429, -0.02961467, -0.0027020446, + 0.030189712, -0.007919898, -0.0016279246, -0.016623946, 0.004920531, + -0.0009050379, 0.023877317, 0.0081355395, 0.011566188, 0.0051329043, + -0.008070193, -0.02791568, -0.004485982, 0.02247892, 0.022439713, + -0.0054400293, 0.009488195, -0.0070050587, 0.012886171, 0.0015478762, + 0.0008748155, -0.0022217536, -0.0031186233, 0.020322511, 0.023315344, + -0.010690556, 0.0056883427, -0.019956576, 0.0075147552, 0.0154869305, + 0.0018982925, -0.01684612, -0.015120994, 0.00660645, -0.028647553, + 0.0068416945, -0.0074886167, -0.0012775083, -0.0005934206, + 0.001881956, 0.0038259905, 0.014493676, -0.008854342, -0.0074886167, + 0.012722807, 0.0024471963, 0.026360454, -0.010167791, -0.008318507, + 0.015813658, -0.00926602, 0.000111393994, 0.02200843, 0.019185496, + -0.007874156, 0.002105765, 0.0020665578, 0.014376054, 0.00736446, + 0.0009393444, -0.013748734, 0.020100337, 0.0005697328, 0.018937184, + 0.0013142653, 0.011272132, 0.004110244, -0.003143128, -0.0012366673, + 0.02004806, -0.006730607, -0.019237773, -0.014794266, -0.016127318, + -0.004106977, 0.016493253, -0.013186761, -0.0022070508, -0.019198565, + -0.026020657, 0.004316083, -0.0016238405, 0.00085684547, 0.014624367, + -0.0033097595, -0.016976813, 0.0078088106, -0.009004637, -0.020152614, + 0.0012619888, -0.009207209, -0.017434232, -0.012905775, -0.03719477, + -0.0013346858, -0.00086419686, -0.0125986505, -0.009729974, + 0.0037345067, 0.043102022, -0.027445193, -0.013709527, 0.036645867, + 0.015774451, -0.001210529, -0.009743043, -0.0028360032, -0.002192348, + -0.00040555172, -0.023602866, -0.029222596, -0.032908093, -0.01312795, + -0.0040089586, 0.009814924, 0.030137435, -0.002965061, 0.015408515, + -0.0023148714, -0.007887226, -0.014362984, -0.0048453836, + -0.010651348, 0.004028562, 0.021315766, -0.008854342, 0.003161098, + 0.005070826, -0.0014057494, 0.03565261, -0.013983979, -0.024190977, + 0.014768127, 0.0010831049, -0.013539628, -0.0035646078, 0.016623946, + -0.0038390597, 0.014467537, -0.013892495, 0.010461846, -0.012245784, + 0.0020567558, -0.01913322, -0.009586214, 0.005861509, -0.0024635328, + -0.0076781195, -0.00083519967, -0.007351391, 0.0004623208, + 0.019002529, -0.015016441, -0.0052243886, 0.020100337, 0.024975125, + 0.00752129, 0.022400504, 0.026504215, 0.01580059, 0.018858768, + -0.0052276556, -0.00044394232, -0.012304595, -0.013500421, 0.00720763, + -0.001960371, 0.023602866, -0.0027984295, -0.014376054, 0.012383009, + 0.02460919, 0.014454468, 0.017813237, 0.007874156, -0.003796585, + 0.031156829, 0.0019064607, -0.024569983, 0.013722597, -0.018153034, + 0.01594435, -0.005400822, 0.012304595, 0.0009066715, 0.015264755, + -0.018244518, -0.024151769, -0.019237773, 0.0061098225, -0.0029911993, + 0.0015225548, -0.0017904721, 0.013134485, 0.001434338, -0.0050054803, + 0.009703836, 0.00006279313, 0.0012897607, -0.015238617, 0.0021433388, + 0.010461846, -0.016427908, -0.007913364, -0.013866357, -0.0030957523, + 0.0154869305, 0.018571246, -0.013591905, 0.012735876, 0.028725969, + 0.008377318, 0.024975125, 0.0006113907, -0.012337267, 0.033012647, + 0.006031408, 0.0024096225, 0.013343591, -0.010951939, -0.008076728, + 0.0032640176, -0.012585581, -0.018414417, 0.0014857978, 0.003404511, + 0.0069658514, -0.019094013, 0.009292158, -0.004254005, -0.018610455, + -0.009468592, 0.0058582416, -0.012317664, -0.008828203, 0.023302276, + 0.023655143, 0.031209106, 0.007018128, -0.0002805781, 0.0025321457, + -0.004799641, -0.0070246626, 0.012278456, -0.016166525, -0.004623208, + -0.018897975, 0.0028245677, 0.003348967, -0.010775506, 0.02006113, + 0.022505058, 0.00013201873, 0.015055648, -0.0020861614, -0.01312795, + 0.003626686, -0.020897554, 0.0056066606, -0.00037267467, -0.007423271, + -0.017538784, -0.012997259, 0.0174473, 0.0011639702, -0.0062731868, + 0.021747047, 0.0039534145, 0.01806155, -0.019655986, -0.013539628, + -0.008318507, 0.0040743044, 0.021773186, 0.0031169897, 0.017029088, + 0.0006126159, 0.016297217, 0.0053616143, 0.009174536, -0.028307756, + 0.012840429, 0.011082631, 0.02612521, -0.0021596754, -0.006008537, + 0.0019636382, -0.023498312, 0.018950252, 0.01929005, 0.011363616, + 0.0059693293, -0.006848229, -0.013408937, 0.011357082, -0.03614924, + -0.008305438, 0.00051786465, 0.028490724, -0.013892495, 0.020989038, + 0.023459105, -0.01564376, 0.0070115933, -0.005815767, -0.0018019076 + ] + } + ] + } ] diff --git a/src/scrapers/markitdown/examples/collboard-fra1-cdn-digitaloceanspaces-com-ptbk-user-knowledge-source-b-d-10-simple-pdf.knowledge.json b/src/scrapers/markitdown/examples/collboard-fra1-cdn-digitaloceanspaces-com-ptbk-user-knowledge-source-b-d-10-simple-pdf.knowledge.json index 4e4185016f..02ddbf26c2 100644 --- a/src/scrapers/markitdown/examples/collboard-fra1-cdn-digitaloceanspaces-com-ptbk-user-knowledge-source-b-d-10-simple-pdf.knowledge.json +++ b/src/scrapers/markitdown/examples/collboard-fra1-cdn-digitaloceanspaces-com-ptbk-user-knowledge-source-b-d-10-simple-pdf.knowledge.json @@ -1,18 +1,636 @@ [ - { - "name": "springfield-illinois-historic-capital-city", - "title": "Springfield: Illinois' Historic Capital City", - "content": "Springfield is the county seat of Sangamon County in Illinois, USA. As of 2019, it had a population of 10,566, making it the sixth most populous city in the state.", - "keywords": [ - "Springfield: Illinois' Historic Capital City" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.022805648, 0.035417367, -0.0029928328, -0.024086423, 0.016179595, -0.0070115933, 0.011461635, 0.0027347174, -0.056981448, 0.017042158, -0.029248733, 0.01291231, -0.008736719, 0.018858768, 0.011193718, 0.009886804, -0.016336424, 0.033953626, 0.003763912, -0.0007968091, 0.012670531, -0.0023491778, -0.049244516, 0.03596627, 0.023171583, 0.0093379, -0.03126138, 0.008860877, -0.034188867, -0.0043977653, -0.021616356, -0.0042866776, 0.049793422, 0.0024504636, 0.015983557, -0.016349493, 0.0077042575, -0.020740723, 0.024190977, 0.02930101, 0.0013836952, -0.0013395868, 0.01671543, 0.00820742, 0.03293423, -0.036985666, -0.004815978, -0.024360877, 0.029248733, 0.016937604, 0.024870573, -0.012500632, -0.019708263, 0.037168633, 0.010141652, -0.01806155, 0.011677276, 0.022596542, -0.0067959526, -0.043232713, -0.0032231766, -0.0049597383, -0.00013263135, 0.00044802643, 0.0061326935, 0.0029536255, 0.0051002316, 0.004289945, 0.053688023, 0.011376685, 0.013317453, 0.056563236, 0.008070193, 0.043781616, 0.05980438, -0.005485771, -0.03147049, -0.002639966, -0.04252698, 0.029562393, 0.013173693, -0.013604974, -0.0145720905, 0.002955259, 0.017251264, 0.008638701, -0.00064978126, -0.07783979, -0.00006636672, 0.014206154, 0.017133642, -0.0077696033, -0.0059431912, -0.028752107, -0.043781616, -0.028124787, 0.041559864, -0.013435075, -0.00046354602, 0.01974747, 0.014781197, -0.009239881, -0.026504215, 0.01914629, 0.008187816, 0.015238617, -0.0075539625, -0.0068220906, -0.020204889, 0.0046591484, 0.018649662, -0.0037933176, -0.012964586, 0.006345067, -0.016205734, 0.00068653823, -0.014545952, -0.0107689705, 0.042370148, 0.02429553, -0.0038848016, 0.0049139964, -0.0276543, 0.011402824, -0.0046983557, -0.0016556965, 0.015277823, -0.027288362, 0.007978709, -0.0412462, 0.03463322, 0.0062960577, -0.018139966, 0.018715007, 0.00956661, -0.034136593, -0.0057275505, 0.030921584, 0.03052951, 0.03873693, -0.06984148, -0.0030941188, 0.026033726, -0.0000384927, 0.0048355814, -0.010357293, -0.01927698, 0.0032362456, 0.013604974, -0.028516863, -0.01152698, -0.010494519, -0.027419055, 0.004874789, 0.0025615513, 0.016637014, -0.0039762855, -0.013278246, 0.043337267, -0.053583473, -0.019512225, -0.009710371, 0.03450253, -0.026138278, -0.00872365, -0.013088743, -0.01488575, -0.0041233134, -0.00736446, 0.008096332, 0.022727232, -0.009488195, -0.061895445, -0.048800167, 0.0021188343, -0.018545108, -0.03688111, -0.015656829, -0.00064365513, 0.029666945, -0.0039664838, 0.01146817, 0.02883052, -0.008867411, -0.024988195, -0.019329257, 0.008436129, 0.011291737, 0.036489036, -0.010043633, -0.0066129845, 0.041978076, 0.010389966, -0.0023687815, 0.04022681, -0.003298324, -0.023511382, -0.03293423, -0.0040677697, 0.029091904, -0.05755649, 0.021603288, -0.024073355, -0.03207167, 0.0006881719, 0.023001686, -0.00842306, -0.0133566605, 0.020034991, 0.026360454, -0.006952782, -0.0030843168, -0.01944688, 0.03492074, -0.020714587, 0.023158515, -0.013944772, -0.0011966431, -0.015617621, 0.038841482, 0.020518549, 0.028647553, 0.0004202545, -0.014807335, -0.032698985, -0.025236508, -0.053034566, 0.016297217, -0.009050379, -0.037011802, 0.024413152, 0.019394603, 0.004022028, 0.06017032, -0.021433389, 0.043467958, 0.018453624, -0.00009761014, -0.014493676, 0.03267285, 0.018937184, 0.018963322, 0.0011705048, -0.030581785, 0.0012186973, -0.02080607, 0.019721331, -0.0033358978, 0.013435075, 0.021747047, 0.008717116, -0.0030124367, 0.057190552, 0.0014637436, -0.0112263905, 0.01715978, -0.015774451, 0.013173693, 0.0088478075, 0.00023606133, -0.016833052, 0.0037018338, -0.03842327, 0.0297715, 0.004678752, -0.01504258, 0.022870993, 0.00047947405, 0.00565567, -0.032123946, -0.0045578624, -0.010442242, 0.0061196247, 0.007959106, 0.0054106237, 0.019930437, 0.0479376, 0.014206154, -0.022047639, -0.03753457, -0.021015177, 0.030712478, -0.0012693402, -0.01987816, 0.007684654, 0.0034273819, -0.017695615, 0.013310919, 0.005208052, -0.045924954, -0.013853287, -0.001886857, 0.04265767, 0.025876896, 0.006122892, -0.011866778, 0.01488575, -0.0015290893, -0.04022681, 0.014062393, 0.03322175, 0.036541313, -0.023916526, -0.029327149, 0.022191398, -0.03902445, 0.018100757, -0.0052701305, -0.039677907, -0.06372512, -0.0072795106, -0.048800167, -0.010082841, 0.016022764, 0.0050316188, 0.011762225, 0.018113827, 0.00926602, 0.034084316, -0.02305396, -0.0050316188, 0.010965008, 0.024047216, 0.023615934, -0.010305016, 0.04145531, -0.008638701, 0.00018337637, 0.010461846, 0.011376685, -0.016140386, 0.01594435, -0.0068547637, -0.04116779, 0.009194139, -0.013683389, 0.024060285, 0.0058419053, -0.00070736714, 0.031784147, -0.004071037, 0.008939291, 0.0021449726, 0.041873522, 0.028412309, -0.008717116, 0.0077696033, -0.010200463, -0.023511382, 0.028046373, 0.0006722438, 0.010690556, 0.058079254, -0.0054596327, 0.016336424, -0.0018884906, -0.023459105, -0.01989123, 0.015317031, -0.038658515, -0.020126475, 0.005887647, -0.015251686, 0.011513912, -0.022910202, 0.024948986, -0.012977655, 0.0074298056, 0.038789205, -0.0012358505, -0.0005852524, 0.016780775, -0.0132129, 0.009788785, 0.000096793316, 0.012180438, -0.016323354, 0.008769392, -0.021002106, -0.0015642126, 0.026896289, 0.018244518, -0.014689713, -0.02548482, -0.010108979, -0.023733556, -0.010801644, -0.01382715, -0.028124787, 0.018610455, 0.0053714165, -0.028360032, 0.008730185, 0.0055576516, 0.009305228, 0.032123946, -0.028438447, -0.006828625, 0.030294266, 0.0009360771, 0.01714671, -0.027419055, 0.0023916524, 0.03340472, -0.020126475, -0.0104683805, -0.009520868, -0.061895445, 0.011585792, -0.011219856, -0.015852867, 0.015552276, 0.01487268, -0.007841484, -0.02535413, -0.05739966, -0.010102444, 0.04461804, 0.019852022, 0.019329257, -0.009520868, -0.030634062, 0.0014457735, 0.004613406, -0.0034731238, 0.01441526, 0.0006849046, 0.026242832, -0.026974704, -0.032881953, -0.036410622, 0.004335687, -0.011324409, -0.0025958577, -0.009527403, 0.05206745, -0.04859106, 0.005400822, 0.046892073, 0.0262559, -0.0318887, 0.01684612, 0.043311127, -0.03175801, -0.017551854, 0.008854342, 0.021747047, 0.017303541, 0.017538784, 0.055047214, -0.00926602, 0.017839376, 0.0013812446, 0.013297849, 0.017551854, 0.009625421, -0.021943085, 0.011742622, -0.018388279, 0.022099914, 0.029118042, -0.019224703, 0.011455101, -0.035286676, -0.032986507, 0.035417367, 0.049897972, 0.016088111, 0.0034437182, -0.007939503, -0.012383009, -0.026190555, -0.050943505, 0.023746626, -0.0406973, 0.0007825147, 0.0039730184, 0.021302696, 0.019512225, -0.021851601, -0.044722594, 0.010880059, -0.052041315, 0.011063026, 0.002115567, 0.04101096, -0.026504215, -0.0094555225, 0.036096964, -0.014349915, 0.069893755, 0.007606239, -0.008024451, -0.013376264, 0.014284569, -0.006959317, -0.03949494, -0.058236085, 0.037560705, 0.039129004, -0.0017643338, 0.02217833, 0.023811972, 0.003102287, -0.006518233, -0.009448987, -0.017525716, -0.03978246, 0.0035188657, 0.0002311604, 0.06267959, 0.011121837, -0.036384486, -0.024504635, 0.010625211, -0.014140809, -0.0012824093, -0.014532883, 0.028203202, 0.008429595, -0.009945615, 0.02443929, 0.0035646078, 0.0058680438, -0.023406828, 0.030503372, 0.0012366673, -0.0046166736, 0.019773608, -0.0031055543, -0.042239457, -0.0021417052, 0.0068416945, 0.028673692, 0.04521922, -0.023197722, -0.017486509, -0.025001263, -0.0058549745, 0.042840637, -0.016297217, -0.04610792, 0.0023720488, 0.0273145, 0.014781197, 0.01641484, -0.0041396497, -0.021080522, 0.009729974, -0.019107081, 0.046682965, -0.010566399, 0.015081787, -0.0052831997, -0.007018128, -0.012873102, -0.012775084, -0.010612141, -0.031548902, -0.021381112, 0.01807462, 0.010533726, 0.031627316, 0.003239513, -0.005554384, 0.020283304, 0.0073840637, -0.00076576986, 0.0024586318, 0.0029209526, 0.023171583, 0.017813237, 0.027967958, -0.009703836, 0.00398282, 0.010599072, -0.014898819, -0.01487268, -0.0037443084, 0.0025484823, -0.001069219, 0.018022344, 0.02565472, 0.021969223, 0.0053158724, 0.015748313, -0.006345067, -0.04009612, -0.021250421, 0.04328499, 0.046343166, 0.02869983, 0.00019777284, 0.005492306, -0.0043324195, 0.006730607, -0.015918212, 0.009122259, -0.031496625, -0.038240302, 0.016650084, -0.0012170636, 0.025602443, -0.022910202, -0.020518549, 0.005825569, 0.014938027, 0.009207209, 0.013918634, -0.0031643654, -0.011422427, 0.011383221, 0.00486172, -0.015369308, -0.030555647, -0.0074036676, 0.023171583, 0.034842327, 0.005208052, -0.015317031, -0.0009769181, -0.034241144, -0.011709949, -0.012644392, 0.016493253, -0.025942242, 0.009547006, -0.01701602, 0.00014753833, 0.008070193, -0.0029209526, 0.021341905, -0.014911888, -0.004110244, 0.000940978, -0.002011014, 0.0034241145, 0.012794687, -0.02596838, 0.0001425353, 0.014480607, 0.053243674, 0.0243086, 0.030634062, -0.009076517, 0.0004770236, -0.020309443, 0.019303119, -0.002390019, 0.0069266437, 0.032594435, -0.016506324, 0.0057504214, 0.0005652403, -0.0053126053, 0.018309863, 0.04119393, -0.031182967, 0.026295109, 0.003384907, -0.033012647, 0.0022397237, -0.0003771672, 0.0138140805, 0.015134064, 0.030921584, -0.043494094, 0.030059021, 0.024831364, 0.0048780562, 0.008527613, 0.019525293, -0.00090258743, 0.008227023, -0.017081365, 0.014010117, -0.009187605, -0.0069266437, -0.019956576, 0.0007616858, -0.007567032, -0.010572934, -0.009697301, -0.016048903, 0.013317453, -0.023615934, 0.035155986, 0.04116779, 0.0036070824, -0.005266863, 0.018192243, 0.03782209, -0.0033979763, -0.008429595, 0.04114165, -0.0056524025, -0.004610139, 0.042448565, 0.0076454463, -0.014049325, -0.01684612, 0.02475295, 0.0057079466, -0.04767622, 0.010017496, 0.013271711, -0.007998313, -0.03356155, -0.051387858, -0.00364629, -0.0026889753, -0.0013820615, 0.016937604, -0.02126349, 0.054263067, -0.04132462, 0.028464586, -0.023276137, -0.006482293, 0.039677907, 0.023027824, 0.0153301, -0.034084316, -0.0036299534, -0.038789205, 0.00697892, -0.029091904, -0.023249999, 0.014768127, 0.012010539, -0.037116356, 0.00516231, 0.030294266, -0.008913153, 0.01236994, 0.010455311, -0.007795742, -0.029353287, 0.021080522, 0.0021482399, -0.006459422, -0.021028245, 0.043833893, -0.0010071405, 0.008828203, 0.016532462, -0.0021433388, 0.0028033303, -0.057922427, 0.00271348, -0.033378582, -0.024426222, 0.049270656, 0.024687605, -0.002324673, -0.0021678435, -0.0014506745, 0.0034927274, 0.018989459, 0.053086843, 0.03141821, 0.0118863825, -0.00029548508, 0.013618044, 0.01640177, 0.0005403272, 0.0063483343, -0.0034829257, 0.0064626895, 0.0034633218, 0.049715005, -0.0021008642, -0.023419898, 0.0026121943, 0.039887015, 0.004338954, 0.007795742, -0.022152191, 0.008200885, -0.030059021, 0.03309106, -0.0005047955, -0.0015846331, -0.0006211925, 0.0032836213, -0.012624789, -0.0036560916, -0.0047898395, 0.011938659, -0.011383221, -0.015238617, -0.019629847, -0.038501684, 0.015016441, 0.015892074, -0.020231027, 0.004672217, -0.02885666, 0.04900927, 0.009716905, 0.0115335155, 0.023589797, -0.000035659352, 0.018518971, -0.040435918, 0.0077630687, 0.0075408933, 0.006518233, -0.014402191, 0.016950674, 0.042317875, -0.025249578, 0.029091904, -0.009827993, -0.025314923, -0.018414417, 0.02217833, -0.016271079, 0.0026791736, -0.035129845, 0.03037268, -0.01957757, 0.034580942, -0.012539839, 0.016584737, 0.010932335, 0.010494519, -0.023511382, 0.0048584524, 0.028543001, -0.01411467, 0.0027216482, -0.024086423, -0.012735876, -0.0022413575, 0.015212478, 0.0054563656, 0.003613617, 0.025615513, 0.009867201, -0.0045349915, -0.011337479, 0.020296372, -0.013591905, 0.025458684, 0.045663573, -0.0040350966, -0.04268381, 0.011350548, -0.032986507, -0.031052275, 0.009827993, -0.017656406, 0.021302696, -0.022674957, 0.01624494, 0.037247047, 0.007939503, -0.007730396, 0.009520868, 0.006893971, -0.0012276822, 0.030477233, 0.0003344883, 0.002997734, 0.006162099, 0.011559654, 0.004165788, -0.010148186, -0.018623523, 0.007338322, 0.0066554593, -0.004930333, -0.024426222, -0.009984822, 0.0022511592, -0.012428751, -0.006116357, -0.020701516, 0.024635328, -0.04085413, -0.0048029087, -0.019303119, -0.024687605, 0.0115335155, 0.014898819, -0.009448987, -0.03309106, 0.010984612, -0.0176172, 0.005701412, 0.012402614, 0.00444024, -0.015081787, 0.009958684, 0.012860033, -0.023981871, -0.03526054, -0.011579257, 0.011520446, 0.017395025, -0.008919688, 0.0028343697, -0.029091904, -0.0044239038, 0.01959064, 0.023903456, 0.0049597383, -0.019185496, 0.0074951514, -0.0088086, 0.02383811, 0.0040677697, -0.016127318, -0.01382715, 0.017970067, -0.018884907, -0.0017757693, -0.009762647, 0.026033726, 0.032516018, 0.013291314, -0.034816187, -0.0015829995, -0.0046036043, 0.0007796558, 0.022112984, -0.029536255, -0.013395868, -0.0021939818, 0.019629847, 0.010716694, 0.0026677381, 0.0032819875, 0.003979553, -0.02883052, -0.034737773, -0.006309127, -0.017120572, 0.01670236, -0.0074298056, 0.019054806, -0.026046794, -0.007913364, -0.018963322, 0.037639122, -0.01853204, 0.021211213, 0.03536509, -0.0045023183, 0.0061424957, 0.01821838, -0.008057124, 0.0063417996, 0.04294519, 0.022126053, -0.023498312, 0.030869307, -0.009168001, 0.026556492, -0.007919898, 0.011892917, -0.025236508, 0.0011582525, 0.010344224, -0.019956576, -0.017525716, -0.052903876, -0.014990303, 0.010376897, 0.012435286, -0.0019799746, -0.002046954, -0.023106238, -0.0015070351, 0.034397975, -0.01078204, 0.0030042685, 0.03008516, 0.034685496, 0.0024096225, 0.0137618035, 0.006207841, 0.051152613, 0.026504215, -0.01853204, -0.017342748, -0.009390176, -0.00805059, -0.008253162, 0.03536509, -0.008971964, -0.009893338, 0.0016017864, 0.057086002, -0.009658094, 0.003959949, 0.010089376, -0.011624999, 0.015656829, 0.028752107, -0.0030598124, -0.024021078, 0.0059791314, -0.014284569, 0.028569138, -0.0419258, -0.011794899, -0.012213111, 0.006015071, 0.011023819, -0.0220215, 0.036384486, -0.036227655, -0.0055674533, -0.036227655, -0.016140386, -0.023432966, 0.001388596, -0.0071553537, 0.0024422954, 0.0005007114, -0.0032966903, -0.0241387, -0.000017880726, -0.0011084264, 0.015918212, 0.006282989, 0.022962477, 0.034946878, 0.016833052, -0.0058909147, -0.0012742411, -0.0031153562, -0.028673692, -0.0046199406, -0.010076307, 0.05102192, 0.013787942, 0.018819561, 0.01578752, -0.03586172, 0.02279258, 0.0013902297, -0.0014735454, 0.0002234006, -0.003626686, 0.0043487563, 0.023354553, 0.02170784, -0.007710792, 0.0072795106, -0.0041167787, 0.0075735664, -0.04132462, -0.0016556965, 0.024792157, 0.027209949, -0.0047767702, -0.020021921, 0.0013036466, -0.0102135325, 0.00697892, -0.0025697195, 0.022570403, -0.018649662, -0.015303962, 0.01123946, 0.0042964797, -0.00083070714, 0.009893338, -0.009122259, 0.034345698, -0.017813237, 0.02566779, 0.002718381, -0.019094013, -0.016179595, -0.0074298056, -0.02473988, 0.0096058175, 0.009847596, -0.03248988, 0.025432546, 0.009396711, -0.042161044, 0.016061973, 0.002380217, -0.012383009, 0.01382715, -0.018153034, -0.016114248, 0.011579257, 0.006224178, 0.0473887, -0.024635328, 0.015591484, 0.052616354, -0.004979342, -0.003509064, -0.013709527, -0.0060869516, 0.003486193, -0.04540219, -0.0034731238, -0.003531935, -0.0352344, -0.017120572, -0.047702357, -0.011781829, 0.0008617464, 0.019603709, -0.023289206, 0.007998313, 0.022335159, 0.010435708, -0.021851601, 0.0035286676, -0.0033685707, -0.021158937, -0.021158937, -0.012592115, -0.028464586, 0.0031300588, 0.026817873, -0.020779932, -0.018584317, -0.005698145, 0.016427908, -0.0046591484, -0.015264755, 0.011102234, -0.011794899, -0.042788364, 0.016114248, 0.0004937684, 0.018741146, -0.020766862, 0.032280773, -0.012539839, 0.0017479974, -0.0040775714, -0.003613617, -0.018701939, 0.0013910466, -0.009305228, 0.024177907, -0.003750843, 0.0077369306, -0.017774029, -0.02794182, -0.0026121943, 0.020884484, -0.009965219, -0.0029225862, -0.022152191, -0.0022691293, -0.013709527, -0.024347806, 0.03052951, -0.021250421, -0.0020404193, -0.008377318, 0.015774451, 0.006789418, 0.017970067, 0.01593128, 0.004655881, -0.0005223572, 0.030973861, 0.002233189, 0.012102023, -0.021289628, -0.022714164, 0.011108768, -0.012860033, 0.0035842115, -0.013552697, 0.0012260487, 0.011899452, -0.004884591, -0.0032444138, -0.007939503, 0.010684022, -0.018492833, -0.016911466, -0.020322511, -0.02443929, -0.03991315, -0.000039743456, -0.001101075, -0.009429384, 0.032594435, 0.017774029, 0.004695088, -0.019538363, -0.008115935, -0.001064318, 0.015578414, 0.029536255, -0.030032882, 0.03186256, -0.0126966685, -0.0004431255, 0.0153301, -0.011330944, 0.008383852, -0.0036528246, -0.0045741987, -0.0150295105, 0.0125986505, 0.014323777, 0.016885327, -0.0132129, 0.015317031, -0.014807335, -0.0262559, 0.015879005, 0.037299324, 0.0028294686, -0.0031349598, 0.0046068714, -0.033195615, 0.011749156, 0.01236994, 0.008952361, -0.028203202, 0.011899452, -0.012820826, 0.0013926801, -0.005861509, 0.015317031, -0.013905564, 0.016075041, -0.0019309653, -0.028647553, -0.026791736, 0.022570403, 0.010108979, 0.020335581, 0.015774451, -0.012507167, 0.016192663, -0.000824581, -0.007586635, 0.020832209, -0.03384907, -0.035731025, 0.010082841, -0.039573353, 0.014362984, 0.017029088, 0.015356239, -0.031784147, -0.03826644, -0.0015241883, -0.03207167, 0.010540261, -0.026608767, -0.0019587371, 0.0070704045, -0.005469435, -0.0054171584, 0.026674114, 0.020793, 0.020675378, -0.019708263, -0.0046820194, 0.019642916, 0.002522344, 0.0028588742, 0.020217959, -0.019172428, -0.016493253, 0.032751262, 0.01989123, -0.017342748, -0.014702782, 0.019995783, 0.009599282, 0.012657462, 0.00035307097, 0.0023001686, 0.008148609, -0.010507588, -0.0015437921, 0.030294266, -0.008932756, 0.016323354, 0.0038684653, 0.030973861, 0.013232504, 0.0047277613, -0.000090360845, -0.02794182, -0.0014523081, 0.022505058, -0.038344856, -0.0000059793865, -0.016349493, 0.024099493, -0.034659356, 0.0039403457, -0.020714587, -0.011951728, -0.01032462, -0.023785833, 0.00016785676, -0.0012350336, 0.0065247677, 0.013774873, 0.012075884, -0.023746626, -0.009886804, 0.015107925, -0.012206576, 0.029954467, 0.003257483, -0.013997048, 0.003888069, -0.052747045, -0.013774873, -0.015591484, -0.0041363826, 0.016833052, 0.0040808385, 0.016963743, 0.012690134, -0.012735876, 0.012232714, -0.01123946, -0.0021874472, -0.012618254, -0.008606028, -0.011821036, -0.024112562, 0.015826728, -0.023877317, 0.0406973, 0.010409569, 0.013644181, 0.009664629, -0.00011997062, 0.003348967, 0.003590746, 0.0037998522, -0.0030679805, -0.014820404, 0.008227023, -0.01640177, -0.005966062, 0.020440133, -0.015682967, 0.023289206, -0.03141821, 0.00972344, -0.004316083, 0.0038717326, 0.020701516, 0.010710159, 0.019708263, 0.03144435, 0.031182967, -0.010474916, -0.0066521917, -0.014637436, -0.0075408933, -0.013787942, -0.0012235981, -0.028516863, 0.022674957, 0.01472892, 0.018754214, 0.024426222, 0.0030500104, -0.003522133, -0.00531914, -0.030241989, -0.011134907, -0.00093526026, -0.014650505, -0.004652614, 0.013918634, 0.011245995, 0.020596964, 0.02490978, 0.029222596, -0.0017528983, -0.0050381534, -0.0013044635, 0.011605396, 0.009514334, 0.012284991, 0.007697723, -0.011500843, -0.0014065661, 0.010056702, -0.008102867, 0.0016393602, -0.008396922, -0.0071096118, -0.016153457, 0.009971754, -0.015696036, 0.049715005, -0.0038684653, 0.0061686337, -0.013193296, -0.017682545, 0.009233347, 0.00265957, -0.018649662, 0.012958052, -0.02548482, -0.007018128, -0.017852444, -0.008161677, 0.020257166, 0.016022764, -0.021903878, 0.011840641, -0.018336002, -0.011821036, 0.011807967, 0.0041167787, -0.036018547, -0.017342748, -0.023419898, 0.0007702624, 0.007789207, -0.006893971, 0.012389544, -0.009592748, -0.007181492, 0.0071357503, -0.0060673477, 0.0009115725, -0.0056458684, -0.011017284, -0.004711425, -0.008501475, -0.0014065661, -0.00037308308, 0.020610033, -0.0086909775, 0.0047244937, -0.010527192, -0.027811129, -0.012186972, -0.0037541103, -0.02050548, 0.0025419476, -0.005322407, -0.004450042, 0.034267284, 0.009736509, 0.0013877793, 0.025772342, 0.013062605, -0.0032215428, 0.020714587, -0.007410202, 0.0073579256, 0.013801011, 0.004280143, 0.0014294371, 0.008860877, 0.00720763, -0.009926011, 0.0031300588, -0.015578414, -0.004737563, 0.013252107, 0.022988616, 0.0028670426, 0.024583051, 0.040409777, -0.002197249, -0.028020235, -0.014663574, -0.0069658514, 0.014101601, 0.007057335, 0.011317874, 0.0016434443, 0.027680438, -0.01518634, -0.002087795, 0.007560497, 0.023759695, -0.03157504, 0.015003372, 0.030895445, -0.014023187, -0.018884907, 0.0057504214, 0.024321668, 0.01031155, 0.013219435, 0.03431956, -0.0023949197, 0.018309863, -0.023328414, -0.0127489455, -0.00872365, 0.0023410097, 0.006227445, -0.03978246, -0.008782461, -0.017564923, 0.015134064, 0.016493253, -0.007978709, 0.02187774, -0.037586845, 0.009945615, 0.011219856, -0.0022233874, 0.014898819, -0.0073448564, 0.009429384, 0.062313657, 0.029065765, -0.013944772, 0.017826306, -0.011461635, 0.0075735664, 0.0114812385, -0.0022952675, -0.025942242, -0.002621996, 0.031235242, -0.019094013, 0.00035102892, -0.0028637752, 0.0044696457, 0.0005223572, 0.007874156, -0.0176172, -0.010069772, -0.011415893, 0.011579257, 0.017826306, -0.0053518126, -0.014532883, 0.021916946, 0.0015707472, -0.0110760955, -0.0220215, -0.008684443, -0.026229763, 0.0021939818, 0.007272976, 0.0013379531, 0.0059693293, 0.017669477, -0.007834949, 0.031679593, -0.0012709738, 0.007351391, -0.0076454463, 0.018244518, 0.004129848, 0.009533937, 0.0039501474, -0.007527824, 0.008939291, -0.010174325, 0.010808178, -0.013219435, 0.010246205, 0.0007812895, 0.011572723, -0.0031856026, -0.022962477, -0.012768549, -0.01564376, -0.0059693293, 0.005518444, 0.012193507, -0.015957419, -0.0006113907, -0.021694772, -0.013918634, 0.015447723, 0.011500843, 0.0039305435, -0.0032885221, 0.017303541, 0.000052582865, -0.012918844, 0.006227445, -0.0005382852, 0.013578836, -0.023759695, -0.017839376, 0.023197722, -0.009527403, -0.011004215, -0.014389123, 0.002489671, 0.021054383, -0.044461213, -0.0017349282, -0.0060183387, -0.008481871, 0.0061653666, -0.0036005478, -0.013016863, -0.016075041, -0.0029111507, -0.01671543, -0.0042703412, -0.0027918948, -0.011794899, 0.008272765, 0.0011500843, -0.009899873, -0.0022821985, 0.0053714165, -0.011337479, -0.016937604, -0.008821669, 0.03526054, -0.025576307, 0.013997048, -0.0023230396, -0.0013265177, 0.00067551114, 0.013343591, 0.009547006, 0.008802066, 0.016127318, 0.007076939, -0.0071161464, -0.008645236, 0.024556912, -0.012559443, 0.0071422844, 0.0031905037, 0.048303537, -0.0097561125, -0.020832209, 0.034058176, 0.019695193, -0.00489766, 0.004234401, -0.0080832625, -0.003750843, -0.014794266, -0.0019097279, -0.0075997044, 0.01252677, 0.017695615, -0.0004729395, -0.021786256, -0.0051459735, -0.01791779, -0.0009826359, -0.003247681, 0.016584737, 0.0023312077, 0.0070311967, -0.014467537, 0.015526137, 0.018492833, 0.019172428, -0.014598229, -0.014376054, -0.018466694, 0.010442242, -0.002690609, 0.0065541733, -0.0029585264, -0.0032182755, 0.009952149, -0.016336424, -0.0000847452, 0.005492306, -0.0039860876, 0.0048486507, -0.0054334947, 0.034554806, 0.006541104, 0.0047310283, -0.0029111507, -0.017421164, -0.0044990513, -0.0072403033, -0.015055648, 0.0036724282, -0.0029568928, -0.0056687393, -0.0094555225, -0.007076939, 0.00918107, 0.0072599067, -0.023328414, 0.0016164891, -0.02656956, -0.0066587264, -0.0055903243, 0.014741989, -0.008187816, -0.008030986, -0.018715007, -0.018989459, 0.018584317, 0.013435075, 0.0012799589, -0.004384696, -0.005995468, -0.00007565806, 0.0041363826, 0.022374367, 0.024190977, 0.020610033, 0.014820404, -0.003763912, 0.012690134, 0.012977655, -0.0122915255, -0.013323988, 0.02490978, 0.0145198135, -0.0022854658, -0.0064267493, 0.0126966685, -0.01534317, -0.029640809, -0.014349915, -0.0045970697, 0.0020502212, -0.020113405, -0.0035155984, -0.018087689, 0.006730607, -0.01714671, -0.0042180647, 0.00017571867, -0.0074755475, 0.007841484, 0.014180016, -0.008658305, -0.02170784, 0.022701096, -0.0012987457, -0.0052015176, 0.009867201, 0.0036495572, 0.0014384221, -0.00084622676, 0.010435708, 0.0028980817, -0.003362036, -0.030189712, -0.032881953, 0.023955733, 0.007253372, 0.02488364, -0.0035613405, 0.007848018, -0.0002004275, 0.0002891547, 0.0150295105, 0.012938448, 0.03113069, -0.0042017284, 0.015839797, 0.012892706, 0.017395025, 0.029693084, -0.011147976, 0.01943381, 0.009416315, 0.000097303826, 0.010305016, 0.013591905, -0.030921584, 0.010265809, -0.033195615, -0.0060510114, -0.012180438, -0.0044761803, -0.026412731, 0.009867201, -0.009285623, -0.0050806277, 0.003705101, 0.023067031, -0.0054302276, 0.009409781, -0.01823145, -0.021773186, 0.008514544, -0.013337057, -0.01152698, 0.024818296, 0.017342748, 0.0273145, -0.011977866, -0.005966062, 0.011860244, -0.021119729, -0.021668633, -0.00039922138, 0.0001421269, 0.01776096, -0.009547006, 0.01929005, 0.0009246416, -0.005573988, -0.001927698, -0.019459948, 0.017395025, 0.008991567, 0.008285834, -0.0007906829, -0.013644181, 0.013265177, 0.007730396, -0.0016230238, -0.04009612, -0.005737352, 0.014402191, -0.012539839, 0.004695088, 0.0035646078, 0.005217854, -0.026661044, 0.021446457, -0.0000255129, -0.00081804645, 0.005698145, -0.020152614, -0.017042158, 0.003362036, 0.0024047217, 0.006815556, -0.014650505, 0.014088532, 0.00421153, 0.0015054015, 0.004044899, 0.016885327, -0.00146211, 0.013487352, 0.002095963, -0.02200843, 0.025733136, 0.012330733, 0.012565978, 0.0085929595, -0.003183969, -0.0005758589, -0.00045578624, -0.0068612983, 0.00027608557, -0.005136172, 0.0015633958, 0.012559443, -0.004760434, -0.019512225, -0.009736509, -0.014689713, -0.003937078, 0.006224178, -0.0073840637, 0.014598229, -0.032280773, 0.007848018, -0.00466895, -0.009919477, -0.011860244, 0.0002234006, -0.01487268, -0.006031408, -0.012840429, -0.011932124, -0.0018525505, 0.011115303, -0.0046166736, -0.010037099, 0.017355816, -0.015068717, 0.025406407, -0.019825885, 0.008566821, 0.016140386, 0.0013510223, -0.0068547637, -0.008403457, -0.02338069, -0.014441399, 0.007919898, -0.022805648, -0.0138140805, 0.008717116, 0.004554595, -0.0057406193, 0.004002424, 0.0055347807, -0.004172323, -0.027602023, -0.012905775, 0.012186972, 0.010749367, 0.0016663152, 0.0015731977, 0.0024766019, -0.018192243, 0.013644181, -0.03445025, 0.0043716272, 0.007913364, 0.014062393, -0.005864776, 0.021812394, -0.00057871785, 0.014951095, 0.018179173, -0.0054596327, 0.011938659, -0.025981449, -0.007331787, 0.009396711, -0.009612352, -0.0020600231, -0.015408515, 0.027105395, 0.011520446, 0.008658305, 0.009710371, 0.008200885, 0.04325885, -0.010592537, -0.006080417, 0.021067452, -0.03068634, 0.0103965, 0.02822934, -0.013631112, 0.020413995, 0.0077238614, -0.009468592, 0.0020763595, 0.00531914, 0.007397133, -0.03113069, -0.016271079, -0.0047767702, 0.009331365, 0.015617621, -0.01624494, -0.004819245, -0.013435075, 0.001009591, 0.003878267, 0.015983557, 0.028752107, -0.017956998, 0.021616356, -0.017512647, -0.021851601, 0.009462057, -0.0059007164, 0.013983979, 0.007501686, 0.020100337, 0.003603815, 0.028307756, -0.007815345, 0.012873102, 0.028752107, 0.003531935, -0.0063581364, -0.006972386, 0.013108347, -0.016519392, -0.015774451, -0.011585792, 0.019028667, -0.01987816, -0.0020387857, -0.0033914417, 0.0016434443, -0.010298482, 0.0036626263, -0.0038488614, 0.0016532461, 0.012128161, -0.0138140805, 0.008285834, -0.006008537, -0.021694772, -0.011742622, 0.014689713, 0.0007224784, -0.013043001, -0.0048682545, -0.0068416945, -0.017395025, -0.003404511, 0.0069331783, -0.012409148, -0.0009246416, 0.013552697, 0.008161677, 0.008527613, 0.0002001212, 0.018244518, 0.005505375, 0.004280143, 0.022714164, -0.017564923, 0.008318507, 0.0022266547, -0.0029405563, 0.036985666, 0.012363406, 0.0006085318, -0.028072512, -0.014258431, 0.005760223, 0.03207167, -0.006224178, -0.004378162, 0.002064924, 0.00903731, -0.0043912306, -0.008762858, -0.013983979, -0.02217833, -0.011043423, -0.0029993674, 0.003018971, 0.012637857, 0.018270656, 0.0376914, 0.017695615, 0.018022344, -0.02157715, -0.02974536, 0.0050838953, 0.010474916, 0.016349493, -0.021211213, -0.014402191, 0.001932599, -0.0025305122, 0.009468592, 0.026465008, 0.0038586634, 0.009272554, 0.011500843, -0.0052243886, -0.010108979, 0.014663574, -0.0019848756, 0.02778499, 0.00039186998, -0.017538784, 0.030320404, 0.019473018, -0.02352445, 0.0036724282, 0.013016863, 0.0072337687, 0.018754214, -0.0089458255, -0.0029127845, 0.005237458, 0.005093697, 0.01336973, 0.016480185, -0.010801644, 0.012559443, 0.015656829, -0.0057275505, 0.019028667, -0.022975547, -0.018100757, -0.019773608, 0.0029928328, -0.012428751, 0.020727655, -0.02275337, 0.017290471, 0.0031267917, 0.027863406, -0.00993908, 0.00034122705, -0.008403457, -0.0018100758, -0.0052113193, -0.008246627, -0.00082907354, -0.0078088106, 0.020766862, -0.018688869, -0.008331576, -0.022844855, 0.02961467, 0.0072860452, 0.0039534145, -0.021903878, -0.028412309, 0.009716905, -0.004338954, -0.009390176, 0.0102135325, -0.01563069, 0.000013694518, -0.00706387, -0.017878583, 0.004518655, -0.01684612, 0.0062339795, 0.02734064, -0.00061915047, -0.017029088, -0.030424956, 0.010592537, -0.019054806, 0.0038161888, -0.008632166, -0.020244097, 0.013918634, 0.013539628, 0.0092006745, -0.0015650294, 0.005636066, 0.0007049167, -0.0020779932, -0.020766862, 0.006939713, -0.00074371573, -0.020884484, 0.00041351575, 0.008978499, 0.022805648, 0.0049499366, 0.017538784, 0.012794687, -0.014336846, 0.03382293, -0.00410371, -0.0027902613, -0.017120572, -0.026922427, 0.008586424, -0.0011966431, 0.026334316, 0.002011014, -0.009814924, 0.0039272765, -0.009390176, -0.019995783, 0.0028441714, -0.009377107, 0.011317874, 0.01048145, -0.0040547005, 0.023563659, 0.021930017, 0.0064169476, 0.0056033935, 0.025092747, 0.020518549, -0.024413152, -0.010331155, 0.03199325, 0.013709527, -0.008181281, 0.01442833, 0.0008707314, -0.0007543344, -0.011925589, 0.018296795, 0.0017610665, 0.00796564, -0.009272554, -0.0041396497, 0.0050414205, -0.028046373, 0.01640177, -0.035338953, -0.012820826, -0.026765598, 0.011004215, 0.013931703, 0.01504258, 0.0038488614, 0.02049241, -0.026242832, -0.0015388911, 0.019773608, -0.010069772, 0.0010496152, -0.004760434, -0.01914629, 0.0029928328, 0.007684654, -0.008285834, -0.0017888384, -0.00972344, 0.01412774, -0.02157715, -0.0029764965, -0.01261172, 0.011592327, 0.013526559, 0.013892495, 0.012252318, 0.005573988, -0.024021078, 0.0074494095, 0.034816187, -0.00903731, -0.0012383009, -0.008481871, -0.016950674, 0.0062960577, -0.006345067, 0.004649346, -0.004299747, 0.005792896, -0.029065765, -0.020374788, 0.035809442, 0.008455733, 0.008749789, -0.012160834, 0.011461635, 0.0027347174, -0.0005235824, 0.0077499994, -0.018963322, -0.005933389, -0.020596964, 0.010579469, 0.003545004, 0.031548902, -0.006959317, 0.0040547005, -0.023119308, 0.01791779, 0.007090008, -0.0012432019, 0.003247681, -0.013840219, 0.01927698, -0.0041167787, -0.00588438, -0.004946669, 0.00812247, 0.021812394, -0.008279299, 0.021629425, -0.006809022, -0.0019015598, -0.01684612, 0.0039893547, -0.0012235981, 0.019760538, 0.0006538654, -0.00000608468, -0.010474916, 0.008599494, -0.035417367, -0.007188027, -0.00041821247, 0.03330017, 0.008194351, -0.03139207, 0.005155775, -0.029065765, -0.01101075, 0.027131533, -0.006008537, -0.004652614, 0.0054563656, 0.019616779, 0.008971964, -0.00683516, 0.001351839, 0.01085392, -0.000532159, 0.0028425378, -0.019760538, 0.010644814, 0.013023397, 0.010507588, -0.0049760747, 0.038031194, -0.002535413, 0.017133642, 0.0085406825, -0.0063385325, 0.008357715, 0.009645025, -0.0048911255, -0.013029932, 0.02383811, -0.0011664206, -0.0016017864, 0.0013257008, -0.003061446, 0.015918212, -0.013252107, 0.0052733975, -0.0064430856, 0.019094013, -0.01351349, 0.02200843, -0.0044043, -0.008416526, -0.0017202254, 0.016258009, -0.010801644, -0.0034241145, -0.012017074, -0.0048780562, 0.009520868, -0.0031774344, 0.013879426, -0.005230923, -0.0068678325, 0.014807335, -0.014180016, 0.015696036, 0.023929594, -0.0012252318, 0.022988616, 0.0052505266, -0.00019164667, -0.016506324, 0.012474493, -0.021629425, -0.047101177, -0.004022028, 0.0073579256, -0.029379426, 0.010540261, -0.011455101, 0.027079256, 0.003384907, 0.0066521917, 0.0045349915, -0.019224703, 0.016963743, 0.022831786, -0.0055576516, 0.015081787, 0.0069462475, -0.034528665, 0.01989123, 0.0019358663, -0.020975968, 0.013186761, -0.0006346701, -0.012383009, 0.0020338849, -0.024726812, -0.0021907145, 0.017303541, -0.005743887, 0.006410413, 0.0069462475, -0.012200042, -0.004554595, 0.0036332207, 0.0012374842, 0.011585792, -0.010801644, -0.021054383, 0.0058909147, -0.009906407, -0.008279299, 0.028150925, 0.025903035, -0.00015203084, -0.01654553, -0.0071553537, 0.009965219, 0.006939713, 0.0008617464, -0.004672217, 0.0061424957, 0.0142192235, -0.019512225, -0.014964164, -0.006116357, -0.011860244, -0.022988616, -0.017695615, 0.0075735664, 0.0021319033, 0.012265387, -0.013487352, 0.010697091, 0.0083381105, 0.037168633, 0.009246416, 0.006354869, 0.000010152832, -0.014715851, 0.00013559232, 0.028124787, -0.003914207, 0.014075463, -0.0039501474, -0.025314923, 0.021067452, -0.0033342643, -0.02186467, -0.0024161572, -0.0054204254, 0.00805059, 0.008220488, 0.03081703, -0.002563185, -0.0062143756, 0.0037116355, -0.015199409, -0.010167791, 0.011474704, -0.00796564, -0.0033685707, 0.00683516, 0.0264258, 0.0049041947, 0.005342011, 0.016676221, -0.013029932, -0.021694772, -0.008671374, -0.006387542, 0.0061915047, 0.021067452, -0.0028490725, -0.004185392, -0.011925589, 0.015669897, -0.0031807018, 0.009305228, 0.008854342, 0.029248733, -0.02399494, -0.023067031, -0.013297849, 0.0049499366, 0.0037541103, -0.0070508006, -0.02339376, -0.0004778404, -0.00027955705, 0.0032999576, 0.0102135325, -0.018545108, -0.02064924, 0.023811972, -0.01032462, -0.020335581, 0.013036466, 0.030137435, 0.03021585, 0.005345278, -0.008344646, 0.016741568, 0.016480185, 0.0077499994, 0.0017398292, 0.011827571, -0.012265387, 0.0188457, 0.008429595, -0.0019015598, 0.011892917, 0.013239038, 0.0104683805, 0.018871836, 0.0020583894, -0.012278456, -0.007272976, -0.0041951938, -0.02791568, -0.020675378, 0.04451349, -0.0075735664, 0.013350125, 0.0033342643, -0.0073252525, 0.0034600548, -0.01367032, -0.008514544, -0.008553752, -0.0061424957, -0.011082631, 0.020178752, -0.010429173, -0.026282039, 0.0142715005, 0.015356239, -0.024726812, -0.016336424, -0.007377529, 0.006207841, 0.007919898, -0.010886593, 0.0016614143, -0.00942285, 0.0067044683, -0.021132799, 0.0060608136, 0.015120994, 0.028203202, -0.014924957, -0.0030091694, -0.009658094, 0.021930017, -0.0004239302, -0.027079256, -0.005554384, 0.021250421, -0.015970489, -0.014898819, 0.011847175, -0.0040775714, 0.020570826, -0.0068416945, -0.0041461843, 0.006015071, 0.021211213, 0.006469224, 0.01236994, 0.019838953, -0.007815345, -0.0010398134, 0.00006963401, -0.0072403033, 0.00021421137, 0.013258642, 0.02186467, -0.014846542, 0.0063810074, 0.0047473647, -0.019224703, -0.0008919688, -0.0056491354, 0.0071749575, 0.006599915, -0.014768127, 0.008416526, -0.00013600073, -0.01836214, 0.021812394, -0.0004075938, -0.016336424, 0.027131533, 0.012925379, -0.009886804, -0.011102234, -0.025942242, 0.00034796583, -0.000068000365, -0.006057546, 0.022413574, -0.0026791736, 0.01236994, 0.0006857214, 0.00012977247, 0.0073252525, -0.01623187, -0.016467115, 0.0017316609, 0.009631956, -0.019995783, 0.003394709, -0.0154346535, -0.015539207, 0.010089376, -0.0026203624, 0.0010929068, 0.007625843, 0.036985666, 0.0069462475, 0.006998524, 0.010265809, -0.005008748, 0.00095486396, 0.007351391, 0.010487984, 0.01504258, -0.021185074, 0.0005627898, 0.009187605, -0.009083052, 0.0041690553, -0.006711003, -0.030581785, 0.007377529, -0.011664207, 0.012128161, 0.0040775714, -0.011638069, -0.0043748943, 0.0062111085, 0.023981871, -0.004071037, -0.00011762225, 0.014898819, 0.01093887, 0.011389755, -0.0055119097, -0.0075408933, 0.0056491354, -0.02126349, -0.00611309, 0.0041788574, -0.012650927, -0.0038749997, 0.00294219, -0.0035352022, 0.01610118, 0.0060477443, 0.0037214374, -0.002179279, -0.009631956, -0.012128161, 0.007057335, -0.0133566605, 0.0046395445, 0.005243992, -0.0057667578, -0.0011460001, -0.020675378, -0.0077173267, 0.0054988405, 0.0010153088, -0.034737773, -0.008971964, -0.015826728, -0.0010332789, 0.011925589, -0.019303119, 0.0103965, -0.002407989, 0.01472892, 0.004093908, 0.0044075674, 0.010416104, -0.011677276, 0.0176172, 0.025066609, -0.016427908, 0.009187605, 0.0072403033, -0.01382715, -0.00835118, -0.007181492, -0.016663153, -0.0013861456, 0.012422217, -0.002489671, -0.008573355, -0.0014245362, 0.0051884484, -0.00094261166, -0.0032819875, 0.01185371, -0.006697934, 0.025131956, -0.01957757, 0.015003372, 0.0137618035, 0.009239881, -0.01594435, 0.0104683805, 0.0073840637, -0.012833895, 0.013853287, 0.01944688, -0.0058680438, 0.0092006745, 0.024543844, 0.010579469, 0.04631703, -0.0050838953, -0.015264755, -0.012762015, -0.009814924, -0.021943085, 0.005253794, 0.010788575, 0.0014833473, 0.014245362, -0.010971542, 0.007651981, -0.0029536255, 0.0066031828, -0.012200042, -0.0074298056, 0.014624367, -0.008854342, -0.006331998, 0.0060510114, 0.015016441, -0.0034600548, 0.016898397, 0.009860666, 0.0026350652, -0.0039958893, 0.01791779, 0.0114812385, 0.017669477, -0.012938448, 0.019120151, -0.00805059, -0.016480185, 0.0062470487, 0.020845277, 0.03873693, 0.0074951514, 0.017029088, 0.0230801, 0.01101075, -0.010657883, 0.011742622, 0.016937604, -0.0019064607, -0.014206154, 0.013997048, 0.008560286, 0.002759222, 0.007246838, 0.018022344, 0.009599282, 0.0023459105, 0.017839376, -0.011487774, -0.003509064, 0.016192663, -0.0022625946, -0.005952993, -0.002901349, 0.0011672375, 0.006697934, -0.0015797322, -0.017747892, -0.0025158094, -0.011579257, 0.0017316609, -0.021015177, -0.0073905983, 0.0043422217, -0.006243781, 0.025785413, -0.0029519917, -0.0029046163, -0.0031169897, -0.0019391335, 0.0002082894, -0.024465429, -0.02961467, -0.0027020446, 0.030189712, -0.007919898, -0.0016279246, -0.016623946, 0.004920531, -0.0009050379, 0.023877317, 0.0081355395, 0.011566188, 0.0051329043, -0.008070193, -0.02791568, -0.004485982, 0.02247892, 0.022439713, -0.0054400293, 0.009488195, -0.0070050587, 0.012886171, 0.0015478762, 0.0008748155, -0.0022217536, -0.0031186233, 0.020322511, 0.023315344, -0.010690556, 0.0056883427, -0.019956576, 0.0075147552, 0.0154869305, 0.0018982925, -0.01684612, -0.015120994, 0.00660645, -0.028647553, 0.0068416945, -0.0074886167, -0.0012775083, -0.0005934206, 0.001881956, 0.0038259905, 0.014493676, -0.008854342, -0.0074886167, 0.012722807, 0.0024471963, 0.026360454, -0.010167791, -0.008318507, 0.015813658, -0.00926602, 0.000111393994, 0.02200843, 0.019185496, -0.007874156, 0.002105765, 0.0020665578, 0.014376054, 0.00736446, 0.0009393444, -0.013748734, 0.020100337, 0.0005697328, 0.018937184, 0.0013142653, 0.011272132, 0.004110244, -0.003143128, -0.0012366673, 0.02004806, -0.006730607, -0.019237773, -0.014794266, -0.016127318, -0.004106977, 0.016493253, -0.013186761, -0.0022070508, -0.019198565, -0.026020657, 0.004316083, -0.0016238405, 0.00085684547, 0.014624367, -0.0033097595, -0.016976813, 0.0078088106, -0.009004637, -0.020152614, 0.0012619888, -0.009207209, -0.017434232, -0.012905775, -0.03719477, -0.0013346858, -0.00086419686, -0.0125986505, -0.009729974, 0.0037345067, 0.043102022, -0.027445193, -0.013709527, 0.036645867, 0.015774451, -0.001210529, -0.009743043, -0.0028360032, -0.002192348, -0.00040555172, -0.023602866, -0.029222596, -0.032908093, -0.01312795, -0.0040089586, 0.009814924, 0.030137435, -0.002965061, 0.015408515, -0.0023148714, -0.007887226, -0.014362984, -0.0048453836, -0.010651348, 0.004028562, 0.021315766, -0.008854342, 0.003161098, 0.005070826, -0.0014057494, 0.03565261, -0.013983979, -0.024190977, 0.014768127, 0.0010831049, -0.013539628, -0.0035646078, 0.016623946, -0.0038390597, 0.014467537, -0.013892495, 0.010461846, -0.012245784, 0.0020567558, -0.01913322, -0.009586214, 0.005861509, -0.0024635328, -0.0076781195, -0.00083519967, -0.007351391, 0.0004623208, 0.019002529, -0.015016441, -0.0052243886, 0.020100337, 0.024975125, 0.00752129, 0.022400504, 0.026504215, 0.01580059, 0.018858768, -0.0052276556, -0.00044394232, -0.012304595, -0.013500421, 0.00720763, -0.001960371, 0.023602866, -0.0027984295, -0.014376054, 0.012383009, 0.02460919, 0.014454468, 0.017813237, 0.007874156, -0.003796585, 0.031156829, 0.0019064607, -0.024569983, 0.013722597, -0.018153034, 0.01594435, -0.005400822, 0.012304595, 0.0009066715, 0.015264755, -0.018244518, -0.024151769, -0.019237773, 0.0061098225, -0.0029911993, 0.0015225548, -0.0017904721, 0.013134485, 0.001434338, -0.0050054803, 0.009703836, 0.00006279313, 0.0012897607, -0.015238617, 0.0021433388, 0.010461846, -0.016427908, -0.007913364, -0.013866357, -0.0030957523, 0.0154869305, 0.018571246, -0.013591905, 0.012735876, 0.028725969, 0.008377318, 0.024975125, 0.0006113907, -0.012337267, 0.033012647, 0.006031408, 0.0024096225, 0.013343591, -0.010951939, -0.008076728, 0.0032640176, -0.012585581, -0.018414417, 0.0014857978, 0.003404511, 0.0069658514, -0.019094013, 0.009292158, -0.004254005, -0.018610455, -0.009468592, 0.0058582416, -0.012317664, -0.008828203, 0.023302276, 0.023655143, 0.031209106, 0.007018128, -0.0002805781, 0.0025321457, -0.004799641, -0.0070246626, 0.012278456, -0.016166525, -0.004623208, -0.018897975, 0.0028245677, 0.003348967, -0.010775506, 0.02006113, 0.022505058, 0.00013201873, 0.015055648, -0.0020861614, -0.01312795, 0.003626686, -0.020897554, 0.0056066606, -0.00037267467, -0.007423271, -0.017538784, -0.012997259, 0.0174473, 0.0011639702, -0.0062731868, 0.021747047, 0.0039534145, 0.01806155, -0.019655986, -0.013539628, -0.008318507, 0.0040743044, 0.021773186, 0.0031169897, 0.017029088, 0.0006126159, 0.016297217, 0.0053616143, 0.009174536, -0.028307756, 0.012840429, 0.011082631, 0.02612521, -0.0021596754, -0.006008537, 0.0019636382, -0.023498312, 0.018950252, 0.01929005, 0.011363616, 0.0059693293, -0.006848229, -0.013408937, 0.011357082, -0.03614924, -0.008305438, 0.00051786465, 0.028490724, -0.013892495, 0.020989038, 0.023459105, -0.01564376, 0.0070115933, -0.005815767, -0.0018019076 - ] - } - ] - } + { + "name": "springfield-illinois-historic-capital-city", + "title": "Springfield: Illinois' Historic Capital City", + "content": "Springfield is the county seat of Sangamon County in Illinois, USA. As of 2019, it had a population of 10,566, making it the sixth most populous city in the state.", + "keywords": ["Springfield: Illinois' Historic Capital City"], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.022805648, 0.035417367, -0.0029928328, -0.024086423, 0.016179595, + -0.0070115933, 0.011461635, 0.0027347174, -0.056981448, 0.017042158, + -0.029248733, 0.01291231, -0.008736719, 0.018858768, 0.011193718, + 0.009886804, -0.016336424, 0.033953626, 0.003763912, -0.0007968091, + 0.012670531, -0.0023491778, -0.049244516, 0.03596627, 0.023171583, + 0.0093379, -0.03126138, 0.008860877, -0.034188867, -0.0043977653, + -0.021616356, -0.0042866776, 0.049793422, 0.0024504636, 0.015983557, + -0.016349493, 0.0077042575, -0.020740723, 0.024190977, 0.02930101, + 0.0013836952, -0.0013395868, 0.01671543, 0.00820742, 0.03293423, + -0.036985666, -0.004815978, -0.024360877, 0.029248733, 0.016937604, + 0.024870573, -0.012500632, -0.019708263, 0.037168633, 0.010141652, + -0.01806155, 0.011677276, 0.022596542, -0.0067959526, -0.043232713, + -0.0032231766, -0.0049597383, -0.00013263135, 0.00044802643, + 0.0061326935, 0.0029536255, 0.0051002316, 0.004289945, 0.053688023, + 0.011376685, 0.013317453, 0.056563236, 0.008070193, 0.043781616, + 0.05980438, -0.005485771, -0.03147049, -0.002639966, -0.04252698, + 0.029562393, 0.013173693, -0.013604974, -0.0145720905, 0.002955259, + 0.017251264, 0.008638701, -0.00064978126, -0.07783979, -0.00006636672, + 0.014206154, 0.017133642, -0.0077696033, -0.0059431912, -0.028752107, + -0.043781616, -0.028124787, 0.041559864, -0.013435075, -0.00046354602, + 0.01974747, 0.014781197, -0.009239881, -0.026504215, 0.01914629, + 0.008187816, 0.015238617, -0.0075539625, -0.0068220906, -0.020204889, + 0.0046591484, 0.018649662, -0.0037933176, -0.012964586, 0.006345067, + -0.016205734, 0.00068653823, -0.014545952, -0.0107689705, 0.042370148, + 0.02429553, -0.0038848016, 0.0049139964, -0.0276543, 0.011402824, + -0.0046983557, -0.0016556965, 0.015277823, -0.027288362, 0.007978709, + -0.0412462, 0.03463322, 0.0062960577, -0.018139966, 0.018715007, + 0.00956661, -0.034136593, -0.0057275505, 0.030921584, 0.03052951, + 0.03873693, -0.06984148, -0.0030941188, 0.026033726, -0.0000384927, + 0.0048355814, -0.010357293, -0.01927698, 0.0032362456, 0.013604974, + -0.028516863, -0.01152698, -0.010494519, -0.027419055, 0.004874789, + 0.0025615513, 0.016637014, -0.0039762855, -0.013278246, 0.043337267, + -0.053583473, -0.019512225, -0.009710371, 0.03450253, -0.026138278, + -0.00872365, -0.013088743, -0.01488575, -0.0041233134, -0.00736446, + 0.008096332, 0.022727232, -0.009488195, -0.061895445, -0.048800167, + 0.0021188343, -0.018545108, -0.03688111, -0.015656829, -0.00064365513, + 0.029666945, -0.0039664838, 0.01146817, 0.02883052, -0.008867411, + -0.024988195, -0.019329257, 0.008436129, 0.011291737, 0.036489036, + -0.010043633, -0.0066129845, 0.041978076, 0.010389966, -0.0023687815, + 0.04022681, -0.003298324, -0.023511382, -0.03293423, -0.0040677697, + 0.029091904, -0.05755649, 0.021603288, -0.024073355, -0.03207167, + 0.0006881719, 0.023001686, -0.00842306, -0.0133566605, 0.020034991, + 0.026360454, -0.006952782, -0.0030843168, -0.01944688, 0.03492074, + -0.020714587, 0.023158515, -0.013944772, -0.0011966431, -0.015617621, + 0.038841482, 0.020518549, 0.028647553, 0.0004202545, -0.014807335, + -0.032698985, -0.025236508, -0.053034566, 0.016297217, -0.009050379, + -0.037011802, 0.024413152, 0.019394603, 0.004022028, 0.06017032, + -0.021433389, 0.043467958, 0.018453624, -0.00009761014, -0.014493676, + 0.03267285, 0.018937184, 0.018963322, 0.0011705048, -0.030581785, + 0.0012186973, -0.02080607, 0.019721331, -0.0033358978, 0.013435075, + 0.021747047, 0.008717116, -0.0030124367, 0.057190552, 0.0014637436, + -0.0112263905, 0.01715978, -0.015774451, 0.013173693, 0.0088478075, + 0.00023606133, -0.016833052, 0.0037018338, -0.03842327, 0.0297715, + 0.004678752, -0.01504258, 0.022870993, 0.00047947405, 0.00565567, + -0.032123946, -0.0045578624, -0.010442242, 0.0061196247, 0.007959106, + 0.0054106237, 0.019930437, 0.0479376, 0.014206154, -0.022047639, + -0.03753457, -0.021015177, 0.030712478, -0.0012693402, -0.01987816, + 0.007684654, 0.0034273819, -0.017695615, 0.013310919, 0.005208052, + -0.045924954, -0.013853287, -0.001886857, 0.04265767, 0.025876896, + 0.006122892, -0.011866778, 0.01488575, -0.0015290893, -0.04022681, + 0.014062393, 0.03322175, 0.036541313, -0.023916526, -0.029327149, + 0.022191398, -0.03902445, 0.018100757, -0.0052701305, -0.039677907, + -0.06372512, -0.0072795106, -0.048800167, -0.010082841, 0.016022764, + 0.0050316188, 0.011762225, 0.018113827, 0.00926602, 0.034084316, + -0.02305396, -0.0050316188, 0.010965008, 0.024047216, 0.023615934, + -0.010305016, 0.04145531, -0.008638701, 0.00018337637, 0.010461846, + 0.011376685, -0.016140386, 0.01594435, -0.0068547637, -0.04116779, + 0.009194139, -0.013683389, 0.024060285, 0.0058419053, -0.00070736714, + 0.031784147, -0.004071037, 0.008939291, 0.0021449726, 0.041873522, + 0.028412309, -0.008717116, 0.0077696033, -0.010200463, -0.023511382, + 0.028046373, 0.0006722438, 0.010690556, 0.058079254, -0.0054596327, + 0.016336424, -0.0018884906, -0.023459105, -0.01989123, 0.015317031, + -0.038658515, -0.020126475, 0.005887647, -0.015251686, 0.011513912, + -0.022910202, 0.024948986, -0.012977655, 0.0074298056, 0.038789205, + -0.0012358505, -0.0005852524, 0.016780775, -0.0132129, 0.009788785, + 0.000096793316, 0.012180438, -0.016323354, 0.008769392, -0.021002106, + -0.0015642126, 0.026896289, 0.018244518, -0.014689713, -0.02548482, + -0.010108979, -0.023733556, -0.010801644, -0.01382715, -0.028124787, + 0.018610455, 0.0053714165, -0.028360032, 0.008730185, 0.0055576516, + 0.009305228, 0.032123946, -0.028438447, -0.006828625, 0.030294266, + 0.0009360771, 0.01714671, -0.027419055, 0.0023916524, 0.03340472, + -0.020126475, -0.0104683805, -0.009520868, -0.061895445, 0.011585792, + -0.011219856, -0.015852867, 0.015552276, 0.01487268, -0.007841484, + -0.02535413, -0.05739966, -0.010102444, 0.04461804, 0.019852022, + 0.019329257, -0.009520868, -0.030634062, 0.0014457735, 0.004613406, + -0.0034731238, 0.01441526, 0.0006849046, 0.026242832, -0.026974704, + -0.032881953, -0.036410622, 0.004335687, -0.011324409, -0.0025958577, + -0.009527403, 0.05206745, -0.04859106, 0.005400822, 0.046892073, + 0.0262559, -0.0318887, 0.01684612, 0.043311127, -0.03175801, + -0.017551854, 0.008854342, 0.021747047, 0.017303541, 0.017538784, + 0.055047214, -0.00926602, 0.017839376, 0.0013812446, 0.013297849, + 0.017551854, 0.009625421, -0.021943085, 0.011742622, -0.018388279, + 0.022099914, 0.029118042, -0.019224703, 0.011455101, -0.035286676, + -0.032986507, 0.035417367, 0.049897972, 0.016088111, 0.0034437182, + -0.007939503, -0.012383009, -0.026190555, -0.050943505, 0.023746626, + -0.0406973, 0.0007825147, 0.0039730184, 0.021302696, 0.019512225, + -0.021851601, -0.044722594, 0.010880059, -0.052041315, 0.011063026, + 0.002115567, 0.04101096, -0.026504215, -0.0094555225, 0.036096964, + -0.014349915, 0.069893755, 0.007606239, -0.008024451, -0.013376264, + 0.014284569, -0.006959317, -0.03949494, -0.058236085, 0.037560705, + 0.039129004, -0.0017643338, 0.02217833, 0.023811972, 0.003102287, + -0.006518233, -0.009448987, -0.017525716, -0.03978246, 0.0035188657, + 0.0002311604, 0.06267959, 0.011121837, -0.036384486, -0.024504635, + 0.010625211, -0.014140809, -0.0012824093, -0.014532883, 0.028203202, + 0.008429595, -0.009945615, 0.02443929, 0.0035646078, 0.0058680438, + -0.023406828, 0.030503372, 0.0012366673, -0.0046166736, 0.019773608, + -0.0031055543, -0.042239457, -0.0021417052, 0.0068416945, 0.028673692, + 0.04521922, -0.023197722, -0.017486509, -0.025001263, -0.0058549745, + 0.042840637, -0.016297217, -0.04610792, 0.0023720488, 0.0273145, + 0.014781197, 0.01641484, -0.0041396497, -0.021080522, 0.009729974, + -0.019107081, 0.046682965, -0.010566399, 0.015081787, -0.0052831997, + -0.007018128, -0.012873102, -0.012775084, -0.010612141, -0.031548902, + -0.021381112, 0.01807462, 0.010533726, 0.031627316, 0.003239513, + -0.005554384, 0.020283304, 0.0073840637, -0.00076576986, 0.0024586318, + 0.0029209526, 0.023171583, 0.017813237, 0.027967958, -0.009703836, + 0.00398282, 0.010599072, -0.014898819, -0.01487268, -0.0037443084, + 0.0025484823, -0.001069219, 0.018022344, 0.02565472, 0.021969223, + 0.0053158724, 0.015748313, -0.006345067, -0.04009612, -0.021250421, + 0.04328499, 0.046343166, 0.02869983, 0.00019777284, 0.005492306, + -0.0043324195, 0.006730607, -0.015918212, 0.009122259, -0.031496625, + -0.038240302, 0.016650084, -0.0012170636, 0.025602443, -0.022910202, + -0.020518549, 0.005825569, 0.014938027, 0.009207209, 0.013918634, + -0.0031643654, -0.011422427, 0.011383221, 0.00486172, -0.015369308, + -0.030555647, -0.0074036676, 0.023171583, 0.034842327, 0.005208052, + -0.015317031, -0.0009769181, -0.034241144, -0.011709949, -0.012644392, + 0.016493253, -0.025942242, 0.009547006, -0.01701602, 0.00014753833, + 0.008070193, -0.0029209526, 0.021341905, -0.014911888, -0.004110244, + 0.000940978, -0.002011014, 0.0034241145, 0.012794687, -0.02596838, + 0.0001425353, 0.014480607, 0.053243674, 0.0243086, 0.030634062, + -0.009076517, 0.0004770236, -0.020309443, 0.019303119, -0.002390019, + 0.0069266437, 0.032594435, -0.016506324, 0.0057504214, 0.0005652403, + -0.0053126053, 0.018309863, 0.04119393, -0.031182967, 0.026295109, + 0.003384907, -0.033012647, 0.0022397237, -0.0003771672, 0.0138140805, + 0.015134064, 0.030921584, -0.043494094, 0.030059021, 0.024831364, + 0.0048780562, 0.008527613, 0.019525293, -0.00090258743, 0.008227023, + -0.017081365, 0.014010117, -0.009187605, -0.0069266437, -0.019956576, + 0.0007616858, -0.007567032, -0.010572934, -0.009697301, -0.016048903, + 0.013317453, -0.023615934, 0.035155986, 0.04116779, 0.0036070824, + -0.005266863, 0.018192243, 0.03782209, -0.0033979763, -0.008429595, + 0.04114165, -0.0056524025, -0.004610139, 0.042448565, 0.0076454463, + -0.014049325, -0.01684612, 0.02475295, 0.0057079466, -0.04767622, + 0.010017496, 0.013271711, -0.007998313, -0.03356155, -0.051387858, + -0.00364629, -0.0026889753, -0.0013820615, 0.016937604, -0.02126349, + 0.054263067, -0.04132462, 0.028464586, -0.023276137, -0.006482293, + 0.039677907, 0.023027824, 0.0153301, -0.034084316, -0.0036299534, + -0.038789205, 0.00697892, -0.029091904, -0.023249999, 0.014768127, + 0.012010539, -0.037116356, 0.00516231, 0.030294266, -0.008913153, + 0.01236994, 0.010455311, -0.007795742, -0.029353287, 0.021080522, + 0.0021482399, -0.006459422, -0.021028245, 0.043833893, -0.0010071405, + 0.008828203, 0.016532462, -0.0021433388, 0.0028033303, -0.057922427, + 0.00271348, -0.033378582, -0.024426222, 0.049270656, 0.024687605, + -0.002324673, -0.0021678435, -0.0014506745, 0.0034927274, 0.018989459, + 0.053086843, 0.03141821, 0.0118863825, -0.00029548508, 0.013618044, + 0.01640177, 0.0005403272, 0.0063483343, -0.0034829257, 0.0064626895, + 0.0034633218, 0.049715005, -0.0021008642, -0.023419898, 0.0026121943, + 0.039887015, 0.004338954, 0.007795742, -0.022152191, 0.008200885, + -0.030059021, 0.03309106, -0.0005047955, -0.0015846331, -0.0006211925, + 0.0032836213, -0.012624789, -0.0036560916, -0.0047898395, 0.011938659, + -0.011383221, -0.015238617, -0.019629847, -0.038501684, 0.015016441, + 0.015892074, -0.020231027, 0.004672217, -0.02885666, 0.04900927, + 0.009716905, 0.0115335155, 0.023589797, -0.000035659352, 0.018518971, + -0.040435918, 0.0077630687, 0.0075408933, 0.006518233, -0.014402191, + 0.016950674, 0.042317875, -0.025249578, 0.029091904, -0.009827993, + -0.025314923, -0.018414417, 0.02217833, -0.016271079, 0.0026791736, + -0.035129845, 0.03037268, -0.01957757, 0.034580942, -0.012539839, + 0.016584737, 0.010932335, 0.010494519, -0.023511382, 0.0048584524, + 0.028543001, -0.01411467, 0.0027216482, -0.024086423, -0.012735876, + -0.0022413575, 0.015212478, 0.0054563656, 0.003613617, 0.025615513, + 0.009867201, -0.0045349915, -0.011337479, 0.020296372, -0.013591905, + 0.025458684, 0.045663573, -0.0040350966, -0.04268381, 0.011350548, + -0.032986507, -0.031052275, 0.009827993, -0.017656406, 0.021302696, + -0.022674957, 0.01624494, 0.037247047, 0.007939503, -0.007730396, + 0.009520868, 0.006893971, -0.0012276822, 0.030477233, 0.0003344883, + 0.002997734, 0.006162099, 0.011559654, 0.004165788, -0.010148186, + -0.018623523, 0.007338322, 0.0066554593, -0.004930333, -0.024426222, + -0.009984822, 0.0022511592, -0.012428751, -0.006116357, -0.020701516, + 0.024635328, -0.04085413, -0.0048029087, -0.019303119, -0.024687605, + 0.0115335155, 0.014898819, -0.009448987, -0.03309106, 0.010984612, + -0.0176172, 0.005701412, 0.012402614, 0.00444024, -0.015081787, + 0.009958684, 0.012860033, -0.023981871, -0.03526054, -0.011579257, + 0.011520446, 0.017395025, -0.008919688, 0.0028343697, -0.029091904, + -0.0044239038, 0.01959064, 0.023903456, 0.0049597383, -0.019185496, + 0.0074951514, -0.0088086, 0.02383811, 0.0040677697, -0.016127318, + -0.01382715, 0.017970067, -0.018884907, -0.0017757693, -0.009762647, + 0.026033726, 0.032516018, 0.013291314, -0.034816187, -0.0015829995, + -0.0046036043, 0.0007796558, 0.022112984, -0.029536255, -0.013395868, + -0.0021939818, 0.019629847, 0.010716694, 0.0026677381, 0.0032819875, + 0.003979553, -0.02883052, -0.034737773, -0.006309127, -0.017120572, + 0.01670236, -0.0074298056, 0.019054806, -0.026046794, -0.007913364, + -0.018963322, 0.037639122, -0.01853204, 0.021211213, 0.03536509, + -0.0045023183, 0.0061424957, 0.01821838, -0.008057124, 0.0063417996, + 0.04294519, 0.022126053, -0.023498312, 0.030869307, -0.009168001, + 0.026556492, -0.007919898, 0.011892917, -0.025236508, 0.0011582525, + 0.010344224, -0.019956576, -0.017525716, -0.052903876, -0.014990303, + 0.010376897, 0.012435286, -0.0019799746, -0.002046954, -0.023106238, + -0.0015070351, 0.034397975, -0.01078204, 0.0030042685, 0.03008516, + 0.034685496, 0.0024096225, 0.0137618035, 0.006207841, 0.051152613, + 0.026504215, -0.01853204, -0.017342748, -0.009390176, -0.00805059, + -0.008253162, 0.03536509, -0.008971964, -0.009893338, 0.0016017864, + 0.057086002, -0.009658094, 0.003959949, 0.010089376, -0.011624999, + 0.015656829, 0.028752107, -0.0030598124, -0.024021078, 0.0059791314, + -0.014284569, 0.028569138, -0.0419258, -0.011794899, -0.012213111, + 0.006015071, 0.011023819, -0.0220215, 0.036384486, -0.036227655, + -0.0055674533, -0.036227655, -0.016140386, -0.023432966, 0.001388596, + -0.0071553537, 0.0024422954, 0.0005007114, -0.0032966903, -0.0241387, + -0.000017880726, -0.0011084264, 0.015918212, 0.006282989, 0.022962477, + 0.034946878, 0.016833052, -0.0058909147, -0.0012742411, -0.0031153562, + -0.028673692, -0.0046199406, -0.010076307, 0.05102192, 0.013787942, + 0.018819561, 0.01578752, -0.03586172, 0.02279258, 0.0013902297, + -0.0014735454, 0.0002234006, -0.003626686, 0.0043487563, 0.023354553, + 0.02170784, -0.007710792, 0.0072795106, -0.0041167787, 0.0075735664, + -0.04132462, -0.0016556965, 0.024792157, 0.027209949, -0.0047767702, + -0.020021921, 0.0013036466, -0.0102135325, 0.00697892, -0.0025697195, + 0.022570403, -0.018649662, -0.015303962, 0.01123946, 0.0042964797, + -0.00083070714, 0.009893338, -0.009122259, 0.034345698, -0.017813237, + 0.02566779, 0.002718381, -0.019094013, -0.016179595, -0.0074298056, + -0.02473988, 0.0096058175, 0.009847596, -0.03248988, 0.025432546, + 0.009396711, -0.042161044, 0.016061973, 0.002380217, -0.012383009, + 0.01382715, -0.018153034, -0.016114248, 0.011579257, 0.006224178, + 0.0473887, -0.024635328, 0.015591484, 0.052616354, -0.004979342, + -0.003509064, -0.013709527, -0.0060869516, 0.003486193, -0.04540219, + -0.0034731238, -0.003531935, -0.0352344, -0.017120572, -0.047702357, + -0.011781829, 0.0008617464, 0.019603709, -0.023289206, 0.007998313, + 0.022335159, 0.010435708, -0.021851601, 0.0035286676, -0.0033685707, + -0.021158937, -0.021158937, -0.012592115, -0.028464586, 0.0031300588, + 0.026817873, -0.020779932, -0.018584317, -0.005698145, 0.016427908, + -0.0046591484, -0.015264755, 0.011102234, -0.011794899, -0.042788364, + 0.016114248, 0.0004937684, 0.018741146, -0.020766862, 0.032280773, + -0.012539839, 0.0017479974, -0.0040775714, -0.003613617, -0.018701939, + 0.0013910466, -0.009305228, 0.024177907, -0.003750843, 0.0077369306, + -0.017774029, -0.02794182, -0.0026121943, 0.020884484, -0.009965219, + -0.0029225862, -0.022152191, -0.0022691293, -0.013709527, + -0.024347806, 0.03052951, -0.021250421, -0.0020404193, -0.008377318, + 0.015774451, 0.006789418, 0.017970067, 0.01593128, 0.004655881, + -0.0005223572, 0.030973861, 0.002233189, 0.012102023, -0.021289628, + -0.022714164, 0.011108768, -0.012860033, 0.0035842115, -0.013552697, + 0.0012260487, 0.011899452, -0.004884591, -0.0032444138, -0.007939503, + 0.010684022, -0.018492833, -0.016911466, -0.020322511, -0.02443929, + -0.03991315, -0.000039743456, -0.001101075, -0.009429384, 0.032594435, + 0.017774029, 0.004695088, -0.019538363, -0.008115935, -0.001064318, + 0.015578414, 0.029536255, -0.030032882, 0.03186256, -0.0126966685, + -0.0004431255, 0.0153301, -0.011330944, 0.008383852, -0.0036528246, + -0.0045741987, -0.0150295105, 0.0125986505, 0.014323777, 0.016885327, + -0.0132129, 0.015317031, -0.014807335, -0.0262559, 0.015879005, + 0.037299324, 0.0028294686, -0.0031349598, 0.0046068714, -0.033195615, + 0.011749156, 0.01236994, 0.008952361, -0.028203202, 0.011899452, + -0.012820826, 0.0013926801, -0.005861509, 0.015317031, -0.013905564, + 0.016075041, -0.0019309653, -0.028647553, -0.026791736, 0.022570403, + 0.010108979, 0.020335581, 0.015774451, -0.012507167, 0.016192663, + -0.000824581, -0.007586635, 0.020832209, -0.03384907, -0.035731025, + 0.010082841, -0.039573353, 0.014362984, 0.017029088, 0.015356239, + -0.031784147, -0.03826644, -0.0015241883, -0.03207167, 0.010540261, + -0.026608767, -0.0019587371, 0.0070704045, -0.005469435, + -0.0054171584, 0.026674114, 0.020793, 0.020675378, -0.019708263, + -0.0046820194, 0.019642916, 0.002522344, 0.0028588742, 0.020217959, + -0.019172428, -0.016493253, 0.032751262, 0.01989123, -0.017342748, + -0.014702782, 0.019995783, 0.009599282, 0.012657462, 0.00035307097, + 0.0023001686, 0.008148609, -0.010507588, -0.0015437921, 0.030294266, + -0.008932756, 0.016323354, 0.0038684653, 0.030973861, 0.013232504, + 0.0047277613, -0.000090360845, -0.02794182, -0.0014523081, + 0.022505058, -0.038344856, -0.0000059793865, -0.016349493, + 0.024099493, -0.034659356, 0.0039403457, -0.020714587, -0.011951728, + -0.01032462, -0.023785833, 0.00016785676, -0.0012350336, 0.0065247677, + 0.013774873, 0.012075884, -0.023746626, -0.009886804, 0.015107925, + -0.012206576, 0.029954467, 0.003257483, -0.013997048, 0.003888069, + -0.052747045, -0.013774873, -0.015591484, -0.0041363826, 0.016833052, + 0.0040808385, 0.016963743, 0.012690134, -0.012735876, 0.012232714, + -0.01123946, -0.0021874472, -0.012618254, -0.008606028, -0.011821036, + -0.024112562, 0.015826728, -0.023877317, 0.0406973, 0.010409569, + 0.013644181, 0.009664629, -0.00011997062, 0.003348967, 0.003590746, + 0.0037998522, -0.0030679805, -0.014820404, 0.008227023, -0.01640177, + -0.005966062, 0.020440133, -0.015682967, 0.023289206, -0.03141821, + 0.00972344, -0.004316083, 0.0038717326, 0.020701516, 0.010710159, + 0.019708263, 0.03144435, 0.031182967, -0.010474916, -0.0066521917, + -0.014637436, -0.0075408933, -0.013787942, -0.0012235981, + -0.028516863, 0.022674957, 0.01472892, 0.018754214, 0.024426222, + 0.0030500104, -0.003522133, -0.00531914, -0.030241989, -0.011134907, + -0.00093526026, -0.014650505, -0.004652614, 0.013918634, 0.011245995, + 0.020596964, 0.02490978, 0.029222596, -0.0017528983, -0.0050381534, + -0.0013044635, 0.011605396, 0.009514334, 0.012284991, 0.007697723, + -0.011500843, -0.0014065661, 0.010056702, -0.008102867, 0.0016393602, + -0.008396922, -0.0071096118, -0.016153457, 0.009971754, -0.015696036, + 0.049715005, -0.0038684653, 0.0061686337, -0.013193296, -0.017682545, + 0.009233347, 0.00265957, -0.018649662, 0.012958052, -0.02548482, + -0.007018128, -0.017852444, -0.008161677, 0.020257166, 0.016022764, + -0.021903878, 0.011840641, -0.018336002, -0.011821036, 0.011807967, + 0.0041167787, -0.036018547, -0.017342748, -0.023419898, 0.0007702624, + 0.007789207, -0.006893971, 0.012389544, -0.009592748, -0.007181492, + 0.0071357503, -0.0060673477, 0.0009115725, -0.0056458684, + -0.011017284, -0.004711425, -0.008501475, -0.0014065661, + -0.00037308308, 0.020610033, -0.0086909775, 0.0047244937, + -0.010527192, -0.027811129, -0.012186972, -0.0037541103, -0.02050548, + 0.0025419476, -0.005322407, -0.004450042, 0.034267284, 0.009736509, + 0.0013877793, 0.025772342, 0.013062605, -0.0032215428, 0.020714587, + -0.007410202, 0.0073579256, 0.013801011, 0.004280143, 0.0014294371, + 0.008860877, 0.00720763, -0.009926011, 0.0031300588, -0.015578414, + -0.004737563, 0.013252107, 0.022988616, 0.0028670426, 0.024583051, + 0.040409777, -0.002197249, -0.028020235, -0.014663574, -0.0069658514, + 0.014101601, 0.007057335, 0.011317874, 0.0016434443, 0.027680438, + -0.01518634, -0.002087795, 0.007560497, 0.023759695, -0.03157504, + 0.015003372, 0.030895445, -0.014023187, -0.018884907, 0.0057504214, + 0.024321668, 0.01031155, 0.013219435, 0.03431956, -0.0023949197, + 0.018309863, -0.023328414, -0.0127489455, -0.00872365, 0.0023410097, + 0.006227445, -0.03978246, -0.008782461, -0.017564923, 0.015134064, + 0.016493253, -0.007978709, 0.02187774, -0.037586845, 0.009945615, + 0.011219856, -0.0022233874, 0.014898819, -0.0073448564, 0.009429384, + 0.062313657, 0.029065765, -0.013944772, 0.017826306, -0.011461635, + 0.0075735664, 0.0114812385, -0.0022952675, -0.025942242, -0.002621996, + 0.031235242, -0.019094013, 0.00035102892, -0.0028637752, 0.0044696457, + 0.0005223572, 0.007874156, -0.0176172, -0.010069772, -0.011415893, + 0.011579257, 0.017826306, -0.0053518126, -0.014532883, 0.021916946, + 0.0015707472, -0.0110760955, -0.0220215, -0.008684443, -0.026229763, + 0.0021939818, 0.007272976, 0.0013379531, 0.0059693293, 0.017669477, + -0.007834949, 0.031679593, -0.0012709738, 0.007351391, -0.0076454463, + 0.018244518, 0.004129848, 0.009533937, 0.0039501474, -0.007527824, + 0.008939291, -0.010174325, 0.010808178, -0.013219435, 0.010246205, + 0.0007812895, 0.011572723, -0.0031856026, -0.022962477, -0.012768549, + -0.01564376, -0.0059693293, 0.005518444, 0.012193507, -0.015957419, + -0.0006113907, -0.021694772, -0.013918634, 0.015447723, 0.011500843, + 0.0039305435, -0.0032885221, 0.017303541, 0.000052582865, + -0.012918844, 0.006227445, -0.0005382852, 0.013578836, -0.023759695, + -0.017839376, 0.023197722, -0.009527403, -0.011004215, -0.014389123, + 0.002489671, 0.021054383, -0.044461213, -0.0017349282, -0.0060183387, + -0.008481871, 0.0061653666, -0.0036005478, -0.013016863, -0.016075041, + -0.0029111507, -0.01671543, -0.0042703412, -0.0027918948, + -0.011794899, 0.008272765, 0.0011500843, -0.009899873, -0.0022821985, + 0.0053714165, -0.011337479, -0.016937604, -0.008821669, 0.03526054, + -0.025576307, 0.013997048, -0.0023230396, -0.0013265177, + 0.00067551114, 0.013343591, 0.009547006, 0.008802066, 0.016127318, + 0.007076939, -0.0071161464, -0.008645236, 0.024556912, -0.012559443, + 0.0071422844, 0.0031905037, 0.048303537, -0.0097561125, -0.020832209, + 0.034058176, 0.019695193, -0.00489766, 0.004234401, -0.0080832625, + -0.003750843, -0.014794266, -0.0019097279, -0.0075997044, 0.01252677, + 0.017695615, -0.0004729395, -0.021786256, -0.0051459735, -0.01791779, + -0.0009826359, -0.003247681, 0.016584737, 0.0023312077, 0.0070311967, + -0.014467537, 0.015526137, 0.018492833, 0.019172428, -0.014598229, + -0.014376054, -0.018466694, 0.010442242, -0.002690609, 0.0065541733, + -0.0029585264, -0.0032182755, 0.009952149, -0.016336424, + -0.0000847452, 0.005492306, -0.0039860876, 0.0048486507, + -0.0054334947, 0.034554806, 0.006541104, 0.0047310283, -0.0029111507, + -0.017421164, -0.0044990513, -0.0072403033, -0.015055648, + 0.0036724282, -0.0029568928, -0.0056687393, -0.0094555225, + -0.007076939, 0.00918107, 0.0072599067, -0.023328414, 0.0016164891, + -0.02656956, -0.0066587264, -0.0055903243, 0.014741989, -0.008187816, + -0.008030986, -0.018715007, -0.018989459, 0.018584317, 0.013435075, + 0.0012799589, -0.004384696, -0.005995468, -0.00007565806, + 0.0041363826, 0.022374367, 0.024190977, 0.020610033, 0.014820404, + -0.003763912, 0.012690134, 0.012977655, -0.0122915255, -0.013323988, + 0.02490978, 0.0145198135, -0.0022854658, -0.0064267493, 0.0126966685, + -0.01534317, -0.029640809, -0.014349915, -0.0045970697, 0.0020502212, + -0.020113405, -0.0035155984, -0.018087689, 0.006730607, -0.01714671, + -0.0042180647, 0.00017571867, -0.0074755475, 0.007841484, 0.014180016, + -0.008658305, -0.02170784, 0.022701096, -0.0012987457, -0.0052015176, + 0.009867201, 0.0036495572, 0.0014384221, -0.00084622676, 0.010435708, + 0.0028980817, -0.003362036, -0.030189712, -0.032881953, 0.023955733, + 0.007253372, 0.02488364, -0.0035613405, 0.007848018, -0.0002004275, + 0.0002891547, 0.0150295105, 0.012938448, 0.03113069, -0.0042017284, + 0.015839797, 0.012892706, 0.017395025, 0.029693084, -0.011147976, + 0.01943381, 0.009416315, 0.000097303826, 0.010305016, 0.013591905, + -0.030921584, 0.010265809, -0.033195615, -0.0060510114, -0.012180438, + -0.0044761803, -0.026412731, 0.009867201, -0.009285623, -0.0050806277, + 0.003705101, 0.023067031, -0.0054302276, 0.009409781, -0.01823145, + -0.021773186, 0.008514544, -0.013337057, -0.01152698, 0.024818296, + 0.017342748, 0.0273145, -0.011977866, -0.005966062, 0.011860244, + -0.021119729, -0.021668633, -0.00039922138, 0.0001421269, 0.01776096, + -0.009547006, 0.01929005, 0.0009246416, -0.005573988, -0.001927698, + -0.019459948, 0.017395025, 0.008991567, 0.008285834, -0.0007906829, + -0.013644181, 0.013265177, 0.007730396, -0.0016230238, -0.04009612, + -0.005737352, 0.014402191, -0.012539839, 0.004695088, 0.0035646078, + 0.005217854, -0.026661044, 0.021446457, -0.0000255129, -0.00081804645, + 0.005698145, -0.020152614, -0.017042158, 0.003362036, 0.0024047217, + 0.006815556, -0.014650505, 0.014088532, 0.00421153, 0.0015054015, + 0.004044899, 0.016885327, -0.00146211, 0.013487352, 0.002095963, + -0.02200843, 0.025733136, 0.012330733, 0.012565978, 0.0085929595, + -0.003183969, -0.0005758589, -0.00045578624, -0.0068612983, + 0.00027608557, -0.005136172, 0.0015633958, 0.012559443, -0.004760434, + -0.019512225, -0.009736509, -0.014689713, -0.003937078, 0.006224178, + -0.0073840637, 0.014598229, -0.032280773, 0.007848018, -0.00466895, + -0.009919477, -0.011860244, 0.0002234006, -0.01487268, -0.006031408, + -0.012840429, -0.011932124, -0.0018525505, 0.011115303, -0.0046166736, + -0.010037099, 0.017355816, -0.015068717, 0.025406407, -0.019825885, + 0.008566821, 0.016140386, 0.0013510223, -0.0068547637, -0.008403457, + -0.02338069, -0.014441399, 0.007919898, -0.022805648, -0.0138140805, + 0.008717116, 0.004554595, -0.0057406193, 0.004002424, 0.0055347807, + -0.004172323, -0.027602023, -0.012905775, 0.012186972, 0.010749367, + 0.0016663152, 0.0015731977, 0.0024766019, -0.018192243, 0.013644181, + -0.03445025, 0.0043716272, 0.007913364, 0.014062393, -0.005864776, + 0.021812394, -0.00057871785, 0.014951095, 0.018179173, -0.0054596327, + 0.011938659, -0.025981449, -0.007331787, 0.009396711, -0.009612352, + -0.0020600231, -0.015408515, 0.027105395, 0.011520446, 0.008658305, + 0.009710371, 0.008200885, 0.04325885, -0.010592537, -0.006080417, + 0.021067452, -0.03068634, 0.0103965, 0.02822934, -0.013631112, + 0.020413995, 0.0077238614, -0.009468592, 0.0020763595, 0.00531914, + 0.007397133, -0.03113069, -0.016271079, -0.0047767702, 0.009331365, + 0.015617621, -0.01624494, -0.004819245, -0.013435075, 0.001009591, + 0.003878267, 0.015983557, 0.028752107, -0.017956998, 0.021616356, + -0.017512647, -0.021851601, 0.009462057, -0.0059007164, 0.013983979, + 0.007501686, 0.020100337, 0.003603815, 0.028307756, -0.007815345, + 0.012873102, 0.028752107, 0.003531935, -0.0063581364, -0.006972386, + 0.013108347, -0.016519392, -0.015774451, -0.011585792, 0.019028667, + -0.01987816, -0.0020387857, -0.0033914417, 0.0016434443, -0.010298482, + 0.0036626263, -0.0038488614, 0.0016532461, 0.012128161, -0.0138140805, + 0.008285834, -0.006008537, -0.021694772, -0.011742622, 0.014689713, + 0.0007224784, -0.013043001, -0.0048682545, -0.0068416945, + -0.017395025, -0.003404511, 0.0069331783, -0.012409148, -0.0009246416, + 0.013552697, 0.008161677, 0.008527613, 0.0002001212, 0.018244518, + 0.005505375, 0.004280143, 0.022714164, -0.017564923, 0.008318507, + 0.0022266547, -0.0029405563, 0.036985666, 0.012363406, 0.0006085318, + -0.028072512, -0.014258431, 0.005760223, 0.03207167, -0.006224178, + -0.004378162, 0.002064924, 0.00903731, -0.0043912306, -0.008762858, + -0.013983979, -0.02217833, -0.011043423, -0.0029993674, 0.003018971, + 0.012637857, 0.018270656, 0.0376914, 0.017695615, 0.018022344, + -0.02157715, -0.02974536, 0.0050838953, 0.010474916, 0.016349493, + -0.021211213, -0.014402191, 0.001932599, -0.0025305122, 0.009468592, + 0.026465008, 0.0038586634, 0.009272554, 0.011500843, -0.0052243886, + -0.010108979, 0.014663574, -0.0019848756, 0.02778499, 0.00039186998, + -0.017538784, 0.030320404, 0.019473018, -0.02352445, 0.0036724282, + 0.013016863, 0.0072337687, 0.018754214, -0.0089458255, -0.0029127845, + 0.005237458, 0.005093697, 0.01336973, 0.016480185, -0.010801644, + 0.012559443, 0.015656829, -0.0057275505, 0.019028667, -0.022975547, + -0.018100757, -0.019773608, 0.0029928328, -0.012428751, 0.020727655, + -0.02275337, 0.017290471, 0.0031267917, 0.027863406, -0.00993908, + 0.00034122705, -0.008403457, -0.0018100758, -0.0052113193, + -0.008246627, -0.00082907354, -0.0078088106, 0.020766862, + -0.018688869, -0.008331576, -0.022844855, 0.02961467, 0.0072860452, + 0.0039534145, -0.021903878, -0.028412309, 0.009716905, -0.004338954, + -0.009390176, 0.0102135325, -0.01563069, 0.000013694518, -0.00706387, + -0.017878583, 0.004518655, -0.01684612, 0.0062339795, 0.02734064, + -0.00061915047, -0.017029088, -0.030424956, 0.010592537, -0.019054806, + 0.0038161888, -0.008632166, -0.020244097, 0.013918634, 0.013539628, + 0.0092006745, -0.0015650294, 0.005636066, 0.0007049167, -0.0020779932, + -0.020766862, 0.006939713, -0.00074371573, -0.020884484, + 0.00041351575, 0.008978499, 0.022805648, 0.0049499366, 0.017538784, + 0.012794687, -0.014336846, 0.03382293, -0.00410371, -0.0027902613, + -0.017120572, -0.026922427, 0.008586424, -0.0011966431, 0.026334316, + 0.002011014, -0.009814924, 0.0039272765, -0.009390176, -0.019995783, + 0.0028441714, -0.009377107, 0.011317874, 0.01048145, -0.0040547005, + 0.023563659, 0.021930017, 0.0064169476, 0.0056033935, 0.025092747, + 0.020518549, -0.024413152, -0.010331155, 0.03199325, 0.013709527, + -0.008181281, 0.01442833, 0.0008707314, -0.0007543344, -0.011925589, + 0.018296795, 0.0017610665, 0.00796564, -0.009272554, -0.0041396497, + 0.0050414205, -0.028046373, 0.01640177, -0.035338953, -0.012820826, + -0.026765598, 0.011004215, 0.013931703, 0.01504258, 0.0038488614, + 0.02049241, -0.026242832, -0.0015388911, 0.019773608, -0.010069772, + 0.0010496152, -0.004760434, -0.01914629, 0.0029928328, 0.007684654, + -0.008285834, -0.0017888384, -0.00972344, 0.01412774, -0.02157715, + -0.0029764965, -0.01261172, 0.011592327, 0.013526559, 0.013892495, + 0.012252318, 0.005573988, -0.024021078, 0.0074494095, 0.034816187, + -0.00903731, -0.0012383009, -0.008481871, -0.016950674, 0.0062960577, + -0.006345067, 0.004649346, -0.004299747, 0.005792896, -0.029065765, + -0.020374788, 0.035809442, 0.008455733, 0.008749789, -0.012160834, + 0.011461635, 0.0027347174, -0.0005235824, 0.0077499994, -0.018963322, + -0.005933389, -0.020596964, 0.010579469, 0.003545004, 0.031548902, + -0.006959317, 0.0040547005, -0.023119308, 0.01791779, 0.007090008, + -0.0012432019, 0.003247681, -0.013840219, 0.01927698, -0.0041167787, + -0.00588438, -0.004946669, 0.00812247, 0.021812394, -0.008279299, + 0.021629425, -0.006809022, -0.0019015598, -0.01684612, 0.0039893547, + -0.0012235981, 0.019760538, 0.0006538654, -0.00000608468, + -0.010474916, 0.008599494, -0.035417367, -0.007188027, -0.00041821247, + 0.03330017, 0.008194351, -0.03139207, 0.005155775, -0.029065765, + -0.01101075, 0.027131533, -0.006008537, -0.004652614, 0.0054563656, + 0.019616779, 0.008971964, -0.00683516, 0.001351839, 0.01085392, + -0.000532159, 0.0028425378, -0.019760538, 0.010644814, 0.013023397, + 0.010507588, -0.0049760747, 0.038031194, -0.002535413, 0.017133642, + 0.0085406825, -0.0063385325, 0.008357715, 0.009645025, -0.0048911255, + -0.013029932, 0.02383811, -0.0011664206, -0.0016017864, 0.0013257008, + -0.003061446, 0.015918212, -0.013252107, 0.0052733975, -0.0064430856, + 0.019094013, -0.01351349, 0.02200843, -0.0044043, -0.008416526, + -0.0017202254, 0.016258009, -0.010801644, -0.0034241145, -0.012017074, + -0.0048780562, 0.009520868, -0.0031774344, 0.013879426, -0.005230923, + -0.0068678325, 0.014807335, -0.014180016, 0.015696036, 0.023929594, + -0.0012252318, 0.022988616, 0.0052505266, -0.00019164667, + -0.016506324, 0.012474493, -0.021629425, -0.047101177, -0.004022028, + 0.0073579256, -0.029379426, 0.010540261, -0.011455101, 0.027079256, + 0.003384907, 0.0066521917, 0.0045349915, -0.019224703, 0.016963743, + 0.022831786, -0.0055576516, 0.015081787, 0.0069462475, -0.034528665, + 0.01989123, 0.0019358663, -0.020975968, 0.013186761, -0.0006346701, + -0.012383009, 0.0020338849, -0.024726812, -0.0021907145, 0.017303541, + -0.005743887, 0.006410413, 0.0069462475, -0.012200042, -0.004554595, + 0.0036332207, 0.0012374842, 0.011585792, -0.010801644, -0.021054383, + 0.0058909147, -0.009906407, -0.008279299, 0.028150925, 0.025903035, + -0.00015203084, -0.01654553, -0.0071553537, 0.009965219, 0.006939713, + 0.0008617464, -0.004672217, 0.0061424957, 0.0142192235, -0.019512225, + -0.014964164, -0.006116357, -0.011860244, -0.022988616, -0.017695615, + 0.0075735664, 0.0021319033, 0.012265387, -0.013487352, 0.010697091, + 0.0083381105, 0.037168633, 0.009246416, 0.006354869, 0.000010152832, + -0.014715851, 0.00013559232, 0.028124787, -0.003914207, 0.014075463, + -0.0039501474, -0.025314923, 0.021067452, -0.0033342643, -0.02186467, + -0.0024161572, -0.0054204254, 0.00805059, 0.008220488, 0.03081703, + -0.002563185, -0.0062143756, 0.0037116355, -0.015199409, -0.010167791, + 0.011474704, -0.00796564, -0.0033685707, 0.00683516, 0.0264258, + 0.0049041947, 0.005342011, 0.016676221, -0.013029932, -0.021694772, + -0.008671374, -0.006387542, 0.0061915047, 0.021067452, -0.0028490725, + -0.004185392, -0.011925589, 0.015669897, -0.0031807018, 0.009305228, + 0.008854342, 0.029248733, -0.02399494, -0.023067031, -0.013297849, + 0.0049499366, 0.0037541103, -0.0070508006, -0.02339376, -0.0004778404, + -0.00027955705, 0.0032999576, 0.0102135325, -0.018545108, -0.02064924, + 0.023811972, -0.01032462, -0.020335581, 0.013036466, 0.030137435, + 0.03021585, 0.005345278, -0.008344646, 0.016741568, 0.016480185, + 0.0077499994, 0.0017398292, 0.011827571, -0.012265387, 0.0188457, + 0.008429595, -0.0019015598, 0.011892917, 0.013239038, 0.0104683805, + 0.018871836, 0.0020583894, -0.012278456, -0.007272976, -0.0041951938, + -0.02791568, -0.020675378, 0.04451349, -0.0075735664, 0.013350125, + 0.0033342643, -0.0073252525, 0.0034600548, -0.01367032, -0.008514544, + -0.008553752, -0.0061424957, -0.011082631, 0.020178752, -0.010429173, + -0.026282039, 0.0142715005, 0.015356239, -0.024726812, -0.016336424, + -0.007377529, 0.006207841, 0.007919898, -0.010886593, 0.0016614143, + -0.00942285, 0.0067044683, -0.021132799, 0.0060608136, 0.015120994, + 0.028203202, -0.014924957, -0.0030091694, -0.009658094, 0.021930017, + -0.0004239302, -0.027079256, -0.005554384, 0.021250421, -0.015970489, + -0.014898819, 0.011847175, -0.0040775714, 0.020570826, -0.0068416945, + -0.0041461843, 0.006015071, 0.021211213, 0.006469224, 0.01236994, + 0.019838953, -0.007815345, -0.0010398134, 0.00006963401, + -0.0072403033, 0.00021421137, 0.013258642, 0.02186467, -0.014846542, + 0.0063810074, 0.0047473647, -0.019224703, -0.0008919688, + -0.0056491354, 0.0071749575, 0.006599915, -0.014768127, 0.008416526, + -0.00013600073, -0.01836214, 0.021812394, -0.0004075938, -0.016336424, + 0.027131533, 0.012925379, -0.009886804, -0.011102234, -0.025942242, + 0.00034796583, -0.000068000365, -0.006057546, 0.022413574, + -0.0026791736, 0.01236994, 0.0006857214, 0.00012977247, 0.0073252525, + -0.01623187, -0.016467115, 0.0017316609, 0.009631956, -0.019995783, + 0.003394709, -0.0154346535, -0.015539207, 0.010089376, -0.0026203624, + 0.0010929068, 0.007625843, 0.036985666, 0.0069462475, 0.006998524, + 0.010265809, -0.005008748, 0.00095486396, 0.007351391, 0.010487984, + 0.01504258, -0.021185074, 0.0005627898, 0.009187605, -0.009083052, + 0.0041690553, -0.006711003, -0.030581785, 0.007377529, -0.011664207, + 0.012128161, 0.0040775714, -0.011638069, -0.0043748943, 0.0062111085, + 0.023981871, -0.004071037, -0.00011762225, 0.014898819, 0.01093887, + 0.011389755, -0.0055119097, -0.0075408933, 0.0056491354, -0.02126349, + -0.00611309, 0.0041788574, -0.012650927, -0.0038749997, 0.00294219, + -0.0035352022, 0.01610118, 0.0060477443, 0.0037214374, -0.002179279, + -0.009631956, -0.012128161, 0.007057335, -0.0133566605, 0.0046395445, + 0.005243992, -0.0057667578, -0.0011460001, -0.020675378, + -0.0077173267, 0.0054988405, 0.0010153088, -0.034737773, -0.008971964, + -0.015826728, -0.0010332789, 0.011925589, -0.019303119, 0.0103965, + -0.002407989, 0.01472892, 0.004093908, 0.0044075674, 0.010416104, + -0.011677276, 0.0176172, 0.025066609, -0.016427908, 0.009187605, + 0.0072403033, -0.01382715, -0.00835118, -0.007181492, -0.016663153, + -0.0013861456, 0.012422217, -0.002489671, -0.008573355, -0.0014245362, + 0.0051884484, -0.00094261166, -0.0032819875, 0.01185371, -0.006697934, + 0.025131956, -0.01957757, 0.015003372, 0.0137618035, 0.009239881, + -0.01594435, 0.0104683805, 0.0073840637, -0.012833895, 0.013853287, + 0.01944688, -0.0058680438, 0.0092006745, 0.024543844, 0.010579469, + 0.04631703, -0.0050838953, -0.015264755, -0.012762015, -0.009814924, + -0.021943085, 0.005253794, 0.010788575, 0.0014833473, 0.014245362, + -0.010971542, 0.007651981, -0.0029536255, 0.0066031828, -0.012200042, + -0.0074298056, 0.014624367, -0.008854342, -0.006331998, 0.0060510114, + 0.015016441, -0.0034600548, 0.016898397, 0.009860666, 0.0026350652, + -0.0039958893, 0.01791779, 0.0114812385, 0.017669477, -0.012938448, + 0.019120151, -0.00805059, -0.016480185, 0.0062470487, 0.020845277, + 0.03873693, 0.0074951514, 0.017029088, 0.0230801, 0.01101075, + -0.010657883, 0.011742622, 0.016937604, -0.0019064607, -0.014206154, + 0.013997048, 0.008560286, 0.002759222, 0.007246838, 0.018022344, + 0.009599282, 0.0023459105, 0.017839376, -0.011487774, -0.003509064, + 0.016192663, -0.0022625946, -0.005952993, -0.002901349, 0.0011672375, + 0.006697934, -0.0015797322, -0.017747892, -0.0025158094, -0.011579257, + 0.0017316609, -0.021015177, -0.0073905983, 0.0043422217, -0.006243781, + 0.025785413, -0.0029519917, -0.0029046163, -0.0031169897, + -0.0019391335, 0.0002082894, -0.024465429, -0.02961467, -0.0027020446, + 0.030189712, -0.007919898, -0.0016279246, -0.016623946, 0.004920531, + -0.0009050379, 0.023877317, 0.0081355395, 0.011566188, 0.0051329043, + -0.008070193, -0.02791568, -0.004485982, 0.02247892, 0.022439713, + -0.0054400293, 0.009488195, -0.0070050587, 0.012886171, 0.0015478762, + 0.0008748155, -0.0022217536, -0.0031186233, 0.020322511, 0.023315344, + -0.010690556, 0.0056883427, -0.019956576, 0.0075147552, 0.0154869305, + 0.0018982925, -0.01684612, -0.015120994, 0.00660645, -0.028647553, + 0.0068416945, -0.0074886167, -0.0012775083, -0.0005934206, + 0.001881956, 0.0038259905, 0.014493676, -0.008854342, -0.0074886167, + 0.012722807, 0.0024471963, 0.026360454, -0.010167791, -0.008318507, + 0.015813658, -0.00926602, 0.000111393994, 0.02200843, 0.019185496, + -0.007874156, 0.002105765, 0.0020665578, 0.014376054, 0.00736446, + 0.0009393444, -0.013748734, 0.020100337, 0.0005697328, 0.018937184, + 0.0013142653, 0.011272132, 0.004110244, -0.003143128, -0.0012366673, + 0.02004806, -0.006730607, -0.019237773, -0.014794266, -0.016127318, + -0.004106977, 0.016493253, -0.013186761, -0.0022070508, -0.019198565, + -0.026020657, 0.004316083, -0.0016238405, 0.00085684547, 0.014624367, + -0.0033097595, -0.016976813, 0.0078088106, -0.009004637, -0.020152614, + 0.0012619888, -0.009207209, -0.017434232, -0.012905775, -0.03719477, + -0.0013346858, -0.00086419686, -0.0125986505, -0.009729974, + 0.0037345067, 0.043102022, -0.027445193, -0.013709527, 0.036645867, + 0.015774451, -0.001210529, -0.009743043, -0.0028360032, -0.002192348, + -0.00040555172, -0.023602866, -0.029222596, -0.032908093, -0.01312795, + -0.0040089586, 0.009814924, 0.030137435, -0.002965061, 0.015408515, + -0.0023148714, -0.007887226, -0.014362984, -0.0048453836, + -0.010651348, 0.004028562, 0.021315766, -0.008854342, 0.003161098, + 0.005070826, -0.0014057494, 0.03565261, -0.013983979, -0.024190977, + 0.014768127, 0.0010831049, -0.013539628, -0.0035646078, 0.016623946, + -0.0038390597, 0.014467537, -0.013892495, 0.010461846, -0.012245784, + 0.0020567558, -0.01913322, -0.009586214, 0.005861509, -0.0024635328, + -0.0076781195, -0.00083519967, -0.007351391, 0.0004623208, + 0.019002529, -0.015016441, -0.0052243886, 0.020100337, 0.024975125, + 0.00752129, 0.022400504, 0.026504215, 0.01580059, 0.018858768, + -0.0052276556, -0.00044394232, -0.012304595, -0.013500421, 0.00720763, + -0.001960371, 0.023602866, -0.0027984295, -0.014376054, 0.012383009, + 0.02460919, 0.014454468, 0.017813237, 0.007874156, -0.003796585, + 0.031156829, 0.0019064607, -0.024569983, 0.013722597, -0.018153034, + 0.01594435, -0.005400822, 0.012304595, 0.0009066715, 0.015264755, + -0.018244518, -0.024151769, -0.019237773, 0.0061098225, -0.0029911993, + 0.0015225548, -0.0017904721, 0.013134485, 0.001434338, -0.0050054803, + 0.009703836, 0.00006279313, 0.0012897607, -0.015238617, 0.0021433388, + 0.010461846, -0.016427908, -0.007913364, -0.013866357, -0.0030957523, + 0.0154869305, 0.018571246, -0.013591905, 0.012735876, 0.028725969, + 0.008377318, 0.024975125, 0.0006113907, -0.012337267, 0.033012647, + 0.006031408, 0.0024096225, 0.013343591, -0.010951939, -0.008076728, + 0.0032640176, -0.012585581, -0.018414417, 0.0014857978, 0.003404511, + 0.0069658514, -0.019094013, 0.009292158, -0.004254005, -0.018610455, + -0.009468592, 0.0058582416, -0.012317664, -0.008828203, 0.023302276, + 0.023655143, 0.031209106, 0.007018128, -0.0002805781, 0.0025321457, + -0.004799641, -0.0070246626, 0.012278456, -0.016166525, -0.004623208, + -0.018897975, 0.0028245677, 0.003348967, -0.010775506, 0.02006113, + 0.022505058, 0.00013201873, 0.015055648, -0.0020861614, -0.01312795, + 0.003626686, -0.020897554, 0.0056066606, -0.00037267467, -0.007423271, + -0.017538784, -0.012997259, 0.0174473, 0.0011639702, -0.0062731868, + 0.021747047, 0.0039534145, 0.01806155, -0.019655986, -0.013539628, + -0.008318507, 0.0040743044, 0.021773186, 0.0031169897, 0.017029088, + 0.0006126159, 0.016297217, 0.0053616143, 0.009174536, -0.028307756, + 0.012840429, 0.011082631, 0.02612521, -0.0021596754, -0.006008537, + 0.0019636382, -0.023498312, 0.018950252, 0.01929005, 0.011363616, + 0.0059693293, -0.006848229, -0.013408937, 0.011357082, -0.03614924, + -0.008305438, 0.00051786465, 0.028490724, -0.013892495, 0.020989038, + 0.023459105, -0.01564376, 0.0070115933, -0.005815767, -0.0018019076 + ] + } + ] + } ] diff --git a/src/scrapers/markitdown/playground/markitdown-scraper-playground.ts b/src/scrapers/markitdown/playground/markitdown-scraper-playground.ts index 31447ffa9a..d2b784e036 100644 --- a/src/scrapers/markitdown/playground/markitdown-scraper-playground.ts +++ b/src/scrapers/markitdown/playground/markitdown-scraper-playground.ts @@ -1,88 +1,93 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { writeFile } from 'fs/promises'; -import { join } from 'path'; -import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode'; -import { usageToHuman } from '../../../execution/utils/usageToHuman'; -import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground'; -import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson'; -import { titleToName } from '../../../utils/normalization/titleToName'; -import { isValidUrl } from '../../../utils/validators/url/isValidUrl'; -import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode'; -import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler'; -import { MarkitdownScraper } from '../MarkitdownScraper'; +import { join } from "path"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { writeFile } from "fs/promises"; +import { $provideExecutablesForNode } from "../../../executables/$provideExecutablesForNode"; +import { usageToHuman } from "../../../execution/utils/usageToHuman"; +import { $provideLlmToolsForTestingAndScriptsAndPlayground } from "../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground"; +import { stringifyPipelineJson } from "../../../utils/editable/utils/stringifyPipelineJson"; +import { titleToName } from "../../../utils/normalization/titleToName"; +import { isValidUrl } from "../../../utils/validators/url/isValidUrl"; +import { $provideFilesystemForNode } from "../../_common/register/$provideFilesystemForNode"; +import { makeKnowledgeSourceHandler } from "../../_common/utils/makeKnowledgeSourceHandler"; +import { MarkitdownScraper } from "../MarkitdownScraper"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 Scrape knowledge by Markitdown (playground)`); + console.info(`🧸 Scrape knowledge by Markitdown (playground)`); - // Do here stuff you want to test - //========================================> + // Do here stuff you want to test + //========================================> - // const example = '10-simple.pdf'; - // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/4/c/pravidla-pro-zadatele-a-prijemce-3-0.pdf'; - const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/b/d/10-simple.pdf'; - // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/4/3/10-simple.odt'; - // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/4/3/10-simple.odt'; - // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/7/f/10-simple.docx'; - // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/2/9/10-simple.rtf'; - // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/f/3/10-simple.doc'; + // const example = '10-simple.pdf'; + // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/4/c/pravidla-pro-zadatele-a-prijemce-3-0.pdf'; + const example = + "https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/b/d/10-simple.pdf"; + // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/4/3/10-simple.odt'; + // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/4/3/10-simple.odt'; + // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/7/f/10-simple.docx'; + // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/2/9/10-simple.rtf'; + // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/f/3/10-simple.doc'; - // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them + // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them - const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true }); - const rootDirname = isValidUrl(example) ? null : join(__dirname, '..', 'examples'); + const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ + isCacheReloaded: true, + }); + const rootDirname = isValidUrl(example) + ? null + : join(__dirname, "..", "examples"); - const markitdownScraper = new MarkitdownScraper( - { - fs: $provideFilesystemForNode(), - llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), - executables: await $provideExecutablesForNode(), - }, - { - rootDirname, - }, - ); + const markitdownScraper = new MarkitdownScraper( + { + fs: $provideFilesystemForNode(), + llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), + executables: await $provideExecutablesForNode(), + }, + { + rootDirname, + }, + ); - const knowledge = await markitdownScraper.scrape( - await makeKnowledgeSourceHandler( - { knowledgeSourceContent: example }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ); + const knowledge = await markitdownScraper.scrape( + await makeKnowledgeSourceHandler( + { knowledgeSourceContent: example }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ); - console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); - console.info(colors.bgGreen(' Knowledge: ')); - console.info(knowledge); + console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); + console.info(colors.bgGreen(" Knowledge: ")); + console.info(knowledge); - await writeFile( - join( - __dirname, - `../examples/${ - isValidUrl(example) ? titleToName(example) : example - }.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, - ), - stringifyPipelineJson(knowledge), - 'utf-8', - ); - /**/ + await writeFile( + join( + __dirname, + `../examples/${ + isValidUrl(example) ? titleToName(example) : example + }.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, + ), + stringifyPipelineJson(knowledge), + "utf-8", + ); + /**/ - //========================================/ + //========================================/ } /** diff --git a/src/scrapers/markitdown/playground/tsconfig.json b/src/scrapers/markitdown/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/scrapers/markitdown/playground/tsconfig.json +++ b/src/scrapers/markitdown/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/scrapers/markitdown/register-constructor.ts b/src/scrapers/markitdown/register-constructor.ts index c2b9a7a99c..3c3f43446f 100644 --- a/src/scrapers/markitdown/register-constructor.ts +++ b/src/scrapers/markitdown/register-constructor.ts @@ -1,6 +1,6 @@ -import type { Registration } from '../../utils/$Register'; -import { $scrapersRegister } from '../_common/register/$scrapersRegister'; -import { createMarkitdownScraper } from './createMarkitdownScraper'; +import type { Registration } from "../../utils/$Register"; +import { $scrapersRegister } from "../_common/register/$scrapersRegister"; +import { createMarkitdownScraper } from "./createMarkitdownScraper"; /** * Registration of known scraper @@ -12,7 +12,8 @@ import { createMarkitdownScraper } from './createMarkitdownScraper'; * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _MarkitdownScraperRegistration: Registration = $scrapersRegister.register(createMarkitdownScraper); +export const _MarkitdownScraperRegistration: Registration = + $scrapersRegister.register(createMarkitdownScraper); /** * TODO: [🧠][🌜] Export only from `@promptbook/markitdown` or `@promptbook/pdf` NOT both diff --git a/src/scrapers/markitdown/register-metadata.ts b/src/scrapers/markitdown/register-metadata.ts index d9fa8e953c..b5b719046b 100644 --- a/src/scrapers/markitdown/register-metadata.ts +++ b/src/scrapers/markitdown/register-metadata.ts @@ -1,8 +1,8 @@ -import type { Registration } from '../../utils/$Register'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import { $deepFreeze } from '../../utils/serialization/$deepFreeze'; -import { $scrapersMetadataRegister } from '../_common/register/$scrapersMetadataRegister'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; +import type { Registration } from "../../utils/$Register"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import { $deepFreeze } from "../../utils/serialization/$deepFreeze"; +import { $scrapersMetadataRegister } from "../_common/register/$scrapersMetadataRegister"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; keepTypeImported(); @@ -12,19 +12,19 @@ keepTypeImported(); * @private within the scraper directory */ export const markitdownScraperMetadata = $deepFreeze({ - title: 'Markitdown scraper', - packageName: '@promptbook/markitdown', // <- [🌜] - className: 'MarkitdownScraper', - mimeTypes: [ - 'application/pdf', - // TODO: [💵] Make priority for scrapers and than analyze which mime-types can Markitdown scrape and allow all mime types here: - // 'text/html', - // 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - ], - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@', - isAvilableInBrowser: false, - // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server - requiredExecutables: [], + title: "Markitdown scraper", + packageName: "@promptbook/markitdown", // <- [🌜] + className: "MarkitdownScraper", + mimeTypes: [ + "application/pdf", + // TODO: [💵] Make priority for scrapers and than analyze which mime-types can Markitdown scrape and allow all mime types here: + // 'text/html', + // 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + ], + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@", + isAvilableInBrowser: false, + // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server + requiredExecutables: [], }) satisfies ScraperAndConverterMetadata; /* <- Note: [🤛] */ /** @@ -37,7 +37,7 @@ export const markitdownScraperMetadata = $deepFreeze({ * @public exported from `@promptbook/cli` */ export const _MarkitdownScraperMetadataRegistration: Registration = - $scrapersMetadataRegister.register(markitdownScraperMetadata); + $scrapersMetadataRegister.register(markitdownScraperMetadata); /** * Note: [💞] Ignore a discrepancy between file name and entity name diff --git a/src/scrapers/pdf/PdfScraper.ts b/src/scrapers/pdf/PdfScraper.ts index 4d7479e044..8431f5b504 100644 --- a/src/scrapers/pdf/PdfScraper.ts +++ b/src/scrapers/pdf/PdfScraper.ts @@ -1,17 +1,17 @@ -import type { KnowledgePiecePreparedJson } from '../../pipeline/PipelineJson/KnowledgePieceJson'; -import type { Scraper } from '../_common/Scraper'; -import type { ScraperSourceHandler } from '../_common/Scraper'; // TODO: [🏳‍🌈] Finally take pick of .json vs .ts // import PipelineCollection from '../../../books/books'; -import { NotYetImplementedError } from '../../errors/NotYetImplementedError'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { TODO_USE } from '../../utils/organization/TODO_USE'; -import type { Converter } from '../_common/Converter'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; -import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource'; -import { MarkdownScraper } from '../markdown/MarkdownScraper'; -import { pdfScraperMetadata } from './register-metadata'; +import { NotYetImplementedError } from "../../errors/NotYetImplementedError"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { KnowledgePiecePreparedJson } from "../../pipeline/PipelineJson/KnowledgePieceJson"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { TODO_USE } from "../../utils/organization/TODO_USE"; +import type { Converter } from "../_common/Converter"; +import type { Scraper } from "../_common/Scraper"; +import type { ScraperSourceHandler } from "../_common/Scraper"; +import type { ScraperIntermediateSource } from "../_common/ScraperIntermediateSource"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; +import { MarkdownScraper } from "../markdown/MarkdownScraper"; +import { pdfScraperMetadata } from "./register-metadata"; /** * Scraper for .pdf files @@ -20,44 +20,48 @@ import { pdfScraperMetadata } from './register-metadata'; * @public exported from `@promptbook/pdf` */ export class PdfScraper implements Converter, Scraper { - /** - * Metadata of the scraper which includes title, mime types, etc. - */ - public get metadata(): ScraperAndConverterMetadata { - return pdfScraperMetadata; - } + /** + * Metadata of the scraper which includes title, mime types, etc. + */ + public get metadata(): ScraperAndConverterMetadata { + return pdfScraperMetadata; + } - /** - * Markdown scraper is used internally - */ - private readonly markdownScraper: MarkdownScraper; + /** + * Markdown scraper is used internally + */ + private readonly markdownScraper: MarkdownScraper; - public constructor( - private readonly tools: Pick, - private readonly options: PrepareAndScrapeOptions, - ) { - this.markdownScraper = new MarkdownScraper(tools, options); - } + public constructor( + private readonly tools: Pick, + private readonly options: PrepareAndScrapeOptions, + ) { + this.markdownScraper = new MarkdownScraper(tools, options); + } - /** - * Converts the `.pdf` file to `.md` file and returns intermediate source - */ - public async $convert(source: ScraperSourceHandler): Promise { - TODO_USE(source); - TODO_USE(this.options); - throw new NotYetImplementedError('PDF conversion not yet implemented'); - } + /** + * Converts the `.pdf` file to `.md` file and returns intermediate source + */ + public async $convert( + source: ScraperSourceHandler, + ): Promise { + TODO_USE(source); + TODO_USE(this.options); + throw new NotYetImplementedError("PDF conversion not yet implemented"); + } - /** - * Scrapes the `.pdf` file and returns the knowledge pieces or `null` if it can't scrape it - */ - public async scrape( - source: ScraperSourceHandler, - ): Promise> | null> { - TODO_USE(source); - TODO_USE(this.options); + /** + * Scrapes the `.pdf` file and returns the knowledge pieces or `null` if it can't scrape it + */ + public async scrape( + source: ScraperSourceHandler, + ): Promise + > | null> { + TODO_USE(source); + TODO_USE(this.options); - /* + /* const { cacheDirname = SCRAPE_CACHE_DIRNAME, intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, @@ -65,8 +69,8 @@ export class PdfScraper implements Converter, Scraper { } = options; */ - throw new NotYetImplementedError('PDF scraping not yet implemented'); - } + throw new NotYetImplementedError("PDF scraping not yet implemented"); + } } /** diff --git a/src/scrapers/pdf/createPdfScraper.ts b/src/scrapers/pdf/createPdfScraper.ts index d107d44aba..acb9459226 100644 --- a/src/scrapers/pdf/createPdfScraper.ts +++ b/src/scrapers/pdf/createPdfScraper.ts @@ -1,9 +1,9 @@ -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import type { ScraperConstructor } from '../_common/register/ScraperConstructor'; -import { PdfScraper } from './PdfScraper'; -import { pdfScraperMetadata } from './register-metadata'; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import type { ScraperConstructor } from "../_common/register/ScraperConstructor"; +import { PdfScraper } from "./PdfScraper"; +import { pdfScraperMetadata } from "./register-metadata"; keepTypeImported(); @@ -13,10 +13,13 @@ keepTypeImported(); * @public exported from `@promptbook/pdf` */ export const createPdfScraper = Object.assign( - (tools: Pick, options: PrepareAndScrapeOptions): PdfScraper => { - return new PdfScraper(tools, options); - }, - pdfScraperMetadata, + ( + tools: Pick, + options: PrepareAndScrapeOptions, + ): PdfScraper => { + return new PdfScraper(tools, options); + }, + pdfScraperMetadata, ) satisfies ScraperConstructor; /* <- Note: [🤛] */ /** diff --git a/src/scrapers/pdf/lib/pdf2md/models/Annotation.js b/src/scrapers/pdf/lib/pdf2md/models/Annotation.js index b7210e6c82..8d902f2e27 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/Annotation.js +++ b/src/scrapers/pdf/lib/pdf2md/models/Annotation.js @@ -1,34 +1,34 @@ // Annotation for a text item class Annotation { - constructor(options) { - this.category = options.category; - this.color = options.color; - } + constructor(options) { + this.category = options.category; + this.color = options.color; + } } exports.default = Annotation; exports.ADDED_ANNOTATION = new Annotation({ - category: 'Added', - color: 'green', + category: "Added", + color: "green", }); exports.REMOVED_ANNOTATION = new Annotation({ - category: 'Removed', - color: 'red', + category: "Removed", + color: "red", }); exports.UNCHANGED_ANNOTATION = new Annotation({ - category: 'Unchanged', - color: 'brown', + category: "Unchanged", + color: "brown", }); exports.DETECTED_ANNOTATION = new Annotation({ - category: 'Detected', - color: 'green', + category: "Detected", + color: "green", }); exports.MODIFIED_ANNOTATION = new Annotation({ - category: 'Modified', - color: 'green', + category: "Modified", + color: "green", }); diff --git a/src/scrapers/pdf/lib/pdf2md/models/HeadlineFinder.js b/src/scrapers/pdf/lib/pdf2md/models/HeadlineFinder.js index 428aacefb6..f2fb8da2ec 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/HeadlineFinder.js +++ b/src/scrapers/pdf/lib/pdf2md/models/HeadlineFinder.js @@ -1,40 +1,40 @@ -const { normalizedCharCodeArray } = require('../util/string-functions'); +const { normalizedCharCodeArray } = require("../util/string-functions"); module.exports = class HeadlineFinder { - constructor(options) { - this.headlineCharCodes = normalizedCharCodeArray(options.headline); - this.stackedLineItems = []; - this.stackedChars = 0; - } + constructor(options) { + this.headlineCharCodes = normalizedCharCodeArray(options.headline); + this.stackedLineItems = []; + this.stackedChars = 0; + } - consume(lineItem) { - // TODO avoid join - const normalizedCharCodes = normalizedCharCodeArray(lineItem.text()); - const matchAll = this.matchAll(normalizedCharCodes); - if (matchAll) { - this.stackedLineItems.push(lineItem); - this.stackedChars += normalizedCharCodes.length; - if (this.stackedChars === this.headlineCharCodes.length) { - return this.stackedLineItems; - } - } else { - if (this.stackedChars > 0) { - this.stackedChars = 0; - this.stackedLineItems = []; - this.consume(lineItem); // test again without stack - } - } - return null; - } + consume(lineItem) { + // TODO avoid join + const normalizedCharCodes = normalizedCharCodeArray(lineItem.text()); + const matchAll = this.matchAll(normalizedCharCodes); + if (matchAll) { + this.stackedLineItems.push(lineItem); + this.stackedChars += normalizedCharCodes.length; + if (this.stackedChars === this.headlineCharCodes.length) { + return this.stackedLineItems; + } + } else { + if (this.stackedChars > 0) { + this.stackedChars = 0; + this.stackedLineItems = []; + this.consume(lineItem); // test again without stack + } + } + return null; + } - matchAll(normalizedCharCodes) { - for (var i = 0; i < normalizedCharCodes.length; i++) { - const headlineChar = this.headlineCharCodes[this.stackedChars + i]; - const textItemChar = normalizedCharCodes[i]; - if (textItemChar !== headlineChar) { - return false; - } - } - return true; - } + matchAll(normalizedCharCodes) { + for (var i = 0; i < normalizedCharCodes.length; i++) { + const headlineChar = this.headlineCharCodes[this.stackedChars + i]; + const textItemChar = normalizedCharCodes[i]; + if (textItemChar !== headlineChar) { + return false; + } + } + return true; + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/LineConverter.js b/src/scrapers/pdf/lib/pdf2md/models/LineConverter.js index 123ba1649b..2f9c5a0413 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/LineConverter.js +++ b/src/scrapers/pdf/lib/pdf2md/models/LineConverter.js @@ -1,167 +1,167 @@ -const TextItem = require('./TextItem'); -const Word = require('./Word'); -const WordType = require('./markdown/WordType'); -const WordFormat = require('./markdown/WordFormat'); -const LineItem = require('./LineItem'); -const StashingStream = require('./StashingStream'); -const ParsedElements = require('./ParsedElements'); -const { isNumber, isListItemCharacter } = require('../util/string-functions'); -const { sortByX } = require('../util/page-item-functions'); +const TextItem = require("./TextItem"); +const Word = require("./Word"); +const WordType = require("./markdown/WordType"); +const WordFormat = require("./markdown/WordFormat"); +const LineItem = require("./LineItem"); +const StashingStream = require("./StashingStream"); +const ParsedElements = require("./ParsedElements"); +const { isNumber, isListItemCharacter } = require("../util/string-functions"); +const { sortByX } = require("../util/page-item-functions"); // Converts text items which have been grouped to a line (through TextItemLineGrouper) to a single LineItem doing inline transformations like // 'whitespace removal', bold/emphasis annotation, link-detection, etc.. module.exports = class LineConverter { - constructor(fontToFormats) { - this.fontToFormats = fontToFormats; - } + constructor(fontToFormats) { + this.fontToFormats = fontToFormats; + } - // returns a CombineResult - compact(textItems /*: TextItem[] */) /*: LineItem */ { - // we can't trust order of occurence, esp. footnoteLinks like to come last - sortByX(textItems); + // returns a CombineResult + compact(textItems /*: TextItem[] */) /*: LineItem */ { + // we can't trust order of occurence, esp. footnoteLinks like to come last + sortByX(textItems); - const wordStream = new WordDetectionStream(this.fontToFormats); - wordStream.consumeAll(textItems.map((item) => new TextItem({ ...item }))); - const words = wordStream.complete(); + const wordStream = new WordDetectionStream(this.fontToFormats); + wordStream.consumeAll(textItems.map((item) => new TextItem({ ...item }))); + const words = wordStream.complete(); - var maxHeight = 0; - var widthSum = 0; - textItems.forEach((item) => { - maxHeight = Math.max(maxHeight, item.height); - widthSum += item.width; - }); - return new LineItem({ - x: textItems[0].x, - y: textItems[0].y, - height: maxHeight, - width: widthSum, - words: words, - parsedElements: new ParsedElements({ - footnoteLinks: wordStream.footnoteLinks, - footnotes: wordStream.footnotes, - containLinks: wordStream.containLinks, - formattedWords: wordStream.formattedWords, - }), - }); - } + var maxHeight = 0; + var widthSum = 0; + textItems.forEach((item) => { + maxHeight = Math.max(maxHeight, item.height); + widthSum += item.width; + }); + return new LineItem({ + x: textItems[0].x, + y: textItems[0].y, + height: maxHeight, + width: widthSum, + words: words, + parsedElements: new ParsedElements({ + footnoteLinks: wordStream.footnoteLinks, + footnotes: wordStream.footnotes, + containLinks: wordStream.containLinks, + formattedWords: wordStream.formattedWords, + }), + }); + } }; class WordDetectionStream extends StashingStream { - constructor(fontToFormats) { - super(); - this.fontToFormats = fontToFormats; - this.footnoteLinks = []; - this.footnotes = []; - this.formattedWords = 0; - this.containLinks = false; - this.stashedNumber = false; - } + constructor(fontToFormats) { + super(); + this.fontToFormats = fontToFormats; + this.footnoteLinks = []; + this.footnotes = []; + this.formattedWords = 0; + this.containLinks = false; + this.stashedNumber = false; + } - shouldStash(item) { - // eslint-disable-line no-unused-vars - if (!this.firstY) { - this.firstY = item.y; - } - this.currentItem = item; - return true; - } + shouldStash(item) { + // eslint-disable-line no-unused-vars + if (!this.firstY) { + this.firstY = item.y; + } + this.currentItem = item; + return true; + } - onPushOnStash(item) { - // eslint-disable-line no-unused-vars - this.stashedNumber = isNumber(item.text.trim()); - } + onPushOnStash(item) { + // eslint-disable-line no-unused-vars + this.stashedNumber = isNumber(item.text.trim()); + } - doMatchesStash(lastItem, item) { - const lastItemFormat = this.fontToFormats.get(lastItem.font); - const itemFormat = this.fontToFormats.get(item.font); - if (lastItemFormat !== itemFormat) { - return false; - } - const itemIsANumber = isNumber(item.text.trim()); - return this.stashedNumber === itemIsANumber; - } + doMatchesStash(lastItem, item) { + const lastItemFormat = this.fontToFormats.get(lastItem.font); + const itemFormat = this.fontToFormats.get(item.font); + if (lastItemFormat !== itemFormat) { + return false; + } + const itemIsANumber = isNumber(item.text.trim()); + return this.stashedNumber === itemIsANumber; + } - doFlushStash(stash, results) { - if (this.stashedNumber) { - const joinedNumber = stash - .map((item) => item.text) - .join('') - .trim(); - if (stash[0].y > this.firstY) { - // footnote link - results.push( - new Word({ - string: `${joinedNumber}`, - type: WordType.FOOTNOTE_LINK, - }), - ); - this.footnoteLinks.push(parseInt(joinedNumber)); - } else if (this.currentItem && this.currentItem.y < stash[0].y) { - // footnote - results.push( - new Word({ - string: `${joinedNumber}`, - type: WordType.FOOTNOTE, - }), - ); - this.footnotes.push(joinedNumber); - } else { - this.copyStashItemsAsText(stash, results); - } - } else { - this.copyStashItemsAsText(stash, results); - } - } + doFlushStash(stash, results) { + if (this.stashedNumber) { + const joinedNumber = stash + .map((item) => item.text) + .join("") + .trim(); + if (stash[0].y > this.firstY) { + // footnote link + results.push( + new Word({ + string: `${joinedNumber}`, + type: WordType.FOOTNOTE_LINK, + }), + ); + this.footnoteLinks.push(Number.parseInt(joinedNumber)); + } else if (this.currentItem && this.currentItem.y < stash[0].y) { + // footnote + results.push( + new Word({ + string: `${joinedNumber}`, + type: WordType.FOOTNOTE, + }), + ); + this.footnotes.push(joinedNumber); + } else { + this.copyStashItemsAsText(stash, results); + } + } else { + this.copyStashItemsAsText(stash, results); + } + } - copyStashItemsAsText(stash, results) { - const format = this.fontToFormats.get(stash[0].font); - results.push(...this.itemsToWords(stash, format)); - } + copyStashItemsAsText(stash, results) { + const format = this.fontToFormats.get(stash[0].font); + results.push(...this.itemsToWords(stash, format)); + } - itemsToWords(items, formatName) { - const combinedText = combineText(items); - const words = combinedText.split(' '); - const format = formatName ? WordFormat.enumValueOf(formatName) : null; - return words - .filter((w) => w.trim().length > 0) - .map((word) => { - var type = null; - if (word.startsWith('http:')) { - this.containLinks = true; - type = WordType.LINK; - } else if (word.startsWith('www.')) { - this.containLinks = true; - word = `http://${word}`; - type = WordType.LINK; - } + itemsToWords(items, formatName) { + const combinedText = combineText(items); + const words = combinedText.split(" "); + const format = formatName ? WordFormat.enumValueOf(formatName) : null; + return words + .filter((w) => w.trim().length > 0) + .map((word) => { + var type = null; + if (word.startsWith("http:")) { + this.containLinks = true; + type = WordType.LINK; + } else if (word.startsWith("www.")) { + this.containLinks = true; + word = `http://${word}`; + type = WordType.LINK; + } - if (format) { - this.formattedWords++; - } - return new Word({ string: word, type, format }); - }); - } + if (format) { + this.formattedWords++; + } + return new Word({ string: word, type, format }); + }); + } } function combineText(textItems) { - var text = ''; - var lastItem; - textItems.forEach((textItem) => { - var textToAdd = textItem.text; - if (!text.endsWith(' ') && !textToAdd.startsWith(' ')) { - if (lastItem) { - const xDistance = textItem.x - lastItem.x - lastItem.width; - if (xDistance > 5) { - text += ' '; - } - } else { - if (isListItemCharacter(textItem.text)) { - textToAdd += ' '; - } - } - } - text += textToAdd; - lastItem = textItem; - }); - return text; + var text = ""; + var lastItem; + textItems.forEach((textItem) => { + var textToAdd = textItem.text; + if (!text.endsWith(" ") && !textToAdd.startsWith(" ")) { + if (lastItem) { + const xDistance = textItem.x - lastItem.x - lastItem.width; + if (xDistance > 5) { + text += " "; + } + } else { + if (isListItemCharacter(textItem.text)) { + textToAdd += " "; + } + } + } + text += textToAdd; + lastItem = textItem; + }); + return text; } diff --git a/src/scrapers/pdf/lib/pdf2md/models/LineItem.js b/src/scrapers/pdf/lib/pdf2md/models/LineItem.js index 8e8b803ad7..247c875dd2 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/LineItem.js +++ b/src/scrapers/pdf/lib/pdf2md/models/LineItem.js @@ -1,33 +1,33 @@ -const PageItem = require('./PageItem'); -const Word = require('./Word'); +const PageItem = require("./PageItem"); +const Word = require("./Word"); // A line within a page module.exports = class LineItem extends PageItem { - constructor(options) { - super(options); - this.x = options.x; - this.y = options.y; - this.width = options.width; - this.height = options.height; - this.words = options.words || []; - if (options.text && !options.words) { - this.words = options.text - .split(' ') - .filter((string) => string.trim().length > 0) - .map( - (wordAsString) => - new Word({ - string: wordAsString, - }), - ); - } - } + constructor(options) { + super(options); + this.x = options.x; + this.y = options.y; + this.width = options.width; + this.height = options.height; + this.words = options.words || []; + if (options.text && !options.words) { + this.words = options.text + .split(" ") + .filter((string) => string.trim().length > 0) + .map( + (wordAsString) => + new Word({ + string: wordAsString, + }), + ); + } + } - text() { - return this.wordStrings().join(' '); - } + text() { + return this.wordStrings().join(" "); + } - wordStrings() { - return this.words.map((word) => word.string); - } + wordStrings() { + return this.words.map((word) => word.string); + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/LineItemBlock.js b/src/scrapers/pdf/lib/pdf2md/models/LineItemBlock.js index d15051f5da..c462656282 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/LineItemBlock.js +++ b/src/scrapers/pdf/lib/pdf2md/models/LineItemBlock.js @@ -1,32 +1,34 @@ -const PageItem = require('./PageItem'); -const LineItem = require('./LineItem'); +const PageItem = require("./PageItem"); +const LineItem = require("./LineItem"); // A block of LineItem[] within a Page module.exports = class LineItemBlock extends PageItem { - constructor(options) { - super(options); - this.items = []; - if (options.items) { - options.items.forEach((item) => this.addItem(item)); - } - } + constructor(options) { + super(options); + this.items = []; + if (options.items) { + options.items.forEach((item) => this.addItem(item)); + } + } - addItem(item /*: LineItem */) { - if (this.type && item.type && this.type !== item.type) { - throw new Error(`Adding item of type ${item.type} to block of type ${this.type}`); - } - if (!this.type) { - this.type = item.type; - } - if (item.parsedElements) { - if (this.parsedElements) { - this.parsedElements.add(item.parsedElements); - } else { - this.parsedElements = item.parsedElements; - } - } - const copiedItem = new LineItem({ ...item }); - copiedItem.type = null; - this.items.push(copiedItem); - } + addItem(item /*: LineItem */) { + if (this.type && item.type && this.type !== item.type) { + throw new Error( + `Adding item of type ${item.type} to block of type ${this.type}`, + ); + } + if (!this.type) { + this.type = item.type; + } + if (item.parsedElements) { + if (this.parsedElements) { + this.parsedElements.add(item.parsedElements); + } else { + this.parsedElements = item.parsedElements; + } + } + const copiedItem = new LineItem({ ...item }); + copiedItem.type = null; + this.items.push(copiedItem); + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/Metadata.js b/src/scrapers/pdf/lib/pdf2md/models/Metadata.js index c51179a576..9fffc2f626 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/Metadata.js +++ b/src/scrapers/pdf/lib/pdf2md/models/Metadata.js @@ -1,15 +1,15 @@ // Metadata of the PDF document module.exports = class Metadata { - constructor(originalMetadata) { - if (originalMetadata.metadata) { - this.title = originalMetadata.metadata.get('dc:title'); - this.creator = originalMetadata.metadata.get('xap:creatortool'); - this.producer = originalMetadata.metadata.get('pdf:producer'); - } else { - this.title = originalMetadata.info.Title; - this.author = originalMetadata.info.Author; - this.creator = originalMetadata.info.Creator; - this.producer = originalMetadata.info.Producer; - } - } + constructor(originalMetadata) { + if (originalMetadata.metadata) { + this.title = originalMetadata.metadata.get("dc:title"); + this.creator = originalMetadata.metadata.get("xap:creatortool"); + this.producer = originalMetadata.metadata.get("pdf:producer"); + } else { + this.title = originalMetadata.info.Title; + this.author = originalMetadata.info.Author; + this.creator = originalMetadata.info.Creator; + this.producer = originalMetadata.info.Producer; + } + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/Page.js b/src/scrapers/pdf/lib/pdf2md/models/Page.js index 1c52c695d7..769f3e4c6d 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/Page.js +++ b/src/scrapers/pdf/lib/pdf2md/models/Page.js @@ -1,7 +1,7 @@ // A page which holds PageItems displayable via PdfPageView module.exports = class Page { - constructor(options) { - this.index = options.index; - this.items = options.items || []; // PageItem - } + constructor(options) { + this.index = options.index; + this.items = options.items || []; // PageItem + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/PageItem.js b/src/scrapers/pdf/lib/pdf2md/models/PageItem.js index d5e0004bad..5d3ca16f60 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/PageItem.js +++ b/src/scrapers/pdf/lib/pdf2md/models/PageItem.js @@ -1,11 +1,11 @@ // A abstract PageItem class, can be TextItem, LineItem or LineItemBlock module.exports = class PageItem { - constructor(options) { - if (this.constructor === PageItem) { - throw new TypeError('Can not construct abstract class.'); - } - this.type = options.type; - this.annotation = options.annotation; - this.parsedElements = options.parsedElements; - } + constructor(options) { + if (this.constructor === PageItem) { + throw new TypeError("Can not construct abstract class."); + } + this.type = options.type; + this.annotation = options.annotation; + this.parsedElements = options.parsedElements; + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/ParseResult.js b/src/scrapers/pdf/lib/pdf2md/models/ParseResult.js index 128a75992d..a894a86f87 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/ParseResult.js +++ b/src/scrapers/pdf/lib/pdf2md/models/ParseResult.js @@ -1,8 +1,8 @@ // The result of a PDF parse respectively a Transformation module.exports = class ParseResult { - constructor(options) { - this.pages = options.pages; // like Page[] - this.globals = options.globals; // properties accasable for all the following transformations in debug mode - this.messages = options.messages; // something to show only for the transformation in debug mode - } + constructor(options) { + this.pages = options.pages; // like Page[] + this.globals = options.globals; // properties accasable for all the following transformations in debug mode + this.messages = options.messages; // something to show only for the transformation in debug mode + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/ParsedElements.js b/src/scrapers/pdf/lib/pdf2md/models/ParsedElements.js index bbef3f948d..6e2546c990 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/ParsedElements.js +++ b/src/scrapers/pdf/lib/pdf2md/models/ParsedElements.js @@ -1,15 +1,17 @@ module.exports = class ParsedElements { - constructor(options) { - this.footnoteLinks = options.footnoteLinks || []; - this.footnotes = options.footnotes || []; - this.containLinks = options.containLinks; - this.formattedWords = options.formattedWords; - } + constructor(options) { + this.footnoteLinks = options.footnoteLinks || []; + this.footnotes = options.footnotes || []; + this.containLinks = options.containLinks; + this.formattedWords = options.formattedWords; + } - add(parsedElements) { - this.footnoteLinks = this.footnoteLinks.concat(parsedElements.footnoteLinks); - this.footnotes = this.footnotes.concat(parsedElements.footnotes); - this.containLinks = this.containLinks || parsedElements.containLinks; - this.formattedWords += parsedElements.formattedWords; - } + add(parsedElements) { + this.footnoteLinks = this.footnoteLinks.concat( + parsedElements.footnoteLinks, + ); + this.footnotes = this.footnotes.concat(parsedElements.footnotes); + this.containLinks = this.containLinks || parsedElements.containLinks; + this.formattedWords += parsedElements.formattedWords; + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/StashingStream.js b/src/scrapers/pdf/lib/pdf2md/models/StashingStream.js index 2f9d4adc2d..13be3dea3b 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/StashingStream.js +++ b/src/scrapers/pdf/lib/pdf2md/models/StashingStream.js @@ -1,73 +1,77 @@ // Abstract stream which allows stash items temporarily module.exports = class StashingStream { - constructor() { - if (this.constructor === StashingStream) { - throw new TypeError('Can not construct abstract class.'); - } - this.results = []; - this.stash = []; - } + constructor() { + if (this.constructor === StashingStream) { + throw new TypeError("Can not construct abstract class."); + } + this.results = []; + this.stash = []; + } - consumeAll(items) { - items.forEach((item) => this.consume(item)); - } + consumeAll(items) { + items.forEach((item) => this.consume(item)); + } - consume(item) { - if (this.shouldStash(item)) { - if (!this.matchesStash(item)) { - this.flushStash(); - } - this.pushOnStash(item); - } else { - if (this.stash.length > 0) { - this.flushStash(); - } - this.results.push(item); - } - } + consume(item) { + if (this.shouldStash(item)) { + if (!this.matchesStash(item)) { + this.flushStash(); + } + this.pushOnStash(item); + } else { + if (this.stash.length > 0) { + this.flushStash(); + } + this.results.push(item); + } + } - pushOnStash(item) { - this.onPushOnStash(item); - this.stash.push(item); - } + pushOnStash(item) { + this.onPushOnStash(item); + this.stash.push(item); + } - complete() { - if (this.stash.length > 0) { - this.flushStash(); - } - return this.results; - } + complete() { + if (this.stash.length > 0) { + this.flushStash(); + } + return this.results; + } - // return true if the item matches the items of the stack - matchesStash(item) { - if (this.stash.length === 0) { - return true; - } - const lastItem = this.stash[this.stash.length - 1]; - return this.doMatchesStash(lastItem, item); - } + // return true if the item matches the items of the stack + matchesStash(item) { + if (this.stash.length === 0) { + return true; + } + const lastItem = this.stash[this.stash.length - 1]; + return this.doMatchesStash(lastItem, item); + } - flushStash() { - if (this.stash.length > 0) { - this.doFlushStash(this.stash, this.results); - this.stash = []; - } - } + flushStash() { + if (this.stash.length > 0) { + this.doFlushStash(this.stash, this.results); + this.stash = []; + } + } - onPushOnStash(item) { - // eslint-disable-line no-unused-vars - // sub-classes may override - } + onPushOnStash(item) { + // eslint-disable-line no-unused-vars + // sub-classes may override + } - shouldStash(item) { - throw new TypeError(' Do not call abstract method foo from child.' + item); - } + shouldStash(item) { + throw new TypeError(" Do not call abstract method foo from child." + item); + } - doMatchesStash(lastItem, item) { - throw new TypeError(' Do not call abstract method foo from child.' + lastItem + item); - } + doMatchesStash(lastItem, item) { + throw new TypeError( + " Do not call abstract method foo from child." + lastItem + item, + ); + } - doFlushStash(stash, results) { - throw new TypeError(' Do not call abstract method foo from child.' + stash + results); - } + doFlushStash(stash, results) { + throw new TypeError( + " Do not call abstract method foo from child." + stash + results, + ); + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/TextItem.js b/src/scrapers/pdf/lib/pdf2md/models/TextItem.js index 7039699961..a2f6e04a20 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/TextItem.js +++ b/src/scrapers/pdf/lib/pdf2md/models/TextItem.js @@ -1,18 +1,18 @@ -const PageItem = require('./PageItem'); +const PageItem = require("./PageItem"); // A text item, i.e. a line or a word within a page module.exports = class TextItem extends PageItem { - constructor(options) { - super(options); - this.x = options.x; - this.y = options.y; - this.width = options.width; - this.height = options.height; - this.text = options.text; - this.font = options.font; + constructor(options) { + super(options); + this.x = options.x; + this.y = options.y; + this.width = options.width; + this.height = options.height; + this.text = options.text; + this.font = options.font; - this.lineFormat = options.lineFormat; - this.unopenedFormat = options.unopenedFormat; - this.unclosedFormat = options.unclosedFormat; - } + this.lineFormat = options.lineFormat; + this.unopenedFormat = options.unopenedFormat; + this.unclosedFormat = options.unclosedFormat; + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/TextItemLineGrouper.js b/src/scrapers/pdf/lib/pdf2md/models/TextItemLineGrouper.js index 77016e889e..ca2fbcb282 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/TextItemLineGrouper.js +++ b/src/scrapers/pdf/lib/pdf2md/models/TextItemLineGrouper.js @@ -4,31 +4,34 @@ import TextItem from './TextItem' */ -const { sortByX } = require('../util/page-item-functions'); +const { sortByX } = require("../util/page-item-functions"); // Groups all text items which are on the same y line module.exports = class TextItemLineGrouper { - constructor(options) { - this.mostUsedDistance = options.mostUsedDistance || 12; - } + constructor(options) { + this.mostUsedDistance = options.mostUsedDistance || 12; + } - // returns a CombineResult - group(textItems /*: TextItem[] */) /*: TextItem[][] */ { - const lines = []; - var currentLine = []; - textItems.forEach((item) => { - if (currentLine.length > 0 && Math.abs(currentLine[0].y - item.y) >= this.mostUsedDistance / 2) { - lines.push(currentLine); - currentLine = []; - } - currentLine.push(item); - }); - lines.push(currentLine); + // returns a CombineResult + group(textItems /*: TextItem[] */) /*: TextItem[][] */ { + const lines = []; + var currentLine = []; + textItems.forEach((item) => { + if ( + currentLine.length > 0 && + Math.abs(currentLine[0].y - item.y) >= this.mostUsedDistance / 2 + ) { + lines.push(currentLine); + currentLine = []; + } + currentLine.push(item); + }); + lines.push(currentLine); - lines.forEach((textItems) => { - // we can't trust order of occurence, esp. footnoteLinks like to come last - sortByX(textItems); - }); - return lines; - } + lines.forEach((textItems) => { + // we can't trust order of occurence, esp. footnoteLinks like to come last + sortByX(textItems); + }); + return lines; + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/Word.js b/src/scrapers/pdf/lib/pdf2md/models/Word.js index 5e09936398..95c8ee8d6c 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/Word.js +++ b/src/scrapers/pdf/lib/pdf2md/models/Word.js @@ -1,7 +1,7 @@ module.exports = class Word { - constructor(options) { - this.string = options.string; - this.type = options.type; // WordType - this.format = options.format; // WordFormat - } + constructor(options) { + this.string = options.string; + this.type = options.type; // WordType + this.format = options.format; // WordFormat + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/markdown/BlockType.js b/src/scrapers/pdf/lib/pdf2md/models/markdown/BlockType.js index 192402302c..403a75803e 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/markdown/BlockType.js +++ b/src/scrapers/pdf/lib/pdf2md/models/markdown/BlockType.js @@ -4,63 +4,68 @@ import LineItemBlock from '../LineItemBlock' */ -const { Enum } = require('enumify'); +const { Enum } = require("enumify"); function firstFormat(lineItem) { - if (lineItem.words.length === 0) { - return null; - } - return lineItem.words[0].format; + if (lineItem.words.length === 0) { + return null; + } + return lineItem.words[0].format; } function isPunctationCharacter(string) { - if (string.length !== 1) { - return false; - } - return string[0] === '.' || string[0] === '!' || string[0] === '?'; + if (string.length !== 1) { + return false; + } + return string[0] === "." || string[0] === "!" || string[0] === "?"; } function linesToText(lineItems, disableInlineFormats) { - var text = ''; - var openFormat; - - const closeFormat = () => { - text += openFormat.endSymbol; - openFormat = null; - }; - - lineItems.forEach((line, lineIndex) => { - line.words.forEach((word, i) => { - const wordType = word.type; - const wordFormat = word.format; - if (openFormat && (!wordFormat || wordFormat !== openFormat)) { - closeFormat(); - } - - if (i > 0 && !(wordType && wordType.attachWithoutWhitespace) && !isPunctationCharacter(word.string)) { - text += ' '; - } - - if (wordFormat && !openFormat && !disableInlineFormats) { - openFormat = wordFormat; - text += openFormat.startSymbol; - } - - if (wordType && (!disableInlineFormats || wordType.plainTextFormat)) { - text += wordType.toText(word.string); - } else { - text += word.string; - } - }); - if ( - openFormat && - (lineIndex === lineItems.length - 1 || firstFormat(lineItems[lineIndex + 1]) !== openFormat) - ) { - closeFormat(); - } - text += '\n'; - }); - return text; + var text = ""; + var openFormat; + + const closeFormat = () => { + text += openFormat.endSymbol; + openFormat = null; + }; + + lineItems.forEach((line, lineIndex) => { + line.words.forEach((word, i) => { + const wordType = word.type; + const wordFormat = word.format; + if (openFormat && (!wordFormat || wordFormat !== openFormat)) { + closeFormat(); + } + + if ( + i > 0 && + !(wordType && wordType.attachWithoutWhitespace) && + !isPunctationCharacter(word.string) + ) { + text += " "; + } + + if (wordFormat && !openFormat && !disableInlineFormats) { + openFormat = wordFormat; + text += openFormat.startSymbol; + } + + if (wordType && (!disableInlineFormats || wordType.plainTextFormat)) { + text += wordType.toText(word.string); + } else { + text += word.string; + } + }); + if ( + openFormat && + (lineIndex === lineItems.length - 1 || + firstFormat(lineItems[lineIndex + 1]) !== openFormat) + ) { + closeFormat(); + } + text += "\n"; + }); + return text; } // An Markdown block @@ -69,109 +74,117 @@ class BlockType extends Enum {} module.exports = BlockType; BlockType.initEnum({ - H1: { - headline: true, - headlineLevel: 1, - toText(block /*: LineItemBlock */) /*: string */ { - return '# ' + linesToText(block.items, true); - }, - }, - H2: { - headline: true, - headlineLevel: 2, - toText(block /*: LineItemBlock */) /*: string */ { - return '## ' + linesToText(block.items, true); - }, - }, - H3: { - headline: true, - headlineLevel: 3, - toText(block /*: LineItemBlock */) /*: string */ { - return '### ' + linesToText(block.items, true); - }, - }, - H4: { - headline: true, - headlineLevel: 4, - toText(block /*: LineItemBlock */) /*: string */ { - return '#### ' + linesToText(block.items, true); - }, - }, - H5: { - headline: true, - headlineLevel: 5, - toText(block /*: LineItemBlock */) /*: string */ { - return '##### ' + linesToText(block.items, true); - }, - }, - H6: { - headline: true, - headlineLevel: 6, - toText(block /*: LineItemBlock */) /*: string */ { - return '###### ' + linesToText(block.items, true); - }, - }, - TOC: { - mergeToBlock: true, - toText(block /*: LineItemBlock */) /*: string */ { - return linesToText(block.items, true); - }, - }, - FOOTNOTES: { - mergeToBlock: true, - mergeFollowingNonTypedItems: true, - toText(block /*: LineItemBlock */) /*: string */ { - return linesToText(block.items, false); - }, - }, - CODE: { - mergeToBlock: true, - toText(block /*: LineItemBlock */) /*: string */ { - return '```\n' + linesToText(block.items, true) + '```'; - }, - }, - LIST: { - mergeToBlock: false, - mergeFollowingNonTypedItemsWithSmallDistance: true, - toText(block /*: LineItemBlock */) /*: string */ { - return linesToText(block.items, false); - }, - }, - PARAGRAPH: { - toText(block /*: LineItemBlock */) /*: string */ { - return linesToText(block.items, false); - }, - }, + H1: { + headline: true, + headlineLevel: 1, + toText(block /*: LineItemBlock */) /*: string */ { + return "# " + linesToText(block.items, true); + }, + }, + H2: { + headline: true, + headlineLevel: 2, + toText(block /*: LineItemBlock */) /*: string */ { + return "## " + linesToText(block.items, true); + }, + }, + H3: { + headline: true, + headlineLevel: 3, + toText(block /*: LineItemBlock */) /*: string */ { + return "### " + linesToText(block.items, true); + }, + }, + H4: { + headline: true, + headlineLevel: 4, + toText(block /*: LineItemBlock */) /*: string */ { + return "#### " + linesToText(block.items, true); + }, + }, + H5: { + headline: true, + headlineLevel: 5, + toText(block /*: LineItemBlock */) /*: string */ { + return "##### " + linesToText(block.items, true); + }, + }, + H6: { + headline: true, + headlineLevel: 6, + toText(block /*: LineItemBlock */) /*: string */ { + return "###### " + linesToText(block.items, true); + }, + }, + TOC: { + mergeToBlock: true, + toText(block /*: LineItemBlock */) /*: string */ { + return linesToText(block.items, true); + }, + }, + FOOTNOTES: { + mergeToBlock: true, + mergeFollowingNonTypedItems: true, + toText(block /*: LineItemBlock */) /*: string */ { + return linesToText(block.items, false); + }, + }, + CODE: { + mergeToBlock: true, + toText(block /*: LineItemBlock */) /*: string */ { + return "```\n" + linesToText(block.items, true) + "```"; + }, + }, + LIST: { + mergeToBlock: false, + mergeFollowingNonTypedItemsWithSmallDistance: true, + toText(block /*: LineItemBlock */) /*: string */ { + return linesToText(block.items, false); + }, + }, + PARAGRAPH: { + toText(block /*: LineItemBlock */) /*: string */ { + return linesToText(block.items, false); + }, + }, }); -module.exports.isHeadline = function isHeadline(type /*: BlockType */) /*: boolean */ { - return type && type.name.length === 2 && type.name[0] === 'H'; +module.exports.isHeadline = function isHeadline( + type /*: BlockType */, +) /*: boolean */ { + return type && type.name.length === 2 && type.name[0] === "H"; }; -module.exports.blockToText = function blockToText(block /*: LineItemBlock */) /*: string */ { - if (!block.type) { - return linesToText(block.items, false); - } - return block.type.toText(block); +module.exports.blockToText = function blockToText( + block /*: LineItemBlock */, +) /*: string */ { + if (!block.type) { + return linesToText(block.items, false); + } + return block.type.toText(block); }; module.exports.headlineByLevel = function headlineByLevel(level) { - if (level === 1) { - return BlockType.H1; - } else if (level === 2) { - return BlockType.H2; - } else if (level === 3) { - return BlockType.H3; - } else if (level === 4) { - return BlockType.H4; - } else if (level === 5) { - return BlockType.H5; - } else if (level === 6) { - return BlockType.H6; - } else { - // if level is >= 6, just use BlockType H6 - // eslint-disable-next-line no-console - console.warn('Unsupported headline level: ' + level + ' (supported are 1-6), defaulting to level 6'); - return BlockType.H6; - } + if (level === 1) { + return BlockType.H1; + } else if (level === 2) { + return BlockType.H2; + } else if (level === 3) { + return BlockType.H3; + } else if (level === 4) { + return BlockType.H4; + } else if (level === 5) { + return BlockType.H5; + } else if (level === 6) { + return BlockType.H6; + } else { + // if level is >= 6, just use BlockType H6 + // eslint-disable-next-line no-console + console.warn( + "Unsupported headline level: " + + level + + " (supported are 1-6), defaulting to level 6", + ); + return BlockType.H6; + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/markdown/WordFormat.js b/src/scrapers/pdf/lib/pdf2md/models/markdown/WordFormat.js index b38c36c36f..2f87c11d14 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/markdown/WordFormat.js +++ b/src/scrapers/pdf/lib/pdf2md/models/markdown/WordFormat.js @@ -1,4 +1,4 @@ -const { Enum } = require('enumify'); +const { Enum } = require("enumify"); // The format of a word element class WordFormat extends Enum {} @@ -6,16 +6,16 @@ class WordFormat extends Enum {} module.exports = WordFormat; WordFormat.initEnum({ - BOLD: { - startSymbol: '**', - endSymbol: '**', - }, - OBLIQUE: { - startSymbol: '_', - endSymbol: '_', - }, - BOLD_OBLIQUE: { - startSymbol: '**_', - endSymbol: '_**', - }, + BOLD: { + startSymbol: "**", + endSymbol: "**", + }, + OBLIQUE: { + startSymbol: "_", + endSymbol: "_", + }, + BOLD_OBLIQUE: { + startSymbol: "**_", + endSymbol: "_**", + }, }); diff --git a/src/scrapers/pdf/lib/pdf2md/models/markdown/WordType.js b/src/scrapers/pdf/lib/pdf2md/models/markdown/WordType.js index 07be4ae30b..de6d66b1a2 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/markdown/WordType.js +++ b/src/scrapers/pdf/lib/pdf2md/models/markdown/WordType.js @@ -1,4 +1,4 @@ -const { Enum } = require('enumify'); +const { Enum } = require("enumify"); // An Markdown word element class WordType extends Enum {} @@ -6,22 +6,22 @@ class WordType extends Enum {} module.exports = WordType; WordType.initEnum({ - LINK: { - toText(string) { - return `[${string}](${string})`; - }, - }, - FOOTNOTE_LINK: { - attachWithoutWhitespace: true, - plainTextFormat: true, - toText(string) { - return `^${string}`; - // return `[${string}](#${string})` - }, - }, - FOOTNOTE: { - toText(string) { - return `(^${string})`; - }, - }, + LINK: { + toText(string) { + return `[${string}](${string})`; + }, + }, + FOOTNOTE_LINK: { + attachWithoutWhitespace: true, + plainTextFormat: true, + toText(string) { + return `^${string}`; + // return `[${string}](#${string})` + }, + }, + FOOTNOTE: { + toText(string) { + return `(^${string})`; + }, + }, }); diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/ToLineItemBlockTransformation.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/ToLineItemBlockTransformation.js index 2c7cc04cbc..6db9ce4de2 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/ToLineItemBlockTransformation.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/ToLineItemBlockTransformation.js @@ -4,26 +4,28 @@ import ParseResult from '../../ParseResult' */ -const Transformation = require('./Transformation'); -const LineItemBlock = require('../LineItemBlock'); -const { REMOVED_ANNOTATION } = require('../Annotation'); +const Transformation = require("./Transformation"); +const LineItemBlock = require("../LineItemBlock"); +const { REMOVED_ANNOTATION } = require("../Annotation"); // Abstract class for transformations producing LineItemBlock(s) to be shown in the LineItemBlockPageView module.exports = class ToLineItemBlockTransformation extends Transformation { - constructor(name) { - super(name, LineItemBlock.name); - if (this.constructor === ToLineItemBlockTransformation) { - throw new TypeError('Can not construct abstract class.'); - } - } + constructor(name) { + super(name, LineItemBlock.name); + if (this.constructor === ToLineItemBlockTransformation) { + throw new TypeError("Can not construct abstract class."); + } + } - completeTransform(parseResult /*: ParseResult */) /*: ParseResult */ { - // The usual cleanup - parseResult.messages = []; - parseResult.pages.forEach((page) => { - page.items = page.items.filter((item) => !item.annotation || item.annotation !== REMOVED_ANNOTATION); - page.items.forEach((item) => (item.annotation = null)); - }); - return parseResult; - } + completeTransform(parseResult /*: ParseResult */) /*: ParseResult */ { + // The usual cleanup + parseResult.messages = []; + parseResult.pages.forEach((page) => { + page.items = page.items.filter( + (item) => !item.annotation || item.annotation !== REMOVED_ANNOTATION, + ); + page.items.forEach((item) => (item.annotation = null)); + }); + return parseResult; + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/ToLineItemTransformation.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/ToLineItemTransformation.js index 246538985f..19884c1979 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/ToLineItemTransformation.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/ToLineItemTransformation.js @@ -4,26 +4,28 @@ import ParseResult from '../ParseResult' */ -const Transformation = require('./Transformation'); -const LineItem = require('../LineItem'); -const { REMOVED_ANNOTATION } = require('../Annotation'); +const Transformation = require("./Transformation"); +const LineItem = require("../LineItem"); +const { REMOVED_ANNOTATION } = require("../Annotation"); // Abstract class for transformations producing LineItem(s) to be shown in the LineItemPageView module.exports = class ToLineItemTransformation extends Transformation { - constructor(name) { - super(name, LineItem.name); - if (this.constructor === ToLineItemTransformation) { - throw new TypeError('Can not construct abstract class.'); - } - } + constructor(name) { + super(name, LineItem.name); + if (this.constructor === ToLineItemTransformation) { + throw new TypeError("Can not construct abstract class."); + } + } - completeTransform(parseResult /*: ParseResult */) /*: ParseResult */ { - // The usual cleanup - parseResult.messages = []; - parseResult.pages.forEach((page) => { - page.items = page.items.filter((item) => !item.annotation || item.annotation !== REMOVED_ANNOTATION); - page.items.forEach((item) => (item.annotation = null)); - }); - return parseResult; - } + completeTransform(parseResult /*: ParseResult */) /*: ParseResult */ { + // The usual cleanup + parseResult.messages = []; + parseResult.pages.forEach((page) => { + page.items = page.items.filter( + (item) => !item.annotation || item.annotation !== REMOVED_ANNOTATION, + ); + page.items.forEach((item) => (item.annotation = null)); + }); + return parseResult; + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/ToMarkdown.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/ToMarkdown.js index 71cf22ade4..0a4efba1ec 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/ToMarkdown.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/ToMarkdown.js @@ -1,42 +1,42 @@ // @flow -const Transformation = require('./Transformation'); -const ParseResult = require('../ParseResult'); +const Transformation = require("./Transformation"); +const ParseResult = require("../ParseResult"); module.exports = class ToMarkdown extends Transformation { - constructor() { - super('To Markdown', 'String'); - } + constructor() { + super("To Markdown", "String"); + } - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - parseResult.pages.forEach((page) => { - var text = ''; - page.items.forEach((block) => { - // Concatenate all words in the same block, unless it's a Table of Contents block - let concatText; - if (block.category === 'TOC') { - concatText = block.text; - } else { - concatText = block.text.replace(/(\r\n|\n|\r)/gm, ' '); - } + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + parseResult.pages.forEach((page) => { + var text = ""; + page.items.forEach((block) => { + // Concatenate all words in the same block, unless it's a Table of Contents block + let concatText; + if (block.category === "TOC") { + concatText = block.text; + } else { + concatText = block.text.replace(/(\r\n|\n|\r)/gm, " "); + } - // Concatenate words that were previously broken up by newline - if (block.category !== 'LIST') { - concatText = concatText.split('- ').join(''); - } + // Concatenate words that were previously broken up by newline + if (block.category !== "LIST") { + concatText = concatText.split("- ").join(""); + } - // Assume there are no code blocks in our documents - if (block.category === 'CODE') { - concatText = concatText.split('`').join(''); - } + // Assume there are no code blocks in our documents + if (block.category === "CODE") { + concatText = concatText.split("`").join(""); + } - text += concatText + '\n\n'; - }); + text += concatText + "\n\n"; + }); - page.items = [text]; - }); - return new ParseResult({ - ...parseResult, - }); - } + page.items = [text]; + }); + return new ParseResult({ + ...parseResult, + }); + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/ToTextBlocks.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/ToTextBlocks.js index 6a6c4a5634..d9c742286d 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/ToTextBlocks.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/ToTextBlocks.js @@ -1,27 +1,27 @@ -const Transformation = require('./Transformation'); -const ParseResult = require('../ParseResult'); -const { blockToText } = require('../markdown/BlockType'); +const Transformation = require("./Transformation"); +const ParseResult = require("../ParseResult"); +const { blockToText } = require("../markdown/BlockType"); module.exports = class ToTextBlocks extends Transformation { - constructor() { - super('To Text Blocks', 'TextBlock'); - } + constructor() { + super("To Text Blocks", "TextBlock"); + } - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - parseResult.pages.forEach((page) => { - const textItems = []; - page.items.forEach((block) => { - // TODO category to type (before have no unknowns, have paragraph) - const category = block.type ? block.type.name : 'Unknown'; - textItems.push({ - category: category, - text: blockToText(block), - }); - }); - page.items = textItems; - }); - return new ParseResult({ - ...parseResult, - }); - } + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + parseResult.pages.forEach((page) => { + const textItems = []; + page.items.forEach((block) => { + // TODO category to type (before have no unknowns, have paragraph) + const category = block.type ? block.type.name : "Unknown"; + textItems.push({ + category: category, + text: blockToText(block), + }); + }); + page.items = textItems; + }); + return new ParseResult({ + ...parseResult, + }); + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/ToTextItemTransformation.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/ToTextItemTransformation.js index f77a8334d9..83e8610c30 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/ToTextItemTransformation.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/ToTextItemTransformation.js @@ -4,26 +4,28 @@ import ParseResult from '../ParseResult' */ -const Transformation = require('./Transformation'); -const TextItem = require('../TextItem'); -const { REMOVED_ANNOTATION } = require('../Annotation'); +const Transformation = require("./Transformation"); +const TextItem = require("../TextItem"); +const { REMOVED_ANNOTATION } = require("../Annotation"); // Abstract class for transformations producing TextItem(s) to be shown in the TextItemPageView module.exports = class ToTextItemTransformation extends Transformation { - constructor(name) { - super(name, TextItem.name); - if (this.constructor === ToTextItemTransformation) { - throw new TypeError('Can not construct abstract class.'); - } - } + constructor(name) { + super(name, TextItem.name); + if (this.constructor === ToTextItemTransformation) { + throw new TypeError("Can not construct abstract class."); + } + } - completeTransform(parseResult /*: ParseResult */) /*: ParseResult */ { - // The usual cleanup - parseResult.messages = []; - parseResult.pages.forEach((page) => { - page.items = page.items.filter((item) => !item.annotation || item.annotation !== REMOVED_ANNOTATION); - page.items.forEach((item) => (item.annotation = null)); - }); - return parseResult; - } + completeTransform(parseResult /*: ParseResult */) /*: ParseResult */ { + // The usual cleanup + parseResult.messages = []; + parseResult.pages.forEach((page) => { + page.items = page.items.filter( + (item) => !item.annotation || item.annotation !== REMOVED_ANNOTATION, + ); + page.items.forEach((item) => (item.annotation = null)); + }); + return parseResult; + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/Transformation.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/Transformation.js index 8b46b538af..82d54f4f07 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/Transformation.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/Transformation.js @@ -6,27 +6,27 @@ import ParseResult from '../../ParseResult' // A transformation from an PdfPage to an PdfPage module.exports = class Transformation { - constructor(name, itemType) { - if (this.constructor === Transformation) { - throw new TypeError('Can not construct abstract class.'); - } - if (this.transform === Transformation.prototype.transform) { - throw new TypeError("Please implement abstract method 'transform()'."); - } - this.name = name; - this.itemType = itemType; - } + constructor(name, itemType) { + if (this.constructor === Transformation) { + throw new TypeError("Can not construct abstract class."); + } + if (this.transform === Transformation.prototype.transform) { + throw new TypeError("Please implement abstract method 'transform()'."); + } + this.name = name; + this.itemType = itemType; + } - // Transform an incoming ParseResult into an outgoing ParseResult - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - // eslint-disable-line no-unused-vars - throw new TypeError('Do not call abstract method foo from child.'); - } + // Transform an incoming ParseResult into an outgoing ParseResult + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + // eslint-disable-line no-unused-vars + throw new TypeError("Do not call abstract method foo from child."); + } - // Sometimes the transform() does only visualize a change. This methods then does the actual change. - completeTransform(parseResult /*: ParseResult */) /*: ParseResult */ { - // eslint-disable-line no-unused-vars - parseResult.messages = []; - return parseResult; - } + // Sometimes the transform() does only visualize a change. This methods then does the actual change. + completeTransform(parseResult /*: ParseResult */) /*: ParseResult */ { + // eslint-disable-line no-unused-vars + parseResult.messages = []; + return parseResult; + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item-block/DetectCodeQuoteBlocks.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item-block/DetectCodeQuoteBlocks.js index a4df164bca..e3c9da33c6 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item-block/DetectCodeQuoteBlocks.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item-block/DetectCodeQuoteBlocks.js @@ -1,49 +1,54 @@ // @flow -const ToLineItemBlockTransformation = require('../ToLineItemBlockTransformation'); -const ParseResult = require('../../ParseResult'); -const { DETECTED_ANNOTATION } = require('../../Annotation'); -const BlockType = require('../../markdown/BlockType'); -const { minXFromBlocks } = require('../../../util/page-item-functions'); +const ToLineItemBlockTransformation = require("../ToLineItemBlockTransformation"); +const ParseResult = require("../../ParseResult"); +const { DETECTED_ANNOTATION } = require("../../Annotation"); +const BlockType = require("../../markdown/BlockType"); +const { minXFromBlocks } = require("../../../util/page-item-functions"); // Detect items which are code/quote blocks -module.exports = class DetectCodeQuoteBlocks extends ToLineItemBlockTransformation { - constructor() { - super('$1'); - } +module.exports = class DetectCodeQuoteBlocks extends ( + ToLineItemBlockTransformation +) { + constructor() { + super("$1"); + } - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - const { mostUsedHeight } = parseResult.globals; - var foundCodeItems = 0; - parseResult.pages.forEach((page) => { - var minX = minXFromBlocks(page.items); - page.items.forEach((block) => { - if (!block.type && looksLikeCodeBlock(minX, block.items, mostUsedHeight)) { - block.annotation = DETECTED_ANNOTATION; - block.type = BlockType.CODE; - foundCodeItems++; - } - }); - }); + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + const { mostUsedHeight } = parseResult.globals; + var foundCodeItems = 0; + parseResult.pages.forEach((page) => { + var minX = minXFromBlocks(page.items); + page.items.forEach((block) => { + if ( + !block.type && + looksLikeCodeBlock(minX, block.items, mostUsedHeight) + ) { + block.annotation = DETECTED_ANNOTATION; + block.type = BlockType.CODE; + foundCodeItems++; + } + }); + }); - return new ParseResult({ - ...parseResult, - messages: ['Detected ' + foundCodeItems + ' code/quote items.'], - }); - } + return new ParseResult({ + ...parseResult, + messages: ["Detected " + foundCodeItems + " code/quote items."], + }); + } }; function looksLikeCodeBlock(minX, items, mostUsedHeight) { - if (items.length === 0) { - return false; - } - if (items.length === 1) { - return items[0].x > minX && items[0].height <= mostUsedHeight + 1; - } - for (var item of items) { - if (item.x === minX) { - return false; - } - } - return true; + if (items.length === 0) { + return false; + } + if (items.length === 1) { + return items[0].x > minX && items[0].height <= mostUsedHeight + 1; + } + for (var item of items) { + if (item.x === minX) { + return false; + } + } + return true; } diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item-block/DetectListLevels.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item-block/DetectListLevels.js index d2e09c177e..607d0d569f 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item-block/DetectListLevels.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item-block/DetectListLevels.js @@ -1,59 +1,66 @@ // @flow -const ToLineItemBlockTransformation = require('../ToLineItemBlockTransformation'); -const ParseResult = require('../../ParseResult'); -const Word = require('../../Word'); -const { MODIFIED_ANNOTATION, UNCHANGED_ANNOTATION } = require('../../Annotation'); -const BlockType = require('../../markdown/BlockType'); +const ToLineItemBlockTransformation = require("../ToLineItemBlockTransformation"); +const ParseResult = require("../../ParseResult"); +const Word = require("../../Word"); +const { + MODIFIED_ANNOTATION, + UNCHANGED_ANNOTATION, +} = require("../../Annotation"); +const BlockType = require("../../markdown/BlockType"); // Cares for proper sub-item spacing/leveling module.exports = class DetectListLevels extends ToLineItemBlockTransformation { - constructor() { - super('Level Lists'); - } + constructor() { + super("Level Lists"); + } - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - var listBlocks = 0; - var modifiedBlocks = 0; - parseResult.pages.forEach((page) => { - page.items - .filter((block) => block.type === BlockType.LIST) - .forEach((listBlock) => { - var lastItemX; - var currentLevel = 0; - const xByLevel = {}; - var modifiedBlock = false; - listBlock.items.forEach((item) => { - const isListItem = true; - if (lastItemX && isListItem) { - if (item.x > lastItemX) { - currentLevel++; - xByLevel[item.x] = currentLevel; - } else if (item.x < lastItemX) { - currentLevel = xByLevel[item.x]; - } - } else { - xByLevel[item.x] = 0; - } - if (currentLevel > 0) { - item.words = [new Word({ string: ' '.repeat(currentLevel * 3) })].concat(item.words); - modifiedBlock = true; - } - lastItemX = item.x; - }); - listBlocks++; - if (modifiedBlock) { - modifiedBlocks++; - listBlock.annotation = MODIFIED_ANNOTATION; - } else { - listBlock.annotation = UNCHANGED_ANNOTATION; - } - }); - }); + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + var listBlocks = 0; + var modifiedBlocks = 0; + parseResult.pages.forEach((page) => { + page.items + .filter((block) => block.type === BlockType.LIST) + .forEach((listBlock) => { + var lastItemX; + var currentLevel = 0; + const xByLevel = {}; + var modifiedBlock = false; + listBlock.items.forEach((item) => { + const isListItem = true; + if (lastItemX && isListItem) { + if (item.x > lastItemX) { + currentLevel++; + xByLevel[item.x] = currentLevel; + } else if (item.x < lastItemX) { + currentLevel = xByLevel[item.x]; + } + } else { + xByLevel[item.x] = 0; + } + if (currentLevel > 0) { + item.words = [ + new Word({ string: " ".repeat(currentLevel * 3) }), + ].concat(item.words); + modifiedBlock = true; + } + lastItemX = item.x; + }); + listBlocks++; + if (modifiedBlock) { + modifiedBlocks++; + listBlock.annotation = MODIFIED_ANNOTATION; + } else { + listBlock.annotation = UNCHANGED_ANNOTATION; + } + }); + }); - return new ParseResult({ - ...parseResult, - messages: ['Modified ' + modifiedBlocks + ' / ' + listBlocks + ' list blocks.'], - }); - } + return new ParseResult({ + ...parseResult, + messages: [ + "Modified " + modifiedBlocks + " / " + listBlocks + " list blocks.", + ], + }); + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item-block/GatherBlocks.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item-block/GatherBlocks.js index b67e7d68f0..8c8f310894 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item-block/GatherBlocks.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item-block/GatherBlocks.js @@ -1,92 +1,105 @@ // @flow -const ToLineItemBlockTransformation = require('../ToLineItemBlockTransformation'); -const ParseResult = require('../../ParseResult'); -const LineItemBlock = require('../../LineItemBlock'); -const { DETECTED_ANNOTATION } = require('../../Annotation'); -const { minXFromPageItems } = require('../../../util/page-item-functions'); +const ToLineItemBlockTransformation = require("../ToLineItemBlockTransformation"); +const ParseResult = require("../../ParseResult"); +const LineItemBlock = require("../../LineItemBlock"); +const { DETECTED_ANNOTATION } = require("../../Annotation"); +const { minXFromPageItems } = require("../../../util/page-item-functions"); // Gathers lines to blocks module.exports = class GatherBlocks extends ToLineItemBlockTransformation { - constructor() { - super('Gather Blocks'); - } + constructor() { + super("Gather Blocks"); + } - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - const { mostUsedDistance } = parseResult.globals; - var createdBlocks = 0; - var lineItemCount = 0; - parseResult.pages.map((page) => { - lineItemCount += page.items.length; - const blocks = []; - var stashedBlock = new LineItemBlock({}); - const flushStashedItems = () => { - if (stashedBlock.items.length > 1) { - stashedBlock.annotation = DETECTED_ANNOTATION; - } + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + const { mostUsedDistance } = parseResult.globals; + var createdBlocks = 0; + var lineItemCount = 0; + parseResult.pages.map((page) => { + lineItemCount += page.items.length; + const blocks = []; + var stashedBlock = new LineItemBlock({}); + const flushStashedItems = () => { + if (stashedBlock.items.length > 1) { + stashedBlock.annotation = DETECTED_ANNOTATION; + } - blocks.push(stashedBlock); - stashedBlock = new LineItemBlock({}); - createdBlocks++; - }; + blocks.push(stashedBlock); + stashedBlock = new LineItemBlock({}); + createdBlocks++; + }; - var minX = minXFromPageItems(page.items); - page.items.forEach((item) => { - if (stashedBlock.items.length > 0 && shouldFlushBlock(stashedBlock, item, minX, mostUsedDistance)) { - flushStashedItems(); - } - stashedBlock.addItem(item); - }); - if (stashedBlock.items.length > 0) { - flushStashedItems(); - } - page.items = blocks; - }); + var minX = minXFromPageItems(page.items); + page.items.forEach((item) => { + if ( + stashedBlock.items.length > 0 && + shouldFlushBlock(stashedBlock, item, minX, mostUsedDistance) + ) { + flushStashedItems(); + } + stashedBlock.addItem(item); + }); + if (stashedBlock.items.length > 0) { + flushStashedItems(); + } + page.items = blocks; + }); - return new ParseResult({ - ...parseResult, - messages: ['Gathered ' + createdBlocks + ' blocks out of ' + lineItemCount + ' line items'], - }); - } + return new ParseResult({ + ...parseResult, + messages: [ + "Gathered " + + createdBlocks + + " blocks out of " + + lineItemCount + + " line items", + ], + }); + } }; function shouldFlushBlock(stashedBlock, item, minX, mostUsedDistance) { - if (stashedBlock.type && stashedBlock.type.mergeFollowingNonTypedItems && !item.type) { - return false; - } - const lastItem = stashedBlock.items[stashedBlock.items.length - 1]; - const hasBigDistance = bigDistance(lastItem, item, minX, mostUsedDistance); - if ( - stashedBlock.type && - stashedBlock.type.mergeFollowingNonTypedItemsWithSmallDistance && - !item.type && - !hasBigDistance - ) { - return false; - } - if (item.type !== stashedBlock.type) { - return true; - } - if (item.type) { - return !item.type.mergeToBlock; - } else { - return hasBigDistance; - } + if ( + stashedBlock.type && + stashedBlock.type.mergeFollowingNonTypedItems && + !item.type + ) { + return false; + } + const lastItem = stashedBlock.items[stashedBlock.items.length - 1]; + const hasBigDistance = bigDistance(lastItem, item, minX, mostUsedDistance); + if ( + stashedBlock.type && + stashedBlock.type.mergeFollowingNonTypedItemsWithSmallDistance && + !item.type && + !hasBigDistance + ) { + return false; + } + if (item.type !== stashedBlock.type) { + return true; + } + if (item.type) { + return !item.type.mergeToBlock; + } else { + return hasBigDistance; + } } function bigDistance(lastItem, item, minX, mostUsedDistance) { - const distance = lastItem.y - item.y; - if (distance < 0 - mostUsedDistance / 2) { - // distance is negative - and not only a bit - return true; - } - var allowedDisctance = mostUsedDistance + 1; - if (lastItem.x > minX && item.x > minX) { - // intended elements like lists often have greater spacing - allowedDisctance = mostUsedDistance + mostUsedDistance / 2; - } - if (distance > allowedDisctance) { - return true; - } - return false; + const distance = lastItem.y - item.y; + if (distance < 0 - mostUsedDistance / 2) { + // distance is negative - and not only a bit + return true; + } + var allowedDisctance = mostUsedDistance + 1; + if (lastItem.x > minX && item.x > minX) { + // intended elements like lists often have greater spacing + allowedDisctance = mostUsedDistance + mostUsedDistance / 2; + } + if (distance > allowedDisctance) { + return true; + } + return false; } diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/CompactLines.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/CompactLines.js index 13f93a0cbc..6d235ca2f0 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/CompactLines.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/CompactLines.js @@ -1,87 +1,91 @@ // @flow -const ToLineItemTransformation = require('../ToLineItemTransformation'); -const ParseResult = require('../../ParseResult'); -const LineItem = require('../../LineItem'); -const TextItemLineGrouper = require('../../TextItemLineGrouper'); -const LineConverter = require('../../LineConverter'); -const BlockType = require('../../markdown/BlockType'); -const { REMOVED_ANNOTATION, ADDED_ANNOTATION } = require('../../Annotation'); +const ToLineItemTransformation = require("../ToLineItemTransformation"); +const ParseResult = require("../../ParseResult"); +const LineItem = require("../../LineItem"); +const TextItemLineGrouper = require("../../TextItemLineGrouper"); +const LineConverter = require("../../LineConverter"); +const BlockType = require("../../markdown/BlockType"); +const { REMOVED_ANNOTATION, ADDED_ANNOTATION } = require("../../Annotation"); // gathers text items on the same y line to one line item module.exports = class CompactLines extends ToLineItemTransformation { - constructor() { - super('Compact To Lines'); - } + constructor() { + super("Compact To Lines"); + } - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - const { mostUsedDistance, fontToFormats } = parseResult.globals; - const foundFootnotes = []; - const foundFootnoteLinks = []; - var linkCount = 0; - var formattedWords = 0; + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + const { mostUsedDistance, fontToFormats } = parseResult.globals; + const foundFootnotes = []; + const foundFootnoteLinks = []; + var linkCount = 0; + var formattedWords = 0; - const lineGrouper = new TextItemLineGrouper({ - mostUsedDistance: mostUsedDistance, - }); - const lineCompactor = new LineConverter(fontToFormats); + const lineGrouper = new TextItemLineGrouper({ + mostUsedDistance: mostUsedDistance, + }); + const lineCompactor = new LineConverter(fontToFormats); - parseResult.pages.forEach((page) => { - if (page.items.length > 0) { - const lineItems = []; - const textItemsGroupedByLine = lineGrouper.group(page.items); - textItemsGroupedByLine.forEach((lineTextItems) => { - const lineItem = lineCompactor.compact(lineTextItems); - if (lineTextItems.length > 1) { - lineItem.annotation = ADDED_ANNOTATION; - lineTextItems.forEach((item) => { - item.annotation = REMOVED_ANNOTATION; - lineItems.push( - new LineItem({ - ...item, - }), - ); - }); - } - if (lineItem.words.length === 0) { - lineItem.annotation = REMOVED_ANNOTATION; - } - lineItems.push(lineItem); + parseResult.pages.forEach((page) => { + if (page.items.length > 0) { + const lineItems = []; + const textItemsGroupedByLine = lineGrouper.group(page.items); + textItemsGroupedByLine.forEach((lineTextItems) => { + const lineItem = lineCompactor.compact(lineTextItems); + if (lineTextItems.length > 1) { + lineItem.annotation = ADDED_ANNOTATION; + lineTextItems.forEach((item) => { + item.annotation = REMOVED_ANNOTATION; + lineItems.push( + new LineItem({ + ...item, + }), + ); + }); + } + if (lineItem.words.length === 0) { + lineItem.annotation = REMOVED_ANNOTATION; + } + lineItems.push(lineItem); - if (lineItem.parsedElements.formattedWords) { - formattedWords += lineItem.parsedElements.formattedWords; - } - if (lineItem.parsedElements.containLinks > 0) { - linkCount++; - } - if (lineItem.parsedElements.footnoteLinks.length > 0) { - const footnoteLinks = lineItem.parsedElements.footnoteLinks.map((footnoteLink) => ({ - footnoteLink, - page: page.index + 1, - })); - foundFootnoteLinks.push.apply(foundFootnoteLinks, footnoteLinks); - } - if (lineItem.parsedElements.footnotes.length > 0) { - lineItem.type = BlockType.FOOTNOTES; - const footnotes = lineItem.parsedElements.footnotes.map((footnote) => ({ - footnote, - page: page.index + 1, - })); - foundFootnotes.push.apply(foundFootnotes, footnotes); - } - }); - page.items = lineItems; - } - }); + if (lineItem.parsedElements.formattedWords) { + formattedWords += lineItem.parsedElements.formattedWords; + } + if (lineItem.parsedElements.containLinks > 0) { + linkCount++; + } + if (lineItem.parsedElements.footnoteLinks.length > 0) { + const footnoteLinks = lineItem.parsedElements.footnoteLinks.map( + (footnoteLink) => ({ + footnoteLink, + page: page.index + 1, + }), + ); + foundFootnoteLinks.push.apply(foundFootnoteLinks, footnoteLinks); + } + if (lineItem.parsedElements.footnotes.length > 0) { + lineItem.type = BlockType.FOOTNOTES; + const footnotes = lineItem.parsedElements.footnotes.map( + (footnote) => ({ + footnote, + page: page.index + 1, + }), + ); + foundFootnotes.push.apply(foundFootnotes, footnotes); + } + }); + page.items = lineItems; + } + }); - return new ParseResult({ - ...parseResult, - messages: [ - 'Detected ' + formattedWords + ' formatted words', - 'Found ' + linkCount + ' links', - 'Detected ' + foundFootnoteLinks.length + ' footnotes links', - 'Detected ' + foundFootnotes.length + ' footnotes', - ], - }); - } + return new ParseResult({ + ...parseResult, + messages: [ + "Detected " + formattedWords + " formatted words", + "Found " + linkCount + " links", + "Detected " + foundFootnoteLinks.length + " footnotes links", + "Detected " + foundFootnotes.length + " footnotes", + ], + }); + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/DetectHeaders.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/DetectHeaders.js index 615cc8c2ee..c9243e0f86 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/DetectHeaders.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/DetectHeaders.js @@ -1,140 +1,164 @@ // @flow -const ToLineItemTransformation = require('../ToLineItemTransformation'); -const ParseResult = require('../../ParseResult'); -const { DETECTED_ANNOTATION } = require('../../Annotation'); -const BlockType = require('../../markdown/BlockType'); -const { headlineByLevel } = require('../../markdown/BlockType'); -const { isListItem } = require('../../../util/string-functions'); +const ToLineItemTransformation = require("../ToLineItemTransformation"); +const ParseResult = require("../../ParseResult"); +const { DETECTED_ANNOTATION } = require("../../Annotation"); +const BlockType = require("../../markdown/BlockType"); +const { headlineByLevel } = require("../../markdown/BlockType"); +const { isListItem } = require("../../../util/string-functions"); // Detect headlines based on heights module.exports = class DetectHeaders extends ToLineItemTransformation { - constructor() { - super('Detect Headers'); - } + constructor() { + super("Detect Headers"); + } - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - const { tocPages, headlineTypeToHeightRange, mostUsedHeight, mostUsedDistance, mostUsedFont, maxHeight } = - parseResult.globals; - const hasToc = tocPages.length > 0; - var detectedHeaders = 0; + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + const { + tocPages, + headlineTypeToHeightRange, + mostUsedHeight, + mostUsedDistance, + mostUsedFont, + maxHeight, + } = parseResult.globals; + const hasToc = tocPages.length > 0; + var detectedHeaders = 0; - // Handle title pages - const pagesWithMaxHeight = findPagesWithMaxHeight(parseResult.pages, maxHeight); - const min2ndLevelHeaderHeigthOnMaxPage = mostUsedHeight + (maxHeight - mostUsedHeight) / 4; - pagesWithMaxHeight.forEach((titlePage) => { - titlePage.items.forEach((item) => { - const height = item.height; - if (!item.type && height > min2ndLevelHeaderHeigthOnMaxPage) { - if (height === maxHeight) { - item.type = BlockType.H1; - } else { - item.type = BlockType.H2; - } - item.annotation = DETECTED_ANNOTATION; - detectedHeaders++; - } - }); - }); + // Handle title pages + const pagesWithMaxHeight = findPagesWithMaxHeight( + parseResult.pages, + maxHeight, + ); + const min2ndLevelHeaderHeigthOnMaxPage = + mostUsedHeight + (maxHeight - mostUsedHeight) / 4; + pagesWithMaxHeight.forEach((titlePage) => { + titlePage.items.forEach((item) => { + const height = item.height; + if (!item.type && height > min2ndLevelHeaderHeigthOnMaxPage) { + if (height === maxHeight) { + item.type = BlockType.H1; + } else { + item.type = BlockType.H2; + } + item.annotation = DETECTED_ANNOTATION; + detectedHeaders++; + } + }); + }); - if (hasToc) { - // Use existing headline heights to find additional headlines - const headlineTypes = Object.keys(headlineTypeToHeightRange); - headlineTypes.forEach((headlineType) => { - var range = headlineTypeToHeightRange[headlineType]; - if (range.max > mostUsedHeight) { - // use only very clear headlines, only use max - parseResult.pages.forEach((page) => { - page.items.forEach((item) => { - if (!item.type && item.height === range.max) { - item.annotation = DETECTED_ANNOTATION; - item.type = BlockType.enumValueOf(headlineType); - detectedHeaders++; - } - }); - }); - } - }); - } else { - // Categorize headlines by the text heights - const heights = []; - var lastHeight; - parseResult.pages.forEach((page) => { - page.items.forEach((item) => { - if (!item.type && item.height > mostUsedHeight && !isListItem(item.text())) { - if (!heights.includes(item.height) && (!lastHeight || lastHeight > item.height)) { - heights.push(item.height); - } - } - }); - }); - heights.sort((a, b) => b - a); + if (hasToc) { + // Use existing headline heights to find additional headlines + const headlineTypes = Object.keys(headlineTypeToHeightRange); + headlineTypes.forEach((headlineType) => { + var range = headlineTypeToHeightRange[headlineType]; + if (range.max > mostUsedHeight) { + // use only very clear headlines, only use max + parseResult.pages.forEach((page) => { + page.items.forEach((item) => { + if (!item.type && item.height === range.max) { + item.annotation = DETECTED_ANNOTATION; + item.type = BlockType.enumValueOf(headlineType); + detectedHeaders++; + } + }); + }); + } + }); + } else { + // Categorize headlines by the text heights + const heights = []; + var lastHeight; + parseResult.pages.forEach((page) => { + page.items.forEach((item) => { + if ( + !item.type && + item.height > mostUsedHeight && + !isListItem(item.text()) + ) { + if ( + !heights.includes(item.height) && + (!lastHeight || lastHeight > item.height) + ) { + heights.push(item.height); + } + } + }); + }); + heights.sort((a, b) => b - a); - heights.forEach((height, i) => { - const headlineLevel = i + 2; - if (headlineLevel <= 6) { - const headlineType = headlineByLevel(2 + i); - parseResult.pages.forEach((page) => { - page.items.forEach((item) => { - if (!item.type && item.height === height && !isListItem(item.text())) { - detectedHeaders++; - item.annotation = DETECTED_ANNOTATION; - item.type = headlineType; - } - }); - }); - } - }); - } + heights.forEach((height, i) => { + const headlineLevel = i + 2; + if (headlineLevel <= 6) { + const headlineType = headlineByLevel(2 + i); + parseResult.pages.forEach((page) => { + page.items.forEach((item) => { + if ( + !item.type && + item.height === height && + !isListItem(item.text()) + ) { + detectedHeaders++; + item.annotation = DETECTED_ANNOTATION; + item.type = headlineType; + } + }); + }); + } + }); + } - // find headlines which have paragraph height - var smallesHeadlineLevel = 1; - parseResult.pages.forEach((page) => { - page.items.forEach((item) => { - if (item.type && item.type.headline) { - smallesHeadlineLevel = Math.max(smallesHeadlineLevel, item.type.headlineLevel); - } - }); - }); - if (smallesHeadlineLevel < 6) { - const nextHeadlineType = headlineByLevel(smallesHeadlineLevel + 1); - parseResult.pages.forEach((page) => { - var lastItem; - page.items.forEach((item) => { - if ( - !item.type && - item.height === mostUsedHeight && - item.font !== mostUsedFont && - (!lastItem || - lastItem.y < item.y || - (lastItem.type && lastItem.type.headline) || - lastItem.y - item.y > mostUsedDistance * 2) && - item.text() === item.text().toUpperCase() - ) { - detectedHeaders++; - item.annotation = DETECTED_ANNOTATION; - item.type = nextHeadlineType; - } - lastItem = item; - }); - }); - } + // find headlines which have paragraph height + var smallesHeadlineLevel = 1; + parseResult.pages.forEach((page) => { + page.items.forEach((item) => { + if (item.type && item.type.headline) { + smallesHeadlineLevel = Math.max( + smallesHeadlineLevel, + item.type.headlineLevel, + ); + } + }); + }); + if (smallesHeadlineLevel < 6) { + const nextHeadlineType = headlineByLevel(smallesHeadlineLevel + 1); + parseResult.pages.forEach((page) => { + var lastItem; + page.items.forEach((item) => { + if ( + !item.type && + item.height === mostUsedHeight && + item.font !== mostUsedFont && + (!lastItem || + lastItem.y < item.y || + (lastItem.type && lastItem.type.headline) || + lastItem.y - item.y > mostUsedDistance * 2) && + item.text() === item.text().toUpperCase() + ) { + detectedHeaders++; + item.annotation = DETECTED_ANNOTATION; + item.type = nextHeadlineType; + } + lastItem = item; + }); + }); + } - return new ParseResult({ - ...parseResult, - messages: ['Detected ' + detectedHeaders + ' headlines.'], - }); - } + return new ParseResult({ + ...parseResult, + messages: ["Detected " + detectedHeaders + " headlines."], + }); + } }; function findPagesWithMaxHeight(pages, maxHeight) { - const maxHeaderPagesSet = new Set(); - pages.forEach((page) => { - page.items.forEach((item) => { - if (!item.type && item.height === maxHeight) { - maxHeaderPagesSet.add(page); - } - }); - }); - return maxHeaderPagesSet; + const maxHeaderPagesSet = new Set(); + pages.forEach((page) => { + page.items.forEach((item) => { + if (!item.type && item.height === maxHeight) { + maxHeaderPagesSet.add(page); + } + }); + }); + return maxHeaderPagesSet; } diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/DetectListItems.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/DetectListItems.js index 509adb0ba1..3d0878067d 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/DetectListItems.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/DetectListItems.js @@ -1,68 +1,75 @@ // @flow -const ToLineItemTransformation = require('../ToLineItemTransformation'); -const ParseResult = require('../../ParseResult'); -const LineItem = require('../../LineItem'); -const Word = require('../../Word'); -const { REMOVED_ANNOTATION, ADDED_ANNOTATION, DETECTED_ANNOTATION } = require('../../Annotation'); -const BlockType = require('../../markdown/BlockType'); -const { isListItemCharacter, isNumberedListItem } = require('../../../util/string-functions'); +const ToLineItemTransformation = require("../ToLineItemTransformation"); +const ParseResult = require("../../ParseResult"); +const LineItem = require("../../LineItem"); +const Word = require("../../Word"); +const { + REMOVED_ANNOTATION, + ADDED_ANNOTATION, + DETECTED_ANNOTATION, +} = require("../../Annotation"); +const BlockType = require("../../markdown/BlockType"); +const { + isListItemCharacter, + isNumberedListItem, +} = require("../../../util/string-functions"); // Detect items starting with -, •, etc... module.exports = class DetectListItems extends ToLineItemTransformation { - constructor() { - super('Detect List Items'); - } + constructor() { + super("Detect List Items"); + } - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - var foundListItems = 0; - var foundNumberedItems = 0; - parseResult.pages.forEach((page) => { - const newItems = []; - page.items.forEach((item) => { - newItems.push(item); - if (!item.type) { - var text = item.text(); - if (isListItemCharacter(item.words[0].string)) { - foundListItems++; - if (item.words[0].string === '-') { - item.annotation = DETECTED_ANNOTATION; - item.type = BlockType.LIST; - } else { - item.annotation = REMOVED_ANNOTATION; - const newWords = item.words.map( - (word) => - new Word({ - ...word, - }), - ); - newWords[0].string = '-'; - newItems.push( - new LineItem({ - ...item, - words: newWords, - annotation: ADDED_ANNOTATION, - type: BlockType.LIST, - }), - ); - } - } else if (isNumberedListItem(text)) { - // TODO check that starts with 1 (kala chakra) - foundNumberedItems++; - item.annotation = DETECTED_ANNOTATION; - item.type = BlockType.LIST; - } - } - }); - page.items = newItems; - }); + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + var foundListItems = 0; + var foundNumberedItems = 0; + parseResult.pages.forEach((page) => { + const newItems = []; + page.items.forEach((item) => { + newItems.push(item); + if (!item.type) { + var text = item.text(); + if (isListItemCharacter(item.words[0].string)) { + foundListItems++; + if (item.words[0].string === "-") { + item.annotation = DETECTED_ANNOTATION; + item.type = BlockType.LIST; + } else { + item.annotation = REMOVED_ANNOTATION; + const newWords = item.words.map( + (word) => + new Word({ + ...word, + }), + ); + newWords[0].string = "-"; + newItems.push( + new LineItem({ + ...item, + words: newWords, + annotation: ADDED_ANNOTATION, + type: BlockType.LIST, + }), + ); + } + } else if (isNumberedListItem(text)) { + // TODO check that starts with 1 (kala chakra) + foundNumberedItems++; + item.annotation = DETECTED_ANNOTATION; + item.type = BlockType.LIST; + } + } + }); + page.items = newItems; + }); - return new ParseResult({ - ...parseResult, - messages: [ - 'Detected ' + foundListItems + ' plain list items.', - 'Detected ' + foundNumberedItems + ' numbered list items.', - ], - }); - } + return new ParseResult({ + ...parseResult, + messages: [ + "Detected " + foundListItems + " plain list items.", + "Detected " + foundNumberedItems + " numbered list items.", + ], + }); + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/DetectTOC.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/DetectTOC.js index 55b6e7bd33..2e0986c20b 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/DetectTOC.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/DetectTOC.js @@ -1,393 +1,442 @@ // @flow -const ToLineItemTransformation = require('../ToLineItemTransformation'); -const ParseResult = require('../../ParseResult'); -const LineItem = require('../../LineItem'); -const Word = require('../../Word'); -const HeadlineFinder = require('../../HeadlineFinder'); -const { REMOVED_ANNOTATION, ADDED_ANNOTATION } = require('../../Annotation'); -const BlockType = require('../../markdown/BlockType'); -const { headlineByLevel } = require('../../markdown/BlockType'); -const { isDigit, isNumber, wordMatch, hasOnly } = require('../../../util/string-functions'); +const ToLineItemTransformation = require("../ToLineItemTransformation"); +const ParseResult = require("../../ParseResult"); +const LineItem = require("../../LineItem"); +const Word = require("../../Word"); +const HeadlineFinder = require("../../HeadlineFinder"); +const { REMOVED_ANNOTATION, ADDED_ANNOTATION } = require("../../Annotation"); +const BlockType = require("../../markdown/BlockType"); +const { headlineByLevel } = require("../../markdown/BlockType"); +const { + isDigit, + isNumber, + wordMatch, + hasOnly, +} = require("../../../util/string-functions"); // Detect table of contents pages plus linked headlines module.exports = class DetectTOC extends ToLineItemTransformation { - constructor() { - super('Detect TOC'); - } + constructor() { + super("Detect TOC"); + } - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - const tocPages = []; - const maxPagesToEvaluate = Math.min(20, parseResult.pages.length); - const linkLeveler = new LinkLeveler(); + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + const tocPages = []; + const maxPagesToEvaluate = Math.min(20, parseResult.pages.length); + const linkLeveler = new LinkLeveler(); - var tocLinks = []; - var lastTocPage; - var headlineItem; - parseResult.pages.slice(0, maxPagesToEvaluate).forEach((page) => { - var lineItemsWithDigits = 0; - const unknownLines = new Set(); - const pageTocLinks = []; - var lastWordsWithoutNumber; - var lastLine; - // find lines with words containing only "." ... - const tocLines = page.items.filter((line) => line.words.includes((word) => hasOnly(word.string, '.'))); - // ... and ending with a number per page - tocLines.forEach((line) => { - var words = line.words.filter((word) => !hasOnly(word.string, '.')); - const digits = []; - while (words.length > 0 && isNumber(words[words.length - 1].string)) { - const lastWord = words.pop(); - digits.unshift(lastWord.string); - } + var tocLinks = []; + var lastTocPage; + var headlineItem; + parseResult.pages.slice(0, maxPagesToEvaluate).forEach((page) => { + var lineItemsWithDigits = 0; + const unknownLines = new Set(); + const pageTocLinks = []; + var lastWordsWithoutNumber; + var lastLine; + // find lines with words containing only "." ... + const tocLines = page.items.filter((line) => + line.words.includes((word) => hasOnly(word.string, ".")), + ); + // ... and ending with a number per page + tocLines.forEach((line) => { + var words = line.words.filter((word) => !hasOnly(word.string, ".")); + const digits = []; + while (words.length > 0 && isNumber(words[words.length - 1].string)) { + const lastWord = words.pop(); + digits.unshift(lastWord.string); + } - if (digits.length === 0 && words.length > 0) { - const lastWord = words[words.length - 1]; - while (isDigit(lastWord.string.charCodeAt(lastWord.string.length - 1))) { - digits.unshift(lastWord.string.charAt(lastWord.string.length - 1)); - lastWord.string = lastWord.string.substring(0, lastWord.string.length - 1); - } - } - var endsWithDigit = digits.length > 0; - if (endsWithDigit) { - endsWithDigit = true; - if (lastWordsWithoutNumber) { - // 2-line item ? - words.push(...lastWordsWithoutNumber); - lastWordsWithoutNumber = null; - } - pageTocLinks.push( - new TocLink({ - pageNumber: parseInt(digits.join('')), - lineItem: new LineItem({ ...line, words }), - }), - ); - lineItemsWithDigits++; - } else { - if (!headlineItem) { - headlineItem = line; - } else { - if (lastWordsWithoutNumber) { - unknownLines.add(lastLine); - } - lastWordsWithoutNumber = words; - lastLine = line; - } - } - }); + if (digits.length === 0 && words.length > 0) { + const lastWord = words[words.length - 1]; + while ( + isDigit(lastWord.string.charCodeAt(lastWord.string.length - 1)) + ) { + digits.unshift(lastWord.string.charAt(lastWord.string.length - 1)); + lastWord.string = lastWord.string.substring( + 0, + lastWord.string.length - 1, + ); + } + } + var endsWithDigit = digits.length > 0; + if (endsWithDigit) { + endsWithDigit = true; + if (lastWordsWithoutNumber) { + // 2-line item ? + words.push(...lastWordsWithoutNumber); + lastWordsWithoutNumber = null; + } + pageTocLinks.push( + new TocLink({ + pageNumber: Number.parseInt(digits.join("")), + lineItem: new LineItem({ ...line, words }), + }), + ); + lineItemsWithDigits++; + } else { + if (!headlineItem) { + headlineItem = line; + } else { + if (lastWordsWithoutNumber) { + unknownLines.add(lastLine); + } + lastWordsWithoutNumber = words; + lastLine = line; + } + } + }); - // page has been processed - if ((lineItemsWithDigits * 100) / page.items.length > 75) { - tocPages.push(page.index + 1); - lastTocPage = page; - linkLeveler.levelPageItems(pageTocLinks); - tocLinks.push(...pageTocLinks); + // page has been processed + if ((lineItemsWithDigits * 100) / page.items.length > 75) { + tocPages.push(page.index + 1); + lastTocPage = page; + linkLeveler.levelPageItems(pageTocLinks); + tocLinks.push(...pageTocLinks); - const newBlocks = []; - page.items.forEach((line) => { - if (!unknownLines.has(line)) { - line.annotation = REMOVED_ANNOTATION; - } - newBlocks.push(line); - if (line === headlineItem) { - newBlocks.push( - new LineItem({ - ...line, - type: BlockType.H2, - annotation: ADDED_ANNOTATION, - }), - ); - } - }); - page.items = newBlocks; - } else { - headlineItem = null; - } - }); + const newBlocks = []; + page.items.forEach((line) => { + if (!unknownLines.has(line)) { + line.annotation = REMOVED_ANNOTATION; + } + newBlocks.push(line); + if (line === headlineItem) { + newBlocks.push( + new LineItem({ + ...line, + type: BlockType.H2, + annotation: ADDED_ANNOTATION, + }), + ); + } + }); + page.items = newBlocks; + } else { + headlineItem = null; + } + }); - // all pages have been processed - var foundHeadlines = tocLinks.length; - const notFoundHeadlines = []; - const foundBySize = []; - const headlineTypeToHeightRange = {}; // H1={min:23, max:25} + // all pages have been processed + var foundHeadlines = tocLinks.length; + const notFoundHeadlines = []; + const foundBySize = []; + const headlineTypeToHeightRange = {}; // H1={min:23, max:25} - if (tocPages.length > 0) { - // Add TOC items - tocLinks.forEach((tocLink) => { - lastTocPage.items.push( - new LineItem({ - words: [ - new Word({ - string: ' '.repeat(tocLink.level * 3) + '-', - }), - ].concat(tocLink.lineItem.words), - type: BlockType.TOC, - annotation: ADDED_ANNOTATION, - }), - ); - }); + if (tocPages.length > 0) { + // Add TOC items + tocLinks.forEach((tocLink) => { + lastTocPage.items.push( + new LineItem({ + words: [ + new Word({ + string: " ".repeat(tocLink.level * 3) + "-", + }), + ].concat(tocLink.lineItem.words), + type: BlockType.TOC, + annotation: ADDED_ANNOTATION, + }), + ); + }); - // Add linked headers - const pageMapping = detectPageMappingNumber( - parseResult.pages.filter((page) => page.index > lastTocPage.index), - tocLinks, - ); - tocLinks.forEach((tocLink) => { - var linkedPage = parseResult.pages[tocLink.pageNumber + pageMapping]; - var foundHealineItems; - if (linkedPage) { - foundHealineItems = findHeadlineItems(linkedPage, tocLink.lineItem.text()); - if (!foundHealineItems) { - // pages are off by 1 ? - linkedPage = parseResult.pages[tocLink.pageNumber + pageMapping + 1]; - if (linkedPage) { - foundHealineItems = findHeadlineItems(linkedPage, tocLink.lineItem.text()); - } - } - } - if (foundHealineItems) { - addHeadlineItems(linkedPage, tocLink, foundHealineItems, headlineTypeToHeightRange); - } else { - notFoundHeadlines.push(tocLink); - } - }); + // Add linked headers + const pageMapping = detectPageMappingNumber( + parseResult.pages.filter((page) => page.index > lastTocPage.index), + tocLinks, + ); + tocLinks.forEach((tocLink) => { + var linkedPage = parseResult.pages[tocLink.pageNumber + pageMapping]; + var foundHealineItems; + if (linkedPage) { + foundHealineItems = findHeadlineItems( + linkedPage, + tocLink.lineItem.text(), + ); + if (!foundHealineItems) { + // pages are off by 1 ? + linkedPage = + parseResult.pages[tocLink.pageNumber + pageMapping + 1]; + if (linkedPage) { + foundHealineItems = findHeadlineItems( + linkedPage, + tocLink.lineItem.text(), + ); + } + } + } + if (foundHealineItems) { + addHeadlineItems( + linkedPage, + tocLink, + foundHealineItems, + headlineTypeToHeightRange, + ); + } else { + notFoundHeadlines.push(tocLink); + } + }); - // Try to find linked headers by height - var fromPage = lastTocPage.index + 2; - var lastNotFound = []; - const rollupLastNotFound = (currentPageNumber) => { - if (lastNotFound.length > 0) { - lastNotFound.forEach((notFoundTocLink) => { - const headlineType = headlineByLevel(notFoundTocLink.level + 2); - const heightRange = headlineTypeToHeightRange[headlineType.name]; - if (heightRange) { - const [pageIndex, lineIndex] = findPageAndLineFromHeadline( - parseResult.pages, - notFoundTocLink, - heightRange, - fromPage, - currentPageNumber, - ); - if (lineIndex > -1) { - const page = parseResult.pages[pageIndex]; - page.items[lineIndex].annotation = REMOVED_ANNOTATION; - page.items.splice( - lineIndex + 1, - 0, - new LineItem({ - ...notFoundTocLink.lineItem, - type: headlineType, - annotation: ADDED_ANNOTATION, - }), - ); - foundBySize.push(notFoundTocLink); - } - } - }); - lastNotFound = []; - } - }; - if (notFoundHeadlines.length > 0) { - tocLinks.forEach((tocLink) => { - if (notFoundHeadlines.includes(tocLink)) { - lastNotFound.push(tocLink); - } else { - rollupLastNotFound(tocLink.pageNumber); - fromPage = tocLink.pageNumber; - } - }); - if (lastNotFound.length > 0) { - rollupLastNotFound(parseResult.pages.length); - } - } - } + // Try to find linked headers by height + var fromPage = lastTocPage.index + 2; + var lastNotFound = []; + const rollupLastNotFound = (currentPageNumber) => { + if (lastNotFound.length > 0) { + lastNotFound.forEach((notFoundTocLink) => { + const headlineType = headlineByLevel(notFoundTocLink.level + 2); + const heightRange = headlineTypeToHeightRange[headlineType.name]; + if (heightRange) { + const [pageIndex, lineIndex] = findPageAndLineFromHeadline( + parseResult.pages, + notFoundTocLink, + heightRange, + fromPage, + currentPageNumber, + ); + if (lineIndex > -1) { + const page = parseResult.pages[pageIndex]; + page.items[lineIndex].annotation = REMOVED_ANNOTATION; + page.items.splice( + lineIndex + 1, + 0, + new LineItem({ + ...notFoundTocLink.lineItem, + type: headlineType, + annotation: ADDED_ANNOTATION, + }), + ); + foundBySize.push(notFoundTocLink); + } + } + }); + lastNotFound = []; + } + }; + if (notFoundHeadlines.length > 0) { + tocLinks.forEach((tocLink) => { + if (notFoundHeadlines.includes(tocLink)) { + lastNotFound.push(tocLink); + } else { + rollupLastNotFound(tocLink.pageNumber); + fromPage = tocLink.pageNumber; + } + }); + if (lastNotFound.length > 0) { + rollupLastNotFound(parseResult.pages.length); + } + } + } - const messages = []; - messages.push('Detected ' + tocPages.length + ' table of content pages'); - if (tocPages.length > 0) { - messages.push('TOC headline heights: ' + JSON.stringify(headlineTypeToHeightRange)); - messages.push( - 'Found TOC headlines: ' + - (foundHeadlines - notFoundHeadlines.length + foundBySize.length) + - '/' + - foundHeadlines, - ); - } - if (notFoundHeadlines.length > 0) { - messages.push('Found TOC headlines (by size): ' + foundBySize.map((tocLink) => tocLink.lineItem.text())); - messages.push( - 'Missing TOC headlines: ' + - notFoundHeadlines - .filter((fTocLink) => !foundBySize.includes(fTocLink)) - .map((tocLink) => tocLink.lineItem.text() + '=>' + tocLink.pageNumber), - ); - } - return new ParseResult({ - ...parseResult, - globals: { - ...parseResult.globals, - tocPages, - headlineTypeToHeightRange, - }, - messages, - }); - } + const messages = []; + messages.push("Detected " + tocPages.length + " table of content pages"); + if (tocPages.length > 0) { + messages.push( + "TOC headline heights: " + JSON.stringify(headlineTypeToHeightRange), + ); + messages.push( + "Found TOC headlines: " + + (foundHeadlines - notFoundHeadlines.length + foundBySize.length) + + "/" + + foundHeadlines, + ); + } + if (notFoundHeadlines.length > 0) { + messages.push( + "Found TOC headlines (by size): " + + foundBySize.map((tocLink) => tocLink.lineItem.text()), + ); + messages.push( + "Missing TOC headlines: " + + notFoundHeadlines + .filter((fTocLink) => !foundBySize.includes(fTocLink)) + .map( + (tocLink) => tocLink.lineItem.text() + "=>" + tocLink.pageNumber, + ), + ); + } + return new ParseResult({ + ...parseResult, + globals: { + ...parseResult.globals, + tocPages, + headlineTypeToHeightRange, + }, + messages, + }); + } }; // Find out how the TOC page link actualy translates to the page.index function detectPageMappingNumber(pages, tocLinks) { - for (var tocLink of tocLinks) { - const page = findPageWithHeadline(pages, tocLink.lineItem.text()); - if (page) { - return page.index - tocLink.pageNumber; - } - } - return null; + for (var tocLink of tocLinks) { + const page = findPageWithHeadline(pages, tocLink.lineItem.text()); + if (page) { + return page.index - tocLink.pageNumber; + } + } + return null; } function findPageWithHeadline(pages, headline) { - for (var page of pages) { - if (findHeadlineItems(page, headline)) { - return page; - } - } - return null; + for (var page of pages) { + if (findHeadlineItems(page, headline)) { + return page; + } + } + return null; } function findHeadlineItems(page, headline) { - const headlineFinder = new HeadlineFinder({ headline }); - var lineIndex = 0; - for (var line of page.items) { - const headlineItems = headlineFinder.consume(line); - if (headlineItems) { - return { lineIndex, headlineItems }; - } - lineIndex++; - } - return null; + const headlineFinder = new HeadlineFinder({ headline }); + var lineIndex = 0; + for (var line of page.items) { + const headlineItems = headlineFinder.consume(line); + if (headlineItems) { + return { lineIndex, headlineItems }; + } + lineIndex++; + } + return null; } -function addHeadlineItems(page, tocLink, foundItems, headlineTypeToHeightRange) { - foundItems.headlineItems.forEach((item) => (item.annotation = REMOVED_ANNOTATION)); - const headlineType = headlineByLevel(tocLink.level + 2); - const headlineHeight = foundItems.headlineItems.reduce((max, item) => Math.max(max, item.height), 0); - page.items.splice( - foundItems.lineIndex + 1, - 0, - new LineItem({ - ...foundItems.headlineItems[0], - words: tocLink.lineItem.words, - height: headlineHeight, - type: headlineType, - annotation: ADDED_ANNOTATION, - }), - ); - var range = headlineTypeToHeightRange[headlineType.name]; - if (range) { - range.min = Math.min(range.min, headlineHeight); - range.max = Math.max(range.max, headlineHeight); - } else { - range = { - min: headlineHeight, - max: headlineHeight, - }; - headlineTypeToHeightRange[headlineType.name] = range; - } +function addHeadlineItems( + page, + tocLink, + foundItems, + headlineTypeToHeightRange, +) { + foundItems.headlineItems.forEach( + (item) => (item.annotation = REMOVED_ANNOTATION), + ); + const headlineType = headlineByLevel(tocLink.level + 2); + const headlineHeight = foundItems.headlineItems.reduce( + (max, item) => Math.max(max, item.height), + 0, + ); + page.items.splice( + foundItems.lineIndex + 1, + 0, + new LineItem({ + ...foundItems.headlineItems[0], + words: tocLink.lineItem.words, + height: headlineHeight, + type: headlineType, + annotation: ADDED_ANNOTATION, + }), + ); + var range = headlineTypeToHeightRange[headlineType.name]; + if (range) { + range.min = Math.min(range.min, headlineHeight); + range.max = Math.max(range.max, headlineHeight); + } else { + range = { + min: headlineHeight, + max: headlineHeight, + }; + headlineTypeToHeightRange[headlineType.name] = range; + } } -function findPageAndLineFromHeadline(pages, tocLink, heightRange, fromPage, toPage) { - const linkText = tocLink.lineItem.text().toUpperCase(); - for (var i = fromPage; i <= toPage; i++) { - const page = pages[i - 1]; - if (page) { - const lineIndex = page.items.findIndex((line) => { - if ( - !line.type && - !line.annotation && - line.height >= heightRange.min && - line.height <= heightRange.max - ) { - const match = wordMatch(linkText, line.text()); - return match >= 0.5; - } - return false; - }); - if (lineIndex > -1) return [i - 1, lineIndex]; - } - } - return [-1, -1]; +function findPageAndLineFromHeadline( + pages, + tocLink, + heightRange, + fromPage, + toPage, +) { + const linkText = tocLink.lineItem.text().toUpperCase(); + for (var i = fromPage; i <= toPage; i++) { + const page = pages[i - 1]; + if (page) { + const lineIndex = page.items.findIndex((line) => { + if ( + !line.type && + !line.annotation && + line.height >= heightRange.min && + line.height <= heightRange.max + ) { + const match = wordMatch(linkText, line.text()); + return match >= 0.5; + } + return false; + }); + if (lineIndex > -1) return [i - 1, lineIndex]; + } + } + return [-1, -1]; } class LinkLeveler { - constructor() { - this.levelByMethod = null; - this.uniqueFonts = []; - } + constructor() { + this.levelByMethod = null; + this.uniqueFonts = []; + } - levelPageItems(tocLinks /*: TocLink[] */) { - if (!this.levelByMethod) { - const uniqueX = this.calculateUniqueX(tocLinks); - if (uniqueX.length > 1) { - this.levelByMethod = this.levelByXDiff; - } else { - const uniqueFonts = this.calculateUniqueFonts(tocLinks); - if (uniqueFonts.length > 1) { - this.uniqueFonts = uniqueFonts; - this.levelByMethod = this.levelByFont; - } else { - this.levelByMethod = this.levelToZero; - } - } - } - this.levelByMethod(tocLinks); - } + levelPageItems(tocLinks /*: TocLink[] */) { + if (!this.levelByMethod) { + const uniqueX = this.calculateUniqueX(tocLinks); + if (uniqueX.length > 1) { + this.levelByMethod = this.levelByXDiff; + } else { + const uniqueFonts = this.calculateUniqueFonts(tocLinks); + if (uniqueFonts.length > 1) { + this.uniqueFonts = uniqueFonts; + this.levelByMethod = this.levelByFont; + } else { + this.levelByMethod = this.levelToZero; + } + } + } + this.levelByMethod(tocLinks); + } - levelByXDiff(tocLinks) { - const uniqueX = this.calculateUniqueX(tocLinks); - tocLinks.forEach((link) => { - link.level = uniqueX.indexOf(link.lineItem.x); - }); - } + levelByXDiff(tocLinks) { + const uniqueX = this.calculateUniqueX(tocLinks); + tocLinks.forEach((link) => { + link.level = uniqueX.indexOf(link.lineItem.x); + }); + } - levelByFont(tocLinks) { - tocLinks.forEach((link) => { - link.level = this.uniqueFonts.indexOf(link.lineItem.font); - }); - } + levelByFont(tocLinks) { + tocLinks.forEach((link) => { + link.level = this.uniqueFonts.indexOf(link.lineItem.font); + }); + } - levelToZero(tocLinks) { - tocLinks.forEach((link) => { - link.level = 0; - }); - } + levelToZero(tocLinks) { + tocLinks.forEach((link) => { + link.level = 0; + }); + } - calculateUniqueX(tocLinks) { - var uniqueX = tocLinks.reduce(function (uniquesArray, link) { - if (uniquesArray.indexOf(link.lineItem.x) < 0) uniquesArray.push(link.lineItem.x); - return uniquesArray; - }, []); + calculateUniqueX(tocLinks) { + var uniqueX = tocLinks.reduce((uniquesArray, link) => { + if (uniquesArray.indexOf(link.lineItem.x) < 0) + uniquesArray.push(link.lineItem.x); + return uniquesArray; + }, []); - uniqueX.sort((a, b) => { - return a - b; - }); + uniqueX.sort((a, b) => { + return a - b; + }); - return uniqueX; - } + return uniqueX; + } - calculateUniqueFonts(tocLinks) { - var uniqueFont = tocLinks.reduce(function (uniquesArray, link) { - if (uniquesArray.indexOf(link.lineItem.font) < 0) uniquesArray.push(link.lineItem.font); - return uniquesArray; - }, []); + calculateUniqueFonts(tocLinks) { + var uniqueFont = tocLinks.reduce((uniquesArray, link) => { + if (uniquesArray.indexOf(link.lineItem.font) < 0) + uniquesArray.push(link.lineItem.font); + return uniquesArray; + }, []); - return uniqueFont; - } + return uniqueFont; + } } class TocLink { - constructor(options) { - this.lineItem = options.lineItem; - this.pageNumber = options.pageNumber; - this.level = 0; - } + constructor(options) { + this.lineItem = options.lineItem; + this.pageNumber = options.pageNumber; + this.level = 0; + } } diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/RemoveRepetitiveElements.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/RemoveRepetitiveElements.js index aff7154f6f..eecc098b6e 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/RemoveRepetitiveElements.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/RemoveRepetitiveElements.js @@ -1,111 +1,122 @@ // @flow -const ToLineItemTransformation = require('../ToLineItemTransformation'); -const ParseResult = require('../../ParseResult'); -const { REMOVED_ANNOTATION } = require('../../Annotation'); +const ToLineItemTransformation = require("../ToLineItemTransformation"); +const ParseResult = require("../../ParseResult"); +const { REMOVED_ANNOTATION } = require("../../Annotation"); -const { isDigit } = require('../../../util/string-functions'); +const { isDigit } = require("../../../util/string-functions"); function hashCodeIgnoringSpacesAndNumbers(string) { - var hash = 0; - if (string.trim().length === 0) return hash; - for (var i = 0; i < string.length; i++) { - const charCode = string.charCodeAt(i); - if (!isDigit(charCode) && charCode !== 32 && charCode !== 160) { - hash = (hash << 5) - hash + charCode; - hash |= 0; // Convert to 32bit integer - } - } - return hash; + var hash = 0; + if (string.trim().length === 0) return hash; + for (var i = 0; i < string.length; i++) { + const charCode = string.charCodeAt(i); + if (!isDigit(charCode) && charCode !== 32 && charCode !== 160) { + hash = (hash << 5) - hash + charCode; + hash |= 0; // Convert to 32bit integer + } + } + return hash; } // Remove elements with similar content on same page positions, like page numbers, licenes information, etc... -module.exports = class RemoveRepetitiveElements extends ToLineItemTransformation { - constructor() { - super('Remove Repetitive Elements'); - } +module.exports = class RemoveRepetitiveElements extends ( + ToLineItemTransformation +) { + constructor() { + super("Remove Repetitive Elements"); + } - // The idea is the following: - // - For each page, collect all items of the first, and all items of the last line - // - Calculate how often these items occur accros all pages (hash ignoring numbers, whitespace, upper/lowercase) - // - Delete items occuring on more then 2/3 of all pages - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - // find first and last lines per page - const pageStore = []; - const minLineHashRepetitions = {}; - const maxLineHashRepetitions = {}; - parseResult.pages.forEach((page) => { - const minMaxItems = page.items.reduce( - (itemStore, item) => { - if (item.y < itemStore.minY) { - itemStore.minElements = [item]; - itemStore.minY = item.y; - } else if (item.y === itemStore.minY) { - itemStore.minElements.push(item); - } - if (item.y > itemStore.maxY) { - itemStore.maxElements = [item]; - itemStore.maxY = item.y; - } else if (item.y === itemStore.maxY) { - itemStore.maxElements.push(item); - } - return itemStore; - }, - { - minY: 999, - maxY: 0, - minElements: [], - maxElements: [], - }, - ); + // The idea is the following: + // - For each page, collect all items of the first, and all items of the last line + // - Calculate how often these items occur accros all pages (hash ignoring numbers, whitespace, upper/lowercase) + // - Delete items occuring on more then 2/3 of all pages + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + // find first and last lines per page + const pageStore = []; + const minLineHashRepetitions = {}; + const maxLineHashRepetitions = {}; + parseResult.pages.forEach((page) => { + const minMaxItems = page.items.reduce( + (itemStore, item) => { + if (item.y < itemStore.minY) { + itemStore.minElements = [item]; + itemStore.minY = item.y; + } else if (item.y === itemStore.minY) { + itemStore.minElements.push(item); + } + if (item.y > itemStore.maxY) { + itemStore.maxElements = [item]; + itemStore.maxY = item.y; + } else if (item.y === itemStore.maxY) { + itemStore.maxElements.push(item); + } + return itemStore; + }, + { + minY: 999, + maxY: 0, + minElements: [], + maxElements: [], + }, + ); - const minLineHash = hashCodeIgnoringSpacesAndNumbers( - minMaxItems.minElements.reduce( - (combinedString, item) => combinedString + item.text().toUpperCase(), - '', - ), - ); - const maxLineHash = hashCodeIgnoringSpacesAndNumbers( - minMaxItems.maxElements.reduce( - (combinedString, item) => combinedString + item.text().toUpperCase(), - '', - ), - ); - pageStore.push({ - minElements: minMaxItems.minElements, - maxElements: minMaxItems.maxElements, - minLineHash: minLineHash, - maxLineHash: maxLineHash, - }); - minLineHashRepetitions[minLineHash] = minLineHashRepetitions[minLineHash] - ? minLineHashRepetitions[minLineHash] + 1 - : 1; - maxLineHashRepetitions[maxLineHash] = maxLineHashRepetitions[maxLineHash] - ? maxLineHashRepetitions[maxLineHash] + 1 - : 1; - }); + const minLineHash = hashCodeIgnoringSpacesAndNumbers( + minMaxItems.minElements.reduce( + (combinedString, item) => combinedString + item.text().toUpperCase(), + "", + ), + ); + const maxLineHash = hashCodeIgnoringSpacesAndNumbers( + minMaxItems.maxElements.reduce( + (combinedString, item) => combinedString + item.text().toUpperCase(), + "", + ), + ); + pageStore.push({ + minElements: minMaxItems.minElements, + maxElements: minMaxItems.maxElements, + minLineHash: minLineHash, + maxLineHash: maxLineHash, + }); + minLineHashRepetitions[minLineHash] = minLineHashRepetitions[minLineHash] + ? minLineHashRepetitions[minLineHash] + 1 + : 1; + maxLineHashRepetitions[maxLineHash] = maxLineHashRepetitions[maxLineHash] + ? maxLineHashRepetitions[maxLineHash] + 1 + : 1; + }); - // now annoate all removed items - var removedHeader = 0; - var removedFooter = 0; - parseResult.pages.forEach((page, i) => { - if (minLineHashRepetitions[pageStore[i].minLineHash] >= Math.max(3, (parseResult.pages.length * 2) / 3)) { - pageStore[i].minElements.forEach((item) => { - item.annotation = REMOVED_ANNOTATION; - }); - removedFooter++; - } - if (maxLineHashRepetitions[pageStore[i].maxLineHash] >= Math.max(3, (parseResult.pages.length * 2) / 3)) { - pageStore[i].maxElements.forEach((item) => { - item.annotation = REMOVED_ANNOTATION; - }); - removedHeader++; - } - }); + // now annoate all removed items + var removedHeader = 0; + var removedFooter = 0; + parseResult.pages.forEach((page, i) => { + if ( + minLineHashRepetitions[pageStore[i].minLineHash] >= + Math.max(3, (parseResult.pages.length * 2) / 3) + ) { + pageStore[i].minElements.forEach((item) => { + item.annotation = REMOVED_ANNOTATION; + }); + removedFooter++; + } + if ( + maxLineHashRepetitions[pageStore[i].maxLineHash] >= + Math.max(3, (parseResult.pages.length * 2) / 3) + ) { + pageStore[i].maxElements.forEach((item) => { + item.annotation = REMOVED_ANNOTATION; + }); + removedHeader++; + } + }); - return new ParseResult({ - ...parseResult, - messages: ['Removed Header: ' + removedHeader, 'Removed Footers: ' + removedFooter], - }); - } + return new ParseResult({ + ...parseResult, + messages: [ + "Removed Header: " + removedHeader, + "Removed Footers: " + removedFooter, + ], + }); + } }; diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/VerticalToHorizontal.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/VerticalToHorizontal.js index ffecb3831d..c5c567f6b8 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/VerticalToHorizontal.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/line-item/VerticalToHorizontal.js @@ -1,79 +1,81 @@ // @flow -const ToLineItemTransformation = require('../ToLineItemTransformation'); -const ParseResult = require('../../ParseResult'); -const LineItem = require('../../LineItem'); -const StashingStream = require('../../StashingStream'); -const { REMOVED_ANNOTATION, ADDED_ANNOTATION } = require('../../Annotation'); +const ToLineItemTransformation = require("../ToLineItemTransformation"); +const ParseResult = require("../../ParseResult"); +const LineItem = require("../../LineItem"); +const StashingStream = require("../../StashingStream"); +const { REMOVED_ANNOTATION, ADDED_ANNOTATION } = require("../../Annotation"); // Converts vertical text to horizontal module.exports = class VerticalToHorizontal extends ToLineItemTransformation { - constructor() { - super('Vertical to Horizontal Text'); - } + constructor() { + super("Vertical to Horizontal Text"); + } - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - var foundVerticals = 0; - parseResult.pages.forEach((page) => { - const stream = new VerticalsStream(); - stream.consumeAll(page.items); - page.items = stream.complete(); - foundVerticals += stream.foundVerticals; - }); + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + var foundVerticals = 0; + parseResult.pages.forEach((page) => { + const stream = new VerticalsStream(); + stream.consumeAll(page.items); + page.items = stream.complete(); + foundVerticals += stream.foundVerticals; + }); - return new ParseResult({ - ...parseResult, - messages: ['Converted ' + foundVerticals + ' verticals'], - }); - } + return new ParseResult({ + ...parseResult, + messages: ["Converted " + foundVerticals + " verticals"], + }); + } }; class VerticalsStream extends StashingStream { - constructor() { - super(); - this.foundVerticals = 0; - } + constructor() { + super(); + this.foundVerticals = 0; + } - shouldStash(item) { - return item.words.length === 1 && item.words[0].string.length === 1; - } + shouldStash(item) { + return item.words.length === 1 && item.words[0].string.length === 1; + } - doMatchesStash(lastItem, item) { - return lastItem.y - item.y > 5 && lastItem.words[0].type === item.words[0].type; - } + doMatchesStash(lastItem, item) { + return ( + lastItem.y - item.y > 5 && lastItem.words[0].type === item.words[0].type + ); + } - doFlushStash(stash, results) { - if (stash.length > 5) { - // unite - var combinedWords = []; - var minX = 999; - var maxY = 0; - var sumWidth = 0; - var maxHeight = 0; - stash.forEach((oneCharacterLine) => { - oneCharacterLine.annotation = REMOVED_ANNOTATION; - results.push(oneCharacterLine); - combinedWords.push(oneCharacterLine.words[0]); - minX = Math.min(minX, oneCharacterLine.x); - maxY = Math.max(maxY, oneCharacterLine.y); - sumWidth += oneCharacterLine.width; - maxHeight = Math.max(maxHeight, oneCharacterLine.height); - }); - results.push( - new LineItem({ - ...stash[0], - x: minX, - y: maxY, - width: sumWidth, - height: maxHeight, - words: combinedWords, - annotation: ADDED_ANNOTATION, - }), - ); - this.foundVerticals++; - } else { - // add as singles - results.push(...stash); - } - } + doFlushStash(stash, results) { + if (stash.length > 5) { + // unite + var combinedWords = []; + var minX = 999; + var maxY = 0; + var sumWidth = 0; + var maxHeight = 0; + stash.forEach((oneCharacterLine) => { + oneCharacterLine.annotation = REMOVED_ANNOTATION; + results.push(oneCharacterLine); + combinedWords.push(oneCharacterLine.words[0]); + minX = Math.min(minX, oneCharacterLine.x); + maxY = Math.max(maxY, oneCharacterLine.y); + sumWidth += oneCharacterLine.width; + maxHeight = Math.max(maxHeight, oneCharacterLine.height); + }); + results.push( + new LineItem({ + ...stash[0], + x: minX, + y: maxY, + width: sumWidth, + height: maxHeight, + words: combinedWords, + annotation: ADDED_ANNOTATION, + }), + ); + this.foundVerticals++; + } else { + // add as singles + results.push(...stash); + } + } } diff --git a/src/scrapers/pdf/lib/pdf2md/models/transformations/text-item/CalculateGlobalStats.js b/src/scrapers/pdf/lib/pdf2md/models/transformations/text-item/CalculateGlobalStats.js index 356ddfce18..9ff0d5d22a 100644 --- a/src/scrapers/pdf/lib/pdf2md/models/transformations/text-item/CalculateGlobalStats.js +++ b/src/scrapers/pdf/lib/pdf2md/models/transformations/text-item/CalculateGlobalStats.js @@ -1,117 +1,127 @@ // @flow -const ToTextItemTransformation = require('../ToTextItemTransformation'); -const ParseResult = require('../../ParseResult'); -const WordFormat = require('../../markdown/WordFormat'); +const ToTextItemTransformation = require("../ToTextItemTransformation"); +const ParseResult = require("../../ParseResult"); +const WordFormat = require("../../markdown/WordFormat"); module.exports = class CalculateGlobalStats extends ToTextItemTransformation { - constructor(fontMap) { - super('$1'); - this.fontMap = fontMap; - } + constructor(fontMap) { + super("$1"); + this.fontMap = fontMap; + } - transform(parseResult /*: ParseResult */) /*: ParseResult */ { - // Parse heights - const heightToOccurrence = {}; - const fontToOccurrence = {}; - var maxHeight = 0; - var maxHeightFont; - parseResult.pages.forEach((page) => { - page.items.forEach((item) => { - if (!item.height) return; - heightToOccurrence[item.height] = heightToOccurrence[item.height] - ? heightToOccurrence[item.height] + 1 - : 1; - fontToOccurrence[item.font] = fontToOccurrence[item.font] ? fontToOccurrence[item.font] + 1 : 1; - if (item.height > maxHeight) { - maxHeight = item.height; - maxHeightFont = item.font; - } - }); - }); - const mostUsedHeight = parseInt(getMostUsedKey(heightToOccurrence)); - const mostUsedFont = getMostUsedKey(fontToOccurrence); + transform(parseResult /*: ParseResult */) /*: ParseResult */ { + // Parse heights + const heightToOccurrence = {}; + const fontToOccurrence = {}; + var maxHeight = 0; + var maxHeightFont; + parseResult.pages.forEach((page) => { + page.items.forEach((item) => { + if (!item.height) return; + heightToOccurrence[item.height] = heightToOccurrence[item.height] + ? heightToOccurrence[item.height] + 1 + : 1; + fontToOccurrence[item.font] = fontToOccurrence[item.font] + ? fontToOccurrence[item.font] + 1 + : 1; + if (item.height > maxHeight) { + maxHeight = item.height; + maxHeightFont = item.font; + } + }); + }); + const mostUsedHeight = Number.parseInt(getMostUsedKey(heightToOccurrence)); + const mostUsedFont = getMostUsedKey(fontToOccurrence); - // Parse line distances - const distanceToOccurrence = {}; - parseResult.pages.forEach((page) => { - var lastItemOfMostUsedHeight; - page.items.forEach((item) => { - if (item.height === mostUsedHeight && item.text.trim().length > 0) { - if (lastItemOfMostUsedHeight && item.y !== lastItemOfMostUsedHeight.y) { - const distance = lastItemOfMostUsedHeight.y - item.y; - if (distance > 0) { - distanceToOccurrence[distance] = distanceToOccurrence[distance] - ? distanceToOccurrence[distance] + 1 - : 1; - } - } - lastItemOfMostUsedHeight = item; - } else { - lastItemOfMostUsedHeight = null; - } - }); - }); - const mostUsedDistance = parseInt(getMostUsedKey(distanceToOccurrence)); - const fontIdToName = []; - const fontToFormats = new Map(); - this.fontMap.forEach(function (value, key) { - fontIdToName.push(key + ' = ' + value.name); - const fontName = value.name.toLowerCase(); - var format; - if (key === mostUsedFont) { - format = null; - } else if (fontName.includes('bold') && (fontName.includes('oblique') || fontName.includes('italic'))) { - format = WordFormat.BOLD_OBLIQUE; - } else if (fontName.includes('bold')) { - format = WordFormat.BOLD; - } else if (fontName.includes('oblique') || fontName.includes('italic')) { - format = WordFormat.OBLIQUE; - } else if (fontName === maxHeightFont) { - format = WordFormat.BOLD; - } - if (format) { - fontToFormats.set(key, format.name); - } - }); - fontIdToName.sort(); + // Parse line distances + const distanceToOccurrence = {}; + parseResult.pages.forEach((page) => { + var lastItemOfMostUsedHeight; + page.items.forEach((item) => { + if (item.height === mostUsedHeight && item.text.trim().length > 0) { + if ( + lastItemOfMostUsedHeight && + item.y !== lastItemOfMostUsedHeight.y + ) { + const distance = lastItemOfMostUsedHeight.y - item.y; + if (distance > 0) { + distanceToOccurrence[distance] = distanceToOccurrence[distance] + ? distanceToOccurrence[distance] + 1 + : 1; + } + } + lastItemOfMostUsedHeight = item; + } else { + lastItemOfMostUsedHeight = null; + } + }); + }); + const mostUsedDistance = Number.parseInt( + getMostUsedKey(distanceToOccurrence), + ); + const fontIdToName = []; + const fontToFormats = new Map(); + this.fontMap.forEach((value, key) => { + fontIdToName.push(key + " = " + value.name); + const fontName = value.name.toLowerCase(); + var format; + if (key === mostUsedFont) { + format = null; + } else if ( + fontName.includes("bold") && + (fontName.includes("oblique") || fontName.includes("italic")) + ) { + format = WordFormat.BOLD_OBLIQUE; + } else if (fontName.includes("bold")) { + format = WordFormat.BOLD; + } else if (fontName.includes("oblique") || fontName.includes("italic")) { + format = WordFormat.OBLIQUE; + } else if (fontName === maxHeightFont) { + format = WordFormat.BOLD; + } + if (format) { + fontToFormats.set(key, format.name); + } + }); + fontIdToName.sort(); - // Make a copy of the originals so all following transformation don't modify them - const newPages = parseResult.pages.map((page) => { - return { - ...page, - items: page.items.map((textItem) => ({ ...textItem })), - }; - }); - return new ParseResult({ - ...parseResult, - pages: newPages, - globals: { - mostUsedHeight, - mostUsedFont, - mostUsedDistance, - maxHeight, - maxHeightFont, - fontToFormats, - }, - messages: [ - 'Items per height: ' + JSON.stringify(heightToOccurrence), - 'Items per font: ' + JSON.stringify(fontToOccurrence), - 'Items per distance: ' + JSON.stringify(distanceToOccurrence), - 'Fonts:' + JSON.stringify(fontIdToName), - ], - }); - } + // Make a copy of the originals so all following transformation don't modify them + const newPages = parseResult.pages.map((page) => { + return { + ...page, + items: page.items.map((textItem) => ({ ...textItem })), + }; + }); + return new ParseResult({ + ...parseResult, + pages: newPages, + globals: { + mostUsedHeight, + mostUsedFont, + mostUsedDistance, + maxHeight, + maxHeightFont, + fontToFormats, + }, + messages: [ + "Items per height: " + JSON.stringify(heightToOccurrence), + "Items per font: " + JSON.stringify(fontToOccurrence), + "Items per distance: " + JSON.stringify(distanceToOccurrence), + "Fonts:" + JSON.stringify(fontIdToName), + ], + }); + } }; function getMostUsedKey(keyToOccurrence) { - var maxOccurence = 0; - var maxKey; - Object.keys(keyToOccurrence).map((element) => { - if (!maxKey || keyToOccurrence[element] > maxOccurence) { - maxOccurence = keyToOccurrence[element]; - maxKey = element; - } - }); - return maxKey; + var maxOccurence = 0; + var maxKey; + Object.keys(keyToOccurrence).map((element) => { + if (!maxKey || keyToOccurrence[element] > maxOccurence) { + maxOccurence = keyToOccurrence[element]; + maxKey = element; + } + }); + return maxKey; } diff --git a/src/scrapers/pdf/lib/pdf2md/pdf2md-cli.js b/src/scrapers/pdf/lib/pdf2md/pdf2md-cli.js index 8e715d649d..b29de70e7c 100644 --- a/src/scrapers/pdf/lib/pdf2md/pdf2md-cli.js +++ b/src/scrapers/pdf/lib/pdf2md/pdf2md-cli.js @@ -2,62 +2,77 @@ /* eslint no-console: 0 */ -const fs = require('fs'); -const path = require('path'); +const fs = require("fs"); +const path = require("path"); -const pdf2md = require('./pdf2md'); -const { getFileAndFolderPaths, getAllFileAndFolderPaths } = require('./util/cli'); +const pdf2md = require("./pdf2md"); +const { + getFileAndFolderPaths, + getAllFileAndFolderPaths, +} = require("./util/cli"); -var argv = require('minimist')(process.argv.slice(2)); +var argv = require("minimist")(process.argv.slice(2)); -if (!argv['inputFolderPath']) { - console.log('Please specify inputFolderPath'); -} else if (!argv['outputFolderPath']) { - console.log('Please specify outputFolderPath'); -} else if (typeof argv['recursive'] !== 'boolean' && argv['recursive'] !== undefined) { - console.log('Add tag --recursive for recursive folder conversion, otherwise omit'); +if (!argv["inputFolderPath"]) { + console.log("Please specify inputFolderPath"); +} else if (!argv["outputFolderPath"]) { + console.log("Please specify outputFolderPath"); +} else if ( + typeof argv["recursive"] !== "boolean" && + argv["recursive"] !== undefined +) { + console.log( + "Add tag --recursive for recursive folder conversion, otherwise omit", + ); } else { - const folderPath = argv['inputFolderPath']; - const outputPath = argv['outputFolderPath']; - const recursive = argv['recursive']; - run(folderPath, outputPath, recursive); + const folderPath = argv["inputFolderPath"]; + const outputPath = argv["outputFolderPath"]; + const recursive = argv["recursive"]; + run(folderPath, outputPath, recursive); } function run(folderPath, outputPath, recursive = true) { - var [filenames, folderPaths] = getFileAndFolderPaths(folderPath); - var [allFilePaths] = getAllFileAndFolderPaths(filenames, folderPaths, recursive); - var allOutputPaths = allFilePaths.map((x) => { - const fileNameWithExtension = x.split(folderPath)[1]; - const fileNameWithoutExtension = fileNameWithExtension.slice(0, fileNameWithExtension.indexOf('.pdf')); - return outputPath + fileNameWithoutExtension; - }); - makeOutputDirs(allOutputPaths); - createMarkdownFiles(allFilePaths, allOutputPaths); + var [filenames, folderPaths] = getFileAndFolderPaths(folderPath); + var [allFilePaths] = getAllFileAndFolderPaths( + filenames, + folderPaths, + recursive, + ); + var allOutputPaths = allFilePaths.map((x) => { + const fileNameWithExtension = x.split(folderPath)[1]; + const fileNameWithoutExtension = fileNameWithExtension.slice( + 0, + fileNameWithExtension.indexOf(".pdf"), + ); + return outputPath + fileNameWithoutExtension; + }); + makeOutputDirs(allOutputPaths); + createMarkdownFiles(allFilePaths, allOutputPaths); } function makeOutputDirs(allOutputPaths) { - allOutputPaths.forEach((outputPath) => { - outputPath = outputPath.split('/').slice(0, -1).join('/'); - if (!fs.existsSync(outputPath)) { - fs.mkdirSync(outputPath, { recursive: true }); - } - }); + allOutputPaths.forEach((outputPath) => { + outputPath = outputPath.split("/").slice(0, -1).join("/"); + if (!fs.existsSync(outputPath)) { + fs.mkdirSync(outputPath, { recursive: true }); + } + }); } async function createMarkdownFiles(filenames, allOutputPaths) { - // If outputPath specified, supply callbacks to log progress - for (let i = 0; i < filenames.length; ++i) { - const filename = filenames[i]; - const callbacks = allOutputPaths[i] && {}; - const pdfBuffer = fs.readFileSync(filename); - try { - const text = await pdf2md(new Uint8Array(pdfBuffer), callbacks); - const outputFile = allOutputPaths[i] + '.md'; - console.log(`Writing to ${outputFile}...`); - fs.writeFileSync(path.resolve(outputFile), text); - console.log('Done.'); - } catch (err) { - console.error(err); - } - } + // If outputPath specified, supply callbacks to log progress + for (let i = 0; i < filenames.length; ++i) { + const filename = filenames[i]; + const callbacks = allOutputPaths[i] && {}; + const pdfBuffer = fs.readFileSync(filename); + try { + const text = await pdf2md(new Uint8Array(pdfBuffer), callbacks); + const outputFile = allOutputPaths[i] + ".md"; + console.log(`Writing to ${outputFile}...`); + fs.writeFileSync(path.resolve(outputFile), text); + console.log("Done."); + } catch (err) { + console.error(err); + } + } } diff --git a/src/scrapers/pdf/lib/pdf2md/pdf2md.js b/src/scrapers/pdf/lib/pdf2md/pdf2md.js index 0675b04373..2d07373f60 100644 --- a/src/scrapers/pdf/lib/pdf2md/pdf2md.js +++ b/src/scrapers/pdf/lib/pdf2md/pdf2md.js @@ -1,7 +1,7 @@ -const { parse } = require('./util/pdf'); -const { makeTransformations, transform } = require('./util/transformations'); -if (typeof document === 'undefined') { - require('./util/dom-stubs').setStubs(global); +const { parse } = require("./util/pdf"); +const { makeTransformations, transform } = require("./util/transformations"); +if (typeof document === "undefined") { + require("./util/dom-stubs").setStubs(global); } /** * Reads a @@ -18,11 +18,13 @@ if (typeof document === 'undefined') { * * @returns {Promise} The Markdown text */ -module.exports = async function (pdfBuffer, callbacks) { - const result = await parse(pdfBuffer, callbacks); - const { fonts, pages } = result; - const transformations = makeTransformations(fonts.map); - const parseResult = transform(pages, transformations); - const text = parseResult.pages.map((page) => page.items.join('\n') + '\n').join(''); - return text; +module.exports = async (pdfBuffer, callbacks) => { + const result = await parse(pdfBuffer, callbacks); + const { fonts, pages } = result; + const transformations = makeTransformations(fonts.map); + const parseResult = transform(pages, transformations); + const text = parseResult.pages + .map((page) => page.items.join("\n") + "\n") + .join(""); + return text; }; diff --git a/src/scrapers/pdf/lib/pdf2md/util/cli.js b/src/scrapers/pdf/lib/pdf2md/util/cli.js index 1bdbadc932..15b4479f8a 100644 --- a/src/scrapers/pdf/lib/pdf2md/util/cli.js +++ b/src/scrapers/pdf/lib/pdf2md/util/cli.js @@ -1,40 +1,42 @@ -const fs = require('fs'); +const fs = require("fs"); function getFileAndFolderPaths(folderPath) { - var filenames = []; - var folderPaths = []; - var directoryItems = fs.readdirSync(folderPath); - directoryItems.forEach((directoryItem) => { - const isDirectory = fs.lstatSync(folderPath + '/' + directoryItem).isDirectory(); - if (isDirectory) { - folderPaths.push(folderPath + '/' + directoryItem); - } - const fileExtension = directoryItem.split('.').pop().toLowerCase(); - if (fileExtension === 'pdf') { - filenames.push(folderPath + '/' + directoryItem); - } - }); - return [filenames, folderPaths]; + var filenames = []; + var folderPaths = []; + var directoryItems = fs.readdirSync(folderPath); + directoryItems.forEach((directoryItem) => { + const isDirectory = fs + .lstatSync(folderPath + "/" + directoryItem) + .isDirectory(); + if (isDirectory) { + folderPaths.push(folderPath + "/" + directoryItem); + } + const fileExtension = directoryItem.split(".").pop().toLowerCase(); + if (fileExtension === "pdf") { + filenames.push(folderPath + "/" + directoryItem); + } + }); + return [filenames, folderPaths]; } function getAllFileAndFolderPaths(filenames, folderPaths, recursive) { - var allFolderPaths = folderPaths; - if (recursive) { - while (allFolderPaths.length !== 0) { - var nextFolderPaths = []; - allFolderPaths.forEach((folderPath) => { - const outputArray = getFileAndFolderPaths(folderPath); - filenames = filenames.concat(outputArray[0]); - nextFolderPaths = nextFolderPaths.concat(outputArray[1]); - folderPaths = folderPaths.concat(outputArray[1]); - }); - allFolderPaths = nextFolderPaths; - } - } - return [filenames, folderPaths]; + var allFolderPaths = folderPaths; + if (recursive) { + while (allFolderPaths.length !== 0) { + var nextFolderPaths = []; + allFolderPaths.forEach((folderPath) => { + const outputArray = getFileAndFolderPaths(folderPath); + filenames = filenames.concat(outputArray[0]); + nextFolderPaths = nextFolderPaths.concat(outputArray[1]); + folderPaths = folderPaths.concat(outputArray[1]); + }); + allFolderPaths = nextFolderPaths; + } + } + return [filenames, folderPaths]; } module.exports = { - getFileAndFolderPaths, - getAllFileAndFolderPaths, + getFileAndFolderPaths, + getAllFileAndFolderPaths, }; diff --git a/src/scrapers/pdf/lib/pdf2md/util/dom-stubs.js b/src/scrapers/pdf/lib/pdf2md/util/dom-stubs.js index 140e7f28e7..bbacac1d2e 100644 --- a/src/scrapers/pdf/lib/pdf2md/util/dom-stubs.js +++ b/src/scrapers/pdf/lib/pdf2md/util/dom-stubs.js @@ -7,258 +7,261 @@ * http://creativecommons.org/publicdomain/zero/1.0/ */ function xmlEncode(s) { - var i = 0, - ch; - s = String(s); - while ( - i < s.length && - (ch = s[i]) !== '&' && - ch !== '<' && - ch !== '"' && - ch !== '\n' && - ch !== '\r' && - ch !== '\t' - ) { - i++; - } - if (i >= s.length) { - return s; - } - var buf = s.substring(0, i); - while (i < s.length) { - ch = s[i++]; - switch (ch) { - case '&': - buf += '&'; - break; - case '<': - buf += '<'; - break; - case '"': - buf += '"'; - break; - case '\n': - buf += ' '; - break; - case '\r': - buf += ' '; - break; - case '\t': - buf += ' '; - break; - default: - buf += ch; - break; - } - } - return buf; + var i = 0, + ch; + s = String(s); + while ( + i < s.length && + (ch = s[i]) !== "&" && + ch !== "<" && + ch !== '"' && + ch !== "\n" && + ch !== "\r" && + ch !== "\t" + ) { + i++; + } + if (i >= s.length) { + return s; + } + var buf = s.substring(0, i); + while (i < s.length) { + ch = s[i++]; + switch (ch) { + case "&": + buf += "&"; + break; + case "<": + buf += "<"; + break; + case '"': + buf += """; + break; + case "\n": + buf += " "; + break; + case "\r": + buf += " "; + break; + case "\t": + buf += " "; + break; + default: + buf += ch; + break; + } + } + return buf; } function DOMElement(name) { - this.nodeName = name; - this.childNodes = []; - this.attributes = {}; - this.textContent = ''; + this.nodeName = name; + this.childNodes = []; + this.attributes = {}; + this.textContent = ""; - if (name === 'style') { - this.sheet = { - cssRules: [], - insertRule: function (rule) { - this.cssRules.push(rule); - }, - }; - } + if (name === "style") { + this.sheet = { + cssRules: [], + insertRule: function (rule) { + this.cssRules.push(rule); + }, + }; + } } DOMElement.prototype = { - getAttribute: function DOMElement_getAttribute(name) { - if (name in this.attributes) { - return this.attributes[name]; - } - return null; - }, + getAttribute: function DOMElement_getAttribute(name) { + if (name in this.attributes) { + return this.attributes[name]; + } + return null; + }, - getAttributeNS: function DOMElement_getAttributeNS(NS, name) { - // Fast path - if (name in this.attributes) { - return this.attributes[name]; - } - // Slow path - used by test/unit/display_svg_spec.js - // Assuming that there is only one matching attribute for a given name, - // across all namespaces. - if (NS) { - var suffix = ':' + name; - for (var fullName in this.attributes) { - if (fullName.slice(-suffix.length) === suffix) { - return this.attributes[fullName]; - } - } - } - return null; - }, + getAttributeNS: function DOMElement_getAttributeNS(NS, name) { + // Fast path + if (name in this.attributes) { + return this.attributes[name]; + } + // Slow path - used by test/unit/display_svg_spec.js + // Assuming that there is only one matching attribute for a given name, + // across all namespaces. + if (NS) { + var suffix = ":" + name; + for (var fullName in this.attributes) { + if (fullName.slice(-suffix.length) === suffix) { + return this.attributes[fullName]; + } + } + } + return null; + }, - setAttribute: function DOMElement_setAttribute(name, value) { - value = value || ''; - value = xmlEncode(value); - this.attributes[name] = value; - }, + setAttribute: function DOMElement_setAttribute(name, value) { + value = value || ""; + value = xmlEncode(value); + this.attributes[name] = value; + }, - setAttributeNS: function DOMElement_setAttributeNS(NS, name, value) { - this.setAttribute(name, value); - }, + setAttributeNS: function DOMElement_setAttributeNS(NS, name, value) { + this.setAttribute(name, value); + }, - appendChild: function DOMElement_appendChild(element) { - var childNodes = this.childNodes; - if (!childNodes.includes(element)) { - childNodes.push(element); - } - }, + appendChild: function DOMElement_appendChild(element) { + var childNodes = this.childNodes; + if (!childNodes.includes(element)) { + childNodes.push(element); + } + }, - hasChildNodes: function DOMElement_hasChildNodes() { - return this.childNodes.length !== 0; - }, + hasChildNodes: function DOMElement_hasChildNodes() { + return this.childNodes.length !== 0; + }, - cloneNode: function DOMElement_cloneNode() { - var newNode = new DOMElement(this.nodeName); - newNode.childNodes = this.childNodes; - newNode.attributes = this.attributes; - newNode.textContent = this.textContent; - return newNode; - }, + cloneNode: function DOMElement_cloneNode() { + var newNode = new DOMElement(this.nodeName); + newNode.childNodes = this.childNodes; + newNode.attributes = this.attributes; + newNode.textContent = this.textContent; + return newNode; + }, - // This method is offered for convenience. It is recommended to directly use - // getSerializer because that allows you to process the chunks as they come - // instead of requiring the whole image to fit in memory. - toString: function DOMElement_toString() { - var buf = []; - var serializer = this.getSerializer(); - var chunk; - while ((chunk = serializer.getNext()) !== null) { - buf.push(chunk); - } - return buf.join(''); - }, + // This method is offered for convenience. It is recommended to directly use + // getSerializer because that allows you to process the chunks as they come + // instead of requiring the whole image to fit in memory. + toString: function DOMElement_toString() { + var buf = []; + var serializer = this.getSerializer(); + var chunk; + while ((chunk = serializer.getNext()) !== null) { + buf.push(chunk); + } + return buf.join(""); + }, - getSerializer: function DOMElement_getSerializer() { - return new DOMElementSerializer(this); - }, + getSerializer: function DOMElement_getSerializer() { + return new DOMElementSerializer(this); + }, }; function DOMElementSerializer(node) { - this._node = node; - this._state = 0; - this._loopIndex = 0; - this._attributeKeys = null; - this._childSerializer = null; + this._node = node; + this._state = 0; + this._loopIndex = 0; + this._attributeKeys = null; + this._childSerializer = null; } DOMElementSerializer.prototype = { - /** - * Yields the next chunk in the serialization of the element. - * - * @returns {string|null} null if the element has fully been serialized. - */ - getNext: function DOMElementSerializer_getNext() { - var node = this._node; - switch (this._state) { - case 0: // Start opening tag. - ++this._state; - return '<' + node.nodeName; - case 1: // Add SVG namespace if this is the root element. - ++this._state; - if (node.nodeName === 'svg:svg') { - return ' xmlns:xlink="http://www.w3.org/1999/xlink"' + ' xmlns:svg="http://www.w3.org/2000/svg"'; - } - /* falls through */ - case 2: // Initialize variables for looping over attributes. - ++this._state; - this._loopIndex = 0; - this._attributeKeys = Object.keys(node.attributes); - /* falls through */ - case 3: // Serialize any attributes and end opening tag. - if (this._loopIndex < this._attributeKeys.length) { - var name = this._attributeKeys[this._loopIndex++]; - return ' ' + name + '="' + xmlEncode(node.attributes[name]) + '"'; - } - ++this._state; - return '>'; - case 4: // Serialize textContent for tspan/style elements. - if (node.nodeName === 'svg:tspan' || node.nodeName === 'svg:style') { - this._state = 6; - return xmlEncode(node.textContent); - } - ++this._state; - this._loopIndex = 0; - /* falls through */ - case 5: // Serialize child nodes (only for non-tspan/style elements). - var value; - while (true) { - value = this._childSerializer && this._childSerializer.getNext(); - if (value !== null) { - return value; - } - var nextChild = node.childNodes[this._loopIndex++]; - if (nextChild) { - this._childSerializer = new DOMElementSerializer(nextChild); - } else { - this._childSerializer = null; - ++this._state; - break; - } - } - /* falls through */ - case 6: // Ending tag. - ++this._state; - return ''; - case 7: // Done. - return null; - default: - throw new Error('Unexpected serialization state: ' + this._state); - } - }, + /** + * Yields the next chunk in the serialization of the element. + * + * @returns {string|null} null if the element has fully been serialized. + */ + getNext: function DOMElementSerializer_getNext() { + var node = this._node; + switch (this._state) { + case 0: // Start opening tag. + ++this._state; + return "<" + node.nodeName; + case 1: // Add SVG namespace if this is the root element. + ++this._state; + if (node.nodeName === "svg:svg") { + return ( + ' xmlns:xlink="http://www.w3.org/1999/xlink"' + + ' xmlns:svg="http://www.w3.org/2000/svg"' + ); + } + /* falls through */ + case 2: // Initialize variables for looping over attributes. + ++this._state; + this._loopIndex = 0; + this._attributeKeys = Object.keys(node.attributes); + /* falls through */ + case 3: // Serialize any attributes and end opening tag. + if (this._loopIndex < this._attributeKeys.length) { + var name = this._attributeKeys[this._loopIndex++]; + return " " + name + '="' + xmlEncode(node.attributes[name]) + '"'; + } + ++this._state; + return ">"; + case 4: // Serialize textContent for tspan/style elements. + if (node.nodeName === "svg:tspan" || node.nodeName === "svg:style") { + this._state = 6; + return xmlEncode(node.textContent); + } + ++this._state; + this._loopIndex = 0; + /* falls through */ + case 5: // Serialize child nodes (only for non-tspan/style elements). + var value; + while (true) { + value = this._childSerializer && this._childSerializer.getNext(); + if (value !== null) { + return value; + } + var nextChild = node.childNodes[this._loopIndex++]; + if (nextChild) { + this._childSerializer = new DOMElementSerializer(nextChild); + } else { + this._childSerializer = null; + ++this._state; + break; + } + } + /* falls through */ + case 6: // Ending tag. + ++this._state; + return ""; + case 7: // Done. + return null; + default: + throw new Error("Unexpected serialization state: " + this._state); + } + }, }; const document = { - childNodes: [], + childNodes: [], - get currentScript() { - return { src: '' }; - }, + get currentScript() { + return { src: "" }; + }, - get documentElement() { - return this; - }, + get documentElement() { + return this; + }, - createElementNS: function (NS, element) { - var elObject = new DOMElement(element); - return elObject; - }, + createElementNS: (NS, element) => { + var elObject = new DOMElement(element); + return elObject; + }, - createElement: function (element) { - return this.createElementNS('', element); - }, + createElement: function (element) { + return this.createElementNS("", element); + }, - getElementsByTagName: function (element) { - if (element === 'head') { - return [this.head || (this.head = new DOMElement('head'))]; - } - return []; - }, + getElementsByTagName: function (element) { + if (element === "head") { + return [this.head || (this.head = new DOMElement("head"))]; + } + return []; + }, }; function Image() { - this._src = null; - this.onload = null; + this._src = null; + this.onload = null; } Image.prototype = { - get src() { - return this._src; - }, - set src(value) { - this._src = value; - if (this.onload) { - this.onload(); - } - }, + get src() { + return this._src; + }, + set src(value) { + this._src = value; + if (this.onload) { + this.onload(); + } + }, }; exports.document = document; @@ -266,15 +269,15 @@ exports.Image = Image; var exported_symbols = Object.keys(exports); -exports.setStubs = function (namespace) { - exported_symbols.forEach(function (key) { - console.assert(!(key in namespace), 'property should not be set: ' + key); - namespace[key] = exports[key]; - }); +exports.setStubs = (namespace) => { + exported_symbols.forEach((key) => { + console.assert(!(key in namespace), "property should not be set: " + key); + namespace[key] = exports[key]; + }); }; -exports.unsetStubs = function (namespace) { - exported_symbols.forEach(function (key) { - console.assert(key in namespace, 'property should be set: ' + key); - delete namespace[key]; - }); +exports.unsetStubs = (namespace) => { + exported_symbols.forEach((key) => { + console.assert(key in namespace, "property should be set: " + key); + delete namespace[key]; + }); }; diff --git a/src/scrapers/pdf/lib/pdf2md/util/page-item-functions.js b/src/scrapers/pdf/lib/pdf2md/util/page-item-functions.js index 50f8cf2110..bfb6abb423 100644 --- a/src/scrapers/pdf/lib/pdf2md/util/page-item-functions.js +++ b/src/scrapers/pdf/lib/pdf2md/util/page-item-functions.js @@ -5,30 +5,34 @@ import PageItem from '../models/PageItem' import LineItemBlock from '../models/LineItemBlock' */ -exports.minXFromBlocks = function minXFromBlocks(blocks /*: LineItemBlock[] */) /*: number */ { - var minX = 999; - blocks.forEach((block) => { - block.items.forEach((item) => { - minX = Math.min(minX, item.x); - }); - }); - if (minX === 999) { - return null; - } - return minX; +exports.minXFromBlocks = function minXFromBlocks( + blocks /*: LineItemBlock[] */, +) /*: number */ { + var minX = 999; + blocks.forEach((block) => { + block.items.forEach((item) => { + minX = Math.min(minX, item.x); + }); + }); + if (minX === 999) { + return null; + } + return minX; }; -exports.minXFromPageItems = function minXFromPageItems(items /*: PageItem */) /*: number */ { - var minX = 999; - items.forEach((item) => { - minX = Math.min(minX, item.x); - }); - if (minX === 999) { - return null; - } - return minX; +exports.minXFromPageItems = function minXFromPageItems( + items /*: PageItem */, +) /*: number */ { + var minX = 999; + items.forEach((item) => { + minX = Math.min(minX, item.x); + }); + if (minX === 999) { + return null; + } + return minX; }; exports.sortByX = function sortByX(items /*: PageItem */) { - items.sort((a, b) => a.x - b.x); + items.sort((a, b) => a.x - b.x); }; diff --git a/src/scrapers/pdf/lib/pdf2md/util/page-number-functions.js b/src/scrapers/pdf/lib/pdf2md/util/page-number-functions.js index 9201e9da40..4925c362c2 100644 --- a/src/scrapers/pdf/lib/pdf2md/util/page-number-functions.js +++ b/src/scrapers/pdf/lib/pdf2md/util/page-number-functions.js @@ -1,4 +1,8 @@ -const { removeLeadingWhitespaces, removeTrailingWhitespaces, isNumber } = require('./string-functions'); +const { + removeLeadingWhitespaces, + removeTrailingWhitespaces, + isNumber, +} = require("./string-functions"); /** * Returns an index number for start/end of search @@ -10,7 +14,7 @@ const { removeLeadingWhitespaces, removeTrailingWhitespaces, isNumber } = requir * @returns {number} A range of where to loop and search */ const searchRange = (numerator, denominator, length) => { - return Math.floor((numerator / denominator) * length); + return Math.floor((numerator / denominator) * length); }; /** @@ -23,17 +27,17 @@ const searchRange = (numerator, denominator, length) => { * @returns {object} pageIndexNumMap object */ const searchArea = (range, pageIndexNumMap, pageIndex) => { - for (const { str } of range) { - const trimLeadingWhitespaces = removeLeadingWhitespaces(str); - const trimWhitespaces = removeTrailingWhitespaces(trimLeadingWhitespaces); - if (isNumber(trimWhitespaces)) { - if (!pageIndexNumMap[pageIndex]) { - pageIndexNumMap[pageIndex] = []; - } - pageIndexNumMap[pageIndex].push(Number(trimWhitespaces)); - } - } - return pageIndexNumMap; + for (const { str } of range) { + const trimLeadingWhitespaces = removeLeadingWhitespaces(str); + const trimWhitespaces = removeTrailingWhitespaces(trimLeadingWhitespaces); + if (isNumber(trimWhitespaces)) { + if (!pageIndexNumMap[pageIndex]) { + pageIndexNumMap[pageIndex] = []; + } + pageIndexNumMap[pageIndex].push(Number(trimWhitespaces)); + } + } + return pageIndexNumMap; }; /** @@ -46,11 +50,15 @@ const searchArea = (range, pageIndexNumMap, pageIndex) => { * @returns {object} pageIndexNumMap object */ exports.findPageNumbers = (pageIndexNumMap, pageIndex, items) => { - const topArea = searchRange(1, 6, items.length); - const bottomArea = searchRange(5, 6, items.length); + const topArea = searchRange(1, 6, items.length); + const bottomArea = searchRange(5, 6, items.length); - const topAreaResult = searchArea(items.slice(0, topArea), pageIndexNumMap, pageIndex); - return searchArea(items.slice(bottomArea), topAreaResult, pageIndex); + const topAreaResult = searchArea( + items.slice(0, topArea), + pageIndexNumMap, + pageIndex, + ); + return searchArea(items.slice(bottomArea), topAreaResult, pageIndex); }; /** @@ -61,36 +69,42 @@ exports.findPageNumbers = (pageIndexNumMap, pageIndex, items) => { * @returns {object} For example { pageIndex: 10, pageNum: 3 } */ exports.findFirstPage = (pageIndexNumMap) => { - let counter = 0; - const keys = Object.keys(pageIndexNumMap); - if (keys.length === 0 || keys.length === 1) { - return; - } + let counter = 0; + const keys = Object.keys(pageIndexNumMap); + if (keys.length === 0 || keys.length === 1) { + return; + } - for (let x = 0; x < keys.length - 1; x++) { - const firstPage = pageIndexNumMap[keys[x]]; - const secondPage = pageIndexNumMap[keys[x + 1]]; - const prevCounter = counter; + for (let x = 0; x < keys.length - 1; x++) { + const firstPage = pageIndexNumMap[keys[x]]; + const secondPage = pageIndexNumMap[keys[x + 1]]; + const prevCounter = counter; - for (let y = 0; y < firstPage.length && counter < 2; y++) { - for (let z = 0; z < secondPage.length && counter < 2; z++) { - const pageDifference = keys[x + 1] - keys[x]; - if (firstPage[y] + 1 === secondPage[z]) { - counter++; - } else if (pageDifference > 1 && firstPage[y] + pageDifference === secondPage[z]) { - counter++; - } - } - } + for (let y = 0; y < firstPage.length && counter < 2; y++) { + for (let z = 0; z < secondPage.length && counter < 2; z++) { + const pageDifference = keys[x + 1] - keys[x]; + if (firstPage[y] + 1 === secondPage[z]) { + counter++; + } else if ( + pageDifference > 1 && + firstPage[y] + pageDifference === secondPage[z] + ) { + counter++; + } + } + } - let pageDetails = x > 0 ? Object.entries(pageIndexNumMap)[x - 1] : Object.entries(pageIndexNumMap)[x]; - if (prevCounter === counter) { - counter = 0; - pageDetails = Object.entries(pageIndexNumMap)[x]; - } else if (counter >= 2) { - return { pageIndex: Number(pageDetails[0]), pageNum: pageDetails[1][0] }; - } - } + let pageDetails = + x > 0 + ? Object.entries(pageIndexNumMap)[x - 1] + : Object.entries(pageIndexNumMap)[x]; + if (prevCounter === counter) { + counter = 0; + pageDetails = Object.entries(pageIndexNumMap)[x]; + } else if (counter >= 2) { + return { pageIndex: Number(pageDetails[0]), pageNum: pageDetails[1][0] }; + } + } }; /** @@ -102,15 +116,16 @@ exports.findFirstPage = (pageIndexNumMap) => { * @returns {object} filteredContent - textContent without items that have pageNum */ exports.removePageNumber = (textContent, pageNum) => { - const filteredContent = { items: [...textContent.items] }; - const topArea = searchRange(1, 6, filteredContent.items.length); - const bottomArea = searchRange(5, 6, filteredContent.items.length); + const filteredContent = { items: [...textContent.items] }; + const topArea = searchRange(1, 6, filteredContent.items.length); + const bottomArea = searchRange(5, 6, filteredContent.items.length); - filteredContent.items = filteredContent.items.filter((item, index) => { - const isAtTop = index > 0 && index < topArea; - const isAtBottom = index > bottomArea && index < filteredContent.items.length; + filteredContent.items = filteredContent.items.filter((item, index) => { + const isAtTop = index > 0 && index < topArea; + const isAtBottom = + index > bottomArea && index < filteredContent.items.length; - return isAtTop || isAtBottom ? Number(item.str) !== Number(pageNum) : item; - }); - return filteredContent; + return isAtTop || isAtBottom ? Number(item.str) !== Number(pageNum) : item; + }); + return filteredContent; }; diff --git a/src/scrapers/pdf/lib/pdf2md/util/pdf.js b/src/scrapers/pdf/lib/pdf2md/util/pdf.js index 7fc619af0c..0b2003faa7 100644 --- a/src/scrapers/pdf/lib/pdf2md/util/pdf.js +++ b/src/scrapers/pdf/lib/pdf2md/util/pdf.js @@ -1,109 +1,127 @@ -const path = require('path'); -const pdfjsPromise = import('pdfjs-dist/build/pdf.mjs'); +const path = require("path"); +const pdfjsPromise = import("pdfjs-dist/build/pdf.mjs"); pdfjsPromise.then((pdfjs) => { - pdfjs.GlobalWorkerOptions.workerSrc = `pdfjs-dist/build/pdf.worker.mjs`; + pdfjs.GlobalWorkerOptions.workerSrc = `pdfjs-dist/build/pdf.worker.mjs`; }); -const { findPageNumbers, findFirstPage, removePageNumber } = require('../../lib/util/page-number-functions'); -const TextItem = require('../models/TextItem'); -const Page = require('../models/Page'); +const { + findPageNumbers, + findFirstPage, + removePageNumber, +} = require("../../lib/util/page-number-functions"); +const TextItem = require("../models/TextItem"); +const Page = require("../models/Page"); const NO_OP = () => {}; exports.parse = async function parse(docOptions, callbacks) { - const { metadataParsed, pageParsed, fontParsed, documentParsed } = { - metadataParsed: NO_OP, - pageParsed: NO_OP, - fontParsed: NO_OP, - documentParsed: NO_OP, - ...(callbacks || {}), - }; - const fontDataPath = path.join(path.resolve(require.resolve('pdfjs-dist'), '../../standard_fonts'), '/'); - if (typeof docOptions === 'string' || docOptions instanceof URL) { - docOptions = { url: docOptions }; - } else if (docOptions instanceof ArrayBuffer || ArrayBuffer.isView(docOptions)) { - docOptions = { data: docOptions }; - } - const pdfjs = await pdfjsPromise; - const pdfDocument = await pdfjs.getDocument({ - ...docOptions, - standardFontDataUrl: fontDataPath, - }).promise; - const metadata = await pdfDocument.getMetadata(); - metadataParsed(metadata); + const { metadataParsed, pageParsed, fontParsed, documentParsed } = { + metadataParsed: NO_OP, + pageParsed: NO_OP, + fontParsed: NO_OP, + documentParsed: NO_OP, + ...(callbacks || {}), + }; + const fontDataPath = path.join( + path.resolve(require.resolve("pdfjs-dist"), "../../standard_fonts"), + "/", + ); + if (typeof docOptions === "string" || docOptions instanceof URL) { + docOptions = { url: docOptions }; + } else if ( + docOptions instanceof ArrayBuffer || + ArrayBuffer.isView(docOptions) + ) { + docOptions = { data: docOptions }; + } + const pdfjs = await pdfjsPromise; + const pdfDocument = await pdfjs.getDocument({ + ...docOptions, + standardFontDataUrl: fontDataPath, + }).promise; + const metadata = await pdfDocument.getMetadata(); + metadataParsed(metadata); - const pages = [...Array(pdfDocument.numPages).keys()].map((index) => new Page({ index })); + const pages = [...Array(pdfDocument.numPages).keys()].map( + (index) => new Page({ index }), + ); - documentParsed(pdfDocument, pages); + documentParsed(pdfDocument, pages); - const fonts = { - ids: new Set(), - map: new Map(), - }; + const fonts = { + ids: new Set(), + map: new Map(), + }; - let pageIndexNumMap = {}; - let firstPage; - for (let j = 1; j <= pdfDocument.numPages; j++) { - const page = await pdfDocument.getPage(j); - const textContent = await page.getTextContent(); + let pageIndexNumMap = {}; + let firstPage; + for (let j = 1; j <= pdfDocument.numPages; j++) { + const page = await pdfDocument.getPage(j); + const textContent = await page.getTextContent(); - if (Object.keys(pageIndexNumMap).length < 10) { - pageIndexNumMap = findPageNumbers(pageIndexNumMap, page.pageNumber - 1, textContent.items); - } else { - firstPage = findFirstPage(pageIndexNumMap); - break; - } - } + if (Object.keys(pageIndexNumMap).length < 10) { + pageIndexNumMap = findPageNumbers( + pageIndexNumMap, + page.pageNumber - 1, + textContent.items, + ); + } else { + firstPage = findFirstPage(pageIndexNumMap); + break; + } + } - let pageNum = firstPage ? firstPage.pageNum : 0; - for (let j = 1; j <= pdfDocument.numPages; j++) { - const page = await pdfDocument.getPage(j); + let pageNum = firstPage ? firstPage.pageNum : 0; + for (let j = 1; j <= pdfDocument.numPages; j++) { + const page = await pdfDocument.getPage(j); - // Trigger the font retrieval for the page - await page.getOperatorList(); + // Trigger the font retrieval for the page + await page.getOperatorList(); - const scale = 1.0; - const viewport = page.getViewport({ scale }); - let textContent = await page.getTextContent(); - if (firstPage && page.pageIndex >= firstPage.pageIndex) { - textContent = removePageNumber(textContent, pageNum); - pageNum++; - } - const textItems = textContent.items.map((item) => { - const tx = pdfjs.Util.transform(viewport.transform, item.transform); + const scale = 1.0; + const viewport = page.getViewport({ scale }); + let textContent = await page.getTextContent(); + if (firstPage && page.pageIndex >= firstPage.pageIndex) { + textContent = removePageNumber(textContent, pageNum); + pageNum++; + } + const textItems = textContent.items.map((item) => { + const tx = pdfjs.Util.transform(viewport.transform, item.transform); - const fontHeight = Math.sqrt(tx[2] * tx[2] + tx[3] * tx[3]); - const dividedHeight = item.height / fontHeight; - return new TextItem({ - x: Math.round(item.transform[4]), - y: Math.round(item.transform[5]), - width: Math.round(item.width), - height: Math.round(dividedHeight <= 1 ? item.height : dividedHeight), - text: item.str, - font: item.fontName, - }); - }); - pages[page.pageNumber - 1].items = textItems; - pageParsed(pages); + const fontHeight = Math.sqrt(tx[2] * tx[2] + tx[3] * tx[3]); + const dividedHeight = item.height / fontHeight; + return new TextItem({ + x: Math.round(item.transform[4]), + y: Math.round(item.transform[5]), + width: Math.round(item.width), + height: Math.round(dividedHeight <= 1 ? item.height : dividedHeight), + text: item.str, + font: item.fontName, + }); + }); + pages[page.pageNumber - 1].items = textItems; + pageParsed(pages); - const fontIds = new Set(textItems.map((t) => t.font)); - for (const fontId of fontIds) { - if (!fonts.ids.has(fontId) && fontId.startsWith('g_d')) { - // Depending on which build of pdfjs-dist is used, the - // WorkerTransport containing the font objects is either transport or _transport - const transport = pdfDocument.transport || pdfDocument._transport; // eslint-disable-line no-underscore-dangle - const font = await new Promise((resolve) => transport.commonObjs.get(fontId, resolve)); - fonts.ids.add(fontId); - fonts.map.set(fontId, font); - fontParsed(fonts); - } - } - } - return { - fonts, - metadata, - pages, - pdfDocument, - }; + const fontIds = new Set(textItems.map((t) => t.font)); + for (const fontId of fontIds) { + if (!fonts.ids.has(fontId) && fontId.startsWith("g_d")) { + // Depending on which build of pdfjs-dist is used, the + // WorkerTransport containing the font objects is either transport or _transport + const transport = pdfDocument.transport || pdfDocument._transport; // eslint-disable-line no-underscore-dangle + const font = await new Promise((resolve) => + transport.commonObjs.get(fontId, resolve), + ); + fonts.ids.add(fontId); + fonts.map.set(fontId, font); + fontParsed(fonts); + } + } + } + return { + fonts, + metadata, + pages, + pdfDocument, + }; }; diff --git a/src/scrapers/pdf/lib/pdf2md/util/string-functions.js b/src/scrapers/pdf/lib/pdf2md/util/string-functions.js index cd6f4ad00b..2d3def08dd 100644 --- a/src/scrapers/pdf/lib/pdf2md/util/string-functions.js +++ b/src/scrapers/pdf/lib/pdf2md/util/string-functions.js @@ -5,120 +5,127 @@ const TAB_CHAR_CODE = 9; const DOT_CHAR_CODE = 46; exports.removeLeadingWhitespaces = function removeLeadingWhitespaces(string) { - while (string.charCodeAt(0) === WHITESPACE_CHAR_CODE) { - string = string.substring(1, string.length); - } - return string; + while (string.charCodeAt(0) === WHITESPACE_CHAR_CODE) { + string = string.substring(1, string.length); + } + return string; }; exports.removeTrailingWhitespaces = function removeTrailingWhitespaces(string) { - while (string.charCodeAt(string.length - 1) === WHITESPACE_CHAR_CODE) { - string = string.substring(0, string.length - 1); - } - return string; + while (string.charCodeAt(string.length - 1) === WHITESPACE_CHAR_CODE) { + string = string.substring(0, string.length - 1); + } + return string; }; exports.isDigit = function isDigit(charCode) { - return charCode >= MIN_DIGIT_CHAR_CODE && charCode <= MAX_DIGIT_CHAR_CODE; + return charCode >= MIN_DIGIT_CHAR_CODE && charCode <= MAX_DIGIT_CHAR_CODE; }; exports.isNumber = function isNumber(string) { - for (var i = 0; i < string.length; i++) { - const charCode = string.charCodeAt(i); - if (!exports.isDigit(charCode)) { - return false; - } - } - return true; + for (var i = 0; i < string.length; i++) { + const charCode = string.charCodeAt(i); + if (!exports.isDigit(charCode)) { + return false; + } + } + return true; }; exports.hasOnly = function hasOnly(string, char) { - const charCode = char.charCodeAt(0); - for (var i = 0; i < string.length; i++) { - const aCharCode = string.charCodeAt(i); - if (aCharCode !== charCode) { - return false; - } - } - return true; + const charCode = char.charCodeAt(0); + for (var i = 0; i < string.length; i++) { + const aCharCode = string.charCodeAt(i); + if (aCharCode !== charCode) { + return false; + } + } + return true; }; -exports.hasUpperCaseCharacterInMiddleOfWord = function hasUpperCaseCharacterInMiddleOfWord(text) { - var beginningOfWord = true; - for (var i = 0; i < text.length; i++) { - const character = text.charAt(i); - if (character === ' ') { - beginningOfWord = true; - } else { - if ( - !beginningOfWord && - isNaN(character * 1) && - character === character.toUpperCase() && - character.toUpperCase() !== character.toLowerCase() - ) { - return true; - } - beginningOfWord = false; - } - } - return false; -}; +exports.hasUpperCaseCharacterInMiddleOfWord = + function hasUpperCaseCharacterInMiddleOfWord(text) { + var beginningOfWord = true; + for (var i = 0; i < text.length; i++) { + const character = text.charAt(i); + if (character === " ") { + beginningOfWord = true; + } else { + if ( + !beginningOfWord && + isNaN(character * 1) && + character === character.toUpperCase() && + character.toUpperCase() !== character.toLowerCase() + ) { + return true; + } + beginningOfWord = false; + } + } + return false; + }; // Remove whitespace/dots + to uppercase exports.normalizedCharCodeArray = function normalizedCharCodeArray(string) { - string = string.toUpperCase(); - return exports - .charCodeArray(string) - .filter( - (charCode) => charCode !== WHITESPACE_CHAR_CODE && charCode !== TAB_CHAR_CODE && charCode !== DOT_CHAR_CODE, - ); + string = string.toUpperCase(); + return exports + .charCodeArray(string) + .filter( + (charCode) => + charCode !== WHITESPACE_CHAR_CODE && + charCode !== TAB_CHAR_CODE && + charCode !== DOT_CHAR_CODE, + ); }; exports.charCodeArray = function charCodeArray(string) { - const charCodes = []; - for (var i = 0; i < string.length; i++) { - charCodes.push(string.charCodeAt(i)); - } - return charCodes; + const charCodes = []; + for (var i = 0; i < string.length; i++) { + charCodes.push(string.charCodeAt(i)); + } + return charCodes; }; exports.prefixAfterWhitespace = function prefixAfterWhitespace(prefix, string) { - if (string.charCodeAt(0) === WHITESPACE_CHAR_CODE) { - string = exports.removeLeadingWhitespaces(string); - return ' ' + prefix + string; - } else { - return prefix + string; - } + if (string.charCodeAt(0) === WHITESPACE_CHAR_CODE) { + string = exports.removeLeadingWhitespaces(string); + return " " + prefix + string; + } else { + return prefix + string; + } }; -exports.suffixBeforeWhitespace = function suffixBeforeWhitespace(string, suffix) { - if (string.charCodeAt(string.length - 1) === WHITESPACE_CHAR_CODE) { - string = exports.removeTrailingWhitespaces(string); - return string + suffix + ' '; - } else { - return string + suffix; - } +exports.suffixBeforeWhitespace = function suffixBeforeWhitespace( + string, + suffix, +) { + if (string.charCodeAt(string.length - 1) === WHITESPACE_CHAR_CODE) { + string = exports.removeTrailingWhitespaces(string); + return string + suffix + " "; + } else { + return string + suffix; + } }; exports.isListItemCharacter = function isListItemCharacter(string) { - if (string.length > 1) { - return false; - } - const char = string.charAt(0); - return char === '-' || char === '•' || char === '–'; + if (string.length > 1) { + return false; + } + const char = string.charAt(0); + return char === "-" || char === "•" || char === "–"; }; exports.isListItem = function isListItem(string) { - return /^[\s]*[-•–][\s].*$/g.test(string); + return /^[\s]*[-•–][\s].*$/g.test(string); }; exports.isNumberedListItem = function isNumberedListItem(string) { - return /^[\s]*[\d]*[.][\s].*$/g.test(string); + return /^[\s]*[\d]*[.][\s].*$/g.test(string); }; exports.wordMatch = function wordMatch(string1, string2) { - const words1 = new Set(string1.toUpperCase().split(' ')); - const words2 = new Set(string2.toUpperCase().split(' ')); - const intersection = new Set([...words1].filter((x) => words2.has(x))); - return intersection.size / Math.max(words1.size, words2.size); + const words1 = new Set(string1.toUpperCase().split(" ")); + const words2 = new Set(string2.toUpperCase().split(" ")); + const intersection = new Set([...words1].filter((x) => words2.has(x))); + return intersection.size / Math.max(words1.size, words2.size); }; diff --git a/src/scrapers/pdf/lib/pdf2md/util/transformations.js b/src/scrapers/pdf/lib/pdf2md/util/transformations.js index cd1d56388b..ebc8635619 100644 --- a/src/scrapers/pdf/lib/pdf2md/util/transformations.js +++ b/src/scrapers/pdf/lib/pdf2md/util/transformations.js @@ -1,46 +1,46 @@ -const CalculateGlobalStats = require('../models/transformations/text-item/CalculateGlobalStats'); +const CalculateGlobalStats = require("../models/transformations/text-item/CalculateGlobalStats"); -const CompactLines = require('../models/transformations/line-item/CompactLines'); -const RemoveRepetitiveElements = require('../models/transformations/line-item/RemoveRepetitiveElements'); -const VerticalToHorizontal = require('../models/transformations/line-item/VerticalToHorizontal'); -const DetectTOC = require('../models/transformations/line-item/DetectTOC'); -const DetectListItems = require('../models/transformations/line-item/DetectListItems'); -const DetectHeaders = require('../models/transformations/line-item/DetectHeaders'); +const CompactLines = require("../models/transformations/line-item/CompactLines"); +const RemoveRepetitiveElements = require("../models/transformations/line-item/RemoveRepetitiveElements"); +const VerticalToHorizontal = require("../models/transformations/line-item/VerticalToHorizontal"); +const DetectTOC = require("../models/transformations/line-item/DetectTOC"); +const DetectListItems = require("../models/transformations/line-item/DetectListItems"); +const DetectHeaders = require("../models/transformations/line-item/DetectHeaders"); -const GatherBlocks = require('../models/transformations/line-item-block/GatherBlocks'); -const DetectCodeQuoteBlocks = require('../models/transformations/line-item-block/DetectCodeQuoteBlocks'); -const DetectListLevels = require('../models/transformations/line-item-block/DetectListLevels'); -const ToTextBlocks = require('../models/transformations/ToTextBlocks'); -const ToMarkdown = require('../models/transformations/ToMarkdown'); +const GatherBlocks = require("../models/transformations/line-item-block/GatherBlocks"); +const DetectCodeQuoteBlocks = require("../models/transformations/line-item-block/DetectCodeQuoteBlocks"); +const DetectListLevels = require("../models/transformations/line-item-block/DetectListLevels"); +const ToTextBlocks = require("../models/transformations/ToTextBlocks"); +const ToMarkdown = require("../models/transformations/ToMarkdown"); -const ParseResult = require('../models/ParseResult'); +const ParseResult = require("../models/ParseResult"); exports.makeTransformations = (fontMap) => [ - new CalculateGlobalStats(fontMap), - new CompactLines(), - new RemoveRepetitiveElements(), - new VerticalToHorizontal(), - new DetectTOC(), - new DetectHeaders(), - new DetectListItems(), + new CalculateGlobalStats(fontMap), + new CompactLines(), + new RemoveRepetitiveElements(), + new VerticalToHorizontal(), + new DetectTOC(), + new DetectHeaders(), + new DetectListItems(), - new GatherBlocks(), - new DetectCodeQuoteBlocks(), - new DetectListLevels(), + new GatherBlocks(), + new DetectCodeQuoteBlocks(), + new DetectListLevels(), - new ToTextBlocks(), - new ToMarkdown(), + new ToTextBlocks(), + new ToMarkdown(), ]; exports.transform = (pages, transformations) => { - var parseResult = new ParseResult({ pages }); - let lastTransformation; - transformations.forEach((transformation) => { - if (lastTransformation) { - parseResult = lastTransformation.completeTransform(parseResult); - } - parseResult = transformation.transform(parseResult); - lastTransformation = transformation; - }); - return parseResult; + var parseResult = new ParseResult({ pages }); + let lastTransformation; + transformations.forEach((transformation) => { + if (lastTransformation) { + parseResult = lastTransformation.completeTransform(parseResult); + } + parseResult = transformation.transform(parseResult); + lastTransformation = transformation; + }); + return parseResult; }; diff --git a/src/scrapers/pdf/playground/pdf-scraper-playground.ts b/src/scrapers/pdf/playground/pdf-scraper-playground.ts index 38c8c47276..3f540213c7 100644 --- a/src/scrapers/pdf/playground/pdf-scraper-playground.ts +++ b/src/scrapers/pdf/playground/pdf-scraper-playground.ts @@ -1,72 +1,74 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { writeFile } from 'fs/promises'; -import { join } from 'path'; -import { usageToHuman } from '../../../execution/utils/usageToHuman'; -import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground'; -import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson'; -import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode'; -import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler'; -import { PdfScraper } from '../PdfScraper'; +import { join } from "path"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { writeFile } from "fs/promises"; +import { usageToHuman } from "../../../execution/utils/usageToHuman"; +import { $provideLlmToolsForTestingAndScriptsAndPlayground } from "../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground"; +import { stringifyPipelineJson } from "../../../utils/editable/utils/stringifyPipelineJson"; +import { $provideFilesystemForNode } from "../../_common/register/$provideFilesystemForNode"; +import { makeKnowledgeSourceHandler } from "../../_common/utils/makeKnowledgeSourceHandler"; +import { PdfScraper } from "../PdfScraper"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 Scrape knowledge from PDFs (playground)`); + console.info(`🧸 Scrape knowledge from PDFs (playground)`); - // Do here stuff you want to test - //========================================> + // Do here stuff you want to test + //========================================> - //const example = '10-simple.doc'; - const example = '10-simple.pdf'; - // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them + //const example = '10-simple.doc'; + const example = "10-simple.pdf"; + // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them - const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true }); - const rootDirname = join(__dirname, '..', 'examples'); + const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ + isCacheReloaded: true, + }); + const rootDirname = join(__dirname, "..", "examples"); - const pdfScraper = new PdfScraper( - { llm: await $provideLlmToolsForTestingAndScriptsAndPlayground() }, - { - rootDirname, - }, - ); + const pdfScraper = new PdfScraper( + { llm: await $provideLlmToolsForTestingAndScriptsAndPlayground() }, + { + rootDirname, + }, + ); - const knowledge = await pdfScraper.scrape( - await makeKnowledgeSourceHandler( - { knowledgeSourceContent: example }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ), - ); + const knowledge = await pdfScraper.scrape( + await makeKnowledgeSourceHandler( + { knowledgeSourceContent: example }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ), + ); - console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); - console.info(colors.bgGreen(' Knowledge: ')); - console.info(knowledge); + console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); + console.info(colors.bgGreen(" Knowledge: ")); + console.info(knowledge); - await writeFile( - join( - __dirname, - `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, - ), - stringifyPipelineJson(knowledge), - 'utf-8', - ); - /**/ + await writeFile( + join( + __dirname, + `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, + ), + stringifyPipelineJson(knowledge), + "utf-8", + ); + /**/ - //========================================/ + //========================================/ } /** diff --git a/src/scrapers/pdf/playground/tsconfig.json b/src/scrapers/pdf/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/scrapers/pdf/playground/tsconfig.json +++ b/src/scrapers/pdf/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/scrapers/pdf/register-constructor.ts b/src/scrapers/pdf/register-constructor.ts index 253f8de22c..a10487679f 100644 --- a/src/scrapers/pdf/register-constructor.ts +++ b/src/scrapers/pdf/register-constructor.ts @@ -1,6 +1,6 @@ -import type { Registration } from '../../utils/$Register'; -import { $scrapersRegister } from '../_common/register/$scrapersRegister'; -import { createPdfScraper } from './createPdfScraper'; +import type { Registration } from "../../utils/$Register"; +import { $scrapersRegister } from "../_common/register/$scrapersRegister"; +import { createPdfScraper } from "./createPdfScraper"; /** * Registration of known scraper @@ -11,7 +11,8 @@ import { createPdfScraper } from './createPdfScraper'; * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _PdfScraperRegistration: Registration = $scrapersRegister.register(createPdfScraper); +export const _PdfScraperRegistration: Registration = + $scrapersRegister.register(createPdfScraper); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/scrapers/pdf/register-metadata.ts b/src/scrapers/pdf/register-metadata.ts index d29969a5ff..97f028d257 100644 --- a/src/scrapers/pdf/register-metadata.ts +++ b/src/scrapers/pdf/register-metadata.ts @@ -1,8 +1,8 @@ -import type { Registration } from '../../utils/$Register'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import { $deepFreeze } from '../../utils/serialization/$deepFreeze'; -import { $scrapersMetadataRegister } from '../_common/register/$scrapersMetadataRegister'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; +import type { Registration } from "../../utils/$Register"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import { $deepFreeze } from "../../utils/serialization/$deepFreeze"; +import { $scrapersMetadataRegister } from "../_common/register/$scrapersMetadataRegister"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; keepTypeImported(); @@ -12,14 +12,14 @@ keepTypeImported(); * @private within the scraper directory */ export const pdfScraperMetadata = $deepFreeze({ - title: 'Pdf scraper', - packageName: '@promptbook/pdf', - className: 'PdfScraper', - mimeTypes: ['application/pdf-DISABLED'], // <- TODO: [🌜] Remove `PdfScraper` or make it work or make it as alias of `MarkitdownScraper` - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@', - isAvilableInBrowser: false, - // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server - requiredExecutables: [], + title: "Pdf scraper", + packageName: "@promptbook/pdf", + className: "PdfScraper", + mimeTypes: ["application/pdf-DISABLED"], // <- TODO: [🌜] Remove `PdfScraper` or make it work or make it as alias of `MarkitdownScraper` + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@", + isAvilableInBrowser: false, + // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server + requiredExecutables: [], }) satisfies ScraperAndConverterMetadata; /* <- Note: [🤛] */ /** @@ -31,7 +31,8 @@ export const pdfScraperMetadata = $deepFreeze({ * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _PdfScraperMetadataRegistration: Registration = $scrapersMetadataRegister.register(pdfScraperMetadata); +export const _PdfScraperMetadataRegistration: Registration = + $scrapersMetadataRegister.register(pdfScraperMetadata); /** * Note: [💞] Ignore a discrepancy between file name and entity name diff --git a/src/scrapers/website/WebsiteScraper.ts b/src/scrapers/website/WebsiteScraper.ts index 65777045c6..41a8e01e48 100644 --- a/src/scrapers/website/WebsiteScraper.ts +++ b/src/scrapers/website/WebsiteScraper.ts @@ -1,27 +1,27 @@ -import type { KnowledgePiecePreparedJson } from '../../pipeline/PipelineJson/KnowledgePieceJson'; -import type { string_markdown } from '../../types/typeAliases'; -import type { Converter } from '../_common/Converter'; -import type { Scraper } from '../_common/Scraper'; -import type { ScraperSourceHandler } from '../_common/Scraper'; // TODO: [🏳‍🌈] Finally take pick of .json vs .ts // import PipelineCollection from '../../../books/books'; -import { Readability } from '@mozilla/readability'; -import { JSDOM } from 'jsdom'; -import { Converter as ShowdownConverter } from 'showdown'; -import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from '../../config'; -import { DEFAULT_IS_VERBOSE } from '../../config'; -import { DEFAULT_SCRAPE_CACHE_DIRNAME } from '../../config'; -import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError'; -import { KnowledgeScrapeError } from '../../errors/KnowledgeScrapeError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; -import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource'; -import { getScraperIntermediateSource } from '../_common/utils/getScraperIntermediateSource'; -import { MarkdownScraper } from '../markdown/MarkdownScraper'; -import { websiteScraperMetadata } from './register-metadata'; -import { createShowdownConverter } from './utils/createShowdownConverter'; +import { Readability } from "@mozilla/readability"; +import { JSDOM } from "jsdom"; +import type { Converter as ShowdownConverter } from "showdown"; +import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from "../../config"; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import { DEFAULT_SCRAPE_CACHE_DIRNAME } from "../../config"; +import { EnvironmentMismatchError } from "../../errors/EnvironmentMismatchError"; +import { KnowledgeScrapeError } from "../../errors/KnowledgeScrapeError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { KnowledgePiecePreparedJson } from "../../pipeline/PipelineJson/KnowledgePieceJson"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import type { string_markdown } from "../../types/typeAliases"; +import type { Converter } from "../_common/Converter"; +import type { Scraper } from "../_common/Scraper"; +import type { ScraperSourceHandler } from "../_common/Scraper"; +import type { ScraperIntermediateSource } from "../_common/ScraperIntermediateSource"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; +import { getScraperIntermediateSource } from "../_common/utils/getScraperIntermediateSource"; +import { MarkdownScraper } from "../markdown/MarkdownScraper"; +import { websiteScraperMetadata } from "./register-metadata"; +import { createShowdownConverter } from "./utils/createShowdownConverter"; /** * Scraper for websites @@ -30,113 +30,126 @@ import { createShowdownConverter } from './utils/createShowdownConverter'; * @public exported from `@promptbook/website-crawler` */ export class WebsiteScraper implements Converter, Scraper { - /** - * Metadata of the scraper which includes title, mime types, etc. - */ - public get metadata(): ScraperAndConverterMetadata { - return websiteScraperMetadata; - } - - /** - * Markdown scraper is used internally - */ - private readonly markdownScraper: MarkdownScraper; - - /** - * Showdown converter is used internally - */ - private readonly showdownConverter: ShowdownConverter; - - public constructor( - private readonly tools: Pick, - private readonly options: PrepareAndScrapeOptions, - ) { - this.markdownScraper = new MarkdownScraper(tools, options); - this.showdownConverter = createShowdownConverter(); - } - - /** - * Convert the website to `.md` file and returns intermediate source - * - * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object - */ - public async $convert( - source: ScraperSourceHandler, - ): Promise { - const { - // TODO: [🧠] Maybe in node use headless browser not just JSDOM - rootDirname = process.cwd(), - cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, - intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, - isVerbose = DEFAULT_IS_VERBOSE, - } = this.options; - - if (source.url === null) { - throw new KnowledgeScrapeError('Website scraper requires URL'); - } - - if (this.tools.fs === undefined) { - throw new EnvironmentMismatchError('Can not scrape websites without filesystem tools'); - } - - const jsdom = new JSDOM(await source.asText(), { - url: source.url, - }); - - const reader = new Readability(jsdom.window.document); - const article = reader.parse(); - - // console.log(article); - // await forTime(10000); - - let html = article?.content || article?.textContent || jsdom.window.document.body.innerHTML; - - // Note: Unwrap html such as it is convertable by `markdownConverter` - for (let i = 0; i < 2; i++) { - html = html.replace(/(.*)<\/div>/is, '$1'); - } - - if (html.includes('> | null> { - const cacheFilehandler = await this.$convert(source); - - const markdownSource = { - source: source.source, - filename: cacheFilehandler.filename, - url: null, - mimeType: 'text/markdown', - asText() { - return cacheFilehandler.markdown; - }, - asJson() { - throw new UnexpectedError( - 'Did not expect that `markdownScraper` would need to get the content `asJson`', - ); - }, - /* + /** + * Metadata of the scraper which includes title, mime types, etc. + */ + public get metadata(): ScraperAndConverterMetadata { + return websiteScraperMetadata; + } + + /** + * Markdown scraper is used internally + */ + private readonly markdownScraper: MarkdownScraper; + + /** + * Showdown converter is used internally + */ + private readonly showdownConverter: ShowdownConverter; + + public constructor( + private readonly tools: Pick, + private readonly options: PrepareAndScrapeOptions, + ) { + this.markdownScraper = new MarkdownScraper(tools, options); + this.showdownConverter = createShowdownConverter(); + } + + /** + * Convert the website to `.md` file and returns intermediate source + * + * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object + */ + public async $convert( + source: ScraperSourceHandler, + ): Promise { + const { + // TODO: [🧠] Maybe in node use headless browser not just JSDOM + rootDirname = process.cwd(), + cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, + intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, + isVerbose = DEFAULT_IS_VERBOSE, + } = this.options; + + if (source.url === null) { + throw new KnowledgeScrapeError("Website scraper requires URL"); + } + + if (this.tools.fs === undefined) { + throw new EnvironmentMismatchError( + "Can not scrape websites without filesystem tools", + ); + } + + const jsdom = new JSDOM(await source.asText(), { + url: source.url, + }); + + const reader = new Readability(jsdom.window.document); + const article = reader.parse(); + + // console.log(article); + // await forTime(10000); + + let html = + article?.content || + article?.textContent || + jsdom.window.document.body.innerHTML; + + // Note: Unwrap html such as it is convertable by `markdownConverter` + for (let i = 0; i < 2; i++) { + html = html.replace( + /(.*)<\/div>/is, + "$1", + ); + } + + if (html.includes(" + > | null> { + const cacheFilehandler = await this.$convert(source); + + const markdownSource = { + source: source.source, + filename: cacheFilehandler.filename, + url: null, + mimeType: "text/markdown", + asText() { + return cacheFilehandler.markdown; + }, + asJson() { + throw new UnexpectedError( + "Did not expect that `markdownScraper` would need to get the content `asJson`", + ); + }, + /* TODO: [🥽] > asBlob() { > throw new UnexpectedError( @@ -144,14 +157,14 @@ export class WebsiteScraper implements Converter, Scraper { > ); > }, */ - } satisfies ScraperSourceHandler; + } satisfies ScraperSourceHandler; - const knowledge = this.markdownScraper.scrape(markdownSource); + const knowledge = this.markdownScraper.scrape(markdownSource); - await cacheFilehandler.destroy(); + await cacheFilehandler.destroy(); - return knowledge; - } + return knowledge; + } } /** diff --git a/src/scrapers/website/createWebsiteScraper.ts b/src/scrapers/website/createWebsiteScraper.ts index ebec36de40..ef2063cd0a 100644 --- a/src/scrapers/website/createWebsiteScraper.ts +++ b/src/scrapers/website/createWebsiteScraper.ts @@ -1,9 +1,9 @@ -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import type { ScraperConstructor } from '../_common/register/ScraperConstructor'; -import { websiteScraperMetadata } from './register-metadata'; -import { WebsiteScraper } from './WebsiteScraper'; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { PrepareAndScrapeOptions } from "../../prepare/PrepareAndScrapeOptions"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import type { ScraperConstructor } from "../_common/register/ScraperConstructor"; +import { WebsiteScraper } from "./WebsiteScraper"; +import { websiteScraperMetadata } from "./register-metadata"; keepTypeImported(); @@ -13,10 +13,13 @@ keepTypeImported(); * @public exported from `@promptbook/website-crawler` */ export const createWebsiteScraper = Object.assign( - (tools: Pick, options: PrepareAndScrapeOptions): WebsiteScraper => { - return new WebsiteScraper(tools, options); - }, - websiteScraperMetadata, + ( + tools: Pick, + options: PrepareAndScrapeOptions, + ): WebsiteScraper => { + return new WebsiteScraper(tools, options); + }, + websiteScraperMetadata, ) satisfies ScraperConstructor; /* <- Note: [🤛] */ /** diff --git a/src/scrapers/website/examples/koralkykatlas-cz-cs-blog-prispevek-rijna-zhorseni-kvality-kovove-bizuterie.knowledge.json b/src/scrapers/website/examples/koralkykatlas-cz-cs-blog-prispevek-rijna-zhorseni-kvality-kovove-bizuterie.knowledge.json index f7bb3e5892..a22fa5c874 100644 --- a/src/scrapers/website/examples/koralkykatlas-cz-cs-blog-prispevek-rijna-zhorseni-kvality-kovove-bizuterie.knowledge.json +++ b/src/scrapers/website/examples/koralkykatlas-cz-cs-blog-prispevek-rijna-zhorseni-kvality-kovove-bizuterie.knowledge.json @@ -1,73 +1,1934 @@ [ - { - "name": "narust-podvodne-bizuterie-v-krizi", - "title": "Nárůst podvodné bižuterie v krizi", - "content": "---\n\nKorálky Katlas zaznamenaly zvýšený výskyt podvodné a zdraví škodlivé bižuterie na trhu v době ekonomické krize, kdy lidé více šetří. Zákazníci hlásí problémy a podvody s bižuterií jak v České republice, tak v zahraničí.", - "keywords": [ - "korálky", - "Katlas", - "podvodná bižuterie", - "zdraví škodlivá bižuterie", - "ekonomická krize", - "šetření", - "zákazníci", - "problémy", - "podvody", - "Česká republika", - "zahraničí" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - -0.0037002896, -0.071282186, -0.011276486, -0.0038142868, 0.03477841, -0.01859696, 0.0058847237, -0.0064639533, 0.0043904353, -0.031697404, 0.022922693, 0.019730771, -0.011911173, -0.051613033, 0.021604022, 0.017081104, 0.004362706, -0.04444045, 0.00035835966, 0.019989574, -0.0030517376, -0.016033562, 0.011214866, -0.020716693, 0.02080296, 0.013075794, -0.036257293, -0.009785279, -0.022207899, 0.01820259, -0.0028145001, -0.022294167, -0.008990379, -0.005595109, 0.008830166, 0.0067905397, 0.017672658, -0.036331236, -0.0014873563, -0.011110112, 0.01223776, -0.056542642, -0.022922693, 0.06960612, 0.0066981097, -0.01189885, -0.020876905, -0.017561741, 0.0028915252, -0.0053825197, -0.027827658, 0.013507135, -0.028641043, -0.03988056, 0.023415655, 0.011886526, -0.012287057, 0.0437503, 0.030760776, 0.049567245, -0.02664455, 0.013112767, 0.006285255, 0.029479077, -0.020568805, 0.026102293, -0.026570605, -0.0031380057, 0.0057337545, 0.02486989, -0.011066978, -0.024845243, 0.052746844, -0.002346187, 0.023773052, -0.020987822, -0.010635637, -0.0075114956, -0.011430536, 0.007912027, -0.020445563, 0.06571172, 0.048901748, 0.011017682, -0.035715036, -0.044465095, -0.007788786, -0.055556722, -0.013913829, -0.032781918, -0.04971513, -0.009631229, -0.0583666, 0.014825807, 0.008540552, -0.00047986687, -0.035838276, -0.003943689, -0.045993276, 0.048310194, 0.0105616925, -0.0053856005, 0.017228993, 0.04111296, -0.0011576884, 0.053239804, -0.013901505, 0.026422719, -0.024315309, 0.053584877, 0.014640946, -0.014098689, 0.015626868, -0.015713137, 0.010641798, 0.022614593, 0.025362851, 0.010025597, -0.018190267, 0.010068731, -0.0673385, -0.032017827, 0.054767985, 0.029134003, 0.013482488, -0.0065563833, -0.017253641, 0.025905108, 0.0033336498, 0.014788834, -0.048655264, -0.013679672, 0.025412148, -0.017931461, 0.011325782, -0.004439731, -0.0035524012, 0.0139508005, 0.011418212, -0.017167373, -0.039781965, -0.012792341, 0.017081104, 0.0025526145, -0.062014513, -0.056542642, -0.0013587242, -0.025609331, -0.026989624, -0.014184957, 0.03465517, - -4.0016988e-7, - -0.014147985, -0.019841686, 0.0050497707, -0.0118926875, 0.03401432, 0.0006631868, 0.019878658, 0.035074186, -0.013655024, -0.010537044, 0.025929756, -0.016699059, 0.0009204509, -0.0110731395, 0.049296115, -0.04419397, -0.040521406, 0.03527137, -0.0193364, -0.018227238, 0.0035185101, 0.006420819, 0.008552876, 0.04237001, 0.029898094, 0.012367163, -0.01529412, -0.05028204, 0.00030020563, 0.023144526, 0.037391104, 0.0024694272, -0.025091723, 0.002364673, -0.00016926283, -0.045155242, 0.00954496, 0.0062760115, 0.025683276, -0.036848847, -0.02006352, -0.071725845, -0.024808269, 0.0029993604, 0.0036417504, -0.008300234, 0.0035185101, -0.013753616, -0.005086743, -0.029134003, 0.034729112, -0.03606011, 0.01547898, -0.014912074, 0.028788932, -0.008700765, -0.01801773, 0.020568805, -0.009532636, 0.019582883, 0.028714987, 0.043848895, -0.04301086, 0.010333698, 0.00480329, 0.0032751106, 0.00006893754, 0.0144314375, 0.019040624, -0.009113619, -0.0048464243, -0.0054441397, -0.020088166, 0.00044520554, -0.0054010055, 0.025436796, -0.011566101, 0.0187202, -0.004636916, -0.045919333, -0.05999337, 0.011214866, -0.018954357, 0.006001802, -0.024832917, 0.01865858, 0.03366925, -0.016415607, -0.007702518, 0.012927907, -0.0032504625, -0.030785425, 0.0028037166, 0.010678771, -0.006488601, 0.010493911, 0.0023045933, 0.041926347, -0.0152817955, -0.01667441, 0.001508153, 0.027162159, 0.006907618, 0.04606722, 0.050725702, -0.022836424, -0.00020662004, -0.027112864, 0.046165813, -0.023440303, -0.059451114, 0.029996686, 0.009822251, 0.03618335, -0.005974073, 0.0023199983, -0.025461443, 0.013408544, -0.0016514199, -0.010025597, 0.036331236, -0.0016714464, 0.001520477, 0.040324222, 0.06512017, -0.018239563, -0.029996686, 0.053387694, 0.010746554, 0.03095796, 0.009495664, 0.027088216, -0.0139508005, 0.018313507, -0.028271321, 0.015405036, 0.011997442, 0.037144624, -0.0005915534, -0.05535954, -0.028345266, -0.00020450185, 0.012379487, -0.002162867, 0.017672658, 0.024611086, -0.024031855, -0.002199839, -0.05378206, -0.02740864, -0.034753762, 0.043158747, 0.019841686, 0.009779117, 0.04557426, 0.023440303, -0.06896526, -0.057577863, -0.022811778, 0.0008726953, 0.003999147, -0.0049111256, -0.044046078, -0.019545909, -0.034310095, -0.015651517, 0.071085, -0.026545959, 0.009273832, 0.037612937, -0.009366262, 0.02627483, 0.015934968, -0.009002703, 0.008768546, 0.01275537, 0.030612888, 0.01795611, 0.03366925, -0.00094201794, -0.028197378, 0.013531784, -0.047891177, -0.016132154, -0.0006666529, -0.061718736, 0.01541736, 0.007930513, 0.009723659, -0.010863631, -0.001731526, -0.0075361435, -0.019841686, 0.007332797, -0.012434945, -0.01465327, 0.016773002, 0.0187202, 0.00054495316, 0.022368113, -0.030489648, -0.00010610218, 0.06388777, -0.024845243, 0.011221028, 0.0077641383, 0.017943786, -0.017155048, 0.008158507, -0.002945443, 0.0073697693, 0.003401432, 0.026743142, 0.0425179, -0.014086365, 0.014961371, 0.006537897, -0.01477651, -0.01636631, 0.004726265, -0.0335953, -0.0125889955, -0.012096034, 0.009501826, -0.0039190412, 0.015885673, -0.015577572, -0.00061042455, 0.00017215128, -0.015885673, 0.005145282, 0.014234253, -0.019952603, 0.06955682, 0.0017423095, 0.05264825, 0.006029531, 0.03778547, -0.014912074, 0.0048556672, 0.016341662, 0.034186855, 0.0070185345, -0.022466704, 0.0019348725, 0.06615539, -0.009859223, 0.011701665, -0.0037495857, 0.034975592, -0.029331189, 0.004575296, 0.00302863, -0.01024743, 0.004729346, -0.037834767, -0.015824053, -0.0006277552, -0.008121535, -0.0075114956, -0.018103998, 0.014369817, -0.004763237, 0.0075361435, 0.053091917, -0.0001403784, 0.025436796, 0.034334745, -0.03453193, -0.02652131, 0.041408736, 0.021037117, -0.03854956, -0.013679672, -0.022996638, 0.014628622, 0.0023076744, -0.004356544, -0.011713989, 0.026841735, -0.025707925, -0.003928284, 0.023292415, 0.012780018, -0.007819597, 0.03665166, 0.005148363, -0.006316065, -0.03756364, 0.00045714443, 0.03310234, 0.036725607, -0.001186958, -0.0144314375, 0.021148033, 0.019866334, 0.0056290003, -0.005927858, -0.013556432, 0.020765988, 0.0012185384, -0.0048864773, -0.046880607, -0.013605728, -0.016415607, 0.022429733, 0.035739683, 0.027729064, -0.012040576, 0.043355934, -0.03719392, -0.021295922, 0.0155406, 0.020605776, 0.031007256, 0.022947341, 0.006457791, 0.012478079, -0.031105848, -0.046880607, -0.027211456, -0.0065625454, -0.003047116, -0.06546524, 0.03842632, -0.008632982, 0.014061716, -0.010290564, -0.0012531996, 0.022109307, -0.008133859, 0.02767977, 0.01611983, -0.003077926, -0.012188464, -0.016169125, 0.0045660525, 0.008756222, -0.065317355, -0.016686736, 0.003355217, 0.012354839, -0.014147985, -0.004245628, -0.015700813, -0.048236247, -0.026595253, 0.025030103, -0.008688441, -0.01033986, 0.0139508005, -0.028222026, -0.02906006, -0.06334551, -0.0031580324, 0.022392761, 0.040274926, 0.0152325, 0.004261033, -0.0022137037, 0.014961371, -0.01214533, -0.02753188, -0.008176993, -0.016082857, -0.011775609, -0.0038019628, -0.024549466, 0.016846947, 0.015959617, -0.038894635, -0.019792391, -0.04520454, -0.0077949483, 0.013285303, 0.0052192262, 0.044070728, -0.019348726, 0.0133469235, 0.017352233, 0.013901505, 0.015121583, -0.0007016994, 0.00088424905, -0.0029700908, -0.00088424905, -0.013975449, -0.027383992, 0.00055573665, 0.015084611, 0.0013417787, 0.019262457, 0.02254065, 0.05062711, 0.0070678303, -0.007456037, -0.013063471, -0.017253641, -0.035074186, 0.0112888105, 0.008854815, -0.0036848846, 0.01890506, 0.035567146, 0.043848895, -0.026964976, -0.017746601, -0.009304642, -0.002584965, 0.031031905, -0.00195798, -0.00056266895, 0.023366358, -0.028443858, 0.005450302, 0.052845437, -0.012743046, 0.00009141926, 0.027236104, 0.008694602, -0.021530079, -0.032929804, -0.017413853, 0.0013903045, -0.04771864, -0.011750962, 0.028764283, 0.024450874, -0.026743142, -0.019780066, -0.0007752584, -0.033644598, -0.032560084, -0.0011985118, 0.05314121, -0.009563446, -0.006932266, -0.012471917, 0.02367446, -0.013975449, -0.02086458, -0.014258902, 0.008713088, 0.035345316, -0.019989574, -0.013655024, 0.012669101, 0.021000145, 0.0066981097, 0.008577524, 0.007456037, 0.042443953, -0.009403234, 0.012644454, -0.015762433, -0.019952603, 0.008731575, -0.008337205, 0.00074598886, 0.0028468505, -0.010050246, -0.013433191, 0.019570557, 0.008552876, 0.02563398, 0.02854245, -0.004415083, 0.0190283, 0.018732524, -0.001734607, 0.017352233, -0.01832583, -0.0013818317, -0.024808269, -0.015651517, 0.01667441, 0.0040761726, 0.0021721101, -0.048113007, -0.013691996, 0.009446368, -0.028912172, 0.008170831, 0.027704416, 0.0017669576, 0.07167655, 0.027433287, 0.014912074, 0.011091626, 0.018005406, -0.002178272, 0.0055673798, 0.011584587, 0.011097788, -0.02664455, 0.017808221, 0.021037117, 0.023477275, -0.029232597, 0.010228944, -0.022466704, 0.007979808, -0.03273262, 0.0060233693, -0.0014665595, 0.0077703, -0.00189636, -0.00081877765, -0.018030055, -0.032313604, -0.01229938, 0.0017715791, -0.010882118, 0.0033028398, -0.014603974, -0.005715268, 0.021148033, -0.020593451, 0.02012514, 0.021308245, -0.028887523, -0.024216717, -0.020765988, -0.022010716, 0.013691996, -0.0091998875, -0.017746601, -0.012927907, 0.043454524, 0.005761483, 0.016193774, 0.022096984, -0.002808338, -0.0065502212, 0.008103049, 0.04276438, 0.031376977, -0.0260037, 0.016748356, -0.034088265, -0.011467509, 0.008460445, 0.03719392, -0.0241058, 0.021554727, -0.022084659, -0.02006352, 0.013248331, -0.03310234, -0.009335452, -0.023662135, 0.014862779, 0.026028348, 0.03135233, -0.003153411, -0.006537897, 0.010610988, -0.013839885, -0.005890886, -0.0062051485, -0.040817183, -0.009828413, -0.012262409, 0.0037126136, 0.0046399967, -0.011572263, -0.027630473, 0.025486091, 0.036232643, 0.0070123724, 0.0022953504, -0.010974548, 0.03260938, 0.032954454, 0.021074088, 0.018818792, -0.013605728, 0.0097729545, -0.010512397, 0.038352378, 0.040151685, 0.0035832115, -0.032634027, 0.012490403, -0.03529602, 0.010173486, 0.019065272, 0.012096034, 0.04532778, -0.026447365, -0.009162915, -0.033940375, -0.011344269, -0.027852304, 0.010463101, -0.008213965, 0.034852352, 0.005961749, -0.028443858, -0.007271177, 0.031278387, 0.003943689, -0.015047639, 0.015737785, 0.0069199423, 0.0048957206, -0.04237001, 0.012718398, 0.009625066, -0.005493436, -0.026324125, 0.00053031836, 0.018338155, -0.027704416, -0.029207949, 0.0039775805, 0.02012514, -0.017598713, -0.011331945, 0.005601271, -0.024512494, 0.028468506, -0.012243923, -0.027630473, -0.0010875956, 0.015996588, 0.008842491, 0.004849505, 0.02676779, 0.003432242, 0.018030055, -0.0037434238, 0.023649812, 0.010949899, 0.04362706, -0.0105616925, -0.018646255, 0.008731575, -0.013802912, -0.0032504625, -0.011017682, -0.00050066365, -0.0013541026, 0.018251887, 0.009132105, -0.04426791, -0.0063653607, 0.017364556, -0.00969901, -0.012330191, 0.026200885, 0.00056574994, 0.024056504, 0.0033244067, -0.01287861, 0.0049388544, 0.004285681, 0.014394466, 0.016292365, -0.0049789078, 0.008540552, -0.022294167, 0.01611983, 0.0078011104, -0.0072773388, 0.02906006, 0.017758925, 0.013063471, -0.013839885, -0.0041840076, -0.001945656, 0.003811206, 0.016723707, 0.016809976, 0.017512444, -0.007690194, 0.008232451, 0.0005068257, -0.019693797, -0.0056444053, -0.0033798649, 0.015614544, 0.02024838, 0.0039344463, -0.010167324, 0.0073759314, -0.0241058, -0.012465755, 0.01529412, -0.0044212453, -0.013655024, 0.03095796, 0.032880507, 0.009557284, -0.0031518703, -0.018757172, 0.0041624405, 0.04239466, 0.005595109, -0.0071849087, 0.019003652, -0.023230795, 0.00069361174, 0.003857421, 0.033570655, 0.018744849, -0.020963173, 0.0004401989, 0.017031807, 0.0380566, -0.013272979, 0.004760156, 0.017598713, 0.0022552973, -0.0039498513, -0.046929903, -0.030933313, -0.011467509, -0.011251838, -0.008731575, -0.03184529, 0.02266389, -0.005129877, -0.026052997, -0.0018023892, 0.014061716, -0.028517803, -0.0062883357, 0.019508937, -0.008331044, -0.011985118, -0.004331896, 0.0010899063, -0.021825856, 0.007825758, 0.025362851, -0.021702616, -0.009994787, 0.013186711, 0.0048464243, -0.0055211647, -0.03845097, 0.016317014, -0.0020165192, -0.031943884, -0.012293219, -0.040817183, 0.008571362, 0.014295873, -0.016883919, -0.017734278, 0.000047418627, 0.0074375514, 0.018375127, 0.021517754, 0.0007825758, 0.000057576322, -0.005616676, 0.021061765, -0.01541736, 0.009477179, 0.020704368, -0.006488601, -0.029355837, 0.0007548468, -0.014024745, -0.019730771, 0.0037495857, 0.029503725, 0.005290089, -0.06117648, -0.003860502, 0.008860976, -0.011720152, -0.019619854, -0.006809026, -0.0020581128, -0.012927907, 0.0024755893, -0.005564299, -0.011140922, 0.020729017, -0.008583686, 0.021517754, -0.016846947, 0.0058261845, -0.015922645, -0.013457839, -0.0057121874, -0.014123337, 0.021801207, -0.020236054, -0.022257196, -0.03692279, 0.0021490026, -0.021345219, 0.007265015, 0.005404087, -0.024586437, -0.04012704, -0.0031795995, 0.035000242, 0.017487796, 0.036109403, -0.023748403, 0.009471016, -0.013383895, -0.027507233, -0.002755961, -0.003795801, -0.019287106, -0.01820259, -0.031500217, -0.021604022, -0.024832917, 0.01679765, 0.0054441397, 0.023107555, -0.007948998, 0.029232597, 0.017721953, 0.054324318, 0.010592503, 0.0033120827, 0.030440351, -0.0097729545, 0.015737785, 0.040077742, 0.022676213, -0.0026989623, -0.02435228, -0.005533489, 0.018929709, -0.053979248, -0.018510692, 0.016464902, -0.0075607914, 0.002333863, 0.0050959857, -0.01269375, 0.009840737, -0.0112580005, -0.0020750584, 0.01655117, -0.04872921, -0.00033505954, -0.016711382, -0.0010629474, -0.028764283, 0.01370432, 0.0025649385, -0.009015027, -0.0022398923, -0.00008270579, 0.0029962794, -0.0026881788, -0.008682278, -0.017537093, 0.006414657, 0.008016781, 0.020765988, 0.012219274, 0.0048063714, -0.024623409, -0.017228993, 0.011590749, 0.018399775, 0.0036571554, 0.0011792555, -0.031229088, -0.0043904353, -0.041531976, -0.0005010488, 0.0020442482, 0.009902357, 0.023908615, -0.011492156, -0.0037218567, -0.015515952, 0.0021274355, -0.0013741292, -0.011294972, -0.009415558, -0.024832917, 0.0018979005, -0.010672609, 0.00761625, 0.014394466, -0.015762433, -0.010302888, 0.010265916, 0.013605728, 0.020975497, 0.010352184, -0.020568805, 0.0044027595, 0.03440869, -0.001914846, 0.018449072, -0.010838984, 0.008780871, -0.00067628105, -0.0021705695, 0.012681426, -0.0044551366, 0.013322275, -0.005595109, 0.02334171, 0.0037896389, 0.009397072, 0.004362706, -0.017192021, -0.0066180034, 0.006346875, 0.026866382, -0.005197659, -0.0032412196, -0.006041855, -0.0010590962, 0.019607529, 0.008694602, 0.017426176, -0.009853061, -0.018818792, -0.0013271439, -0.019952603, -0.0066426513, -0.003872826, 0.010518558, 0.010382994, -0.015331091, -0.028764283, -0.006316065, 0.010450777, -0.02024838, -0.0030209275, 0.030465, -0.018091675, -0.017179696, 0.006500925, 0.012471917, -0.0023816186, -0.011485995, -0.016033562, 0.003389108, -0.014221929, 0.012102196, 0.0067843776, -0.0025649385, -0.009846899, -0.005166849, -0.023933263, -0.006140447, 0.0034907812, 0.016526522, -0.010808174, 0.0066241655, 0.0020781395, 0.04212353, 0.013199035, 0.0076655457, -0.013186711, 0.0020411673, 0.016933216, 0.02416742, 0.012151493, 0.0028915252, -0.031056553, 0.01878182, -0.017155048, 0.02429066, -0.015343416, 0.004301086, 0.0329791, -0.007998294, -0.011301135, 0.028517803, 0.00044944193, -0.016908567, 0.010167324, 0.002991658, 0.00611888, -0.0133469235, 0.008084563, 0.025781868, -0.022651564, 0.005579704, -0.034310095, -0.026176237, -0.014801159, 0.011522966, -0.009384748, -0.006476277, -0.028665692, -0.0075731156, 0.01541736, 0.0044027595, 0.020519508, -0.02169029, 0.0032935967, 0.02447552, 0.015885673, 0.006001802, 0.0035616443, 0.024808269, 0.010845145, -0.0055827852, 0.012379487, -0.00342608, 0.028246675, -0.0098037645, -0.029479077, 0.015688488, 0.0020319242, -0.0004502122, 0.012077549, 0.018486043, 0.020938525, -0.012829314, 0.01777125, -0.00022163996, 0.0031888424, -0.011824906, 0.015984265, 0.03260938, -0.02453714, 0.02272551, -0.0026881788, -0.010074894, 0.0041624405, 0.03413756, -0.026299477, 0.011935822, -0.007850407, 0.0054687876, -0.007936674, 0.004760156, -0.012200789, -0.009797603, 0.010463101, 0.010481587, -0.025412148, 0.024586437, -0.027211456, 0.013531784, -0.01033986, -0.021406839, 0.024130449, -0.008047591, 0.01269375, -0.02994739, -0.008392664, 0.0200019, 0.026496662, 0.009557284, -0.009039675, 0.014308197, -0.01257051, -0.013396219, -0.033274878, -0.0004702387, -0.0097421445, -0.0025341285, 0.013334599, 0.027605824, 0.003989904, -0.021221977, -0.017968435, 0.007240367, -0.018609283, 0.010771201, -0.002272243, -0.016156802, -0.018276535, -0.041334793, -0.0031133578, -0.018646255, 0.01618145, -0.00755463, -0.009008865, 0.007973647, -0.026817087, -0.0018686309, -0.037637584, -0.023292415, 0.013544108, 0.016686736, -0.020420916, 0.014295873, -0.01196047, -0.010684933, -0.009236859, 0.013679672, 0.005604352, 0.0026173156, -0.014640946, -0.012293219, 0.014197282, -0.016896244, -0.0111039495, -0.016095182, 0.0009296939, 0.0015759352, -0.0042302227, 0.0075114956, 0.00477248, -0.026299477, -0.00773949, 0.0061127185, -0.013605728, 0.015405036, -0.010672609, -0.04922217, -0.033891078, -0.0053578713, 0.0028591747, -0.0010606367, 0.017327584, 0.022454381, -0.0049326927, 0.024993131, -0.032905158, -0.005669053, -0.018042378, 0.0018439828, 0.004750913, 0.01789449, -0.03135233, 0.00031676606, -0.014813483, -0.027063567, 0.018855764, -0.004732427, -0.024426226, 0.01706878, -0.029355837, 0.0139508005, -0.021788882, 0.015861025, 0.004713941, 0.0048063714, -0.014505382, 0.011757123, 0.017500121, -0.014037069, 0.007850407, 0.0010005571, 0.016896244, 0.02493151, 0.006932266, 0.003134925, 0.011855716, 0.0017299856, -0.022552973, -0.042542547, 0.016538847, 0.022774804, 0.002976253, 0.02753188, 0.013691996, 0.00011341958, 0.014739539, -0.017216668, 0.0073697693, -0.012108359, 0.004606106, 0.03209177, 0.004322653, 0.0069507523, -0.021061765, 0.02144381, -0.012964878, 0.005068257, -0.007271177, -0.014382142, -0.0066549755, -0.0070185345, -0.018584635, 0.016686736, 0.0014711809, 0.0018085513, 0.0012424161, 0.012508889, 0.016218422, -0.019558234, -0.023994884, -0.013494812, -0.007129451, -0.005555056, 0.03221501, -0.00007490699, 0.019545909, 0.0016514199, -0.019102244, -0.019533586, 0.036109403, 0.026669199, 0.0049973936, 0.013618052, 0.011461346, 0.007850407, -0.020297674, 0.018942032, 0.00058346573, 0.026866382, 0.00388515, 0.019361049, -0.0062821736, 0.01878182, -0.032806564, 0.008891786, -0.018486043, 0.0126198055, 0.03258473, 0.008867139, 0.0083803395, -0.017364556, 0.00065663963, -0.0009304642, 0.0045876196, -0.00051029184, -0.0190283, -0.0053147376, 0.013014174, -0.001401088, -0.010647961, 0.011048492, -0.0052654413, -0.004661564, 0.033422764, -0.01477651, 0.010099541, -0.02067972, -0.008540552, 0.00038570358, -0.006969238, 0.019964926, -0.0003591299, 0.0020396267, 0.01263213, -0.035000242, -0.0014827347, 0.028098786, -0.015589897, 0.00038165977, 0.015836377, 0.023008961, -0.0061527714, -0.0034877001, -0.040669296, 0.012478079, -0.013864532, -0.0054318155, 0.000121410936, 0.014209605, -0.0062020672, 0.015047639, 0.010863631, 0.014332846, -0.0053578713, -0.016477227, -0.00025283516, 0.00608807, -0.022811778, 0.013642699, -0.0011191758, -0.021493107, 0.004695455, 0.023464952, 0.016033562, -0.0032350575, 0.01294023, -0.020223731, 0.0025587764, -0.011214866, 0.0025495335, 0.021394514, -0.016206099, 0.018301183, -0.0045075137, -0.009507989, 0.03766223, 0.009834575, 0.013753616, 0.0017515526, -0.011954308, 0.002634261, -0.0016991755, -0.0041624405, 0.011529129, 0.010259754, 0.004180927, -0.007955161, 0.0111655705, 0.02486989, -0.018806469, -0.010937575, -0.008645306, -0.008189317, 0.01994028, 0.014801159, 0.009403234, -0.016452579, -0.002272243, -0.0027836899, -0.012102196, -0.021185005, 0.012434945, -0.006704272, 0.0074930093, -0.0033244067, -0.030884016, -0.001067569, -0.0077641383, -0.050972182, -0.021727262, -0.01370432, -0.005545813, -0.018005406, -0.031426273, -0.0022922694, -0.025412148, 0.0054472205, -0.010272078, -0.020975497, 0.008756222, -0.013987773, -0.0024879132, -0.019743094, -0.0012763073, -0.0066303276, -0.017598713, 0.02740864, 0.028912172, -0.025412148, -0.0012293218, -0.0057121874, -0.0061466093, -0.0096682, -0.026792439, -0.022589944, -0.0076347357, -0.00025764923, -0.016255394, -0.023699107, -0.014357493, 0.017857518, 0.0028437697, -0.0018162538, -0.00090350537, 0.0045290804, 0.0031518703, 0.013790588, 0.017611038, -0.0056444053, 0.034950946, 0.0028129595, -0.023366358, 0.005653648, -0.03347206, 0.017302936, 0.011701665, -0.013063471, -0.027383992, 0.010044083, 0.018214915, -0.018165618, -0.03517278, 0.02385932, 0.008965731, -0.013741292, -0.03817984, -0.019114569, -0.0008996541, 0.0044736224, -0.022836424, -0.021665642, -0.010641798, -0.017167373, 0.006519411, 0.019582883, 0.016822299, -0.0018794144, -0.0005618987, -0.003450728, 0.0041901697, 0.012804666, -0.018880412, 0.0076285736, -0.021456134, -0.009828413, -0.017228993, 0.0075423056, 0.0020642749, -0.020642748, 0.0036540746, -0.012607481, -0.00063083624, 0.037070677, 0.021468459, 0.009464854, -0.032954454, 0.00951415, -0.017820546, -0.02701427, -0.018239563, -0.009316966, 0.01692089, 0.0145546775, 0.0053517097, 0.012243923, 0.029725557, 0.008768546, 0.0027913926, 0.015257147, -0.003469214, -0.019324077, -0.006426981, 0.01214533, -0.007653222, 0.011584587, -0.024832917, -0.003835854, -0.026447365, 0.038155194, -0.003416837, -0.014147985, -0.023132201, -0.0022861073, -0.0035246722, 0.0066611376, 0.01541736, 0.0144314375, 0.004384273, 0.0329791, -0.005305494, 0.009945491, 0.0035369962, 0.008066077, 0.0032042474, 0.014012421, -0.017105753, -0.0074437135, 0.021160357, 0.015873348, 0.021591699, -0.003134925, -0.007301987, -0.022368113, -0.0010290564, 0.027383992, 0.009255346, -0.018744849, -0.014345169, -0.019767743, -0.02006352, 0.026126942, -0.0050805807, 0.028517803, -0.004652321, -0.00095973373, 0.015146231, -0.006402333, -0.009914681, -0.026915679, 0.0021413, 0.0015220175, -0.005573542, -0.008294071, -0.011670855, -0.0022938098, 0.008448122, 0.026545959, -0.010346022, -0.0021721101, 0.0023092148, -0.026496662, -0.0044551366, -0.012108359, -0.017389204, -0.0010467722, 0.01826421, 0.003010144, -0.008719251, -0.017943786, 0.00235543, -0.037760824, -0.012447269, -0.0010136514, -0.0048464243, 0.0040083905, -0.021887476, 0.03630659, -0.020359296, -0.0031010336, 0.012397973, -0.011541453, -0.005068257, 0.0054595447, -0.008170831, 0.028246675, -0.00783192, 0.00049450167, 0.008762385, 0.0016313933, 0.0084419595, 0.011547615, 0.027162159, 0.012952554, -0.0026820167, -0.0056290003, 0.0010259754, 0.0026234777, -0.01006257, 0.031549513, 0.011036168, 0.0009528015, 0.00486491, -0.00007928779, 0.00850358, -0.013309951, 0.015059963, -0.031229088, -0.010580178, -0.0036694796, -0.013482488, -0.0044797845, -0.020199083, 0.02055648, -0.0053609526, 0.008941083, -0.023896292, 0.0044551366, 0.0011214867, -0.013815236, 0.008750061, -0.0055365697, 0.020346971, 0.019348726, 0.023810023, -0.012367163, -0.024438549, -0.004119307, -0.0089534065, -0.002212163, 0.0065687075, -0.011202542, 0.009797603, -0.0015975023, 0.00097205775, -0.029331189, 0.0020488699, -0.017155048, -0.0026481256, 0.0022568377, 0.019780066, 0.019299429, -0.0041624405, -0.019447317, 0.010302888, 0.0310812, 0.0074498756, 0.014172633, -0.0076285736, -0.027950898, 0.0029747125, 0.008121535, 0.023699107, -0.013433191, 0.002367754, -0.00052531174, -0.0036663986, 0.0033305688, -0.0040761726, -0.015861025, -0.02713751, 0.014986019, 0.000040341936, 0.018177943, -0.00077564357, -0.01040148, -0.007419065, -0.016526522, -0.0023970236, -0.01037067, 0.0043134103, -0.0075607914, 0.012108359, 0.0036016975, -0.0076039257, -0.009458692, -0.021172682, 0.019866334, -0.010956062, -0.0076655457, 0.011553777, 0.0010552449, -0.0035770494, 0.0066426513, 0.004673888, 0.019287106, 0.0049727457, -0.014258902, -0.011473671, -0.007930513, 0.01027824, 0.013630376, 0.022947341, 0.0020350053, 0.0075114956, -0.009132105, 0.0069137802, 0.011985118, 0.016501874, -0.016353985, 0.018362803, -0.0012347136, -0.0144314375, -0.008331044, -0.0000048622146, -0.0000992662, -0.0058138603, 0.006048017, 0.00014268914, -0.0035616443, -0.007905864, -0.012231599, 0.0020334648, 0.0068213497, 0.0022999719, -0.03539461, -0.023846995, -0.019163866, -0.003835854, -0.0018824954, 0.0011030006, 0.01453003, -0.02006352, 0.0014827347, 0.0053578713, -0.032781918, -0.002606532, 0.00014808092, -0.012890934, -0.015577572, 0.028074138, 0.001287861, 0.029281892, 0.016045885, 0.0017731197, -0.006796702, 0.025560036, 0.025091723, -0.0062575256, -0.020975497, 0.0075361435, 0.0057738074, -0.0006524033, 0.0022629998, 0.009877709, 0.0037495857, 0.017734278, 0.01024743, -0.006402333, -0.0026296396, -0.006426981, 0.010105704, 0.023625163, -0.020839933, 0.008226289, 0.0035185101, 0.03034176, 0.009224536, -0.0063283886, 0.013987773, 0.011190218, 0.017352233, -0.024586437, -0.021554727, -0.00002479248, -0.013371571, 0.012903258, -0.024500169, -0.005157606, 0.0045691337, -0.016378634, 0.008780871, -0.0015366523, 0.00014249659, 0.011498319, -0.007714842, 0.0066364896, -0.010672609, 0.0044767032, -0.000017655615, -0.016107505, -0.0019564396, -0.005166849, 0.010037921, -0.0029993604, 0.0031179793, -0.018177943, 0.00850358, 0.012903258, 0.0062667686, 0.010228944, 0.03095796, 0.012120683, 0.017549416, -0.020889228, -0.011621559, -0.016292365, 0.0021859745, -0.008879462, 0.0022583783, -0.005746078, 0.004165522, -0.0011368917, -0.002560317, 0.00013238702, 0.006747406, 0.0074806856, 0.013753616, -0.0030933311, 0.002321539, -0.008263261, 0.0044859466, -0.013125091, 0.006155852, 0.011233352, -0.03514813, -0.00079027837, 0.0075238193, -0.022885721, -0.015602221, -0.012780018, -0.017339908, -0.025288908, 0.005709106, -0.022072336, 0.013802912, 0.012564347, 0.0033274877, 0.021665642, -0.017426176, -0.0025156424, -0.026225533, 0.020716693, 0.002786771, 0.011214866, 0.021727262, -0.006735082, -0.005912453, 0.009563446, -0.005715268, -0.028665692, -0.00016656696, -0.018523015, -0.024389252, 0.0125273755, 0.0053024134, 0.03147557, 0.00030328665, -0.0040114713, -0.010604827, -0.0033028398, -0.014739539, -0.0040268763, -0.0050220415, 0.011239515, 0.03221501, -0.02254065, 0.0052931705, 0.0020149788, 0.015552924, 0.010419967, -0.0024078072, -0.03948619, -0.013531784, 0.002780609, 0.040176332, -0.027507233, -0.001354873, 0.016045885, 0.011184056, 0.0061928243, 0.0014480734, -0.021542402, -0.0031888424, 0.013975449, -0.024648057, 0.036700957, 0.007394417, 0.0030394136, 0.017167373, 0.011276486, -0.0023199983, 0.007425227, 0.0019826281, 0.011473671, -0.010087217, -0.00082185864, -0.014912074, -0.003435323, -0.001961061, -0.007813434, 0.018609283, 0.007819597, -0.003869745, 0.016686736, 0.027753713, 0.0012247004, 0.005037447, -0.009791441, 0.0061897435, 0.0007644749, -0.008294071, -0.0043657874, 0.008287909, 0.005206902, 0.0024755893, -0.0029023087, 0.020963173, -0.022355787, 0.0010460019, -0.020889228, 0.016748356, 0.0018609284, -0.009088972, 0.005099067, -0.008706926, -0.020778313, 0.005213064, -0.015959617, -0.0118310675, -0.028443858, -0.018855764, 0.0023199983, 0.038771395, -0.018584635, -0.015269471, -0.017426176, -0.002820662, 0.017155048, -0.008817842, 0.012379487, -0.02514102, -0.002755961, 0.017980758, 0.007912027, 0.0112888105, 0.009649714, -0.0057984553, 0.019780066, -0.0053547905, 0.0057953745, 0.008220127, 0.019693797, -0.0052716034, -0.03270797, 0.0044212453, 0.013371571, -0.00047409, -0.019410346, 0.0049326927, 0.0020411673, -0.0062267156, -0.001685311, 0.00023300116, 0.005188416, -0.027753713, -0.011005358, 0.008041428, -0.031500217, -0.013655024, 0.0070924787, -0.003370622, -0.001973385, 0.014147985, -0.007382093, 0.006066503, -0.011251838, -0.0002899998, 0.023489598, -0.027162159, 0.011664693, 0.00032870495, -0.0152817955, -0.008195479, -0.00997014, 0.01275537, 0.0139508005, 0.023600515, 0.023970235, -0.025338203, -0.0037280186, 0.010580178, 0.007240367, 0.0024817514, -0.012989527, 0.0023816186, 0.003983742, 0.01220695, -0.008016781, -0.010290564, -0.012077549, 0.0058939666, 0.003869745, 0.0030486565, -0.01183723, -0.011855716, 0.009563446, 0.034211505, -0.01655117, -0.0052993326, 0.0027544205, 0.016206099, 0.00043480715, -0.012465755, -0.008047591, 0.0029439023, 0.008035267, -0.00602645, 0.020667396, 0.004279519, 0.0024879132, 0.00021509282, -0.0036171025, -0.0061928243, 0.008361854, 0.015059963, -0.0061312043, 0.023686783, 0.008676116, -0.0032997588, 0.016686736, 0.0013687374, 0.00013951186, 0.010253591, 0.024722002, 0.011664693, 0.0069815624, 0.0035308343, 0.010382994, -0.0021089495, 0.031672753, 0.00302863, -0.0040977397, 0.029750206, 0.0057645645, -0.0032134906, -0.0036201836, 0.0053239805, 0.008737736, -0.016809976, 0.018942032, -0.006371523, -0.0058693187, 0.0070493445, 0.006845998, 0.017265964, -0.0076655457, 0.018473718, -0.011522966, -0.0072095566, -0.024919186, -0.0152817955, 0.023317063, -0.0041994127, 0.001994952, -0.01994028, 0.025781868, -0.0009443287, 0.00394677, 0.00011303445, 0.0030902503, 0.004368868, -0.0032319766, -0.010586341, 0.022737833, 0.001480424, -0.0005545813, -0.008897949, -0.005148363, 0.0011569182, 0.01921316, 0.018214915, -0.0027883116, -0.0025464525, -0.006457791, -0.006876808, 0.006839836, -0.004273357, 0.011060816, 0.022799453, 0.015626868, -0.002193677, 0.0092306975, 0.013285303, 0.007671708, 0.010475424, -0.014283549, -0.029257243, -0.0019903306, -0.017044133, -0.025806516, -0.0028252837, -0.017611038, 0.00064739666, 0.025104046, 0.0041686026, -0.0013571837, 0.00028248984, 0.0014026285, 0.0015343416, 0.0041316305, 0.0045691337, -0.010044083, -0.02012514, -0.0054595447, -0.0026850977, -0.0329791, 0.0026096131, -0.014332846, 0.020667396, -0.028838227, 0.0015990427, -0.018165618, -0.007283501, 0.011720152, 0.010506234, -0.019927954, 0.019052949, 0.017450824, 0.014579326, 0.0077333283, 0.0237977, -0.007776462, -0.012089872, -0.0323629, -0.019533586, -0.009027351, 0.013531784, -0.01208371, 0.008294071, -0.006035693, 0.011479833, 0.0048895585, -0.00605726, 0.024339957, 0.022281844, 0.0067227576, 0.0057676453, -0.014788834, 0.007252691, -0.018153295, 0.009015027, -0.000028186401, -0.0009397072, 0.004544486, 0.0097729545, -0.018744849, -0.009347776, 0.000064845575, 0.021801207, 0.006778216, -0.019348726, -0.013236007, -0.014234253, 0.006870646, -0.009236859, 0.010604827, 0.00758544, 0.0096682, 0.016526522, 0.00050335954, -0.013334599, 0.004285681, 0.0040299576, 0.03655307, -0.021591699, 0.0046245917, -0.014702566, 0.021961419, -0.008830166, -0.008250937, 0.003429161, 0.021702616, 0.010949899, 0.0050590136, -0.007388255, 0.0054841926, 0.00596483, 0.008817842, 0.013581079, -0.0074930093, -0.023058258, -0.014899751, -0.010986872, 0.0030917907, 0.004356544, -0.011603073, -0.0014557759, 0.0008996541, -0.028986115, 0.012330191, -0.00385434, -0.014345169, 0.008435798, -0.018251887, -0.009267669, 0.0035493204, 0.031919234, 0.000341029, -0.0000012305976, 0.0035616443, -0.015984265, 0.007363607, 0.019545909, 0.00296701, 0.015047639, 0.00064585614, -0.0054903547, -0.005237712, -0.002104328, -0.008571362, -0.025165666, -0.0046708067, 0.028197378, 0.016846947, -0.023440303, -0.008515904, -0.009292318, -0.0008518985, -0.0012123763, -0.0187202, 0.019052949, -0.013618052, 0.011985118, 0.0044274074, -0.0041686026, -0.000036370326, 0.00063122134, -0.020420916, 0.0053424663, 0.008029105, 0.0126198055, -0.019607529, -0.0058138603, 0.0050066365, 0.0055827852, 0.017574064, -0.010222781, 0.012552023, 0.032141067, -0.012977202, -0.016477227, -0.016625116, 0.0052469554, 0.0032196525, 0.015306444, 0.0075053335, -0.020457888, 0.018929709, 0.0066426513, 0.008121535, 0.0040392005, 0.012102196, -0.023883969, 0.009883871, 0.0031672753, 0.0031672753, 0.0066611376, -0.015084611, -0.015897997, 0.0048679914, 0.0013386976, -0.0009558825, -0.0041624405, -0.0012485782, -0.0050959857, 0.007240367, -0.013544108, 0.002492535, -0.006932266, -0.014505382, -0.00945253, 0.009791441, 0.020642748, 0.0021736505, -0.013014174, -0.015885673, 0.0048710722, -0.0045845388, -0.017869841, 0.011319621, 0.015269471, 0.005009718, -0.013568755, 0.0028406887, -0.014567002, -0.0000448431, -0.010919089, -0.033940375, 0.00764706, -0.0001549169, 0.024389252, -0.0036294265, -0.0024247526, -0.0016514199, 0.01661279, 0.01685927, -0.032929804, 0.0028360672, 0.011948146, 0.01217614, 0.014369817, -0.012718398, -0.0027051242, -0.0020196002, -0.010518558, -0.003044035, -0.00044828653, 0.011455185, -0.010481587, -0.0040700105, 0.0040761726, 0.001073731, 0.014665594, 0.0065933554, -0.022257196, -0.0084111495, 0.008694602, 0.014295873, 0.016699059, 0.016760679, -0.01269375, 0.011541453, -0.0076285736, -0.041162256, -0.01027824, -0.016353985, -0.004775561, 0.010549368, -0.003040954, 0.023045933, 0.0018516853, 0.009236859, 0.011775609, -0.0026820167, -0.011522966, -0.028714987, -0.0035524012, 0.0012339434, 0.0060541793, -0.003416837, -0.005237712, -0.0062051485, -0.0132236825, -0.023267766, -0.018165618, -0.006932266, -0.024068829, -0.014456086, 0.026964976, -0.010703419, -0.0035801304, 0.009212212, -0.02829597, 0.010358347, 0.0048217764, -0.028665692, 0.009766793, -0.0025110207, 0.015565248, -0.0056752153, 0.006815188, 0.0036109404, -0.013544108, 0.008417312, -0.001364116, -0.002104328, -0.008860976, -0.0075484677, 0.018091675, -0.0053332234, -0.0030532782, 0.008213965, 0.006359199, -0.0044890274, -0.019348726, -0.0032781917, 0.006710434, 0.015392712, 0.022244873, -0.022195576, 0.0018701714, 0.0074868477, -0.0013394678, 0.014998343, 0.0010976088, 0.023711432, 0.023538895, -0.013790588, 0.0008881003, 0.011116274, -0.017635684, 0.010210458, 0.022084659, -0.014221929, -0.023427978, 0.0010968385, 0.017450824, -0.00767787, -0.0016406364, 0.020137463, 0.0004625362, -0.024216717, -0.000694382, 0.012582833, 0.0023230794, 0.00032042473, 0.014037069, -0.018276535, -0.008491256, 0.006433143, -0.008448122, -0.008460445, 0.0017053374, -0.017450824, 0.0019133055, 0.022811778, -0.005958668, -0.017746601, 0.018436747, 0.0018516853, 0.010832821, -0.0054441397, 0.0066303276, -0.0131620625, 0.0017623361, 0.023230795, 0.0051113907, -0.012280894, -0.0022029202, -0.0011692422, -0.0049604215, -0.008368015, 0.028320618, 0.020790637, -0.01655117, -0.014456086, 0.029626966, -0.0018640093, 0.008096887, 0.0072465288, -0.012989527, 0.00060041127, -0.012743046, 0.00058847235, 0.0004147806, 0.016760679, -0.0206181, -0.010869794, -0.006778216, -0.012656777, 0.013938476, -0.0206181, 0.015084611, -0.01446841, 0.022047687, -0.010204296, -0.007955161, -0.0065687075, 0.00035181252, -0.014345169, 0.0045814575, -0.0075977636, 0.00039302098, -0.01033986, -0.018757172, 0.031771347, 0.0036448315, -0.00773949, 0.022355787, -0.0031287628, 0.020790637, -0.004371949, 0.009557284, 0.0012578212, 0.003811206, 0.0017114995, -0.011208704, 0.00016974423, 0.011412051, -0.025732571, 0.022811778, 0.0073697693, 0.00021528537, 0.0084419595, 0.026669199, 0.010210458, 0.019139217, 0.0002936585, 0.005086743, -0.003370622, 0.014899751, 0.010198134, -0.007166423, -0.0057183495, -0.02278713, 0.02187515, -0.010352184, -0.015959617, -0.014850454, 0.007166423, -0.017413853, -0.0003061751, -0.008263261, -0.005290089, -0.006476277, -0.019077597, 0.0003575894, -0.008294071, 0.0032720296, -0.00016300453, -0.004276438, 0.026250182, -0.011547615, 0.010666447, 0.004747832, 0.008103049, 0.00783192, -0.013211359, 0.012077549, -0.0070246966, -0.021714939, 0.0042363848, -0.023354035, 0.00972982, -0.018276535, -0.0018932789, -0.021332894, -0.009963977, -0.017795898, 0.021061765, 0.012077549, 0.011110112, -0.0067535676, -0.000654714, 0.0062790927, -0.015873348, 0.007252691, -0.00477248, 0.0058015366, 0.02373608, 0.008854815, 0.014394466, 0.0062452015, -0.01820259, -0.013236007, -0.013926152, -0.0011184056, 0.006007964, -0.031007256, -0.0044828653, 0.0037988818, 0.016526522, -0.014295873, 0.0036325075, -0.017290613, -0.000875006, 0.017056456, -0.0056813774, 0.011621559, -0.008965731, -0.014875103, 0.008780871, -0.008934921, 0.0038820691, -0.027038919, 0.0058046174, 0.020790637, 0.0030055225, -0.008750061, -0.00025225745, -0.0032196525, -0.021838179, -0.0097113345, 0.010253591, 0.010389157, 0.0145546775, -0.00028403036, -0.015047639, 0.005237712, 0.010543207, 0.0005264671, 0.012065224, -0.014160309, -0.020926202, 0.01263213, -0.012077549, -0.002364673, 0.022417407, -0.0075423056, -0.019250132, 0.017265964, 0.008719251, 0.010037921, -0.008133859, -0.024278337, 0.021677967, -0.012274733, -0.008934921, 0.003838935, -0.019200837, -0.016957864, 0.0011006899, -0.011818743, 0.0012508889, -0.017882166, -0.013716644, 0.0023723755, -0.01758639, -0.0002511021, 0.017204344, 0.0023508086, 0.017216668, 0.008885625, -0.012065224, 0.034433335, 0.021123385, -0.007702518, -0.003959094, 0.0058200224, 0.010352184, -0.026817087, 0.016723707, -0.0077641383, -0.011313458, 0.006390009, -0.033126988, -0.024697354, -0.015528277, 0.0062729307, 0.004590701, 0.015331091, 0.017216668, -0.0053486284, 0.0055365697, -0.0014303576, -0.018486043, -0.0386728, -0.020346971, -0.01189885, 0.03273262, -0.0026696927, -0.015528277, 0.017820546, -0.009039675, 0.012613644, 0.013014174, 0.0035739683, 0.016982513, 0.0007074763, -0.008663792, -0.0057183495, 0.0037187757, -0.0029223354, -0.0033582977, 0.009378586, -0.006035693, 0.012361001, -0.003851259, 0.0013148198, -0.015934968, -0.007776462, 0.027457936, -0.021024793, -0.010296726, 0.022639241, 0.003894393, 0.02049486, 0.00009801454, 0.0028946062, -0.017783573, 0.011153246, 0.009027351, 0.0018994409, -0.0017638766, -0.019952603, -0.024980806, -0.000266122, -0.022959666, 0.008244775, -0.019225486, -0.0145546775, -0.0012347136, 0.011418212, 0.009261508, 0.013088118, -0.0065255733, 0.03477841, 0.013014174, 0.01795611, -0.020913877, 0.0024678868, 0.0024956157, 0.0061219614, -0.008448122, 0.00049796776, -0.023070581, -0.009249183, 0.008016781, 0.0133469235, -0.007727166, 0.00025437566, -0.026447365, -0.02247903, -0.007788786, -0.0062914165, 0.009347776, -0.003777315, -0.0145546775, 0.008368015, 0.013679672, -0.015577572, -0.008127697, 0.00019679933, 0.0062482827, 0.00945253, -0.01541736, 0.029848797, 0.009249183, -0.01655117, 0.015466657, 0.0064947633, -0.010919089, 0.0036756415, 0.015700813, -0.0132236825, -0.024771297, -0.00008140599, 0.010352184, 0.006103475, 0.0039868234, -0.032264307, 0.012416459, -0.0053085755, -0.0044428124, 0.029824149, 0.0023631325, 0.011430536, -0.009021189, 0.031697404, -0.01661279, 0.007289663, 0.008004457, -0.034063615, -0.012182303, 0.01009338, 0.009064323, -0.020913877, -0.006735082, -0.00040438218, 0.0021520834, 0.01208371, -0.0074498756, 0.019188512, -0.007881217, -0.017758925, 0.029528372, -0.0070678303, -0.0022275683, -0.015824053, -0.003829692, 0.0058015366, -0.020876905, -0.00080876437, -0.014382142, -0.0053609526, -0.024339957, -0.007930513, 0.00030116845, 0.014061716, 0.018350478, 0.02018676, 0.004325734, -0.023045933, 0.008164669, 0.008109211, -0.008885625, -0.012243923, -0.03741575, 0.013272979, -0.0020350053, 0.0033244067, -0.027211456, -0.0027220699, -0.023008961, -0.008799356, -0.0010906765, 0.008485094, -0.0044489745, -0.008004457, -0.0355425, -0.00051029184, 0.009088972, 0.013433191, -0.008207804, 0.007265015, 0.0052192262, 0.016686736, 0.00009175624, 0.00041593597, 0.0029716315, 0.02942978, 0.002743637, 0.00464924, 0.0026358017, 0.003463052, -0.007985971, -0.014037069, -0.005866238, 0.0046184296, 0.02526426, 0.029848797, 0.03948619, 0.015022991, -0.008022943, -0.011350431, 0.018067027, 0.024327632, 0.014480734, 0.0035031051, -0.024204392, -0.0036417504, -0.0003548935, 0.024648057, 0.01217614, -0.000058972404, 0.0052870084, -0.012243923, -0.017598713, 0.011245676, -0.015392712, -0.022047687, -0.002389321, -0.012915582, 0.012182303, 0.040546056, -0.004636916, -0.012385649, -0.0097421445, -0.028838227, 0.011048492, 0.005558137, -0.010869794, 0.030760776, -0.008620658, -0.007659384, 0.012441107, -0.0006038774, 0.0063283886, -0.014505382, -0.005157606, -0.014086365, 0.0041131447, -0.024561789, -0.0026049917, 0.004347301, -0.011627721, -0.011664693, -0.018054701, -0.0003055974, 0.0009682065, -0.009908519, -0.018449072, 0.0026958813, -0.008959569, -0.0033244067, -0.009446368, 0.0018208752, -0.0030363325, 0.004020714, -0.003422999, -0.0016390958, 0.0070924787, -0.0200019, -0.009243022, 0.005450302, 0.018855764, 0.020088166, -0.03273262, 0.018806469, 0.0015366523, 0.010524721, 0.0071048024, 0.0013802912, -0.0029439023, 0.027630473, 0.00976063, -0.008016781, -0.03886999, 0.016082857, 0.007881217, -0.008799356, 0.012964878, -0.0010398399, 0.0069938865, 0.02199839, -0.0152201755, -0.013913829, -0.016193774, 0.0032196525, 0.009181402, 0.018214915, -0.0012331732, 0.020605776, -0.018301183, -0.0033459738, 0.010882118, -0.0016437173, -0.01529412, -0.017265964, 0.022589944, -0.021480782, -0.004322653, 0.005425654, 0.010407642, -0.012065224, -0.00013440894, 0.008398825, 0.024771297, 0.00079875113, -0.008195479, -0.006029531, 0.015737785, 0.022873398, -0.020667396, 0.019484289, 0.008343368, -0.0014611677, 0.0062760115, -0.009754469, 0.027901601, 0.008799356, -0.004134712, 0.048679914, 0.01483813, -0.019410346, -0.009415558, -0.010888279, 0.017364556, -0.015651517, -0.0027174484, 0.015503628, 0.022836424, 0.0013225223, -0.015491304, 0.0096682, 0.003370622, 0.007727166, 0.010290564, -0.0036232644, -0.0071048024, -0.025781868, 0.00853439, -0.011301135, 0.030169223, 0.015713137, 0.0066364896, 0.019792391, 0.002746718, 0.0091074575, 0.014998343, 0.0014164931, 0.014320522, -0.008016781, 0.008614496, -0.009809927, -0.020778313, -0.0012824693, -0.0055673798, 0.002963929, 0.023267766, 0.0036848846, -0.011658532, -0.008244775, 0.01370432, 0.004322653, 0.025362851, 0.0016745274, -0.017315261, -0.0054102484, -0.0018532259, 0.0059555867, 0.026718494, -0.022146279, -0.011467509, 0.016896244, 0.009015027, -0.009144429, 0.0022275683, -0.0013140496, 0.027383992, -0.009310803, -0.029281892, -0.008934921, 0.0058847237, 0.004809452, 0.010851308, -0.02156705, 0.007948998, 0.0039344463, 0.028493155, 0.0054071676, -0.016896244, 0.010518558, -0.005071338 - ] - } - ] - }, - { - "name": "dulezite-tipy-pro-nakup-bizuterie", - "title": "Důležité tipy pro nákup bižuterie", - "content": "Korálky Katlas připravily shrnutí důležitých bodů, na které by si zákazníci měli dávat pozor při nákupu hotové bižuterie i bižuterních polotovarů. Tyto informace byly prezentovány během online setkání.", - "keywords": [ - "Korálky Katlas", - "shrnutí", - "důležité body", - "zákazníci", - "nákup", - "hotová bižuterie", - "bižuterní polotovary", - "online setkání" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.0026307285, -0.064501315, -0.009212564, -0.00923262, 0.039096568, -0.02718308, -0.0018585578, 0.02151381, 0.018398385, -0.021981793, -0.010629881, 0.01469464, -0.005495448, -0.048803855, 0.012715745, 0.00067732064, -0.013270638, 0.010041561, -0.0005983487, 0.009861053, 0.0047867894, -0.017248487, -0.018665804, -0.014708011, 0.0042118398, -0.016085217, -0.032063466, -0.0029683441, -0.019267496, 0.03516552, 0.007313893, -0.010349092, -0.004753362, -0.04971308, -0.04639709, -0.019548284, 0.023492705, -0.031368177, 0.0008440393, -0.024308331, 0.0150690265, -0.022382919, -0.016981067, 0.045835514, -0.045648318, -0.01454756, -0.0001768513, -0.031234467, 0.022944499, -0.031608853, -0.024308331, 0.021166166, -0.02152718, -0.009727344, 0.048670147, 0.0453809, 0.014333625, 0.07744437, 0.032491334, 0.02006975, 0.013972611, 0.013905756, 0.0021243049, 0.013284009, -0.0056960117, 0.012829397, -0.030298503, 0.0073406347, 0.0027243248, 0.031956498, -0.027102854, -0.023893833, 0.04187772, -0.046183158, -0.012742487, -0.018024, -0.005381795, -0.017128147, 0.0053650816, 0.014413851, -0.00930616, 0.05819024, 0.05222681, -0.008811436, -0.006992991, -0.037866443, 0.003229077, -0.043803133, 0.002844663, -0.04249278, -0.04219862, -0.011900119, -0.034122586, -0.015216107, 0.010282237, -0.0024151222, -0.022369549, -0.006992991, -0.033373814, 0.027076112, -0.010462744, 0.0010370819, -0.0040781302, 0.0085105905, -0.0058865473, 0.091831475, 0.016887471, 0.024789685, -0.018692546, -0.03634216, 0.022075389, -0.0022329437, 0.027223192, 0.0067389435, -0.011392023, 0.0014031109, 0.010603139, -0.034844615, -0.0194948, 0.01046943, -0.041770753, -0.035085294, 0.062789835, 0.017101405, -0.012842768, -0.00793564, -0.03473765, 0.029977601, 0.03650261, -0.004776761, -0.035967775, -0.026835436, -0.018237934, -0.013745305, 0.0022446432, -0.006952878, 0.026581388, -0.046798218, 0.020671442, -0.0055088187, -0.019120416, 0.008905033, 0.0012886224, 0.0041215857, -0.044979773, -0.072898254, 0.012321303, -0.0030168137, -0.005368424, -0.00027117896, 0.001073852, -0.0024335075, -0.029763667, 0.011759724, 0.010482801, -0.0031755934, 0.0042151823, -0.017970515, -0.0024652632, 0.026501162, -0.00893846, -0.018572208, 0.03358775, 0.010522914, 0.047440022, -0.04869689, 0.032625042, -0.0064681824, -0.0596343, 0.03738509, -0.0031488515, 0.017208373, 0.026046552, 0.011244943, -0.009546837, 0.037331607, 0.032651786, -0.0075144568, -0.031394918, -0.039390728, -0.01933435, 0.031983238, 0.023278771, 0.0020791779, -0.006966249, -0.009299475, 0.023880463, -0.02354619, 0.013671765, 0.0044525163, 0.05380458, -0.018946594, -0.001673872, -0.009800884, -0.034710906, -0.0061573084, -0.04511348, -0.04318807, 0.008229801, 0.006498267, -0.00006675014, -0.03749206, 0.022677079, -0.026073294, 0.029763667, -0.018197821, 0.015456783, 0.0076815933, 0.035379454, 0.014708011, -0.031207725, 0.021848083, -0.0057494952, 0.026140148, -0.02354619, 0.022770677, 0.012100683, -0.0014298527, -0.011505676, 0.020470878, 0.00443246, -0.021339988, 0.00088498776, 0.018197821, -0.032357626, 0.027544094, 0.006461497, 0.036475867, 0.0058598053, 0.03414933, -0.030753115, -0.030218279, -0.046771478, -0.023586303, -0.022235839, -0.0053249686, 0.004780104, -0.021353358, -0.0021878167, -0.016339263, -0.013544741, 0.0064080134, -0.030753115, -0.0058999183, -0.012909623, -0.0012485096, 0.025458232, -0.00240008, 0.021019086, 0.07188206, -0.018852996, -0.0114856195, -0.00086576707, 0.02922883, -0.008991944, 0.056104377, 0.07498412, -0.0251507, 0.0023516105, -0.00094599253, 0.03925702, -0.036663063, -0.019695364, -0.0019103701, 0.014186545, -0.0031956497, -0.008557389, -0.020631328, -0.020818522, 0.06765685, -0.012114053, 0.010549655, 0.040620852, 0.032892462, -0.04072782, 0.030138053, 0.059527334, 0.010268866, -0.011291741, 0.03735835, 0.008049293, -0.0041249287, -0.0035466363, 0.0015777685, -0.0187594, -0.006484896, -0.03390865, 0.031662337, -0.009292789, 0.052012876, -0.020885376, -0.036368903, 0.00023545354, -0.028426575, -0.0077551333, 0.012274504, 0.0074409167, 0.0308066, -0.024976877, -0.0084504215, -0.04816205, -0.034978326, -0.026795322, 0.022302695, 0.021473696, 0.026380824, 0.012668946, 0.030164795, -0.02254337, -0.037144415, -0.010863872, -0.025337892, 0.01309013, 0.004419089, -0.022356177, -0.004235239, -0.019133786, -0.003247462, 0.057762373, -0.041744012, 0.0194948, 0.011398708, -0.006063712, -0.009486668, -0.0020958916, -0.015296332, 0.012401528, 0.0038207402, 0.015911395, 0.020577846, -0.028239382, 0.0024518925, -0.031475145, 0.009633748, -0.02968344, -0.025538456, 0.027945222, -0.066854596, -0.010014819, -0.010242124, 0.01961514, -0.012147481, 0.0024284932, 0.0013521343, -0.02224921, -0.020430764, -0.028774219, 0.012033828, 0.046236638, 0.010496172, 0.0006723066, 0.027102854, -0.01228119, 0.013190412, 0.04540764, -0.015189365, 0.022784047, 0.0034797818, 0.0067155445, 0.013852273, 0.014734753, -0.03417607, -0.018705916, 0.007153442, -0.005655899, 0.047038894, -0.019227382, 0.014280142, -0.005726096, 0.02559194, 0.005953402, 0.017502533, -0.061452746, -0.007848729, 0.010603139, 0.038133863, 0.0049572685, 0.015831169, -0.024187993, -0.0009952978, 0.017783323, -0.031154243, 0.015189365, 0.019722106, -0.0050943205, 0.040246468, 0.0031672365, 0.017114777, -0.0077952463, 0.02255674, -0.013638338, 0.0082431715, 0.0037538856, 0.026688356, 0.019695364, -0.014199916, 0.014293513, 0.09648456, -0.0027794798, 0.0014331954, -0.05744147, 0.032250658, -0.026969144, 0.009139024, -0.00021477039, -0.023853721, 0.03171582, -0.006558436, -0.013437774, 0.03660958, -0.0069328216, -0.05744147, -0.024254847, 0.004482601, 0.00037626602, -0.013484572, 0.0049940385, -0.012047199, 0.0022697137, 0.013772047, -0.040674336, -0.009647119, 0.03388191, 0.040326692, -0.041824237, -0.013023276, -0.0011139647, 0.013564798, -0.031608853, -0.013397661, -0.0061472803, 0.0476807, -0.04551461, 0.016646795, 0.025698908, -0.004405718, 0.0075278278, 0.021126052, -0.009352958, -0.00041324497, -0.053884804, -0.027544094, 0.0046965354, 0.032491334, -0.0044525163, -0.008303341, -0.021206278, -0.005518847, 0.033079654, -0.025311152, -0.032464594, -0.005060893, -0.038882636, -0.032170434, -0.056478765, -0.017556017, 0.033106398, 0.033641234, 0.03709093, 0.0050943205, -0.0116126435, -0.00060210924, 0.0009735701, -0.013377605, 0.007614739, 0.022008533, -0.0083501395, 0.027597578, 0.015135881, 0.05861811, -0.0003760571, -0.04027321, 0.01359154, -0.029736925, 0.0015944821, -0.041075464, 0.00032257344, 0.024789685, -0.03414933, 0.020176718, -0.030860083, 0.017502533, -0.012147481, 0.052761648, -0.007848729, 0.034363262, -0.006966249, 0.0009476639, 0.012568665, 0.028587027, -0.047600474, -0.0066654035, 0.009399757, 0.019588398, -0.036154967, -0.00439569, 0.021928309, -0.04420426, -0.017101405, 0.001215918, 0.005716068, -0.0014047823, -0.030699631, 0.010990896, -0.029790409, -0.032063466, 0.0048970995, 0.007835359, 0.037866443, 0.02180797, -0.017101405, 0.0005615786, 0.0032708612, -0.01587128, 0.022275953, 0.003260833, -0.021072568, -0.006852596, -0.009145709, -0.018090853, 0.004803503, 0.020470878, -0.04460539, 0.0023114977, -0.034951583, -0.019320978, -0.017034551, 0.029870635, 0.009767457, -0.020551104, 0.02283753, 0.033052914, -0.006488239, 0.007333949, 0.020577846, -0.0011114577, -0.022971239, 0.01337092, 0.01715489, -0.026073294, -0.017101405, 0.01859895, -0.00064723607, 0.020898746, 0.025351264, 0.03476439, 0.0149888005, -0.016967697, -0.0055088187, -0.021714373, -0.010422632, 0.0010964154, 0.042118397, -0.009072169, -0.0007608889, 0.022717193, 0.015911395, -0.029309055, -0.0116527565, 0.018064111, -0.0075211423, 0.014427222, 0.01874603, 0.0071935547, 0.017703097, -0.048803855, 0.012301246, 0.028132414, -0.006397985, 0.0065383795, -0.011732982, 0.0049906955, -0.04070108, 0.02397406, -0.015149252, 0.011244943, -0.07022407, 0.00016839002, -0.00016348038, 0.009947964, -0.022048647, -0.0046865074, 0.0036669746, -0.04928521, -0.026514534, 0.02298461, 0.020992344, 0.0025271038, 0.02762432, -0.009299475, 0.0058297208, 0.019575026, -0.014895204, 0.0005126912, -0.006484896, 0.046557542, -0.0047466764, 0.003187293, -0.009078855, -0.0037973411, 0.02109931, 0.009727344, -0.0041717268, 0.025043732, -0.0051544895, 0.023586303, -0.012849454, -0.0016354306, 0.02500362, -0.010148528, -0.0049906955, 0.008744582, -0.023171803, 0.007928955, 0.00183683, -0.02020346, 0.023720011, 0.012635519, -0.01889311, 0.00762811, 0.03385517, -0.0024218077, 0.018090853, -0.03230414, 0.025110587, -0.008544018, -0.0038575104, 0.010496172, 0.008991944, -0.0035031808, -0.012401528, -0.00945324, -0.002876419, -0.034069102, 0.05166523, 0.024388557, 0.030218279, 0.07097284, 0.018130967, 0.019668622, 0.0016462945, -0.010155213, -0.000043638294, 0.059206434, 0.0028229353, 0.0083768815, -0.019949412, -0.0011992044, 0.043562457, 0.041637044, -0.015229478, 0.012956421, -0.023238659, 0.0150556555, -0.019294238, 0.010255495, -0.006237534, 0.017101405, -0.009359644, -0.0007011376, 0.017983887, -0.027022628, -0.013177041, -0.0020307084, -0.0029499591, 0.0027811513, -0.014173174, 0.0020724926, 0.027651062, 0.0051645176, 0.015175994, -0.018144337, -0.027517352, -0.018064111, -0.017034551, -0.026140148, -0.005555617, -0.012655576, -0.028025446, 0.0046330234, 0.019387834, -0.014641157, 0.008249857, 0.028961413, -0.015964879, 0.012769228, 0.014226658, 0.059848234, 0.013217154, -0.007140071, 0.010382519, -0.003210692, -0.040005792, -0.0067122015, -0.005445307, -0.021727744, 0.02124639, 0.007066531, -0.014948688, -0.00734732, 0.013304065, 0.014159804, -0.018398385, 0.006461497, 0.014066207, 0.019147158, -0.009914537, 0.018438498, 0.039176796, -0.0050776065, -0.020577846, -0.017395567, -0.017047923, 0.0064146984, -0.006585178, -0.011164717, -0.019561656, -0.006521666, -0.047466766, -0.0057996362, 0.013865643, -0.005207973, 0.02618026, -0.01801063, -0.022570113, 0.025912842, 0.0041650413, 0.015617234, 0.0003679092, -0.003874224, 0.02426822, 0.012368101, 0.012100683, 0.008697784, -0.026634872, 0.00047550333, -0.039310504, 0.009225935, 0.009112282, -0.020711554, 0.06311074, -0.013150299, -0.009346273, -0.016499715, 0.007715021, -0.038508248, 0.0083835665, -0.002807893, 0.009780828, 0.022596855, -0.017783323, -0.014667898, 0.011385338, 0.013417718, -0.05468706, -0.005318283, -0.003274204, 0.01948143, -0.031689078, 0.00828997, -0.016566569, 0.0023465964, -0.02442867, 0.017368825, -0.0057327817, -0.002192831, -0.01127837, 0.039577924, 0.011111234, -0.00581635, -0.010957468, 0.015376558, -0.00061213743, 0.004516028, -0.016366005, -0.00046171458, 0.0022145587, 0.008818122, -0.011732982, 0.020564474, -0.004442488, 0.0053249686, 0.046156414, 0.01264889, 0.019815702, 0.019254124, 0.0357271, -0.0068726526, -0.00089919433, 0.018278047, 0.04941892, 0.008784695, -0.015470154, -0.011519047, 0.024254847, -0.006016914, 0.019307608, -0.0084370505, -0.024214735, 0.009840997, 0.018839626, -0.0065651215, 0.031876273, 0.027731286, 0.00064096844, -0.0035600073, -0.02601981, -0.015456783, -0.018665804, 0.018291418, 0.006458154, -0.004850301, 0.021781228, -0.014119691, 0.0017933745, 0.02196842, -0.015523638, 0.01700781, 0.005940031, 0.027731286, 0.0021092626, 0.0056960117, -0.024602491, -0.017756581, 0.016446231, 0.031394918, 0.028827703, 0.012582036, 0.025164071, -0.01905356, -0.022449775, -0.017181631, -0.0052915416, 0.03203672, 0.006672089, 0.01469464, -0.016312521, 0.01213411, 0.0083367685, 0.0066888025, -0.0016830645, -0.0012994863, -0.015630605, 0.01657994, 0.042653233, 0.007460973, -0.012040513, -0.018786142, 0.012602092, 0.023145061, -0.008657671, 0.016098587, 0.027891738, -0.03358775, 0.024602491, 0.011432136, 0.027062742, 0.025859358, -0.02412114, 0.0025254325, 0.026394196, 0.022650337, -0.025859358, -0.010482801, 0.01874603, -0.02866725, -0.010790332, -0.0345772, -0.009560208, 0.028346349, -0.015269591, -0.027597578, -0.032785494, 0.008597502, -0.0007416682, -0.020831892, -0.0047199344, 0.016700279, -0.008931775, -0.0019404547, 0.016446231, -0.015563751, -0.021754486, 0.0056960117, 0.00012347208, -0.021032456, -0.010362463, -0.0048737, -0.03535271, -0.0050809495, 0.013999353, -0.0034797818, 0.008176317, -0.03564687, 0.042679977, -0.024655975, -0.023465963, 0.005766209, -0.054045256, -0.006200764, -0.0021326616, -0.0039644777, -0.035834063, 0.01484172, 0.044765837, 0.0172886, 0.011031008, 0.033935394, -0.006080426, 0.016927585, 0.021286504, -0.005589044, -0.0055957297, -0.01992267, -0.0023833665, -0.019882558, -0.022784047, -0.0040480457, -0.025805874, 0.0021059199, 0.015162623, 0.010903985, -0.025845988, -0.0010170256, -0.020256944, -0.01410632, -0.030218279, 0.014667898, 0.0019588398, -0.006137252, 0.0022997982, 0.004950583, 0.021286504, 0.026073294, -0.004312122, 0.045568094, -0.039765116, 0.030913565, -0.0046731364, -0.010803703, 0.016352635, -0.01440048, 0.019280866, -0.018064111, -0.030057827, -0.027463868, 0.015389929, 0.005268142, -0.0035666928, 0.009533466, -0.007815302, -0.02093886, -0.016566569, 0.012842768, -0.0061840503, 0.004509343, -0.014453964, 0.003315988, -0.0046363664, -0.00026532917, 0.015617234, -0.005502133, 0.011719611, -0.017195003, -0.039658148, -0.030405471, 0.006387957, 0.010623195, 0.011860006, 0.04738654, -0.0043789763, 0.02559194, 0.030218279, 0.030592663, 0.010014819, -0.015804427, -0.0012919651, -0.010609824, -0.011759724, 0.03735835, 0.04112895, 0.018251305, -0.0389896, 0.003058598, 0.0035700356, -0.03912331, -0.037545543, 0.0056692697, 0.003068626, 0.021152794, -0.018625692, -0.010583083, 0.0015694117, -0.012876196, -0.002958316, 0.015670719, -0.057548437, -0.02136673, -0.032758754, -0.0042051543, 0.002339911, -0.005602415, -0.007949011, -0.002766109, -0.0059834863, -0.021687632, -0.012561979, -0.019361092, -0.015603863, -0.0023165117, 0.016539827, 0.013009905, 0.012836083, 0.020083122, 0.015938137, -0.010736848, 0.0068024555, 0.022209097, 0.016593311, -0.023332255, -0.0016103601, -0.0070932726, 0.0118265785, -0.042091656, -0.00030836684, -0.0017465764, -0.002321526, 0.015242849, -0.0024853197, -0.025244296, -0.007661537, 0.006939507, 0.0075478842, -0.014266771, -0.008076035, -0.024682717, -0.014320255, -0.035967775, -0.014868462, 0.0064314124, -0.019013448, 0.006153966, -0.0232654, 0.0075278278, 0.019748848, -0.0085105905, -0.0064648394, 0.01789029, -0.002985058, 0.014935317, -0.0065885205, -0.011064436, 0.017609501, 0.008958516, 0.0005795458, 0.0154033, -0.017342083, 0.01039589, 0.011338539, 0.021259762, 0.003371143, 0.023185175, 0.0044525163, -0.02470946, -0.0150155425, -0.0002939513, 0.026554646, 0.0028680624, -0.012715745, 0.005294884, -0.011004266, -0.00013809653, 0.024522267, 0.0011666127, 0.0036903739, 0.01206057, -0.017903661, -0.024401927, -0.012441641, 0.027209822, -0.00982094, -0.011117919, 0.004312122, -0.017636243, -0.0026557988, -0.00056701055, -0.008430365, -0.0062208204, 0.018478611, -0.017328711, -0.028560285, 0.020992344, 0.020524362, -0.01105775, -0.01615207, -0.023867091, 0.0034730963, -0.03794667, 0.003787313, -0.005739467, -0.012100683, -0.010389204, -0.011024323, 0.011873377, 0.0013479558, 0.02254337, 0.005161175, 0.0017365481, -0.0052647996, 0.0115725305, 0.025658794, 0.022944499, -0.004445831, -0.003700402, -0.0072804657, 0.010623195, 0.024589121, 0.0033861853, 0.0052012876, -0.021580664, -0.005652556, -0.024468783, -0.007333949, -0.026220374, 0.00031526122, 0.02006975, 0.00013705193, -0.0038441394, 0.031582113, 0.0083434535, -0.012381472, 0.025498344, -0.020885376, -0.0056726127, -0.00068776665, 0.026407566, 0.0051043485, -0.020310426, -0.021259762, -0.008176317, -0.0075946823, -0.027945222, 0.014935317, -0.013484572, 0.002486991, -0.0009852697, -0.017756581, 0.022730563, 0.01615207, -0.007761819, -0.014948688, -0.014560931, 0.02166089, 0.024482153, -0.007079902, -0.014199916, 0.019227382, 0.028827703, -0.0038474821, 0.020002896, -0.008824807, 0.006992991, -0.00052940485, -0.032758754, 0.012107368, 0.00031150065, -0.00012493452, -0.004258638, 0.023171803, 0.042546265, -0.0045026573, -0.0028195926, 0.00023315541, -0.005652556, -0.006387957, 0.029068379, 0.020604586, -0.0070732166, 0.026862178, -0.012080626, -0.002560531, 0.008838178, 0.009586949, -0.022997981, 0.0149754295, -0.027597578, -0.0062442194, 0.0011657771, 0.012989848, -0.0057996362, -0.005552274, 0.0154033, -0.03430978, 0.0003353176, 0.019788962, -0.026487792, 0.031047275, 0.0020775066, -0.019681994, 0.03037873, -0.0016571584, -0.005665927, -0.016660165, -0.018278047, -0.009032056, 0.01934772, 0.0017022851, 0.011592587, 0.0071267, -0.01294305, -0.007908899, -0.0033210022, -0.009700602, -0.010870557, 0.006448126, 0.03800015, -0.0038407966, -0.016620053, -0.024040913, -0.017221745, 0.005184574, -0.018184451, 0.03519226, -0.005970116, 0.0037973411, -0.020617958, -0.025805874, 0.028346349, -0.032785494, 0.0065049524, 0.001495036, -0.0064915814, -0.010476115, 0.009312846, 0.016620053, -0.005458678, -0.02080515, 0.008971887, 0.006919451, -0.039176796, -0.004044703, -0.0004968132, 0.010389204, -0.002303141, 0.01469464, 0.022035275, 0.012080626, 0.004034675, -0.015242849, 0.0076080533, -0.02369327, 0.011445507, -0.03174256, 0.007333949, -0.0054887626, -0.0031304664, 0.0021092626, 0.0041616987, -0.042679977, -0.0020724926, 0.017088035, 0.00087830226, 0.010924041, -0.010964154, -0.051852427, -0.013136929, 0.008042608, 0.011619329, -0.0023750095, -0.002560531, 0.009861053, 0.0060135713, 0.035539903, -0.0150155425, 0.005024123, -0.013350863, 0.015470154, 0.008664356, 0.0075813113, -0.0055589597, -0.012180908, -0.027651062, -0.030753115, 0.014520818, -0.0034998383, -0.021126052, 0.022316065, -0.02180797, 0.014240029, -0.013985982, 0.0071668127, 0.013217154, 0.012769228, -0.027236562, 0.014092949, 0.008764638, -0.030779857, 0.0036435756, 0.007962382, -0.005903261, 0.02428159, 0.014748124, -0.006728915, 0.020230202, 0.0076347953, -0.014159804, -0.031020533, 0.0025454888, 0.024976877, 0.000036038808, 0.0014231673, 0.00334273, 0.006892709, 0.026808694, -0.018050741, 0.0038441394, -0.004405718, 0.000800166, 0.022864273, 0.022784047, 0.024615863, -0.016192183, -0.004927184, -0.0023549532, -0.021701002, 0.00094348553, 0.013076759, 0.004813531, -0.013798789, 0.00348981, 0.009827626, 0.013731934, 0.010235439, 0.011920175, 0.01469464, 0.014092949, -0.033052914, -0.023399109, -0.029496249, -0.0062542474, 0.0010888942, -0.0023833665, -0.00015428787, 0.0040380177, -0.004743334, -0.017342083, -0.0062542474, 0.03631542, 0.0071333856, -0.016740391, 0.00435892, 0.0005340011, 0.038481507, -0.0070464746, 0.02240966, 0.0015008857, -0.010609824, -0.009145709, 0.027864996, 0.0082565425, 0.004973982, -0.03487136, 0.017248487, -0.014774866, 0.022382919, 0.011946917, 0.00065851776, -0.00034388332, 0.010074988, -0.012007086, -0.005532218, 0.0004813531, 0.0116327, -0.01772984, 0.005940031, 0.011191459, 0.002358296, -0.017983887, -0.0032859035, 0.00027222355, -0.014199916, 0.027009258, 0.01889311, 0.015443412, -0.008985258, 0.011719611, 0.002161075, -0.0018134309, 0.031341437, 0.013578169, -0.002849677, -0.014320255, -0.025057103, 0.0008014195, 0.008704469, -0.0055355607, 0.009433184, 0.022329435, 0.018050741, 0.018117595, 0.0057327817, -0.029041637, 0.010736848, -0.021607406, 0.00045377557, 0.020243572, 0.012114053, 0.015831169, 0.00012284532, 0.0029733581, 0.038909376, -0.008657671, -0.019534914, 0.0073072077, 0.004529399, -0.03460394, 0.001413139, 0.008991944, -0.0011515705, 0.017208373, -0.0058263782, 0.0049840105, -0.017756581, 0.01976222, -0.023907203, 0.016406119, 0.007969068, 0.01714152, 0.0046263384, -0.0030335274, 0.006103825, 0.0058965757, -0.0030552552, 0.032678526, 0.0116995545, 0.018291418, 0.0061573084, -0.028185898, 0.012047199, 0.0013655052, 0.006759, -0.004007933, 0.011452192, -0.029549733, -0.0016128671, 0.017088035, 0.029202089, -0.020671442, -0.01933435, -0.012902937, -0.01700781, 0.010776961, 0.014721382, -0.0015802756, -0.018064111, -0.011264999, 0.00085239613, -0.017783323, 0.00029666728, 0.0059634303, -0.0150155425, -0.0116260145, 0.027597578, -0.013932498, 0.0012393171, -0.0061840503, -0.03233088, -0.004867015, -0.02893467, -0.02543149, -0.0036302046, -0.015456783, 0.013083445, -0.0037639139, 0.01587128, 0.0083635105, 0.0024736202, -0.0028914614, -0.016205555, -0.024401927, 0.0023683242, -0.0007943162, 0.000023359937, -0.010930726, 0.017475791, 0.017609501, -0.0074542877, 0.0151091395, -0.020778408, -0.007888842, 0.008637614, -0.0070331036, -0.009640433, -0.023532819, 0.013304065, -0.018251305, -0.015363187, -0.005060893, -0.0002413033, 0.0010696736, -0.0046263384, 0.0083969375, -0.008096091, -0.013598225, -0.008891662, 0.009861053, -0.0043890043, 0.017609501, 0.0027694518, -0.012475068, 0.018278047, 0.014159804, 0.006484896, -0.014306884, -0.020978972, -0.011819893, 0.0009635419, 0.018973336, -0.014240029, -0.0084236795, 0.018772772, 0.024589121, -0.016686907, -0.014881833, -0.012301246, 0.0013379276, -0.005358396, 0.0028246068, -0.02749061, 0.0006530858, -0.015523638, 0.00334273, 0.0034363263, 0.009145709, -0.031689078, 0.043883357, -0.015991619, -0.0007629781, 0.0028329636, -0.02426822, 0.027758028, -0.0037070874, -0.012896252, -0.016954325, 0.005712725, -0.0044658873, -0.022275953, -0.00024339253, -0.015443412, -0.008610873, 0.035005067, 0.01512251, -0.016339263, -0.032090206, 0.021848083, -0.012996534, -0.0058263782, -0.00064765394, 0.004348892, 0.029469507, 0.025752392, 0.018037371, 0.016071845, 0.009332902, 0.0029198746, -0.0008833164, 0.028560285, -0.013865643, -0.026073294, -0.020925488, 0.01220765, -0.0028229353, 0.016874101, -0.02049762, 0.02382698, 0.008771324, 0.02529778, 0.011238258, -0.02500362, -0.027517352, 0.0063311304, -0.0048335874, 0.021714373, 0.009065484, 0.021834712, 0.0030151424, 0.022717193, -0.010001448, 0.012668946, 0.013518, 0.00027786443, 0.004245267, 0.006214135, -0.003100382, -0.023252029, 0.028613769, 0.014507447, 0.0150690265, -0.005642528, -0.0040246467, -0.024615863, 0.007113329, 0.023038095, -0.0005879026, -0.0045962534, -0.0142534, -0.025097216, 0.007066531, 0.004743334, -0.0117262965, 0.016098587, 0.0078621, 0.007387433, 0.014052836, -0.010108415, -0.016085217, 0.003860853, 0.014708011, 0.0024736202, -0.00037355005, 0.0150690265, -0.0116393855, 0.0050074095, 0.02224921, 0.010342406, -0.008617558, 0.01758276, -0.017997257, -0.023185175, -0.0022429717, -0.0038441394, -0.014012723, 0.0039243647, 0.012053884, 0.0047934745, -0.016713649, 0.005990172, 0.011117919, -0.032812238, -0.009279418, -0.0037338294, -0.0032073492, 0.015951507, -0.013919127, 0.02718308, -0.027945222, -0.010903985, 0.0032591617, -0.0017866892, -0.0015702473, -0.0024702775, 0.0065517505, 0.011458878, 0.0014390452, 0.00960032, -0.0064648394, -0.0051645176, -0.0048335874, 0.004903785, 0.012053884, -0.0061105103, -0.029870635, 0.021152794, -0.00762811, 0.006411356, -0.0076815933, 0.00367366, 0.003274204, -0.017515905, 0.011111234, 0.0021828027, 0.012481754, -0.021152794, -0.014280142, -0.0031956497, 0.00770165, -0.01525622, -0.023479335, -0.010081673, -0.012836083, 0.022489887, 0.009720659, 0.012167537, -0.0025889443, -0.014280142, -0.017810065, -0.008965202, 0.023358997, 0.002395066, 0.01294305, -0.0038207402, 0.022931127, -0.0068726526, -0.005615786, 0.0036669746, 0.008477163, -0.017676355, 0.011465563, -0.014587673, 0.019895928, -0.013023276, -0.0023465964, -0.012087312, -0.0077551333, -0.023960687, 0.011305112, 0.001978896, 0.026715098, 0.014066207, -0.0058497773, -0.007875471, -0.00828997, 0.021834712, 0.0060971393, 0.006829197, -0.015617234, -0.008918404, 0.0039143367, 0.0010487816, 0.014159804, -0.008724525, 0.005692669, -0.0019120415, -0.009339588, -0.017542647, -0.0036703174, -0.0084504215, -0.013705192, 0.010315664, 0.0021543894, -0.0009760771, 0.0016972711, -0.01771647, -0.004582883, -0.012261134, -0.02283753, -0.004509343, 0.006508295, 0.0013613268, 0.00010038636, 0.0077685043, -0.009433184, -0.022342807, -0.011458878, 0.009326217, -0.01962851, -0.00031484338, 0.021594035, 0.019294238, -0.010329035, 0.0029332454, -0.0011198145, 0.021420212, 0.0025154043, -0.0006635319, -0.009205878, -0.008457107, -0.0017022851, -0.0055489317, 0.025418118, 0.006525009, 0.0040848157, 0.003255819, 0.00058330636, 0.0019237411, 0.010349092, -0.020230202, -0.013758676, -0.0059233173, 0.0064915814, 0.010937412, -0.010208697, 0.0009886124, -0.00057286036, 0.015349816, -0.009165766, -0.005268142, -0.0014306884, -0.015510267, 0.011592587, -0.0081963735, -0.017074665, -0.020096492, -0.023024723, 0.01601836, -0.023492705, 0.005070921, -0.006040313, 0.016900843, -0.017355453, -0.008263228, -0.000093335286, -0.01440048, -0.025498344, 0.0003679092, -0.009012, -0.014881833, 0.012067255, 0.0023917232, 0.023572931, 0.009439869, -0.017489163, -0.0022179012, 0.021607406, 0.01629915, -0.027116224, -0.019695364, -0.029309055, 0.0008925089, -0.0078019314, -0.009713973, -0.012755857, 0.0022245867, -0.014534189, 0.0022229154, 0.009065484, 0.011405394, 0.006561779, -0.0013989324, 0.010783646, -0.026140148, -0.010402575, -0.0048101884, 0.014855091, -0.0032073492, 0.013484572, 0.01948143, -0.013230525, 0.022663709, -0.022623597, -0.018050741, -0.000022654829, -0.0013830545, -0.005619129, -0.013203783, -0.0005523861, -0.005940031, 0.0023315542, 0.022182355, -0.015416671, 0.003934393, 0.020256944, 0.0005870669, 0.01845187, 0.002863048, -0.015537009, -0.011291741, 0.0084103085, 0.0023064837, 0.005739467, 0.0033444013, 0.014039465, 0.007848729, -0.014828349, 0.015523638, 0.00472662, 0.007922269, 0.0065283515, 0.025351264, 0.015777685, 0.0013354206, -0.018037371, 0.0016462945, -0.0037538856, -0.01206057, -0.012020457, -0.013611596, 0.0059199748, -0.013658394, -0.01946806, -0.009934593, 0.0020490934, 0.0117931515, 0.013584854, 0.029897377, -0.027971964, 0.0015234492, -0.015363187, 0.0035366083, -0.0001981612, -0.025204184, -0.010001448, -0.031796046, -0.013718563, 0.008517276, -0.019066932, -0.012689003, -0.009727344, 0.007140071, -0.01616544, 0.00031755934, -0.016379377, 0.0117797805, 0.005976801, 0.01069005, 0.01904019, -0.016847359, 0.0040112757, -0.015978249, 0.013558112, 0.011532418, 0.01992267, 0.010977525, -0.0071066436, 0.0072537237, -0.017636243, -0.0073272637, -0.030619405, -0.015684089, -0.018037371, -0.015643977, 0.0052280296, -0.00360012, 0.007013047, -0.017047923, -0.0065016095, -0.01976222, 0.00062968675, -0.0021945022, -0.00011125023, -0.006374586, 0.014681269, 0.027089482, -0.035005067, 0.00886492, 0.012889567, 0.022222469, -0.006401328, -0.00679577, -0.04377639, 0.0025655453, 0.0026691698, 0.00995465, -0.016807245, -0.011318483, 0.024094397, -0.005037494, -0.0116460705, 0.017342083, -0.009092226, -0.013103501, 0.027838254, -0.017515905, 0.010970839, 0.0067021735, 0.01992267, 0.009840997, 0.010349092, -0.005615786, -0.00015731722, -0.006585178, 0.032464594, -0.0020724926, 0.0014382096, 0.0081696315, 0.01264889, -0.0084236795, -0.005639185, 0.013404347, 0.027303418, 0.018986706, 0.003623519, 0.024495525, -0.006334473, 0.000105504914, 0.0016220597, -0.000747518, 0.017114777, -0.00995465, -0.0068392255, 0.004850301, 0.009988077, -0.005248086, -0.005615786, 0.02484317, -0.012628834, 0.0040380177, -0.007434231, 0.00018802856, -0.0013220496, -0.011592587, 0.016406119, 0.004335521, -0.0074810293, 0.012327988, -0.008757953, -0.008263228, -0.015898023, -0.016753763, -0.004144985, 0.02255674, -0.017088035, 0.002715968, -0.034817874, 0.0031889642, 0.037331607, 0.0037505429, -0.0014599373, -0.010843816, 0.011298427, 0.036128223, 0.017662985, 0.002802879, 0.0034430118, 0.0068726526, 0.023666527, -0.01555038, -0.00092259346, -0.0015100782, 0.011164717, -0.00081646175, 0.0067489715, 0.008737896, 0.0047099064, 0.015537009, -0.023345625, 0.0041951262, -0.011244943, -0.005542246, -0.004903785, 0.00030168137, -0.0017783323, -0.028426575, 0.0070331036, -0.006458154, -0.014534189, 0.0036469183, -0.0040480457, -0.006224163, 0.00040196328, -0.000242139, 0.007688279, 0.02691566, -0.009981391, 0.0029817151, 0.029870635, -0.006488239, 0.028025446, -0.00982094, -0.016459603, 0.010442688, -0.0020223516, 0.015430042, 0.020377282, 0.022824159, 0.0007374898, -0.025605312, 0.00041157362, 0.00945324, 0.019936042, -0.0057461527, 0.011458878, -0.0078019314, 0.0009627062, 0.022489887, -0.008771324, -0.015349816, 0.005097663, -0.008818122, 0.005431936, 0.002500362, -0.00821643, -0.02139347, 0.014306884, 0.033748202, -0.01629915, 0.00067648495, 0.013919127, 0.016620053, 0.014146433, -0.0034296408, -0.0010947441, 0.020457506, -0.014306884, 0.009326217, 0.024094397, -0.0053650816, -0.001657994, -0.01076359, -0.010476115, -0.006635319, 0.001992267, -0.013390976, -0.011244943, 0.00056408567, -0.00063386513, -0.002229601, 0.015269591, 0.0022145587, -0.0007717528, -0.0019755533, 0.023947317, 0.0017164918, 0.00028789262, 0.013551427, 0.0046731364, 0.00020965184, 0.041155692, 0.010924041, -0.015643977, 0.025498344, -0.012140795, 0.006595206, 0.0044257743, 0.0062943604, 0.013892385, -0.030779857, 0.00020171286, -0.031635597, -0.010603139, 0.018946594, 0.006508295, 0.015670719, 0.0015167637, 0.015577122, -0.008804751, 0.00236331, -0.005903261, 0.0032357625, 0.008557389, 0.007715021, 0.02500362, -0.027169708, 0.024468783, -0.006608577, -0.0035566646, 0.004679822, -0.010863872, 0.0002584348, 0.014520818, -0.0057962937, 0.016646795, -0.008798065, 0.008577445, -0.010476115, -0.008597502, 0.0018886424, 0.026581388, 0.010349092, -0.016058475, -0.016459603, -0.023613043, -0.00995465, 0.004850301, -0.010997581, 0.0052046305, -0.0030368702, 0.016807245, 0.0037639139, -0.018973336, 0.003586749, 0.008691098, -0.005726096, -0.0051143765, -0.016098587, 0.01890648, -0.023158433, -0.01890648, 0.008657671, -0.012889567, 0.011465563, 0.0047600474, 0.009132338, -0.010496172, -0.006545065, -0.005304912, 0.020778408, 0.00009411874, 0.003068626, -0.01629915, -0.019534914, -0.022182355, -0.010442688, -0.02398743, 0.0073673767, -0.03152863, 0.008136204, -0.024348445, 0.000120024895, -0.006668746, -0.0055255326, 0.016472973, 0.000026350113, -0.008858235, 0.017515905, -0.009974706, 0.010141842, 0.01600499, 0.023372367, 0.0014549232, 0.010489486, -0.020925488, -0.02195505, -0.016686907, 0.009092226, -0.00664869, -0.020564474, -0.01344446, 0.011973659, 0.010623195, -0.0020357226, 0.020002896, 0.01329738, -0.0033009457, -0.0015426698, -0.019695364, 0.016887471, -0.01861232, 0.014026094, -0.014935317, 0.0027142966, -0.017756581, -0.0036402328, -0.017061293, -0.01076359, 0.00930616, 0.007186869, 0.006364558, -0.025123958, -0.013772047, 0.008035922, 0.00345304, -0.014052836, 0.016767133, 0.009399757, 0.007066531, 0.024936765, -0.010335721, -0.008978573, 0.015844539, -0.0053617386, 0.0028530199, -0.025498344, 0.03896286, -0.029068379, 0.011412079, -0.010081673, -0.004044703, 0.019936042, -0.00048678505, 0.00668546, 0.008096091, -0.049231727, -0.020965602, -0.0034797818, -0.002784494, -0.0057494952, 0.011077807, -0.012675632, -0.036074743, -0.011031008, -0.0029599874, 0.0064915814, 0.0023549532, -0.0028279496, 0.004144985, -0.006227506, -0.00023628921, -0.0015226135, -0.0035165518, 0.008042608, -0.010703421, -0.0033143167, -0.00035015095, 0.018425127, -0.027731286, -0.000002350357, 0.01889311, 0.0058531202, 0.007748448, 0.024776313, -0.0033878568, 0.011251628, 0.0014139748, 0.0075679403, -0.003045227, 0.0053784526, -0.0067255725, -0.03663632, 0.007026418, 0.023867091, 0.014681269, -0.009780828, 0.010449373, -0.010903985, 0.0027276676, -0.015242849, -0.005789608, 0.0011732982, 0.007735077, 0.0026090008, 0.0059099463, -0.018050741, -0.012996534, -0.01338429, -0.0228509, -0.0024385215, 0.007333949, 0.0083100265, 0.005298227, -0.010890614, 0.0083969375, 0.0015267918, -0.00010049082, -0.0084972195, 0.009466611, 0.027704546, -0.010349092, -0.0150289135, 0.0033210022, -0.0022429717, 0.0068726526, 0.0072938367, 0.0116661275, -0.021326616, 0.027383644, 0.016713649, 0.0044491733, -0.022302695, -0.0056826407, -0.024308331, -0.007962382, -0.0075946823, -0.0040981867, 0.010783646, -0.007039789, -0.014801608, -0.006508295, 0.016339263, 0.0049472405, -0.0081696315, -0.0084236795, -0.010576397, -0.0017833464, -0.011051065, -0.011318483, -0.012227706, -0.0011682841, 0.018331531, 0.007955697, 0.01571083, 0.0053617386, -0.023706641, -0.010522914, 0.010148528, -0.0034831245, -0.027758028, 0.012735801, 0.023720011, 0.0031471802, -0.0048436155, 0.0050207805, 0.009459926, -0.0023198545, 0.01250181, -0.018518724, -0.025511714, 0.01228119, 0.023800237, 0.004790132, -0.0057695517, -0.0068358826, 0.023800237, 0.0046731364, -0.018090853, -0.0011741339, -0.005010752, -0.00443246, 0.017997257, -0.0031287952, -0.013431089, 0.006337816, -0.010663308, -0.017997257, 0.0012050541, 0.008617558, -0.0075545697, 0.0018050742, 0.019855816, -0.001195026, 0.010342406, -0.012314617, 0.0047867894, -0.018050741, 0.0036435756, 0.0017499191, 0.0061573084, 0.023185175, 0.0030418842, 0.012227706, 0.021192908, -0.017836807, 0.0007074052, -0.0027828226, -0.016352635, 0.0023432537, -0.007908899, 0.009012, -0.0026440993, 0.016833987, -0.011358596, 0.012408214, -0.013491258, 0.011144661, 0.004318807, -0.006548408, -0.016045103, 0.009988077, -0.0011691197, -0.0008544853, -0.014855091, -0.018986706, -0.009546837, -0.018652434, 0.0021059199, -0.010268866, 0.037171155, -0.010977525, -0.0024218077, 0.009078855, -0.022289323, -0.008844864, -0.0069996766, -0.023318883, -0.011198145, -0.011886748, -0.00096019916, 0.0081963735, -0.0048970995, 0.015657347, 0.016646795, 0.0061472803, -0.0118399495, 0.005294884, -0.009118967, -0.0010955797, 0.013678451, -0.008844864, 0.009794199, -0.002849677, 0.0082097445, -0.004863672, -0.011037694, -0.029442765, -0.0032959317, 0.026367454, 0.004445831, -0.017034551, 0.010556341, -0.0011599272, 0.005515504, 0.009038742, 0.015670719, 0.018291418, 0.0024970192, -0.02226258, 0.0011682841, 0.012735801, -0.019000076, 0.013491258, 0.023867091, -0.0266215, -0.039390728, -0.009092226, -0.008898347, -0.0081562605, -0.008015866, 0.020430764, -0.0086443, -0.040192984, -0.013170356, -0.014026094, 0.009065484, -0.009032056, 0.022636967, -0.010369148, -0.00019481848, 0.0043789763, -0.004863672, -0.0024535637, 0.0008118655, -0.0052380576, -0.00019983258, 0.010516228, -0.0041884407, -0.013364234, 0.015964879, -0.0012719088, -0.011291741, 0.017462421, 0.0059066038, 0.0022379577, 0.0064915814, 0.01076359, 0.0012109039, -0.018866368, -0.006939507, -0.0010696736, -0.004813531, -0.012615463, 0.01440048, 0.023238659, -0.030271761, -0.0142534, 0.025110587, 0.0068225116, 0.0039377357, 0.004656423, -0.008430365, 0.0027727943, -0.016740391, 0.018665804, 0.009225935, 0.006327788, -0.021928309, -0.014039465, -0.008035922, -0.000056617486, 0.01815771, -0.015175994, 0.0058063217, 0.02270382, 0.0045494554, 0.011238258, -0.02369327, 0.0057829227, 0.00683254, 0.00011427958, 0.0043656053, -0.021286504, 0.0011407066, -0.03152863, -0.0004721606, 0.00040593275, 0.020671442, 0.009125653, 0.020190088, 0.008483849, 0.015924765, 0.0077083353, 0.018826256, 0.011478934, -0.01366508, 0.001181655, 0.009627062, -0.023064837, 0.022824159, -0.0081830025, 0.009185822, 0.009473297, 0.009419813, -0.0010446031, 0.0012953079, 0.006521666, -0.00032988566, -0.0014273457, -0.015884653, 0.028827703, -0.0011866691, 0.01730197, 0.0077818753, -0.00439569, -0.006080426, 0.012956421, -0.02226258, -0.008557389, -0.005662584, -0.016713649, 0.012902937, 0.004606282, -0.0013696835, -0.007969068, -0.0010412603, 0.003155537, -0.0038775667, -0.016472973, 0.010128472, -0.009640433, -0.018130967, 0.0054887626, 0.010783646, 0.01756939, 0.023867091, 0.01091067, -0.0149888005, -0.023532819, -0.01759613, -0.010268866, -0.0064815534, 0.0014047823, -0.04043366, -0.0013471202, -0.021781228, -0.00806935, -0.015978249, -0.02079178, -0.0032992745, 0.010609824, 0.018558836, -0.0014457307, 0.006026942, 0.009573579, 0.0036435756, -0.010389204, 0.014921946, -0.026059922, 0.011104548, 0.009767457, -0.011184774, -0.0061071673, 0.012461697, -0.014453964, 0.0049906955, -0.006287675, -0.0010980868, -0.00435892, -0.040032532, 0.010115101, -0.009861053, 0.0067723705, 0.0017967173, 0.011966973, -0.005345025, 0.009326217, 0.003787313, 0.003150523, 0.015924765, 0.004602939, -0.021580664, 0.012287875, -0.010990896, 0.00006069144, -0.02659476, -0.00886492, 0.011191459, 0.0013337493, -0.001713149, -0.008109462, -0.023412481, -0.001176641, -0.0062107923, -0.008029237, -0.0083768815, 0.013578169, 0.0031856217, -0.0043288353, -0.009132338, 0.0040480457, 0.0037806274, 0.009466611, -0.004516028, -0.020765038, 0.007447602, 0.009660489, -0.0012359744, 0.03738509, -0.025177442, -0.022423033, 0.008477163, 0.0014407166, 0.002010652, -0.010415946, -0.008724525, 0.0049940385, 0.0056224717, -0.021286504, -0.0024034227, -0.010008133, -0.019387834, -0.0011649413, -0.0044391453, -0.0075946823, -0.022021905, 0.002605658, 0.0069127656, -0.007748448, 0.0014206602, 0.023011353, 0.014921946, -0.008203059, -0.0072670947, 0.004017961, 0.017195003, 0.01787692, -0.0015267918, 0.0150556555, 0.011325168, 0.012455012, -0.009927908, 0.020765038, 0.0016488015, 0.0054386216, 0.0021995162, -0.020109862, -0.030191537, -0.0031170957, 0.014146433, 0.0025220898, 0.019681994, 0.008971887, -0.0072804657, 0.004602939, -0.0022045304, -0.0017733182, -0.029790409, -0.013076759, -0.024615863, 0.027263304, 0.006531694, -0.031341437, 0.021901567, 0.0018919851, 0.0022413004, 0.009152395, -0.0027677803, -0.011926861, 0.005111034, 0.0015535337, -0.0083969375, -0.01585791, -0.011819893, -0.003837454, 0.00172652, -0.006040313, 0.020925488, -0.0030385414, -0.016914213, -0.0055489317, -0.011331854, 0.01787692, -0.024896652, -0.011077807, 0.004877043, -0.007908899, 0.031180985, 0.018852996, -0.0016354306, -0.02500362, -0.017395567, 0.019681994, 0.0041616987, 0.0047834464, 0.0047600474, -0.02021683, 0.017034551, -0.013424403, 0.013758676, -0.016058475, -0.018652434, -0.0007124193, 0.026059922, 0.0027861653, 0.015844539, 0.0020691499, 0.011525732, 0.025351264, 0.006040313, -0.012983163, 0.0047199344, 0.021714373, -0.015844539, -0.003407913, 0.0073406347, -0.015323074, -0.01250181, 0.021500438, 0.01616544, -0.03562013, -0.015443412, -0.012555294, -0.02837309, 0.0034129273, 0.005211316, -0.0053517106, 0.016325893, -0.012408214, 0.0004408225, 0.015537009, -0.014119691, -0.0063043884, -0.0006321938, 0.012628834, 0.026153518, -0.0063077314, 0.03390865, 0.014346996, -0.007788561, 0.012929679, 0.0009560207, -0.013217154, -0.007868786, -0.0035366083, -0.009359644, -0.022931127, 0.01454756, 0.00446923, 0.0007387433, -0.0017967173, -0.025658794, -0.005421908, -0.014186545, -0.016847359, 0.008517276, -0.01440048, 0.032945946, 0.0062341914, 0.0082966555, -0.008577445, 0.004643052, 0.0084370505, -0.018973336, -0.008109462, 0.0015953179, 0.018024, -0.014868462, -0.026688356, 0.001584454, 0.021567293, 0.017756581, -0.001618717, 0.015898023, -0.014734753, -0.014199916, 0.021607406, -0.0023783522, -0.0063679004, -0.01628578, 0.013865643, 0.012695688, 0.010449373, -0.0010320678, -0.002573902, 0.018959964, -0.01381216, -0.00454277, -0.017917031, 0.0039410787, 0.009540151, 0.021767857, -0.0048837285, -0.0044491733, 0.0055723307, -0.0031605512, 0.0062107923, 0.0027076113, -0.018478611, 0.027383644, -0.004482601, 0.001415646, -0.0073072077, -0.011412079, -0.009433184, -0.00800918, 0.011900119, 0.010121786, 0.009473297, -0.012702374, -0.021741115, -0.0049305265, -0.0030652834, -0.013658394, -0.006892709, 0.020591216, -0.001108115, 0.0029198746, 0.00087913795, -0.001250181, 0.0031053962, 0.03631542, 0.012020457, 0.01228119, -0.0061974213, 0.012541923, -0.00013360473, -0.011833264, -0.0037037446, 0.005568988, 0.022677079, 0.0068860236, 0.013919127, 0.005405194, -0.0061706793, -0.016085217, 0.010001448, 0.011264999, 0.023104949, -0.0056859837, -0.028346349, -0.018358273, 0.009439869, 0.034416746, 0.008664356, -0.009185822, -0.010268866, -0.017542647, -0.019427946, 0.013631652, -0.0006154801, -0.008918404, -0.00777519, 0.007287151, 0.0082097445, 0.02136673, -0.009118967, 0.002308155, 0.0015593835, -0.037545543, -0.00014478198, 0.008610873, 0.0005260621, 0.014199916, -0.013745305, -0.010449373, -0.0028714049, 0.0032157062, -0.0005327476, 0.014413851, 0.024589121, -0.023118319, 0.0060837683, -0.0042920653, 0.00023900518, 0.008403623, -0.01046943, -0.008022551, 0.00038859234, -0.00727378, 0.005655899, 0.0035165518, -0.02807893, -0.010603139, -0.020591216, -0.0021894882, -0.008711154, -0.017783323, -0.012013772, 0.009994762, -0.03797341, 0.006892709, 0.009947964, -0.012414899, -0.008637614, 0.0025889443, 0.011452192, -0.0031806075, -0.025391376, 0.018478611, -0.003063612, 0.03024502, 0.0030719687, -0.000847382, -0.007026418, 0.021594035, 0.007989124, 0.022369549, -0.017676355, 0.021126052, 0.0030552552, -0.0046865074, 0.016352635, 0.002881433, 0.0036937166, 0.009914537, 0.007969068, -0.022356177, -0.01032235, 0.0007019733, 0.013838902, 0.0199093, -0.007561255, 0.028480059, -0.0032708612, -0.010810388, 0.0051444615, -0.0014574302, -0.012374786, -0.009219249, 0.006040313, -0.0075278278, -0.0116861835, 0.0116260145, 0.010054931, -0.0011139647, -0.023011353, -0.000012300201, 0.035887547, -0.0038073694, 0.010810388, -0.0038474821, 0.011305112, 0.014266771, -0.015430042, -0.009279418, 0.026875548, -0.005345025, -0.003118767, -0.009981391, 0.030592663, -0.009332902, 0.004529399, 0.050836235, -0.0031354807, -0.006992991, -0.025966326, -0.014801608, 0.016178813, -0.022142243, -0.0052514286, 0.008998629, -0.0047567044, 0.0071066436, 0.007661537, 0.018425127, 0.002568888, 0.0025204183, 0.025565198, -0.0015485196, -0.015978249, -0.011458878, 0.01091067, 0.0011114577, 0.00982094, -0.0015451768, 0.012976478, 0.019521542, 0.0049472405, -0.0020440794, 0.016660165, -0.0026223715, 0.024615863, -0.014133062, 0.022877643, 0.0016220597, -0.027570836, -0.0057728947, -0.0012008757, 0.018866368, 0.0021192906, 0.0031104102, 0.0058865473, 0.00777519, 0.0075077713, 0.021848083, 0.020337168, 0.0057929507, -0.020671442, -0.010081673, -0.0049906955, 0.016031733, 0.020029638, 0.0071200146, 0.0029666729, -0.0058464347, 0.008858235, -0.006769028, -0.022583483, 0.0010563027, 0.030565921, 0.002894804, -0.014012723, -0.018732658, 0.022757305, 0.012742487, 0.012528552, -0.017676355, 0.01469464, 0.001868586, 0.0398186, 0.004492629, -0.0053283116, 0.01615207, -0.010937412 - ] - } - ] - }, - { - "name": "online-marketing-v-koralkach-katlas", - "title": "Online Marketing v Korálkách Katlas", - "content": "Standa, zodpovědný za online aktivity v Korálkách Katlas, vedl prezentaci. Více informací o něm lze najít na stanislavvavrik.cz.\n\n---", - "keywords": [ - "Standa", - "Korálky Katlas", - "online aktivity", - "prezentace", - "stanislavvavrik.cz" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - -0.024927013, -0.041797765, -0.022209998, -0.026238129, 0.035700284, -0.009470053, -0.012976895, 0.0453362, -0.003113902, -0.00042132454, -0.0037990788, 0.032256626, -0.03964943, -0.017960718, 0.020693528, 0.007096616, -0.009138324, 0.03658489, -0.04792684, -0.005287907, 0.0066780066, -0.022462744, -0.06170146, 0.012139676, -0.009288392, -0.02072512, -0.0044032983, -0.016136212, 0.013506081, 0.015006756, 0.010402052, -0.0035917487, 0.013008488, -0.04236644, -0.02202044, -0.017802753, 0.04321946, 0.010307272, -0.026253926, 0.033141237, 0.0068162265, -0.01300059, -0.016586415, 0.00090040534, -0.03250937, 0.026206534, -0.01878214, -0.02650667, 0.0072466834, -0.0066503626, 0.020962069, 0.016902346, 0.014422283, 0.013664046, 0.014959366, -0.007424395, 0.033014864, 0.061353937, 0.0059118723, -0.0038918837, 0.06347068, 0.006508193, -0.014769807, 0.007894344, 0.018608378, 0.016302075, -0.0039491463, 0.0029045972, -0.01165788, 0.062206954, -0.0049877716, 0.0038800363, 0.013055878, 0.001667527, -0.011460423, -0.024958605, 0.023205183, -0.009154121, -0.005746008, -0.028181108, -0.010868051, 0.01938241, -0.03389947, -0.030661173, 0.010654797, -0.079046115, 0.015148926, -0.073675275, 0.0073849037, -0.029539615, -0.011997507, 0.0074480902, -0.029981919, -0.011144491, -0.008016767, -0.012431913, -0.0054577203, -0.0068162265, -0.0010810788, 0.016665397, -0.0082616145, -0.012258151, 0.019745732, -0.0015539891, 0.040281292, 0.06527149, -0.009446358, 0.020740917, 0.007661344, -0.023631692, 0.036332145, -0.0077679707, 0.01868736, 0.0070255315, -0.0057539064, 0.011578897, -0.017550007, -0.07563405, -0.008301105, -0.05816303, -0.036711264, -0.04852711, 0.014935671, -0.008711817, -0.0029618598, -0.0038701633, -0.01952458, 0.017407836, 0.04081838, 0.020804103, -0.060058616, -0.0240582, -0.006776735, -0.017818548, 0.014564452, -0.007696886, -0.0020752763, -0.023031421, 0.008925071, 0.03487886, -0.019319223, -0.02650667, -0.00864863, 0.0021740051, -0.058194622, -0.03418381, 0.00851436, -0.032762118, -0.024263555, 0.002770326, -0.011847439, 0.009896561, -0.035479132, 0.020804103, 0.013577165, -0.015733398, -0.018450411, -0.0012380574, -0.03386788, 0.015046247, -0.024816435, -0.01753421, -0.0051022973, 0.02530613, 0.027707212, -0.028797176, 0.045399386, 0.004584959, -0.03803818, 0.024816435, -0.008316902, 0.020709323, 0.012526692, -0.011262965, -0.056583367, 0.028939344, 0.018624173, -0.044419996, -0.032383, -0.02581162, -0.019208647, 0.017518412, -0.017423633, 0.0133323185, -0.017060312, -0.005588042, 0.064892374, -0.014430181, 0.012013303, -0.018655768, 0.05130731, -0.036521707, -0.0124240145, -0.038101364, -0.016633805, 0.004399349, 0.003882011, -0.059900653, 0.019635156, -0.02919209, -0.026174942, -0.010986526, 0.037121974, -0.022636507, -0.03626896, 0.029223682, -0.007452039, 0.021783492, -0.0017001075, -0.024547894, -0.028007345, 0.038891193, 0.03626896, 0.027091144, -0.009920255, 0.005449822, -0.008253716, 0.008285309, -0.002553123, 0.012968997, 0.049885616, 0.010251984, -0.009138324, -0.0071163615, -0.014872485, -0.015836077, 0.018450411, -0.0161915, 0.0066306167, 0.031719543, -0.056646556, -0.027059551, -0.004154502, 0.019919494, -0.034626115, -0.021009458, 0.024816435, -0.021799289, 0.01600984, -0.036426924, -0.013395504, 0.015622823, -0.027912566, -0.037027195, 0.024737453, 0.011713168, 0.0005143763, 0.012992691, 0.0025412757, 0.0013930614, -0.04103953, 0.009849171, -0.0103151705, 0.003224478, -0.014975163, 0.060437735, 0.032635745, -0.020693528, 0.0060619395, 0.0053945337, 0.0059987535, -0.056804523, -0.02410559, 0.020424984, -0.02511657, 0.006851769, -0.00846697, -0.012471404, -0.012052795, 0.021120034, 0.00020276399, 0.0023122253, 0.010654797, 0.031451, -0.0027071398, 0.033330794, 0.032635745, 0.025211351, 0.002142412, 0.031893305, 0.022478541, -0.022731286, -0.023031421, 0.015994042, -0.017139295, 0.027786193, 0.020140646, -0.004099214, 0.0008352444, 0.040534038, -0.019856308, -0.0023655386, -0.016349467, 0.0086249355, -0.008380089, 0.014880383, -0.020756714, -0.01998268, 0.0029776564, -0.028023142, 0.009778086, -0.028655006, -0.023805454, 0.007890394, 0.02168871, 0.021356983, -0.0049482803, 0.012416116, -0.02585901, -0.030155681, 0.0069189044, 0.009335782, 0.014319604, 0.001019867, -0.015559637, 0.0033705963, 0.0015204214, 0.01957197, -0.018545192, -0.04830596, 0.035542317, 0.012495099, -0.03233561, 0.010275679, 0.032446187, 0.0044230437, 0.004010358, 0.00911463, -0.0145328585, 0.03260415, -0.0017267643, -0.018166073, 0.01059161, 0.020804103, -0.0064292103, -0.032414593, 0.028765582, 0.029492225, -0.0023378946, 0.01563862, -0.0024445215, -0.040281292, 0.00966751, 0.0056038387, -0.052823782, -0.03113507, 0.03154578, 0.027880974, 0.01253459, 0.02211522, 0.0007695899, 0.027043754, 0.055572387, 0.025005994, 0.033457167, -0.029760767, 0.016065128, 0.0036746806, 0.005370839, 0.0070057856, 0.035542317, -0.032856897, -0.009612222, -0.0069189044, 0.002132539, 0.003696401, -0.010939135, -0.028465446, -0.023979217, 0.017360447, -0.0027782244, 0.00629099, -0.06906267, 0.0062475493, -0.0319091, -0.0005003075, 0.0061014313, 0.014398588, -0.052160326, -0.0036351893, 0.012992691, 0.008103648, -0.017913328, 0.011839541, 0.006010601, 0.034784082, 0.044609558, 0.0016033534, 0.027754601, 0.025511485, -0.014509164, -0.0052207713, -0.0015490527, 0.029602801, 0.042808745, -0.004667891, 0.013308623, 0.024848029, 0.028086329, -0.019303428, -0.029160496, -0.005907923, -0.02118322, -0.024326742, -0.0008505474, -0.016128313, 0.024358334, -0.0073059206, -0.0053866357, -0.025511485, 0.01832404, -0.05140209, -0.03775384, 0.016823363, 0.02276288, 0.029429039, 0.014461774, -0.011302457, 0.0014799426, 0.017392041, 0.017091906, -0.008545953, 0.011152389, 0.03576347, -0.048748262, 0.01957197, 0.007858802, 0.011049711, -0.019603562, -0.01188693, 0.004043926, 0.012321336, 0.0073809545, 0.006989989, 0.014856689, -0.028244294, 0.014690825, 0.0057420586, 0.012653065, 0.019619359, -0.019003293, -0.020819899, 0.022162609, 0.023837047, -0.028007345, 0.016302075, -0.024942808, -0.005777601, 0.053360865, 0.029318463, -0.010433645, -0.004146604, -0.03759587, -0.05070704, -0.037248347, -0.055888318, -0.011002322, 0.07386483, 0.008119445, 0.004174248, 0.010575814, -0.010670593, -0.0039017566, -0.01591506, 0.0058447365, 0.011736863, -0.017739566, -0.00070541626, 0.0075902594, 0.014374893, 0.035605505, -0.06369183, 0.08365872, 0.007554717, 0.019919494, 0.017692175, -0.022336371, -0.010678492, -0.015622823, -0.026253926, -0.015614925, -0.0013042056, -0.025701044, 0.01517262, 0.004391451, -0.013845707, -0.0022273185, -0.022841863, -0.01679177, 0.01674438, -0.019350817, 0.017913328, 0.0028888006, 0.015907161, -0.023947623, -0.012945302, -0.011373541, -0.03968102, -0.026822602, -0.0048692976, -0.031245645, 0.010228289, 0.0077126827, 0.015843974, -0.018924309, -0.025969587, 0.02549569, 0.012858421, -0.0048377044, 0.031577375, 0.021214815, 0.03519479, 0.03456293, -0.035700284, 0.0020061664, -0.0009695154, -0.030566392, -0.020804103, 0.0053313477, -0.01008612, -0.0058960756, 0.013142759, -0.020045867, 0.01595455, 0.0092252055, -0.008775003, -0.018766344, 0.027770396, -0.024484707, -0.037690654, 0.030139886, 0.0034870962, 0.0061093294, 0.03525798, -0.014066859, -0.0057302113, 0.00400246, 0.034120623, 0.0017820523, -0.0029934528, -0.037216756, 0.006448956, 0.029097311, 0.028844565, 0.014438079, -0.003579901, 0.017423633, -0.01818187, -0.008853986, -0.00233592, -0.0035443588, -0.011618389, -0.004158451, 0.015235807, 0.026791008, 0.031466797, -0.01928763, -0.035479132, -0.007349361, 0.016870754, -0.015464857, -0.0053945337, 0.021056848, 0.018750547, 0.0014355147, -0.036837637, 0.010307272, 0.03737472, -0.01568601, 0.013956283, 0.0026538263, -0.009912358, -0.026743619, -0.003575952, 0.0045928573, 0.027454466, -0.028402261, -0.0029184192, 0.013616657, 0.015812382, 0.032540966, -0.02257332, 0.0024682165, -0.015733398, -0.029981919, 0.026332907, 0.023615895, -0.031024493, -0.0048732464, -0.012266048, 0.019540377, -0.016159907, -0.010015035, 0.03260415, -0.000788842, -0.0006284079, -0.0019380436, 0.020282816, 0.0012262099, -0.01863997, 0.021767695, 0.021167424, -0.0062159565, 0.018987495, -0.028876157, 0.010670593, 0.038101364, 0.011160288, 0.0101887975, -0.012558285, 0.004213739, 0.03250937, -0.006737244, 0.041639797, 0.007301972, 0.012021202, 0.04612603, 0.03983899, -0.009485849, 0.03465771, -0.003386393, -0.0074362424, 0.024389928, -0.048179585, 0.01887692, -0.01415374, -0.0059592617, -0.010165103, 0.008174733, 0.017091906, -0.0094068665, -0.0036470366, -0.013845707, -0.053076528, 0.032414593, 0.016665397, 0.021783492, 0.07803513, 0.026364502, -0.015077841, -0.00081253686, 0.0051022973, 0.029334258, 0.051654834, -0.0068715145, 0.0073454124, -0.022178406, -0.011744761, 0.03487886, 0.028986733, 0.0072150906, 0.016223093, -0.00023744242, 0.013245437, -0.01813448, -0.0019696367, 0.0002910027, -0.012668861, 0.0061172275, -0.004284824, -0.030645376, -0.025780028, -0.011428829, 0.0024780894, 0.000950757, 0.02007746, -0.027027957, 0.0103704585, -0.009951849, 0.027786193, 0.04043926, -0.021957254, -0.033046458, -0.002655801, 0.009951849, -0.009493748, -0.021278001, -0.007835106, -0.01873475, 0.024595283, 0.019793121, -0.022241592, 0.014469672, 0.01170527, 0.01743943, 0.011365643, -0.019729935, 0.019682545, 0.0014394638, -0.021783492, -0.016120415, 0.020267019, -0.044198845, 0.0040873666, -0.013956283, -0.009888662, -0.010015035, -0.006144872, -0.012447709, -0.008869783, -0.01207649, 0.0048416536, -0.015141027, 0.01887692, -0.0024109539, 0.025669452, -0.007843005, -0.007151904, -0.00550511, -0.034341775, 0.004541518, -0.028212702, 0.0024464962, 0.004237434, 0.024279352, 0.0125503875, -0.011997507, -0.02118322, -0.036237366, 0.032856897, 0.027596634, -0.012100184, -0.013174352, -0.015946653, -0.021499153, 0.009414765, 0.038891193, -0.0038918837, 0.016136212, -0.00342391, -0.008940867, -0.010409949, 0.060311362, 0.016428448, -0.014461774, 0.015227908, -0.0068833623, 0.0061093294, 0.0024168775, -0.017550007, 0.044988677, -0.034815673, -0.013671945, -0.015614925, -0.004956179, -0.0582894, -0.00037442843, 0.022794472, 0.005805245, 0.019856308, -0.029460631, 0.0034910454, -0.0069702435, 0.017565804, -0.03759587, 0.0020298613, 0.0037121975, 0.0025550977, -0.010812763, -0.010141408, 0.021546543, 0.0030467664, -0.034152217, 0.07070552, -0.033646725, -0.009454256, -0.0078074625, -0.0013683792, 0.02211522, -0.03904916, 0.0012795234, -0.005927669, -0.004663942, -0.0029855547, -0.0072111413, -0.0046797385, -0.021041052, 0.011357745, -0.014880383, -0.0041189594, -0.030171478, 0.0031632662, -0.01813448, -0.004229536, -0.015701806, -0.0061883125, 0.047042232, 0.028876157, 0.021499153, -0.011160288, 0.03696401, 0.011373541, -0.0143433, -0.0053471443, -0.0021779542, -0.00078588014, 0.018940106, 0.01327703, -0.0154964505, -0.0074599376, 0.007021582, 0.004201892, 0.0009917293, 0.015054146, 0.012771539, -0.016491635, 0.0019479165, -0.03104029, -0.0143511975, -0.0077521745, 0.021056848, -0.00155004, 0.017170887, 0.0024346488, -0.013111166, 0.012313439, -0.012708353, 0.017091906, -0.008190529, -0.013545572, 0.002572869, 0.026238129, -0.014430181, -0.03806977, 0.028149515, 0.009043545, 0.01253459, 0.01633367, 0.01008612, -0.014698722, -0.020093257, 0.049696058, -0.043409016, 0.010473136, -0.0041939933, 0.0131506575, -0.00008755998, -0.015204214, -0.0066227186, 0.0047903145, 0.027059551, -0.0118790325, 0.0036924519, -0.021988846, 0.036332145, 0.011863235, -0.028560227, -0.036616486, 0.039396685, -0.022747083, 0.039902173, 0.019003293, 0.0035641047, -0.031166663, 0.034152217, -0.009454256, 0.006520041, 0.01952458, 0.030613782, 0.0028256143, -0.005528805, 0.0031375969, -0.008917172, -0.03535276, 0.016112518, -0.01577289, 0.003196834, -0.045999657, 0.01119188, 0.024200369, -0.002049607, -0.029539615, -0.028828768, 0.019966884, 0.0006057003, -0.011428829, -0.03686923, 0.0067135487, 0.006721447, 0.0033548, 0.0094068665, 0.010330967, -0.0073730564, -0.019966884, 0.011294559, -0.03566869, -0.013308623, -0.0066582607, 0.033709913, 0.0047903145, 0.020756714, 0.0041150106, -0.013916791, -0.08220543, 0.03399425, 0.004956179, -0.022968234, 0.01452496, -0.056267437, 0.024832232, 0.01660221, 0.012518794, -0.0018679462, 0.016112518, 0.04713701, 0.02780199, 0.02974497, 0.034310184, 0.019398207, 0.0025471994, 0.017518412, -0.0071361074, -0.028607616, -0.020298611, -0.022131016, 0.014919875, 0.0016931965, -0.03326761, 0.02803894, -0.00018659717, 0.01914546, 0.037722245, -0.015520145, 0.016159907, 0.0018570861, 0.016760178, -0.00142268, -0.0094068665, 0.014438079, -0.014746113, -0.0013209895, 0.011326152, 0.017360447, -0.0013841758, 0.0010761424, 0.022209998, -0.03307805, 0.018718954, -0.0023754116, -0.023584303, 0.022367965, -0.023063015, 0.036332145, 0.0010603458, -0.0003159317, -0.029318463, -0.012621472, -0.028986733, -0.012273947, 0.014106351, 0.021783492, -0.03418381, -0.014145842, 0.000893988, 0.0036332146, 0.015235807, -0.0065516336, 0.008767105, 0.021546543, 0.012826827, 0.007341463, -0.0019380436, 0.022889253, -0.026569856, -0.030724358, -0.01128666, 0.010891746, -0.018339835, 0.015362179, 0.02281027, -0.007933835, 0.03456293, 0.017423633, -0.006050092, 0.002304327, -0.017060312, -0.014556553, -0.0012301591, -0.023552708, 0.0026794958, 0.02391603, -0.004502027, -0.011681575, 0.017565804, 0.020314408, -0.030155681, -0.03298327, 0.031514186, 0.003613469, 0.009872866, -0.035321165, 0.016057229, -0.005039111, -0.013253335, -0.0016586415, 0.039807394, -0.035036825, -0.031766932, -0.0043282644, -0.009248901, -0.00007707006, 0.0031790629, -0.013490284, 0.0018807809, 0.008411681, 0.006504244, -0.011160288, 0.027328093, -0.034531336, 0.023631692, 0.016728584, 0.036995605, 0.033615135, -0.037121974, 0.017218279, -0.027043754, 0.024990197, 0.017913328, 0.010165103, 0.021941457, -0.013363911, -0.04154502, -0.0024050302, -0.01614411, 0.022083627, -0.0014197181, 0.007258531, 0.019256037, -0.00020918135, 0.004833755, -0.015046247, 0.009422664, 0.025937993, 0.003882011, 0.0071953447, -0.04890623, -0.014564452, -0.04745294, -0.020614544, 0.026806805, 0.00074737595, 0.0020041917, -0.03386788, 0.018197667, -0.002280632, -0.046283994, -0.0153147895, 0.019903697, 0.024500504, -0.0075112763, -0.01753421, -0.0049008904, 0.013679843, -0.00025027714, -0.008332699, 0.023094608, 0.013071675, 0.010165103, 0.001267676, 0.014556553, -0.016349467, 0.0007903229, -0.0300767, -0.011807947, -0.0019034885, -0.0064568543, 0.029065717, 0.033457167, 0.044577964, -0.007961479, 0.021072645, 0.022968234, 0.029018328, 0.0030447918, 0.0032264525, 0.0056038387, -0.009880764, -0.007839056, -0.0107337795, 0.011255067, -0.0059987535, -0.032383, 0.013395504, -0.004182146, -0.0026103857, 0.002794021, 0.016412653, 0.0015065994, 0.0067569893, -0.018797936, -0.03737472, 0.020740917, 0.00846697, 0.012929505, -0.0086249355, -0.014469672, -0.013434996, -0.014288012, 0.0025847163, 0.019161258, -0.007921987, -0.02017224, -0.01119188, 0.025132367, 0.0057341605, 0.038227737, 0.009051443, 0.02017224, 0.008751309, 0.016302075, 0.0038938583, 0.0072427345, -0.028876157, 0.022273185, -0.014303808, 0.00400246, -0.0032856897, -0.0018886792, 0.012471404, 0.00911463, 0.012155472, -0.034310184, 0.005047009, 0.005860533, -0.027043754, -0.019619359, -0.009580629, 0.017613193, 0.024184572, 0.01614411, -0.002142412, -0.004446739, -0.019129666, -0.005568296, -0.022873456, 0.014935671, 0.011120796, -0.018260853, -0.008814495, -0.0025689197, -0.0148961805, -0.018276649, 0.025685247, -0.016696991, -0.0042808745, 0.006642464, -0.009872866, 0.02595379, 0.0101335095, 0.012787336, -0.038006585, 0.0013673919, 0.021499153, 0.015946653, -0.00024262568, -0.0120606925, 0.017107703, 0.007835106, -0.015520145, 0.018008107, -0.02142017, 0.014232723, -0.0007843992, -0.034910455, -0.03137202, 0.0109707285, 0.024389928, -0.02585901, 0.02401081, 0.025701044, 0.014185334, -0.0011640108, -0.048779856, -0.013608758, -0.017076109, 0.01947719, 0.016823363, -0.0052207713, 0.022747083, -0.0076810895, 0.006042194, -0.0039392733, -0.009809679, -0.019066479, 0.0019676622, -0.014564452, 0.016570618, -0.0055209068, 0.030866528, -0.0014394638, -0.0152674, 0.024295148, -0.0011057609, 0.001140316, 0.0008357381, -0.02803894, 0.021072645, 0.00513389, -0.015851874, 0.01615201, -0.021641321, 0.0013387606, -0.013956283, -0.0080246655, -0.006824125, 0.018466208, -0.002290505, -0.017834345, -0.004746874, 0.015757093, -0.018655768, -0.00012149796, 0.0017800777, 0.0015036375, 0.010820661, 0.030961307, 0.006820176, -0.013371809, -0.00011902974, -0.024358334, 0.0031060036, 0.009912358, 0.009138324, 0.001977535, -0.0027683517, 0.0011590745, -0.017012922, 0.031340424, -0.028828768, 0.010512628, 0.01105761, -0.014690825, 0.00943846, 0.01522001, 0.02336315, -0.027280703, 0.023015626, -0.0048850942, 0.016997125, -0.015836077, -0.02396342, -0.018482005, -0.013577165, -0.0023793606, 0.03759587, 0.0011185957, 0.02626972, 0.040407665, -0.010291476, -0.0066464134, -0.024500504, 0.0022055982, -0.02415298, 0.016317872, -0.003283715, 0.0010475111, -0.00656743, 0.0076731914, -0.028228499, 0.0042176885, -0.004853501, 0.0051180935, 0.033646725, 0.0045375694, 0.000053498592, -0.024184572, -0.023852844, -0.0009176829, 0.016902346, 0.009612222, -0.009612222, 0.016207296, -0.0044072475, 0.001711955, -0.0023201234, -0.011420932, -0.004229536, 0.008640733, 0.0074480902, -0.030519003, -0.0040913154, 0.008245817, -0.027833583, -0.0019261962, -0.006472651, -0.017581599, 0.035100013, -0.021546543, 0.0057420586, 0.008751309, -0.013284928, 0.009193612, 0.018418819, -0.018545192, 0.0161915, -0.00911463, -0.018671565, -0.010125611, 0.015512247, -0.019082274, -0.008909274, -0.014161639, 0.014880383, 0.016712787, -0.028165312, 0.005797347, -0.03178273, -0.016412653, -0.0038938583, 0.008166835, 0.025132367, -0.019177055, 0.013466589, 0.0374695, -0.02377386, 0.0053273984, 0.015630722, -0.0050351615, 0.019856308, 0.004237434, 0.0047824164, -0.01008612, -0.012202863, -0.022968234, 0.010583712, 0.0018333911, 0.018513598, -0.005362941, -0.023173591, 0.007424395, -0.010433645, 0.017155092, 0.031751137, -0.00453362, -0.0016724635, -0.003252122, -0.020251222, 0.026538264, -0.016412653, -0.022367965, 0.013995775, 0.006172516, -0.00027224427, 0.0141142495, -0.035036825, -0.00091718923, 0.0101887975, 0.01753421, 0.024579488, 0.0008135241, 0.017723769, -0.0074007004, 0.031514186, -0.011081304, -0.012968997, 0.020124849, 0.0007049226, 0.010875949, 0.026980568, -0.0044427896, -0.002821665, -0.03127724, 0.006954447, -0.023789657, 0.023110405, 0.0063265325, 0.024453115, 0.006792532, -0.026080163, -0.025685247, -0.00911463, 0.0029164446, 0.026727822, 0.00010705889, -0.005493263, -0.01832404, 0.004379603, -0.016775973, 0.022494338, -0.0034436556, -0.019319223, 0.016025636, 0.008577546, 0.023236778, -0.018418819, 0.0072940732, -0.013308623, 0.002549174, 0.022510134, 0.015488552, 0.01230554, -0.031845916, -0.01813448, -0.0069860397, -0.007977275, -0.0017425609, -0.0058170925, 0.048369143, -0.008380089, 0.009841273, 0.009612222, -0.02734389, -0.003931375, 0.010259883, 0.0017178786, -0.0001630257, -0.0041110613, 0.00864863, -0.006294939, 0.0097149, 0.011397236, 0.010101916, -0.005777601, 0.013356013, 0.000101999045, -0.026822602, -0.0110418135, 0.0044704336, -0.016475838, 0.021704508, -0.00730987, 0.0004368743, -0.019966884, 0.0076218527, -0.02576423, -0.004778467, 0.011049711, 0.0083248, 0.00072022557, 0.011760558, 0.019082274, 0.0023477676, 0.017139295, 0.043251052, 0.00018869515, 0.017423633, -0.00097198365, -0.03020307, 0.0047666198, -0.02410559, -0.023236778, -0.017818548, 0.005449822, -0.0016379084, -0.0037003502, 0.011736863, 0.021388577, -0.026127553, 0.029160496, 0.001956802, -0.002355666, -0.0043203663, -0.009904459, -0.015480653, -0.012352929, 0.008166835, -0.003396266, -0.006425261, -0.0034179862, 0.01577289, -0.029729174, -0.010567916, 0.0076218527, 0.011318253, -0.009177816, -0.0036016214, 0.013774622, -0.028623413, 0.007104514, -0.011136592, 0.0031612916, -0.028054737, 0.008696021, 0.014651333, 0.025511485, -0.0025590467, 0.0024464962, -0.018797936, -0.01008612, -0.01813448, 0.014422283, -0.009628018, 0.010623204, -0.0076850387, -0.0069741923, 0.0152674, -0.006914955, -0.012345032, 0.037911806, 0.0017326879, 0.020093257, -0.015212112, -0.00934368, -0.010212492, -0.0023438183, -0.019034885, 0.0023754116, -0.0067530405, 0.004644196, -0.008309004, 0.0002964328, 0.01818187, -0.007199294, -0.012313439, -0.011855338, -0.006314685, -0.013237539, 0.014374893, 0.005256314, -0.011713168, -0.0036095197, -0.010820661, 0.015243705, -0.020140646, 0.0065990235, -0.0024682165, 0.005390585, 0.0015085739, -0.027880974, -0.018055497, 0.024358334, 0.007365158, -0.0374695, -0.016633805, -0.03165636, -0.012005405, 0.012850522, -0.0047152806, -0.007854852, -0.013427097, -0.0061685666, -0.017660582, 0.0072506326, 0.009051443, -0.024073996, 0.017107703, -0.005390585, 0.003617418, 0.022336371, -0.008609139, 0.0049877716, -0.021704508, -0.027312296, -0.017360447, 0.007254582, 0.0037418162, -0.018213462, -0.013379708, -0.014430181, -0.0075665643, 0.007846953, 0.00018338849, -0.013671945, -0.014596045, 0.007388853, -0.040692005, -0.0019410055, 0.011942219, -0.014872485, 0.0022352168, 0.017739566, 0.016997125, 0.006772786, -0.0023813352, -0.0147145195, 0.009319985, 0.0151331285, 0.0048179585, -0.03127724, -0.015717603, 0.0040952647, 0.014611841, 0.013687741, -0.013261233, 0.006527939, -0.0043045697, 0.007574463, 0.0036707316, -0.01863997, -0.030708563, -0.017076109, 0.0011037864, -0.021514948, -0.007850903, 0.022209998, 0.007925937, 0.046884265, -0.03044002, 0.018497802, -0.007933835, -0.0053273984, 0.014027368, -0.0057618045, 0.0023398693, -0.015227908, -0.0021029203, 0.03399425, 0.024974402, 0.01452496, -0.026759416, -0.015401671, -0.008996155, 0.005607788, -0.008214224, 0.017107703, -0.0022411405, -0.008846087, 0.018908512, -0.02160973, -0.011294559, 0.034310184, 0.0009892612, -0.010244085, 0.00957273, 0.00038578224, 0.020693528, -0.020061664, 0.006520041, 0.0118790325, 0.0074401917, -0.01596245, -0.027707212, 0.021388577, 0.006097482, 0.013948385, -0.0009828438, -0.002525479, -0.028828768, -0.018513598, 0.0047824164, -0.012597777, 0.00916202, -0.0007399713, 0.020061664, 0.01753421, -0.0068399217, -0.008553851, 0.0020654036, -0.011270864, 0.014398588, -0.0030507157, -0.0013733157, 0.02077251, 0.0082616145, 0.019161258, -0.0075428695, -0.02040919, 0.007933835, -0.03326761, -0.012637269, 0.0058210418, 0.00032234908, 0.006267295, 0.0070887175, -0.020346003, -0.007286175, -0.016057229, -0.012992691, -0.0012331209, -0.009477952, -0.0065358374, 0.011744761, 0.02295244, -0.006480549, 0.0100071365, -0.0012963073, -0.002580767, 0.016349467, -0.018071294, -0.006555583, -0.0023398693, 0.0008742423, -0.029587004, -0.001392074, -0.013040082, 0.023299964, 0.0000945944, 0.0062238546, -0.011760558, 0.007764022, 0.01808709, 0.013727233, 0.016381059, 0.0053195003, -0.0041189594, 0.00036850473, 0.007499429, 0.02123061, 0.0063067866, 0.0043598576, 0.010180899, 0.0036490113, -0.0014414384, -0.0030961307, -0.0038602906, 0.01119188, 0.008917172, 0.009738595, -0.009122528, 0.02720172, -0.00874341, -0.016775973, -0.017186685, 0.0022885303, -0.0037260195, -0.014145842, 0.0034021896, 0.030961307, -0.004032078, -0.014872485, -0.010583712, -0.021862473, 0.025479892, 0.016712787, 0.022920845, -0.036237366, -0.0024800638, -0.0014730316, -0.015212112, -0.0063857697, -0.006583227, 0.017502617, -0.010244085, -0.017597396, 0.0011738838, 0.0008772041, -0.00071084633, 0.0035601554, -0.0033784946, -0.0025610214, -0.007688988, 0.016712787, 0.005611737, 0.014366995, -0.021041052, -0.014335401, -0.019271834, 0.011847439, -0.009477952, 0.021262204, 0.016491635, 0.0014493368, -0.020930476, -0.004889043, 0.008190529, -0.00045193045, 0.015393772, -0.003104029, 0.010125611, 0.015243705, -0.008443275, -0.02803894, -0.010346764, -0.0050904495, -0.0066582607, -0.018482005, -0.006022448, -0.007092667, -0.010441543, 0.01767638, 0.009943951, 0.0024030556, 0.011492016, 0.0032501474, -0.026127553, 0.02295244, -0.017281465, -0.017281465, -0.00985707, -0.0006659248, -0.009233104, -0.0066464134, -0.00060273847, 0.018908512, -0.010425746, 0.022494338, 0.02765982, 0.0012192989, 0.015425365, 0.013506081, -0.005765754, -0.0057697026, -0.025242943, -0.016570618, -0.018276649, -0.007969378, -0.011104999, 0.01938241, -0.00901985, 0.00043070377, 0.0021858525, 0.019003293, -0.01327703, -0.019113868, -0.0094068665, 0.004419095, -0.015164722, 0.013158556, -0.016065128, 0.016033534, 0.01595455, -0.029855546, 0.02142017, 0.019256037, 0.00966751, -0.027091144, -0.02132539, -0.014493367, 0.009090935, 0.0040873666, -0.001639883, -0.023284167, 0.0035305368, -0.011902727, 0.0092252055, 0.0067411927, -0.008956664, 0.016396856, 0.011871134, -0.00012735996, -0.024911216, 0.003806977, 0.0020812, -0.0076455474, 0.009699103, 0.0043006204, 0.020977866, 0.007835106, 0.001436502, -0.009691205, 0.0035680537, -0.00003362945, -0.013016386, 0.0058723805, -0.016033534, 0.010646898, -0.01813448, 0.0013298751, 0.017155092, 0.004269027, 0.010054527, 0.018655768, -0.013845707, 0.028512837, 0.0049482803, -0.010536322, 0.009967646, 0.0035463334, -0.0026834449, 0.0031731392, -0.00088806427, 0.008782902, -0.0009680345, 0.005078602, 0.018513598, 0.0060856347, -0.0038267227, 0.018213462, -0.014501265, 0.0024425471, 0.018150277, -0.011871134, 0.031672154, 0.0061488207, -0.008435377, -0.026633043, 0.008538054, 0.0072703785, -0.0007014671, -0.009501646, -0.010796966, 0.01674438, 0.018055497, -0.008403784, 0.0076929373, -0.003984689, 0.012147574, -0.0029480378, -0.005935567, 0.012155472, -0.013032183, 0.024200369, -0.019729935, -0.017581599, 0.015156824, -0.01868736, -0.016349467, -0.008182631, -0.004450688, -0.0074125477, 0.009683307, -0.025669452, -0.0074007004, -0.0036332146, 0.013498182, 0.015346383, 0.0060540414, -0.013916791, -0.024626877, -0.0029875292, -0.0062159565, 0.0029539615, -0.005232619, -0.008340597, 0.035036825, 0.0017238024, -0.010346764, -0.011373541, 0.005370839, -0.015535941, -0.0063739223, 0.0076850387, 0.021846678, -0.0036806045, 0.0056512286, -0.0017129423, -0.017960718, -0.006859667, 0.0032501474, -0.011397236, -0.0015026502, 0.01128666, 0.039554648, -0.009825476, -0.010947034, 0.007815361, 0.019682545, 0.0063581257, 0.008474868, -0.021041052, -0.012313439, -0.0072150906, 0.03459452, -0.014911977, 0.0013555444, 0.013142759, -0.0044032983, -0.0035937233, 0.014596045, -0.022288982, -0.0031158766, 0.03519479, -0.007132158, -0.01901909, 0.014453876, 0.0015046247, 0.015417468, -0.00045217725, -0.02988714, -0.0037497145, -0.006417363, 0.010931237, 0.0021819035, 0.008403784, 0.027880974, -0.0024247759, -0.011026016, 0.0054379744, 0.009722798, 0.02516396, 0.00809575, 0.0030546647, 0.031340424, 0.0060895835, -0.006046143, 0.00094384595, -0.016554821, 0.009130427, -0.015077841, -0.002572869, -0.003765511, 0.011341948, 0.003414037, -0.016444245, 0.0161915, -0.0025926146, 0.0067096, -0.0153147895, -0.0047824164, 0.012226557, -0.015346383, 0.002016039, 0.0061409227, -0.021941457, -0.006559532, 0.0030605884, 0.008925071, -0.011049711, -0.0049601276, 0.0037398415, 0.010765373, -0.028970938, -0.018213462, -0.010425746, -0.019492986, 0.026980568, 0.020061664, -0.009193612, -0.02137278, 0.0008174733, 0.023505319, -0.0069583957, -0.0032777914, -0.003779333, -0.013837809, 0.004020231, 0.0013545572, 0.008909274, -0.007068972, -0.011247169, 0.005813143, 0.021593932, -0.007179548, 0.016681194, 0.0129532, -0.013592962, 0.00837219, 0.0036095197, -0.0036016214, 0.019587766, 0.008601241, -0.008932969, 0.0039906125, -0.00085647113, 0.009288392, -0.009612222, -0.016854957, 0.009730697, -0.00030260335, 0.008490665, -0.009904459, -0.0034732742, 0.037216756, 0.0035917487, -0.0059197703, 0.003400215, -0.015709704, 0.009896561, -0.0017001075, -0.015456959, 0.012882115, -0.007937784, 0.008569648, 0.004636298, 0.03118246, 0.0022450897, -0.01933502, 0.013735131, 0.015251603, 0.0021601832, -0.011326152, -0.0051022973, 0.008380089, 0.015377976, 0.016365262, -0.010567916, -0.00518128, -0.0145328585, -0.0088065965, 0.023710676, -0.01596245, -0.0118237445, 0.024263555, 0.0053747883, 0.016065128, -0.005323449, 0.0029184192, 0.006263346, 0.011768457, -0.016823363, 0.00851436, 0.009936052, 0.02415298, -0.0031751136, -0.002863131, 0.009628018, -0.019082274, -0.0028394363, -0.029855546, 0.0052168225, -0.0031790629, -0.0063265325, 0.006764888, -0.00846697, -0.0057539064, -0.0054656183, -0.0053826864, 0.0050549074, -0.009011952, 0.007791666, 0.0077600726, 0.0018778191, -0.00053066656, -0.004099214, 0.0013999724, 0.025669452, 0.01068639, 0.01748682, 0.0042808745, 0.016618008, 0.030455817, -0.008782902, -0.002780199, 0.0066740573, -0.013142759, -0.016365262, -0.04407247, 0.002318149, -0.011318253, -0.00024879622, 0.020977866, 0.004561264, -0.018797936, -0.019350817, -0.00029445824, -0.024642672, -0.014864587, -0.009241003, 0.0051101954, 0.023141997, 0.0009413778, -0.008735511, -0.0030191224, 0.019935291, -0.01563862, -0.011026016, 0.0030566393, 0.0002143646, -0.004213739, -0.0059197703, -0.0057420586, -0.0033291304, 0.004004434, 0.0033923166, -0.008538054, -0.00064469816, -0.015843974, 0.026617246, 0.010378357, -0.0062712445, 0.0116973715, -0.0029421141, 0.010402052, -0.009722798, -0.012266048, 0.017723769, 0.0044230437, 0.0088618845, 0.016823363, -0.0059237196, 0.0075152256, -0.0035956977, 0.02281027, -0.015330587, -0.033014864, 0.015946653, 0.013158556, -0.0145881465, 0.0072150906, -0.0053866357, 0.01161049, 0.016775973, -0.004426993, -0.0070255315, -0.0034811725, -0.0124240145, -0.0012133752, -0.0016882601, -0.008735511, -0.022004643, -0.015986145, -0.020946272, 0.00096359174, 0.004269027, 0.010046628, -0.024026606, 0.013071675, -0.015575433, 0.0070808195, -0.0019232343, 0.0038050024, 0.009083036, 0.008309004, 0.0053313477, 0.0012923582, -0.013790419, 0.021135831, 0.001570773, 0.011958015, -0.0062712445, -0.008427478, 0.004079468, -0.008940867, -0.010575814, 0.002770326, -0.005607788, -0.018892717, -0.012471404, -0.013498182, 0.0035423844, -0.016223093, -0.0077047846, -0.007254582, -0.006042194, 0.0026952925, -0.0074954797, 0.0031731392, -0.00629099, 0.010860153, -0.008617037, -0.008135241, -0.027880974, -0.012495099, -0.017313058, 0.001684311, 0.008798698, -0.0009724773, 0.01837143, -0.031751137, -0.01748682, 0.02660145, 0.028212702, 0.007337514, 0.010710085, -0.0011501889, -0.009351579, -0.0054221777, -0.007843005, -0.004513874, 0.0013496208, 0.0018057472, -0.0010534348, -0.03367832, 0.015030451, -0.021767695, 0.0015421417, -0.002835487, -0.0036095197, -0.003358749, -0.00883819, -0.02017224, -0.00707687, -0.038196143, -0.0058170925, -0.005343195, -0.0044664843, -0.011760558, -0.014651333, 0.0024484708, -0.021293797, 0.006907057, -0.0074362424, 0.007068972, -0.00458101, -0.0008757232, -0.013300725, -0.005145738, 0.007606056, 0.0029184192, 0.015472756, 0.001705044, 0.006133024, -0.043882914, -0.0014414384, 0.017060312, -0.007558666, -0.000001770174, 0.012139676, -0.012321336, 0.011112898, 0.026806805, 0.0024109539, 0.007132158, -0.01563862, -0.0007414522, -0.03077175, -0.0057854992, -0.0071755988, -0.029855546, 0.01309537, 0.0065753288, -0.00045933507, -0.0023398693, 0.0009023799, 0.008474868, -0.0033409777, -0.028260091, 0.027549244, 0.001026778, 0.0034298336, -0.00976229, 0.0060619395, -0.0044625355, -0.018102886, 0.002507708, -0.02211522, -0.004584959, 0.0072782766, -0.0013259259, 0.0014068834, 0.00025965637, -0.0062159565, 0.010836458, 0.0034712995, -0.0006126113, 0.011002322, 0.011997507, -0.009549036, -0.017076109, 0.009035647, -0.007290124, -0.0062514986, 0.0029085462, 0.0051299413, -0.0070571247, 0.011365643, -0.0033133337, -0.0069307517, -0.0007932848, 0.0067332946, 0.000047019523, -0.0038662143, -0.008901376, -0.018102886, 0.007471785, -0.00052918564, 0.011231372, 0.005240517, 0.008909274, -0.019034885, -0.015109434, -0.012637269, 0.010465238, -0.013450792, -0.017202482, -0.009604324, 0.0014355147, -0.01938241, 0.004458586, -0.003617418, 0.015780788, -0.020472374, -0.023536913, -0.0064963456, -0.0052800085, 0.019129666, -0.005852635, -0.0030289954, 0.028718192, 0.012818929, -0.0037773584, -0.014264316, -0.010670593, -0.013087471, -0.0061488207, -0.0052049747, -0.0031237747, 0.009809679, 0.014540757, 0.015069943, -0.001147227, -0.0008737486, 0.015907161, 0.008593342, -0.010417848, 0.0046836874, -0.0154964505, -0.020235427, 0.018150277, -0.015109434, 0.000048191927, 0.025716841, 0.008696021, 0.0057065166, 0.011231372, 0.0161915, -0.0027584787, 0.0049759243, 0.011460423, 0.008364292, -0.002873004, -0.013237539, 0.010678492, 0.021451764, 0.001533256, -0.0012716252, 0.017392041, 0.013111166, 0.009454256, 0.015922958, 0.020756714, -0.011073407, -0.000023062397, 0.0020170265, -0.027170127, 0.005694669, -0.009904459, -0.009880764, 0.007313819, -0.0049877716, -0.0007765009, -0.0011245194, 0.018292446, 0.009549036, -0.012084388, -0.014864587, 0.00043613385, -0.010947034, -0.007554717, -0.0037753838, -0.007234836, -0.015227908, 0.0037181212, -0.035321165, 0.0050825514, -0.015054146, 0.028528633, 0.009193612, 0.013182251, 0.020598747, 0.0012084388, -0.0042453324, -0.0051141446, 0.0014868536, 0.005805245, -0.023189388, -0.0074836323, 0.0008609139, 0.012044896, -0.0039886376, -0.017218279, 0.00846697, -0.017929124, 0.009596426, 0.01138144, 0.00043662748, 0.002645928, -0.0031336476, -0.0075428695, 0.0064963456, -0.022794472, -0.014785604, -0.02156234, -0.04423044, -0.011681575, 0.01503835, -0.0076455474, -0.0021601832, 0.027501855, -0.014540757, -0.028275888, 0.013513979, 0.01674438, 0.008893478, -0.031672154, -0.005260263, 0.0010188797, -0.0013269131, -0.0056828214, 0.03320442, -0.0016497559, -0.025432503, -0.019366615, -0.0071874466, -0.02132539, -0.0010085133, 0.007945683, 0.004881145, -0.0021068696, -0.026222331, -0.008036513, 0.0014651333, -0.0072387853, 0.006448956, 0.013956283, 0.0037181212, -0.0017159041, -0.0072506326, -0.031466797, 0.013261233, -0.000046834408, -0.022794472, -0.008830291, 0.0011106974, -0.0024050302, -0.016965533, 0.012226557, -0.017613193, -0.021672916, 0.0053826864, -0.0051891785, -0.0061211768, -0.008846087, 0.002511657, 0.013766725, -0.012179167, -0.0059513636, 0.012044896, -0.0057578553, -0.02410559, -0.010125611, 0.019161258, -0.051338904, 0.0071677007, 0.018118683, -0.00069702434, 0.009849171, 0.012779438, -0.010409949, 0.0043401117, -0.030124089, 0.0060816854, -0.007396751, -0.0013802267, -0.004584959, -0.0059474143, 0.006820176, 0.005212873, 0.011081304, -0.013071675, 0.017518412, 0.028781379, 0.004541518, 0.011183983, -0.007795615, 0.003968892, 0.003358749, -0.0048969416, 0.004541518, -0.012653065, 0.017550007, -0.009785985, 0.0027031908, 0.008356393, 0.0020693527, -0.022146814, 0.004419095, 0.01729726, 0.01947719, -0.014169537, 0.012171269, 0.031940695, -0.011144491, 0.008609139, -0.009454256, 0.0029954275, 0.028323278, 0.011420932, 0.000049210066, 0.0026479026, 0.0092252055, -0.000521781, -0.004991721, 0.01813448, 0.017929124, 0.020440781, -0.013822013, 0.03459452, 0.0022549625, -0.015891366, -0.017976515, 0.009659612, 0.013245437, -0.009896561, -0.03164056, -0.01698133, -0.0116973715, -0.002373437, -0.003937299, 0.0074283443, -0.0013683792, -0.011547304, 0.0043480103, -0.035637096, 0.0037813077, 0.0068083284, 0.0026242076, -0.011270864, 0.012013303, 0.024326742, -0.0152674, 0.02345793, 0.015393772, 0.0065160915, -0.018339835, -0.008909274, 0.013158556, -0.01165788, 0.0062238546, -0.010931237, 0.017281465, -0.013664046, -0.0015944678, -0.010939135, -0.006907057, -0.025669452, -0.016854957, 0.02123061, 0.017044516, 0.004569162, -0.002104895, 0.0074480902, -0.00934368, 0.021514948, -0.010165103, -0.020456579, -0.0009611235, -0.017076109, -0.010583712, 0.007799564, 0.009643815, -0.0065516336, 0.008474868, 0.0036529603, 0.0018373403, -0.0034337828, -0.000078427576, 0.0015283197, -0.0032856897, 0.0013377733, 0.0032027578, 0.0012508921, -0.017170887, 0.007151904, 0.00939107, -0.0019202724, 0.023047218, -0.013395504, -0.022841863, 0.013237539, -0.014043164, 0.0018876919, -0.022826066, -0.018497802, 0.020630341, 0.01952458, 0.013474488, -0.003937299, 0.001378252, 0.01863997, -0.0012331209, -0.018008107, -0.014975163, 0.008190529, 0.00966751, -0.009098833, 0.008348496, -0.004051824, 0.0018580734, 0.009430561, -0.01628628, -0.0044230437, 0.0021483356, 0.0023655386, -0.00874341, 0.0070887175, -0.007013684, -0.015733398, 0.016396856, 0.003613469, -0.022920845, -0.014193232, -0.0039629685, -0.0017287388, -0.0071834973, -0.033804692, 0.0027880974, -0.01138144, -0.0009946913, 0.011073407, -0.0021917762, 0.0021522848, 0.012668861, 0.0019607511, 0.0041505527, -0.0066938032, 0.004711332, 0.0060342955, 0.025100775, 0.0029440885, -0.0011995533, 0.002470191, 0.013893097, 0.012123879, 0.008277411, 0.0032758168, 0.02401081, 0.0023339456, -0.0118790325, 0.020614544, -0.0059118723, 0.0010820661, 0.017913328, -0.0025235044, -0.0058407877, -0.0018886792, 0.015433264, -0.0049759243, 0.028923549, -0.00010156711, -0.009707002, -0.0041308072, 0.007827208, 0.014098452, -0.011973811, -0.0072782766, -0.0049206363, 0.021278001, 0.0005963211, -0.02614335, -0.004478332, 0.009264697, 0.0044309422, 0.008269512, 0.0025649706, -0.011089203, -0.0011906676, 0.0226681, 0.012471404, -0.0033014864, 0.0011511762, -0.010670593, -0.010196696, -0.012818929, 0.007854852, 0.0062396512, -0.023047218, -0.00601455, -0.0133876065, 0.015764993, -0.019872105, -0.0007064036, 0.0009611235, -0.0015737348, 0.026490875, 0.011792151, -0.016807567, -0.029539615, -0.010054527, 0.02040919, -0.00499567, 0.006824125, 0.0018521496, -0.010947034, 0.023789657, 0.0050667548, 0.007037379, -0.010433645, -0.009004054, -0.0026084112, 0.026285518, -0.00458101, -0.004371705, 0.0049759243, 0.0026775212, -0.000063926804, -0.010504729, -0.004663942, 0.0026794958, -0.0051299413, -0.009138324, -0.016949736, -0.002073302, -0.0066740573, -0.014098452, -0.0028433853, 0.004213739, -0.02156234, -0.005291856, -0.04381973, -0.011563101, 0.010338865, -0.01794492, 0.0053352965, 0.003617418, -0.009075139, 0.010212492, -0.012913709, -0.009541137, -0.024453115, 0.0036904772, 0.019745732, 0.036806043, 0.02192566, -0.007420446, 0.015283196, -0.00043045694, 0.01355347, -0.0058091944, -0.00730987, -0.0030921816, -0.020251222, -0.0077403267, -0.0072940732, -0.0036648079, -0.0043282644, -0.004458586, 0.0034594522, -0.03816455, -0.0065437355, -0.002600513, 0.011065508, 0.017692175, 0.0035542317, -0.0031770882, -0.0061132787, 0.0063067866, 0.006982091, -0.017139295, 0.022352168, -0.01068639, -0.014682926, -0.015575433, 0.0024682165, -0.00085992663, -0.032698933, 0.0043322137, 0.0073849037, 0.01327703, -0.011926422, 0.013434996, 0.0062238546, -0.014983062, 0.017407836, 0.0101887975, 0.011065508, -0.025179757, 0.0034890708, -0.0029756818, 0.01036256, 0.019887902, -0.0072980225, -0.011918523, -0.029823953, -0.01161049, -0.0062514986, 0.0061764647, -0.00026878878, 0.011270864, -0.008846087, -0.010836458, -0.013837809, -0.021799289, 0.013087471, 0.022036236, -0.04081838, 0.017328855, -0.0012015278, 0.017091906, -0.033741508, 0.0009862992, -0.0021266153, 0.0034495792, 0.039586242, 0.03803818, 0.018908512, -0.0018521496, -0.021767695, 0.019129666, -0.021815084, -0.019556172, -0.010417848, -0.0012301591, -0.0074836323, -0.003506842, 0.0070808195, 0.014651333, -0.009185715, 0.048495516, 0.009193612, -0.0024306995, 0.0052247206, 0.022857659, -0.005501161, 0.012866319, -0.014840892, -0.010607407, -0.004458586, 0.0010484983, -0.0077798185, 0.002882877, 0.013016386, -0.011018118, 0.0048653483, 0.0033705963, 0.00194298, -0.021830881, -0.010765373, -0.013719334, -0.000696037, 0.025795825, 0.0026794958, 0.0040636715, -0.017076109, -0.025574671, 0.011555202, 0.007839056, -0.027217517, -0.019603562, -0.015101535, -0.016057229, 0.0054103304, 0.020282816, 0.009951849, -0.026174942, 0.013103267, -0.011855338, -0.01952458, -0.018545192, 0.0020298613, 0.04246122, -0.006010601, 0.0054853642, -0.01554384, -0.0029124955, -0.0055169575, -0.0034179862, 0.0056867707, -0.011918523, 0.028165312, -0.007613954, -0.004991721, 0.015654417, -0.007519175, -0.016104618, -0.015417468, 0.015069943, 0.004968026, 0.026332907, -0.011452524, -0.002002217, -0.026948975, -0.007337514, 0.00035690411, -0.005145738, -0.02168871, 0.023394743, -0.013221742, -0.017249871, 0.0017109676, 0.00149179, -0.0049008904, -0.016570618, 0.008301105, -0.006606922, -0.051275715, 0.0019518656, -0.0055524996, 0.019635156, -0.000107922766, -0.01369564, -0.0028749786, -0.008072055, 0.021278001, 0.025969587, -0.022731286, 0.030503206, -0.012755742, -0.0036865282, 0.0134192, -0.0009734646, 0.0028117923, -0.0026360552, 0.0014345275, -0.026822602, -0.038132958, 0.007341463, 0.011247169, 0.017865937, 0.00018881857, 0.019098071, 0.020661933, -0.0071282093, -0.013774622, -0.012408217, -0.017044516, -0.0016112517, -0.0009408841, -0.037848618, -0.016136212, -0.0060619395, 0.011073407, -0.014888282, 0.010141408, 0.025179757, 0.010465238, -0.016712787, -0.004541518, 0.004233485, 0.0069031077, 0.0021700559, -0.0010781169, 0.0003021097, 0.017960718, -0.015630722, 0.0026992415, -0.0079535805, 0.023789657, -0.0003499931, -0.0023063014, 0.03077175, 0.0039945617, -0.019192852, -0.01475401, -0.010646898, 0.0025590467, -0.0013200021, 0.015101535, 0.00013167935, 0.022367965, -0.009912358, 0.012115981, -0.012779438, -0.0003043311, -0.014959366, 0.004695535, -0.01679177, -0.011634185, -0.011942219, 0.009193612, 0.004833755, 0.024374131, -0.0043361625, 0.006227804, 0.0029460632, 0.0007503378, 0.021072645, -0.0004228055, 0.01837143, 0.0057341605, -0.009083036, 0.023521116, 0.014785604, -0.010338865, -0.0020575053, -0.0050035683, 0.0006032321, 0.012747845, -0.00633838, 0.017613193, -0.008028614, -0.0055248556, 0.00014994416, -0.009146223, 0.009296291, -0.023931827, -0.0060342955, -0.021246407, 0.016949736, 0.0055209068, 0.0057539064, -0.008293208, 0.018055497, -0.010947034, -0.008064156, -0.0070018363, 0.011349847, 0.022146814, 0.016412653, 0.0068083284, -0.0068438705, 0.0040281294, -0.0029756818, 0.01059161, -0.021625526, -0.00080068945, -0.013284928, 0.025432503, 0.00212859, -0.02696477, -0.017407836, -0.010402052 - ] - } - ] - } + { + "name": "narust-podvodne-bizuterie-v-krizi", + "title": "Nárůst podvodné bižuterie v krizi", + "content": "---\n\nKorálky Katlas zaznamenaly zvýšený výskyt podvodné a zdraví škodlivé bižuterie na trhu v době ekonomické krize, kdy lidé více šetří. Zákazníci hlásí problémy a podvody s bižuterií jak v České republice, tak v zahraničí.", + "keywords": [ + "korálky", + "Katlas", + "podvodná bižuterie", + "zdraví škodlivá bižuterie", + "ekonomická krize", + "šetření", + "zákazníci", + "problémy", + "podvody", + "Česká republika", + "zahraničí" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + -0.0037002896, -0.071282186, -0.011276486, -0.0038142868, 0.03477841, + -0.01859696, 0.0058847237, -0.0064639533, 0.0043904353, -0.031697404, + 0.022922693, 0.019730771, -0.011911173, -0.051613033, 0.021604022, + 0.017081104, 0.004362706, -0.04444045, 0.00035835966, 0.019989574, + -0.0030517376, -0.016033562, 0.011214866, -0.020716693, 0.02080296, + 0.013075794, -0.036257293, -0.009785279, -0.022207899, 0.01820259, + -0.0028145001, -0.022294167, -0.008990379, -0.005595109, 0.008830166, + 0.0067905397, 0.017672658, -0.036331236, -0.0014873563, -0.011110112, + 0.01223776, -0.056542642, -0.022922693, 0.06960612, 0.0066981097, + -0.01189885, -0.020876905, -0.017561741, 0.0028915252, -0.0053825197, + -0.027827658, 0.013507135, -0.028641043, -0.03988056, 0.023415655, + 0.011886526, -0.012287057, 0.0437503, 0.030760776, 0.049567245, + -0.02664455, 0.013112767, 0.006285255, 0.029479077, -0.020568805, + 0.026102293, -0.026570605, -0.0031380057, 0.0057337545, 0.02486989, + -0.011066978, -0.024845243, 0.052746844, -0.002346187, 0.023773052, + -0.020987822, -0.010635637, -0.0075114956, -0.011430536, 0.007912027, + -0.020445563, 0.06571172, 0.048901748, 0.011017682, -0.035715036, + -0.044465095, -0.007788786, -0.055556722, -0.013913829, -0.032781918, + -0.04971513, -0.009631229, -0.0583666, 0.014825807, 0.008540552, + -0.00047986687, -0.035838276, -0.003943689, -0.045993276, 0.048310194, + 0.0105616925, -0.0053856005, 0.017228993, 0.04111296, -0.0011576884, + 0.053239804, -0.013901505, 0.026422719, -0.024315309, 0.053584877, + 0.014640946, -0.014098689, 0.015626868, -0.015713137, 0.010641798, + 0.022614593, 0.025362851, 0.010025597, -0.018190267, 0.010068731, + -0.0673385, -0.032017827, 0.054767985, 0.029134003, 0.013482488, + -0.0065563833, -0.017253641, 0.025905108, 0.0033336498, 0.014788834, + -0.048655264, -0.013679672, 0.025412148, -0.017931461, 0.011325782, + -0.004439731, -0.0035524012, 0.0139508005, 0.011418212, -0.017167373, + -0.039781965, -0.012792341, 0.017081104, 0.0025526145, -0.062014513, + -0.056542642, -0.0013587242, -0.025609331, -0.026989624, -0.014184957, + 0.03465517, -4.0016988e-7, -0.014147985, -0.019841686, 0.0050497707, + -0.0118926875, 0.03401432, 0.0006631868, 0.019878658, 0.035074186, + -0.013655024, -0.010537044, 0.025929756, -0.016699059, 0.0009204509, + -0.0110731395, 0.049296115, -0.04419397, -0.040521406, 0.03527137, + -0.0193364, -0.018227238, 0.0035185101, 0.006420819, 0.008552876, + 0.04237001, 0.029898094, 0.012367163, -0.01529412, -0.05028204, + 0.00030020563, 0.023144526, 0.037391104, 0.0024694272, -0.025091723, + 0.002364673, -0.00016926283, -0.045155242, 0.00954496, 0.0062760115, + 0.025683276, -0.036848847, -0.02006352, -0.071725845, -0.024808269, + 0.0029993604, 0.0036417504, -0.008300234, 0.0035185101, -0.013753616, + -0.005086743, -0.029134003, 0.034729112, -0.03606011, 0.01547898, + -0.014912074, 0.028788932, -0.008700765, -0.01801773, 0.020568805, + -0.009532636, 0.019582883, 0.028714987, 0.043848895, -0.04301086, + 0.010333698, 0.00480329, 0.0032751106, 0.00006893754, 0.0144314375, + 0.019040624, -0.009113619, -0.0048464243, -0.0054441397, -0.020088166, + 0.00044520554, -0.0054010055, 0.025436796, -0.011566101, 0.0187202, + -0.004636916, -0.045919333, -0.05999337, 0.011214866, -0.018954357, + 0.006001802, -0.024832917, 0.01865858, 0.03366925, -0.016415607, + -0.007702518, 0.012927907, -0.0032504625, -0.030785425, 0.0028037166, + 0.010678771, -0.006488601, 0.010493911, 0.0023045933, 0.041926347, + -0.0152817955, -0.01667441, 0.001508153, 0.027162159, 0.006907618, + 0.04606722, 0.050725702, -0.022836424, -0.00020662004, -0.027112864, + 0.046165813, -0.023440303, -0.059451114, 0.029996686, 0.009822251, + 0.03618335, -0.005974073, 0.0023199983, -0.025461443, 0.013408544, + -0.0016514199, -0.010025597, 0.036331236, -0.0016714464, 0.001520477, + 0.040324222, 0.06512017, -0.018239563, -0.029996686, 0.053387694, + 0.010746554, 0.03095796, 0.009495664, 0.027088216, -0.0139508005, + 0.018313507, -0.028271321, 0.015405036, 0.011997442, 0.037144624, + -0.0005915534, -0.05535954, -0.028345266, -0.00020450185, 0.012379487, + -0.002162867, 0.017672658, 0.024611086, -0.024031855, -0.002199839, + -0.05378206, -0.02740864, -0.034753762, 0.043158747, 0.019841686, + 0.009779117, 0.04557426, 0.023440303, -0.06896526, -0.057577863, + -0.022811778, 0.0008726953, 0.003999147, -0.0049111256, -0.044046078, + -0.019545909, -0.034310095, -0.015651517, 0.071085, -0.026545959, + 0.009273832, 0.037612937, -0.009366262, 0.02627483, 0.015934968, + -0.009002703, 0.008768546, 0.01275537, 0.030612888, 0.01795611, + 0.03366925, -0.00094201794, -0.028197378, 0.013531784, -0.047891177, + -0.016132154, -0.0006666529, -0.061718736, 0.01541736, 0.007930513, + 0.009723659, -0.010863631, -0.001731526, -0.0075361435, -0.019841686, + 0.007332797, -0.012434945, -0.01465327, 0.016773002, 0.0187202, + 0.00054495316, 0.022368113, -0.030489648, -0.00010610218, 0.06388777, + -0.024845243, 0.011221028, 0.0077641383, 0.017943786, -0.017155048, + 0.008158507, -0.002945443, 0.0073697693, 0.003401432, 0.026743142, + 0.0425179, -0.014086365, 0.014961371, 0.006537897, -0.01477651, + -0.01636631, 0.004726265, -0.0335953, -0.0125889955, -0.012096034, + 0.009501826, -0.0039190412, 0.015885673, -0.015577572, -0.00061042455, + 0.00017215128, -0.015885673, 0.005145282, 0.014234253, -0.019952603, + 0.06955682, 0.0017423095, 0.05264825, 0.006029531, 0.03778547, + -0.014912074, 0.0048556672, 0.016341662, 0.034186855, 0.0070185345, + -0.022466704, 0.0019348725, 0.06615539, -0.009859223, 0.011701665, + -0.0037495857, 0.034975592, -0.029331189, 0.004575296, 0.00302863, + -0.01024743, 0.004729346, -0.037834767, -0.015824053, -0.0006277552, + -0.008121535, -0.0075114956, -0.018103998, 0.014369817, -0.004763237, + 0.0075361435, 0.053091917, -0.0001403784, 0.025436796, 0.034334745, + -0.03453193, -0.02652131, 0.041408736, 0.021037117, -0.03854956, + -0.013679672, -0.022996638, 0.014628622, 0.0023076744, -0.004356544, + -0.011713989, 0.026841735, -0.025707925, -0.003928284, 0.023292415, + 0.012780018, -0.007819597, 0.03665166, 0.005148363, -0.006316065, + -0.03756364, 0.00045714443, 0.03310234, 0.036725607, -0.001186958, + -0.0144314375, 0.021148033, 0.019866334, 0.0056290003, -0.005927858, + -0.013556432, 0.020765988, 0.0012185384, -0.0048864773, -0.046880607, + -0.013605728, -0.016415607, 0.022429733, 0.035739683, 0.027729064, + -0.012040576, 0.043355934, -0.03719392, -0.021295922, 0.0155406, + 0.020605776, 0.031007256, 0.022947341, 0.006457791, 0.012478079, + -0.031105848, -0.046880607, -0.027211456, -0.0065625454, -0.003047116, + -0.06546524, 0.03842632, -0.008632982, 0.014061716, -0.010290564, + -0.0012531996, 0.022109307, -0.008133859, 0.02767977, 0.01611983, + -0.003077926, -0.012188464, -0.016169125, 0.0045660525, 0.008756222, + -0.065317355, -0.016686736, 0.003355217, 0.012354839, -0.014147985, + -0.004245628, -0.015700813, -0.048236247, -0.026595253, 0.025030103, + -0.008688441, -0.01033986, 0.0139508005, -0.028222026, -0.02906006, + -0.06334551, -0.0031580324, 0.022392761, 0.040274926, 0.0152325, + 0.004261033, -0.0022137037, 0.014961371, -0.01214533, -0.02753188, + -0.008176993, -0.016082857, -0.011775609, -0.0038019628, -0.024549466, + 0.016846947, 0.015959617, -0.038894635, -0.019792391, -0.04520454, + -0.0077949483, 0.013285303, 0.0052192262, 0.044070728, -0.019348726, + 0.0133469235, 0.017352233, 0.013901505, 0.015121583, -0.0007016994, + 0.00088424905, -0.0029700908, -0.00088424905, -0.013975449, + -0.027383992, 0.00055573665, 0.015084611, 0.0013417787, 0.019262457, + 0.02254065, 0.05062711, 0.0070678303, -0.007456037, -0.013063471, + -0.017253641, -0.035074186, 0.0112888105, 0.008854815, -0.0036848846, + 0.01890506, 0.035567146, 0.043848895, -0.026964976, -0.017746601, + -0.009304642, -0.002584965, 0.031031905, -0.00195798, -0.00056266895, + 0.023366358, -0.028443858, 0.005450302, 0.052845437, -0.012743046, + 0.00009141926, 0.027236104, 0.008694602, -0.021530079, -0.032929804, + -0.017413853, 0.0013903045, -0.04771864, -0.011750962, 0.028764283, + 0.024450874, -0.026743142, -0.019780066, -0.0007752584, -0.033644598, + -0.032560084, -0.0011985118, 0.05314121, -0.009563446, -0.006932266, + -0.012471917, 0.02367446, -0.013975449, -0.02086458, -0.014258902, + 0.008713088, 0.035345316, -0.019989574, -0.013655024, 0.012669101, + 0.021000145, 0.0066981097, 0.008577524, 0.007456037, 0.042443953, + -0.009403234, 0.012644454, -0.015762433, -0.019952603, 0.008731575, + -0.008337205, 0.00074598886, 0.0028468505, -0.010050246, -0.013433191, + 0.019570557, 0.008552876, 0.02563398, 0.02854245, -0.004415083, + 0.0190283, 0.018732524, -0.001734607, 0.017352233, -0.01832583, + -0.0013818317, -0.024808269, -0.015651517, 0.01667441, 0.0040761726, + 0.0021721101, -0.048113007, -0.013691996, 0.009446368, -0.028912172, + 0.008170831, 0.027704416, 0.0017669576, 0.07167655, 0.027433287, + 0.014912074, 0.011091626, 0.018005406, -0.002178272, 0.0055673798, + 0.011584587, 0.011097788, -0.02664455, 0.017808221, 0.021037117, + 0.023477275, -0.029232597, 0.010228944, -0.022466704, 0.007979808, + -0.03273262, 0.0060233693, -0.0014665595, 0.0077703, -0.00189636, + -0.00081877765, -0.018030055, -0.032313604, -0.01229938, 0.0017715791, + -0.010882118, 0.0033028398, -0.014603974, -0.005715268, 0.021148033, + -0.020593451, 0.02012514, 0.021308245, -0.028887523, -0.024216717, + -0.020765988, -0.022010716, 0.013691996, -0.0091998875, -0.017746601, + -0.012927907, 0.043454524, 0.005761483, 0.016193774, 0.022096984, + -0.002808338, -0.0065502212, 0.008103049, 0.04276438, 0.031376977, + -0.0260037, 0.016748356, -0.034088265, -0.011467509, 0.008460445, + 0.03719392, -0.0241058, 0.021554727, -0.022084659, -0.02006352, + 0.013248331, -0.03310234, -0.009335452, -0.023662135, 0.014862779, + 0.026028348, 0.03135233, -0.003153411, -0.006537897, 0.010610988, + -0.013839885, -0.005890886, -0.0062051485, -0.040817183, -0.009828413, + -0.012262409, 0.0037126136, 0.0046399967, -0.011572263, -0.027630473, + 0.025486091, 0.036232643, 0.0070123724, 0.0022953504, -0.010974548, + 0.03260938, 0.032954454, 0.021074088, 0.018818792, -0.013605728, + 0.0097729545, -0.010512397, 0.038352378, 0.040151685, 0.0035832115, + -0.032634027, 0.012490403, -0.03529602, 0.010173486, 0.019065272, + 0.012096034, 0.04532778, -0.026447365, -0.009162915, -0.033940375, + -0.011344269, -0.027852304, 0.010463101, -0.008213965, 0.034852352, + 0.005961749, -0.028443858, -0.007271177, 0.031278387, 0.003943689, + -0.015047639, 0.015737785, 0.0069199423, 0.0048957206, -0.04237001, + 0.012718398, 0.009625066, -0.005493436, -0.026324125, 0.00053031836, + 0.018338155, -0.027704416, -0.029207949, 0.0039775805, 0.02012514, + -0.017598713, -0.011331945, 0.005601271, -0.024512494, 0.028468506, + -0.012243923, -0.027630473, -0.0010875956, 0.015996588, 0.008842491, + 0.004849505, 0.02676779, 0.003432242, 0.018030055, -0.0037434238, + 0.023649812, 0.010949899, 0.04362706, -0.0105616925, -0.018646255, + 0.008731575, -0.013802912, -0.0032504625, -0.011017682, + -0.00050066365, -0.0013541026, 0.018251887, 0.009132105, -0.04426791, + -0.0063653607, 0.017364556, -0.00969901, -0.012330191, 0.026200885, + 0.00056574994, 0.024056504, 0.0033244067, -0.01287861, 0.0049388544, + 0.004285681, 0.014394466, 0.016292365, -0.0049789078, 0.008540552, + -0.022294167, 0.01611983, 0.0078011104, -0.0072773388, 0.02906006, + 0.017758925, 0.013063471, -0.013839885, -0.0041840076, -0.001945656, + 0.003811206, 0.016723707, 0.016809976, 0.017512444, -0.007690194, + 0.008232451, 0.0005068257, -0.019693797, -0.0056444053, -0.0033798649, + 0.015614544, 0.02024838, 0.0039344463, -0.010167324, 0.0073759314, + -0.0241058, -0.012465755, 0.01529412, -0.0044212453, -0.013655024, + 0.03095796, 0.032880507, 0.009557284, -0.0031518703, -0.018757172, + 0.0041624405, 0.04239466, 0.005595109, -0.0071849087, 0.019003652, + -0.023230795, 0.00069361174, 0.003857421, 0.033570655, 0.018744849, + -0.020963173, 0.0004401989, 0.017031807, 0.0380566, -0.013272979, + 0.004760156, 0.017598713, 0.0022552973, -0.0039498513, -0.046929903, + -0.030933313, -0.011467509, -0.011251838, -0.008731575, -0.03184529, + 0.02266389, -0.005129877, -0.026052997, -0.0018023892, 0.014061716, + -0.028517803, -0.0062883357, 0.019508937, -0.008331044, -0.011985118, + -0.004331896, 0.0010899063, -0.021825856, 0.007825758, 0.025362851, + -0.021702616, -0.009994787, 0.013186711, 0.0048464243, -0.0055211647, + -0.03845097, 0.016317014, -0.0020165192, -0.031943884, -0.012293219, + -0.040817183, 0.008571362, 0.014295873, -0.016883919, -0.017734278, + 0.000047418627, 0.0074375514, 0.018375127, 0.021517754, 0.0007825758, + 0.000057576322, -0.005616676, 0.021061765, -0.01541736, 0.009477179, + 0.020704368, -0.006488601, -0.029355837, 0.0007548468, -0.014024745, + -0.019730771, 0.0037495857, 0.029503725, 0.005290089, -0.06117648, + -0.003860502, 0.008860976, -0.011720152, -0.019619854, -0.006809026, + -0.0020581128, -0.012927907, 0.0024755893, -0.005564299, -0.011140922, + 0.020729017, -0.008583686, 0.021517754, -0.016846947, 0.0058261845, + -0.015922645, -0.013457839, -0.0057121874, -0.014123337, 0.021801207, + -0.020236054, -0.022257196, -0.03692279, 0.0021490026, -0.021345219, + 0.007265015, 0.005404087, -0.024586437, -0.04012704, -0.0031795995, + 0.035000242, 0.017487796, 0.036109403, -0.023748403, 0.009471016, + -0.013383895, -0.027507233, -0.002755961, -0.003795801, -0.019287106, + -0.01820259, -0.031500217, -0.021604022, -0.024832917, 0.01679765, + 0.0054441397, 0.023107555, -0.007948998, 0.029232597, 0.017721953, + 0.054324318, 0.010592503, 0.0033120827, 0.030440351, -0.0097729545, + 0.015737785, 0.040077742, 0.022676213, -0.0026989623, -0.02435228, + -0.005533489, 0.018929709, -0.053979248, -0.018510692, 0.016464902, + -0.0075607914, 0.002333863, 0.0050959857, -0.01269375, 0.009840737, + -0.0112580005, -0.0020750584, 0.01655117, -0.04872921, -0.00033505954, + -0.016711382, -0.0010629474, -0.028764283, 0.01370432, 0.0025649385, + -0.009015027, -0.0022398923, -0.00008270579, 0.0029962794, + -0.0026881788, -0.008682278, -0.017537093, 0.006414657, 0.008016781, + 0.020765988, 0.012219274, 0.0048063714, -0.024623409, -0.017228993, + 0.011590749, 0.018399775, 0.0036571554, 0.0011792555, -0.031229088, + -0.0043904353, -0.041531976, -0.0005010488, 0.0020442482, 0.009902357, + 0.023908615, -0.011492156, -0.0037218567, -0.015515952, 0.0021274355, + -0.0013741292, -0.011294972, -0.009415558, -0.024832917, 0.0018979005, + -0.010672609, 0.00761625, 0.014394466, -0.015762433, -0.010302888, + 0.010265916, 0.013605728, 0.020975497, 0.010352184, -0.020568805, + 0.0044027595, 0.03440869, -0.001914846, 0.018449072, -0.010838984, + 0.008780871, -0.00067628105, -0.0021705695, 0.012681426, + -0.0044551366, 0.013322275, -0.005595109, 0.02334171, 0.0037896389, + 0.009397072, 0.004362706, -0.017192021, -0.0066180034, 0.006346875, + 0.026866382, -0.005197659, -0.0032412196, -0.006041855, -0.0010590962, + 0.019607529, 0.008694602, 0.017426176, -0.009853061, -0.018818792, + -0.0013271439, -0.019952603, -0.0066426513, -0.003872826, 0.010518558, + 0.010382994, -0.015331091, -0.028764283, -0.006316065, 0.010450777, + -0.02024838, -0.0030209275, 0.030465, -0.018091675, -0.017179696, + 0.006500925, 0.012471917, -0.0023816186, -0.011485995, -0.016033562, + 0.003389108, -0.014221929, 0.012102196, 0.0067843776, -0.0025649385, + -0.009846899, -0.005166849, -0.023933263, -0.006140447, 0.0034907812, + 0.016526522, -0.010808174, 0.0066241655, 0.0020781395, 0.04212353, + 0.013199035, 0.0076655457, -0.013186711, 0.0020411673, 0.016933216, + 0.02416742, 0.012151493, 0.0028915252, -0.031056553, 0.01878182, + -0.017155048, 0.02429066, -0.015343416, 0.004301086, 0.0329791, + -0.007998294, -0.011301135, 0.028517803, 0.00044944193, -0.016908567, + 0.010167324, 0.002991658, 0.00611888, -0.0133469235, 0.008084563, + 0.025781868, -0.022651564, 0.005579704, -0.034310095, -0.026176237, + -0.014801159, 0.011522966, -0.009384748, -0.006476277, -0.028665692, + -0.0075731156, 0.01541736, 0.0044027595, 0.020519508, -0.02169029, + 0.0032935967, 0.02447552, 0.015885673, 0.006001802, 0.0035616443, + 0.024808269, 0.010845145, -0.0055827852, 0.012379487, -0.00342608, + 0.028246675, -0.0098037645, -0.029479077, 0.015688488, 0.0020319242, + -0.0004502122, 0.012077549, 0.018486043, 0.020938525, -0.012829314, + 0.01777125, -0.00022163996, 0.0031888424, -0.011824906, 0.015984265, + 0.03260938, -0.02453714, 0.02272551, -0.0026881788, -0.010074894, + 0.0041624405, 0.03413756, -0.026299477, 0.011935822, -0.007850407, + 0.0054687876, -0.007936674, 0.004760156, -0.012200789, -0.009797603, + 0.010463101, 0.010481587, -0.025412148, 0.024586437, -0.027211456, + 0.013531784, -0.01033986, -0.021406839, 0.024130449, -0.008047591, + 0.01269375, -0.02994739, -0.008392664, 0.0200019, 0.026496662, + 0.009557284, -0.009039675, 0.014308197, -0.01257051, -0.013396219, + -0.033274878, -0.0004702387, -0.0097421445, -0.0025341285, + 0.013334599, 0.027605824, 0.003989904, -0.021221977, -0.017968435, + 0.007240367, -0.018609283, 0.010771201, -0.002272243, -0.016156802, + -0.018276535, -0.041334793, -0.0031133578, -0.018646255, 0.01618145, + -0.00755463, -0.009008865, 0.007973647, -0.026817087, -0.0018686309, + -0.037637584, -0.023292415, 0.013544108, 0.016686736, -0.020420916, + 0.014295873, -0.01196047, -0.010684933, -0.009236859, 0.013679672, + 0.005604352, 0.0026173156, -0.014640946, -0.012293219, 0.014197282, + -0.016896244, -0.0111039495, -0.016095182, 0.0009296939, 0.0015759352, + -0.0042302227, 0.0075114956, 0.00477248, -0.026299477, -0.00773949, + 0.0061127185, -0.013605728, 0.015405036, -0.010672609, -0.04922217, + -0.033891078, -0.0053578713, 0.0028591747, -0.0010606367, 0.017327584, + 0.022454381, -0.0049326927, 0.024993131, -0.032905158, -0.005669053, + -0.018042378, 0.0018439828, 0.004750913, 0.01789449, -0.03135233, + 0.00031676606, -0.014813483, -0.027063567, 0.018855764, -0.004732427, + -0.024426226, 0.01706878, -0.029355837, 0.0139508005, -0.021788882, + 0.015861025, 0.004713941, 0.0048063714, -0.014505382, 0.011757123, + 0.017500121, -0.014037069, 0.007850407, 0.0010005571, 0.016896244, + 0.02493151, 0.006932266, 0.003134925, 0.011855716, 0.0017299856, + -0.022552973, -0.042542547, 0.016538847, 0.022774804, 0.002976253, + 0.02753188, 0.013691996, 0.00011341958, 0.014739539, -0.017216668, + 0.0073697693, -0.012108359, 0.004606106, 0.03209177, 0.004322653, + 0.0069507523, -0.021061765, 0.02144381, -0.012964878, 0.005068257, + -0.007271177, -0.014382142, -0.0066549755, -0.0070185345, + -0.018584635, 0.016686736, 0.0014711809, 0.0018085513, 0.0012424161, + 0.012508889, 0.016218422, -0.019558234, -0.023994884, -0.013494812, + -0.007129451, -0.005555056, 0.03221501, -0.00007490699, 0.019545909, + 0.0016514199, -0.019102244, -0.019533586, 0.036109403, 0.026669199, + 0.0049973936, 0.013618052, 0.011461346, 0.007850407, -0.020297674, + 0.018942032, 0.00058346573, 0.026866382, 0.00388515, 0.019361049, + -0.0062821736, 0.01878182, -0.032806564, 0.008891786, -0.018486043, + 0.0126198055, 0.03258473, 0.008867139, 0.0083803395, -0.017364556, + 0.00065663963, -0.0009304642, 0.0045876196, -0.00051029184, + -0.0190283, -0.0053147376, 0.013014174, -0.001401088, -0.010647961, + 0.011048492, -0.0052654413, -0.004661564, 0.033422764, -0.01477651, + 0.010099541, -0.02067972, -0.008540552, 0.00038570358, -0.006969238, + 0.019964926, -0.0003591299, 0.0020396267, 0.01263213, -0.035000242, + -0.0014827347, 0.028098786, -0.015589897, 0.00038165977, 0.015836377, + 0.023008961, -0.0061527714, -0.0034877001, -0.040669296, 0.012478079, + -0.013864532, -0.0054318155, 0.000121410936, 0.014209605, + -0.0062020672, 0.015047639, 0.010863631, 0.014332846, -0.0053578713, + -0.016477227, -0.00025283516, 0.00608807, -0.022811778, 0.013642699, + -0.0011191758, -0.021493107, 0.004695455, 0.023464952, 0.016033562, + -0.0032350575, 0.01294023, -0.020223731, 0.0025587764, -0.011214866, + 0.0025495335, 0.021394514, -0.016206099, 0.018301183, -0.0045075137, + -0.009507989, 0.03766223, 0.009834575, 0.013753616, 0.0017515526, + -0.011954308, 0.002634261, -0.0016991755, -0.0041624405, 0.011529129, + 0.010259754, 0.004180927, -0.007955161, 0.0111655705, 0.02486989, + -0.018806469, -0.010937575, -0.008645306, -0.008189317, 0.01994028, + 0.014801159, 0.009403234, -0.016452579, -0.002272243, -0.0027836899, + -0.012102196, -0.021185005, 0.012434945, -0.006704272, 0.0074930093, + -0.0033244067, -0.030884016, -0.001067569, -0.0077641383, + -0.050972182, -0.021727262, -0.01370432, -0.005545813, -0.018005406, + -0.031426273, -0.0022922694, -0.025412148, 0.0054472205, -0.010272078, + -0.020975497, 0.008756222, -0.013987773, -0.0024879132, -0.019743094, + -0.0012763073, -0.0066303276, -0.017598713, 0.02740864, 0.028912172, + -0.025412148, -0.0012293218, -0.0057121874, -0.0061466093, -0.0096682, + -0.026792439, -0.022589944, -0.0076347357, -0.00025764923, + -0.016255394, -0.023699107, -0.014357493, 0.017857518, 0.0028437697, + -0.0018162538, -0.00090350537, 0.0045290804, 0.0031518703, + 0.013790588, 0.017611038, -0.0056444053, 0.034950946, 0.0028129595, + -0.023366358, 0.005653648, -0.03347206, 0.017302936, 0.011701665, + -0.013063471, -0.027383992, 0.010044083, 0.018214915, -0.018165618, + -0.03517278, 0.02385932, 0.008965731, -0.013741292, -0.03817984, + -0.019114569, -0.0008996541, 0.0044736224, -0.022836424, -0.021665642, + -0.010641798, -0.017167373, 0.006519411, 0.019582883, 0.016822299, + -0.0018794144, -0.0005618987, -0.003450728, 0.0041901697, 0.012804666, + -0.018880412, 0.0076285736, -0.021456134, -0.009828413, -0.017228993, + 0.0075423056, 0.0020642749, -0.020642748, 0.0036540746, -0.012607481, + -0.00063083624, 0.037070677, 0.021468459, 0.009464854, -0.032954454, + 0.00951415, -0.017820546, -0.02701427, -0.018239563, -0.009316966, + 0.01692089, 0.0145546775, 0.0053517097, 0.012243923, 0.029725557, + 0.008768546, 0.0027913926, 0.015257147, -0.003469214, -0.019324077, + -0.006426981, 0.01214533, -0.007653222, 0.011584587, -0.024832917, + -0.003835854, -0.026447365, 0.038155194, -0.003416837, -0.014147985, + -0.023132201, -0.0022861073, -0.0035246722, 0.0066611376, 0.01541736, + 0.0144314375, 0.004384273, 0.0329791, -0.005305494, 0.009945491, + 0.0035369962, 0.008066077, 0.0032042474, 0.014012421, -0.017105753, + -0.0074437135, 0.021160357, 0.015873348, 0.021591699, -0.003134925, + -0.007301987, -0.022368113, -0.0010290564, 0.027383992, 0.009255346, + -0.018744849, -0.014345169, -0.019767743, -0.02006352, 0.026126942, + -0.0050805807, 0.028517803, -0.004652321, -0.00095973373, 0.015146231, + -0.006402333, -0.009914681, -0.026915679, 0.0021413, 0.0015220175, + -0.005573542, -0.008294071, -0.011670855, -0.0022938098, 0.008448122, + 0.026545959, -0.010346022, -0.0021721101, 0.0023092148, -0.026496662, + -0.0044551366, -0.012108359, -0.017389204, -0.0010467722, 0.01826421, + 0.003010144, -0.008719251, -0.017943786, 0.00235543, -0.037760824, + -0.012447269, -0.0010136514, -0.0048464243, 0.0040083905, + -0.021887476, 0.03630659, -0.020359296, -0.0031010336, 0.012397973, + -0.011541453, -0.005068257, 0.0054595447, -0.008170831, 0.028246675, + -0.00783192, 0.00049450167, 0.008762385, 0.0016313933, 0.0084419595, + 0.011547615, 0.027162159, 0.012952554, -0.0026820167, -0.0056290003, + 0.0010259754, 0.0026234777, -0.01006257, 0.031549513, 0.011036168, + 0.0009528015, 0.00486491, -0.00007928779, 0.00850358, -0.013309951, + 0.015059963, -0.031229088, -0.010580178, -0.0036694796, -0.013482488, + -0.0044797845, -0.020199083, 0.02055648, -0.0053609526, 0.008941083, + -0.023896292, 0.0044551366, 0.0011214867, -0.013815236, 0.008750061, + -0.0055365697, 0.020346971, 0.019348726, 0.023810023, -0.012367163, + -0.024438549, -0.004119307, -0.0089534065, -0.002212163, 0.0065687075, + -0.011202542, 0.009797603, -0.0015975023, 0.00097205775, -0.029331189, + 0.0020488699, -0.017155048, -0.0026481256, 0.0022568377, 0.019780066, + 0.019299429, -0.0041624405, -0.019447317, 0.010302888, 0.0310812, + 0.0074498756, 0.014172633, -0.0076285736, -0.027950898, 0.0029747125, + 0.008121535, 0.023699107, -0.013433191, 0.002367754, -0.00052531174, + -0.0036663986, 0.0033305688, -0.0040761726, -0.015861025, -0.02713751, + 0.014986019, 0.000040341936, 0.018177943, -0.00077564357, -0.01040148, + -0.007419065, -0.016526522, -0.0023970236, -0.01037067, 0.0043134103, + -0.0075607914, 0.012108359, 0.0036016975, -0.0076039257, -0.009458692, + -0.021172682, 0.019866334, -0.010956062, -0.0076655457, 0.011553777, + 0.0010552449, -0.0035770494, 0.0066426513, 0.004673888, 0.019287106, + 0.0049727457, -0.014258902, -0.011473671, -0.007930513, 0.01027824, + 0.013630376, 0.022947341, 0.0020350053, 0.0075114956, -0.009132105, + 0.0069137802, 0.011985118, 0.016501874, -0.016353985, 0.018362803, + -0.0012347136, -0.0144314375, -0.008331044, -0.0000048622146, + -0.0000992662, -0.0058138603, 0.006048017, 0.00014268914, + -0.0035616443, -0.007905864, -0.012231599, 0.0020334648, 0.0068213497, + 0.0022999719, -0.03539461, -0.023846995, -0.019163866, -0.003835854, + -0.0018824954, 0.0011030006, 0.01453003, -0.02006352, 0.0014827347, + 0.0053578713, -0.032781918, -0.002606532, 0.00014808092, -0.012890934, + -0.015577572, 0.028074138, 0.001287861, 0.029281892, 0.016045885, + 0.0017731197, -0.006796702, 0.025560036, 0.025091723, -0.0062575256, + -0.020975497, 0.0075361435, 0.0057738074, -0.0006524033, 0.0022629998, + 0.009877709, 0.0037495857, 0.017734278, 0.01024743, -0.006402333, + -0.0026296396, -0.006426981, 0.010105704, 0.023625163, -0.020839933, + 0.008226289, 0.0035185101, 0.03034176, 0.009224536, -0.0063283886, + 0.013987773, 0.011190218, 0.017352233, -0.024586437, -0.021554727, + -0.00002479248, -0.013371571, 0.012903258, -0.024500169, -0.005157606, + 0.0045691337, -0.016378634, 0.008780871, -0.0015366523, 0.00014249659, + 0.011498319, -0.007714842, 0.0066364896, -0.010672609, 0.0044767032, + -0.000017655615, -0.016107505, -0.0019564396, -0.005166849, + 0.010037921, -0.0029993604, 0.0031179793, -0.018177943, 0.00850358, + 0.012903258, 0.0062667686, 0.010228944, 0.03095796, 0.012120683, + 0.017549416, -0.020889228, -0.011621559, -0.016292365, 0.0021859745, + -0.008879462, 0.0022583783, -0.005746078, 0.004165522, -0.0011368917, + -0.002560317, 0.00013238702, 0.006747406, 0.0074806856, 0.013753616, + -0.0030933311, 0.002321539, -0.008263261, 0.0044859466, -0.013125091, + 0.006155852, 0.011233352, -0.03514813, -0.00079027837, 0.0075238193, + -0.022885721, -0.015602221, -0.012780018, -0.017339908, -0.025288908, + 0.005709106, -0.022072336, 0.013802912, 0.012564347, 0.0033274877, + 0.021665642, -0.017426176, -0.0025156424, -0.026225533, 0.020716693, + 0.002786771, 0.011214866, 0.021727262, -0.006735082, -0.005912453, + 0.009563446, -0.005715268, -0.028665692, -0.00016656696, -0.018523015, + -0.024389252, 0.0125273755, 0.0053024134, 0.03147557, 0.00030328665, + -0.0040114713, -0.010604827, -0.0033028398, -0.014739539, + -0.0040268763, -0.0050220415, 0.011239515, 0.03221501, -0.02254065, + 0.0052931705, 0.0020149788, 0.015552924, 0.010419967, -0.0024078072, + -0.03948619, -0.013531784, 0.002780609, 0.040176332, -0.027507233, + -0.001354873, 0.016045885, 0.011184056, 0.0061928243, 0.0014480734, + -0.021542402, -0.0031888424, 0.013975449, -0.024648057, 0.036700957, + 0.007394417, 0.0030394136, 0.017167373, 0.011276486, -0.0023199983, + 0.007425227, 0.0019826281, 0.011473671, -0.010087217, -0.00082185864, + -0.014912074, -0.003435323, -0.001961061, -0.007813434, 0.018609283, + 0.007819597, -0.003869745, 0.016686736, 0.027753713, 0.0012247004, + 0.005037447, -0.009791441, 0.0061897435, 0.0007644749, -0.008294071, + -0.0043657874, 0.008287909, 0.005206902, 0.0024755893, -0.0029023087, + 0.020963173, -0.022355787, 0.0010460019, -0.020889228, 0.016748356, + 0.0018609284, -0.009088972, 0.005099067, -0.008706926, -0.020778313, + 0.005213064, -0.015959617, -0.0118310675, -0.028443858, -0.018855764, + 0.0023199983, 0.038771395, -0.018584635, -0.015269471, -0.017426176, + -0.002820662, 0.017155048, -0.008817842, 0.012379487, -0.02514102, + -0.002755961, 0.017980758, 0.007912027, 0.0112888105, 0.009649714, + -0.0057984553, 0.019780066, -0.0053547905, 0.0057953745, 0.008220127, + 0.019693797, -0.0052716034, -0.03270797, 0.0044212453, 0.013371571, + -0.00047409, -0.019410346, 0.0049326927, 0.0020411673, -0.0062267156, + -0.001685311, 0.00023300116, 0.005188416, -0.027753713, -0.011005358, + 0.008041428, -0.031500217, -0.013655024, 0.0070924787, -0.003370622, + -0.001973385, 0.014147985, -0.007382093, 0.006066503, -0.011251838, + -0.0002899998, 0.023489598, -0.027162159, 0.011664693, 0.00032870495, + -0.0152817955, -0.008195479, -0.00997014, 0.01275537, 0.0139508005, + 0.023600515, 0.023970235, -0.025338203, -0.0037280186, 0.010580178, + 0.007240367, 0.0024817514, -0.012989527, 0.0023816186, 0.003983742, + 0.01220695, -0.008016781, -0.010290564, -0.012077549, 0.0058939666, + 0.003869745, 0.0030486565, -0.01183723, -0.011855716, 0.009563446, + 0.034211505, -0.01655117, -0.0052993326, 0.0027544205, 0.016206099, + 0.00043480715, -0.012465755, -0.008047591, 0.0029439023, 0.008035267, + -0.00602645, 0.020667396, 0.004279519, 0.0024879132, 0.00021509282, + -0.0036171025, -0.0061928243, 0.008361854, 0.015059963, -0.0061312043, + 0.023686783, 0.008676116, -0.0032997588, 0.016686736, 0.0013687374, + 0.00013951186, 0.010253591, 0.024722002, 0.011664693, 0.0069815624, + 0.0035308343, 0.010382994, -0.0021089495, 0.031672753, 0.00302863, + -0.0040977397, 0.029750206, 0.0057645645, -0.0032134906, + -0.0036201836, 0.0053239805, 0.008737736, -0.016809976, 0.018942032, + -0.006371523, -0.0058693187, 0.0070493445, 0.006845998, 0.017265964, + -0.0076655457, 0.018473718, -0.011522966, -0.0072095566, -0.024919186, + -0.0152817955, 0.023317063, -0.0041994127, 0.001994952, -0.01994028, + 0.025781868, -0.0009443287, 0.00394677, 0.00011303445, 0.0030902503, + 0.004368868, -0.0032319766, -0.010586341, 0.022737833, 0.001480424, + -0.0005545813, -0.008897949, -0.005148363, 0.0011569182, 0.01921316, + 0.018214915, -0.0027883116, -0.0025464525, -0.006457791, -0.006876808, + 0.006839836, -0.004273357, 0.011060816, 0.022799453, 0.015626868, + -0.002193677, 0.0092306975, 0.013285303, 0.007671708, 0.010475424, + -0.014283549, -0.029257243, -0.0019903306, -0.017044133, -0.025806516, + -0.0028252837, -0.017611038, 0.00064739666, 0.025104046, 0.0041686026, + -0.0013571837, 0.00028248984, 0.0014026285, 0.0015343416, + 0.0041316305, 0.0045691337, -0.010044083, -0.02012514, -0.0054595447, + -0.0026850977, -0.0329791, 0.0026096131, -0.014332846, 0.020667396, + -0.028838227, 0.0015990427, -0.018165618, -0.007283501, 0.011720152, + 0.010506234, -0.019927954, 0.019052949, 0.017450824, 0.014579326, + 0.0077333283, 0.0237977, -0.007776462, -0.012089872, -0.0323629, + -0.019533586, -0.009027351, 0.013531784, -0.01208371, 0.008294071, + -0.006035693, 0.011479833, 0.0048895585, -0.00605726, 0.024339957, + 0.022281844, 0.0067227576, 0.0057676453, -0.014788834, 0.007252691, + -0.018153295, 0.009015027, -0.000028186401, -0.0009397072, + 0.004544486, 0.0097729545, -0.018744849, -0.009347776, 0.000064845575, + 0.021801207, 0.006778216, -0.019348726, -0.013236007, -0.014234253, + 0.006870646, -0.009236859, 0.010604827, 0.00758544, 0.0096682, + 0.016526522, 0.00050335954, -0.013334599, 0.004285681, 0.0040299576, + 0.03655307, -0.021591699, 0.0046245917, -0.014702566, 0.021961419, + -0.008830166, -0.008250937, 0.003429161, 0.021702616, 0.010949899, + 0.0050590136, -0.007388255, 0.0054841926, 0.00596483, 0.008817842, + 0.013581079, -0.0074930093, -0.023058258, -0.014899751, -0.010986872, + 0.0030917907, 0.004356544, -0.011603073, -0.0014557759, 0.0008996541, + -0.028986115, 0.012330191, -0.00385434, -0.014345169, 0.008435798, + -0.018251887, -0.009267669, 0.0035493204, 0.031919234, 0.000341029, + -0.0000012305976, 0.0035616443, -0.015984265, 0.007363607, + 0.019545909, 0.00296701, 0.015047639, 0.00064585614, -0.0054903547, + -0.005237712, -0.002104328, -0.008571362, -0.025165666, -0.0046708067, + 0.028197378, 0.016846947, -0.023440303, -0.008515904, -0.009292318, + -0.0008518985, -0.0012123763, -0.0187202, 0.019052949, -0.013618052, + 0.011985118, 0.0044274074, -0.0041686026, -0.000036370326, + 0.00063122134, -0.020420916, 0.0053424663, 0.008029105, 0.0126198055, + -0.019607529, -0.0058138603, 0.0050066365, 0.0055827852, 0.017574064, + -0.010222781, 0.012552023, 0.032141067, -0.012977202, -0.016477227, + -0.016625116, 0.0052469554, 0.0032196525, 0.015306444, 0.0075053335, + -0.020457888, 0.018929709, 0.0066426513, 0.008121535, 0.0040392005, + 0.012102196, -0.023883969, 0.009883871, 0.0031672753, 0.0031672753, + 0.0066611376, -0.015084611, -0.015897997, 0.0048679914, 0.0013386976, + -0.0009558825, -0.0041624405, -0.0012485782, -0.0050959857, + 0.007240367, -0.013544108, 0.002492535, -0.006932266, -0.014505382, + -0.00945253, 0.009791441, 0.020642748, 0.0021736505, -0.013014174, + -0.015885673, 0.0048710722, -0.0045845388, -0.017869841, 0.011319621, + 0.015269471, 0.005009718, -0.013568755, 0.0028406887, -0.014567002, + -0.0000448431, -0.010919089, -0.033940375, 0.00764706, -0.0001549169, + 0.024389252, -0.0036294265, -0.0024247526, -0.0016514199, 0.01661279, + 0.01685927, -0.032929804, 0.0028360672, 0.011948146, 0.01217614, + 0.014369817, -0.012718398, -0.0027051242, -0.0020196002, -0.010518558, + -0.003044035, -0.00044828653, 0.011455185, -0.010481587, + -0.0040700105, 0.0040761726, 0.001073731, 0.014665594, 0.0065933554, + -0.022257196, -0.0084111495, 0.008694602, 0.014295873, 0.016699059, + 0.016760679, -0.01269375, 0.011541453, -0.0076285736, -0.041162256, + -0.01027824, -0.016353985, -0.004775561, 0.010549368, -0.003040954, + 0.023045933, 0.0018516853, 0.009236859, 0.011775609, -0.0026820167, + -0.011522966, -0.028714987, -0.0035524012, 0.0012339434, 0.0060541793, + -0.003416837, -0.005237712, -0.0062051485, -0.0132236825, + -0.023267766, -0.018165618, -0.006932266, -0.024068829, -0.014456086, + 0.026964976, -0.010703419, -0.0035801304, 0.009212212, -0.02829597, + 0.010358347, 0.0048217764, -0.028665692, 0.009766793, -0.0025110207, + 0.015565248, -0.0056752153, 0.006815188, 0.0036109404, -0.013544108, + 0.008417312, -0.001364116, -0.002104328, -0.008860976, -0.0075484677, + 0.018091675, -0.0053332234, -0.0030532782, 0.008213965, 0.006359199, + -0.0044890274, -0.019348726, -0.0032781917, 0.006710434, 0.015392712, + 0.022244873, -0.022195576, 0.0018701714, 0.0074868477, -0.0013394678, + 0.014998343, 0.0010976088, 0.023711432, 0.023538895, -0.013790588, + 0.0008881003, 0.011116274, -0.017635684, 0.010210458, 0.022084659, + -0.014221929, -0.023427978, 0.0010968385, 0.017450824, -0.00767787, + -0.0016406364, 0.020137463, 0.0004625362, -0.024216717, -0.000694382, + 0.012582833, 0.0023230794, 0.00032042473, 0.014037069, -0.018276535, + -0.008491256, 0.006433143, -0.008448122, -0.008460445, 0.0017053374, + -0.017450824, 0.0019133055, 0.022811778, -0.005958668, -0.017746601, + 0.018436747, 0.0018516853, 0.010832821, -0.0054441397, 0.0066303276, + -0.0131620625, 0.0017623361, 0.023230795, 0.0051113907, -0.012280894, + -0.0022029202, -0.0011692422, -0.0049604215, -0.008368015, + 0.028320618, 0.020790637, -0.01655117, -0.014456086, 0.029626966, + -0.0018640093, 0.008096887, 0.0072465288, -0.012989527, 0.00060041127, + -0.012743046, 0.00058847235, 0.0004147806, 0.016760679, -0.0206181, + -0.010869794, -0.006778216, -0.012656777, 0.013938476, -0.0206181, + 0.015084611, -0.01446841, 0.022047687, -0.010204296, -0.007955161, + -0.0065687075, 0.00035181252, -0.014345169, 0.0045814575, + -0.0075977636, 0.00039302098, -0.01033986, -0.018757172, 0.031771347, + 0.0036448315, -0.00773949, 0.022355787, -0.0031287628, 0.020790637, + -0.004371949, 0.009557284, 0.0012578212, 0.003811206, 0.0017114995, + -0.011208704, 0.00016974423, 0.011412051, -0.025732571, 0.022811778, + 0.0073697693, 0.00021528537, 0.0084419595, 0.026669199, 0.010210458, + 0.019139217, 0.0002936585, 0.005086743, -0.003370622, 0.014899751, + 0.010198134, -0.007166423, -0.0057183495, -0.02278713, 0.02187515, + -0.010352184, -0.015959617, -0.014850454, 0.007166423, -0.017413853, + -0.0003061751, -0.008263261, -0.005290089, -0.006476277, -0.019077597, + 0.0003575894, -0.008294071, 0.0032720296, -0.00016300453, + -0.004276438, 0.026250182, -0.011547615, 0.010666447, 0.004747832, + 0.008103049, 0.00783192, -0.013211359, 0.012077549, -0.0070246966, + -0.021714939, 0.0042363848, -0.023354035, 0.00972982, -0.018276535, + -0.0018932789, -0.021332894, -0.009963977, -0.017795898, 0.021061765, + 0.012077549, 0.011110112, -0.0067535676, -0.000654714, 0.0062790927, + -0.015873348, 0.007252691, -0.00477248, 0.0058015366, 0.02373608, + 0.008854815, 0.014394466, 0.0062452015, -0.01820259, -0.013236007, + -0.013926152, -0.0011184056, 0.006007964, -0.031007256, -0.0044828653, + 0.0037988818, 0.016526522, -0.014295873, 0.0036325075, -0.017290613, + -0.000875006, 0.017056456, -0.0056813774, 0.011621559, -0.008965731, + -0.014875103, 0.008780871, -0.008934921, 0.0038820691, -0.027038919, + 0.0058046174, 0.020790637, 0.0030055225, -0.008750061, -0.00025225745, + -0.0032196525, -0.021838179, -0.0097113345, 0.010253591, 0.010389157, + 0.0145546775, -0.00028403036, -0.015047639, 0.005237712, 0.010543207, + 0.0005264671, 0.012065224, -0.014160309, -0.020926202, 0.01263213, + -0.012077549, -0.002364673, 0.022417407, -0.0075423056, -0.019250132, + 0.017265964, 0.008719251, 0.010037921, -0.008133859, -0.024278337, + 0.021677967, -0.012274733, -0.008934921, 0.003838935, -0.019200837, + -0.016957864, 0.0011006899, -0.011818743, 0.0012508889, -0.017882166, + -0.013716644, 0.0023723755, -0.01758639, -0.0002511021, 0.017204344, + 0.0023508086, 0.017216668, 0.008885625, -0.012065224, 0.034433335, + 0.021123385, -0.007702518, -0.003959094, 0.0058200224, 0.010352184, + -0.026817087, 0.016723707, -0.0077641383, -0.011313458, 0.006390009, + -0.033126988, -0.024697354, -0.015528277, 0.0062729307, 0.004590701, + 0.015331091, 0.017216668, -0.0053486284, 0.0055365697, -0.0014303576, + -0.018486043, -0.0386728, -0.020346971, -0.01189885, 0.03273262, + -0.0026696927, -0.015528277, 0.017820546, -0.009039675, 0.012613644, + 0.013014174, 0.0035739683, 0.016982513, 0.0007074763, -0.008663792, + -0.0057183495, 0.0037187757, -0.0029223354, -0.0033582977, + 0.009378586, -0.006035693, 0.012361001, -0.003851259, 0.0013148198, + -0.015934968, -0.007776462, 0.027457936, -0.021024793, -0.010296726, + 0.022639241, 0.003894393, 0.02049486, 0.00009801454, 0.0028946062, + -0.017783573, 0.011153246, 0.009027351, 0.0018994409, -0.0017638766, + -0.019952603, -0.024980806, -0.000266122, -0.022959666, 0.008244775, + -0.019225486, -0.0145546775, -0.0012347136, 0.011418212, 0.009261508, + 0.013088118, -0.0065255733, 0.03477841, 0.013014174, 0.01795611, + -0.020913877, 0.0024678868, 0.0024956157, 0.0061219614, -0.008448122, + 0.00049796776, -0.023070581, -0.009249183, 0.008016781, 0.0133469235, + -0.007727166, 0.00025437566, -0.026447365, -0.02247903, -0.007788786, + -0.0062914165, 0.009347776, -0.003777315, -0.0145546775, 0.008368015, + 0.013679672, -0.015577572, -0.008127697, 0.00019679933, 0.0062482827, + 0.00945253, -0.01541736, 0.029848797, 0.009249183, -0.01655117, + 0.015466657, 0.0064947633, -0.010919089, 0.0036756415, 0.015700813, + -0.0132236825, -0.024771297, -0.00008140599, 0.010352184, 0.006103475, + 0.0039868234, -0.032264307, 0.012416459, -0.0053085755, -0.0044428124, + 0.029824149, 0.0023631325, 0.011430536, -0.009021189, 0.031697404, + -0.01661279, 0.007289663, 0.008004457, -0.034063615, -0.012182303, + 0.01009338, 0.009064323, -0.020913877, -0.006735082, -0.00040438218, + 0.0021520834, 0.01208371, -0.0074498756, 0.019188512, -0.007881217, + -0.017758925, 0.029528372, -0.0070678303, -0.0022275683, -0.015824053, + -0.003829692, 0.0058015366, -0.020876905, -0.00080876437, + -0.014382142, -0.0053609526, -0.024339957, -0.007930513, + 0.00030116845, 0.014061716, 0.018350478, 0.02018676, 0.004325734, + -0.023045933, 0.008164669, 0.008109211, -0.008885625, -0.012243923, + -0.03741575, 0.013272979, -0.0020350053, 0.0033244067, -0.027211456, + -0.0027220699, -0.023008961, -0.008799356, -0.0010906765, 0.008485094, + -0.0044489745, -0.008004457, -0.0355425, -0.00051029184, 0.009088972, + 0.013433191, -0.008207804, 0.007265015, 0.0052192262, 0.016686736, + 0.00009175624, 0.00041593597, 0.0029716315, 0.02942978, 0.002743637, + 0.00464924, 0.0026358017, 0.003463052, -0.007985971, -0.014037069, + -0.005866238, 0.0046184296, 0.02526426, 0.029848797, 0.03948619, + 0.015022991, -0.008022943, -0.011350431, 0.018067027, 0.024327632, + 0.014480734, 0.0035031051, -0.024204392, -0.0036417504, -0.0003548935, + 0.024648057, 0.01217614, -0.000058972404, 0.0052870084, -0.012243923, + -0.017598713, 0.011245676, -0.015392712, -0.022047687, -0.002389321, + -0.012915582, 0.012182303, 0.040546056, -0.004636916, -0.012385649, + -0.0097421445, -0.028838227, 0.011048492, 0.005558137, -0.010869794, + 0.030760776, -0.008620658, -0.007659384, 0.012441107, -0.0006038774, + 0.0063283886, -0.014505382, -0.005157606, -0.014086365, 0.0041131447, + -0.024561789, -0.0026049917, 0.004347301, -0.011627721, -0.011664693, + -0.018054701, -0.0003055974, 0.0009682065, -0.009908519, -0.018449072, + 0.0026958813, -0.008959569, -0.0033244067, -0.009446368, 0.0018208752, + -0.0030363325, 0.004020714, -0.003422999, -0.0016390958, 0.0070924787, + -0.0200019, -0.009243022, 0.005450302, 0.018855764, 0.020088166, + -0.03273262, 0.018806469, 0.0015366523, 0.010524721, 0.0071048024, + 0.0013802912, -0.0029439023, 0.027630473, 0.00976063, -0.008016781, + -0.03886999, 0.016082857, 0.007881217, -0.008799356, 0.012964878, + -0.0010398399, 0.0069938865, 0.02199839, -0.0152201755, -0.013913829, + -0.016193774, 0.0032196525, 0.009181402, 0.018214915, -0.0012331732, + 0.020605776, -0.018301183, -0.0033459738, 0.010882118, -0.0016437173, + -0.01529412, -0.017265964, 0.022589944, -0.021480782, -0.004322653, + 0.005425654, 0.010407642, -0.012065224, -0.00013440894, 0.008398825, + 0.024771297, 0.00079875113, -0.008195479, -0.006029531, 0.015737785, + 0.022873398, -0.020667396, 0.019484289, 0.008343368, -0.0014611677, + 0.0062760115, -0.009754469, 0.027901601, 0.008799356, -0.004134712, + 0.048679914, 0.01483813, -0.019410346, -0.009415558, -0.010888279, + 0.017364556, -0.015651517, -0.0027174484, 0.015503628, 0.022836424, + 0.0013225223, -0.015491304, 0.0096682, 0.003370622, 0.007727166, + 0.010290564, -0.0036232644, -0.0071048024, -0.025781868, 0.00853439, + -0.011301135, 0.030169223, 0.015713137, 0.0066364896, 0.019792391, + 0.002746718, 0.0091074575, 0.014998343, 0.0014164931, 0.014320522, + -0.008016781, 0.008614496, -0.009809927, -0.020778313, -0.0012824693, + -0.0055673798, 0.002963929, 0.023267766, 0.0036848846, -0.011658532, + -0.008244775, 0.01370432, 0.004322653, 0.025362851, 0.0016745274, + -0.017315261, -0.0054102484, -0.0018532259, 0.0059555867, 0.026718494, + -0.022146279, -0.011467509, 0.016896244, 0.009015027, -0.009144429, + 0.0022275683, -0.0013140496, 0.027383992, -0.009310803, -0.029281892, + -0.008934921, 0.0058847237, 0.004809452, 0.010851308, -0.02156705, + 0.007948998, 0.0039344463, 0.028493155, 0.0054071676, -0.016896244, + 0.010518558, -0.005071338 + ] + } + ] + }, + { + "name": "dulezite-tipy-pro-nakup-bizuterie", + "title": "Důležité tipy pro nákup bižuterie", + "content": "Korálky Katlas připravily shrnutí důležitých bodů, na které by si zákazníci měli dávat pozor při nákupu hotové bižuterie i bižuterních polotovarů. Tyto informace byly prezentovány během online setkání.", + "keywords": [ + "Korálky Katlas", + "shrnutí", + "důležité body", + "zákazníci", + "nákup", + "hotová bižuterie", + "bižuterní polotovary", + "online setkání" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.0026307285, -0.064501315, -0.009212564, -0.00923262, 0.039096568, + -0.02718308, -0.0018585578, 0.02151381, 0.018398385, -0.021981793, + -0.010629881, 0.01469464, -0.005495448, -0.048803855, 0.012715745, + 0.00067732064, -0.013270638, 0.010041561, -0.0005983487, 0.009861053, + 0.0047867894, -0.017248487, -0.018665804, -0.014708011, 0.0042118398, + -0.016085217, -0.032063466, -0.0029683441, -0.019267496, 0.03516552, + 0.007313893, -0.010349092, -0.004753362, -0.04971308, -0.04639709, + -0.019548284, 0.023492705, -0.031368177, 0.0008440393, -0.024308331, + 0.0150690265, -0.022382919, -0.016981067, 0.045835514, -0.045648318, + -0.01454756, -0.0001768513, -0.031234467, 0.022944499, -0.031608853, + -0.024308331, 0.021166166, -0.02152718, -0.009727344, 0.048670147, + 0.0453809, 0.014333625, 0.07744437, 0.032491334, 0.02006975, + 0.013972611, 0.013905756, 0.0021243049, 0.013284009, -0.0056960117, + 0.012829397, -0.030298503, 0.0073406347, 0.0027243248, 0.031956498, + -0.027102854, -0.023893833, 0.04187772, -0.046183158, -0.012742487, + -0.018024, -0.005381795, -0.017128147, 0.0053650816, 0.014413851, + -0.00930616, 0.05819024, 0.05222681, -0.008811436, -0.006992991, + -0.037866443, 0.003229077, -0.043803133, 0.002844663, -0.04249278, + -0.04219862, -0.011900119, -0.034122586, -0.015216107, 0.010282237, + -0.0024151222, -0.022369549, -0.006992991, -0.033373814, 0.027076112, + -0.010462744, 0.0010370819, -0.0040781302, 0.0085105905, + -0.0058865473, 0.091831475, 0.016887471, 0.024789685, -0.018692546, + -0.03634216, 0.022075389, -0.0022329437, 0.027223192, 0.0067389435, + -0.011392023, 0.0014031109, 0.010603139, -0.034844615, -0.0194948, + 0.01046943, -0.041770753, -0.035085294, 0.062789835, 0.017101405, + -0.012842768, -0.00793564, -0.03473765, 0.029977601, 0.03650261, + -0.004776761, -0.035967775, -0.026835436, -0.018237934, -0.013745305, + 0.0022446432, -0.006952878, 0.026581388, -0.046798218, 0.020671442, + -0.0055088187, -0.019120416, 0.008905033, 0.0012886224, 0.0041215857, + -0.044979773, -0.072898254, 0.012321303, -0.0030168137, -0.005368424, + -0.00027117896, 0.001073852, -0.0024335075, -0.029763667, 0.011759724, + 0.010482801, -0.0031755934, 0.0042151823, -0.017970515, -0.0024652632, + 0.026501162, -0.00893846, -0.018572208, 0.03358775, 0.010522914, + 0.047440022, -0.04869689, 0.032625042, -0.0064681824, -0.0596343, + 0.03738509, -0.0031488515, 0.017208373, 0.026046552, 0.011244943, + -0.009546837, 0.037331607, 0.032651786, -0.0075144568, -0.031394918, + -0.039390728, -0.01933435, 0.031983238, 0.023278771, 0.0020791779, + -0.006966249, -0.009299475, 0.023880463, -0.02354619, 0.013671765, + 0.0044525163, 0.05380458, -0.018946594, -0.001673872, -0.009800884, + -0.034710906, -0.0061573084, -0.04511348, -0.04318807, 0.008229801, + 0.006498267, -0.00006675014, -0.03749206, 0.022677079, -0.026073294, + 0.029763667, -0.018197821, 0.015456783, 0.0076815933, 0.035379454, + 0.014708011, -0.031207725, 0.021848083, -0.0057494952, 0.026140148, + -0.02354619, 0.022770677, 0.012100683, -0.0014298527, -0.011505676, + 0.020470878, 0.00443246, -0.021339988, 0.00088498776, 0.018197821, + -0.032357626, 0.027544094, 0.006461497, 0.036475867, 0.0058598053, + 0.03414933, -0.030753115, -0.030218279, -0.046771478, -0.023586303, + -0.022235839, -0.0053249686, 0.004780104, -0.021353358, -0.0021878167, + -0.016339263, -0.013544741, 0.0064080134, -0.030753115, -0.0058999183, + -0.012909623, -0.0012485096, 0.025458232, -0.00240008, 0.021019086, + 0.07188206, -0.018852996, -0.0114856195, -0.00086576707, 0.02922883, + -0.008991944, 0.056104377, 0.07498412, -0.0251507, 0.0023516105, + -0.00094599253, 0.03925702, -0.036663063, -0.019695364, -0.0019103701, + 0.014186545, -0.0031956497, -0.008557389, -0.020631328, -0.020818522, + 0.06765685, -0.012114053, 0.010549655, 0.040620852, 0.032892462, + -0.04072782, 0.030138053, 0.059527334, 0.010268866, -0.011291741, + 0.03735835, 0.008049293, -0.0041249287, -0.0035466363, 0.0015777685, + -0.0187594, -0.006484896, -0.03390865, 0.031662337, -0.009292789, + 0.052012876, -0.020885376, -0.036368903, 0.00023545354, -0.028426575, + -0.0077551333, 0.012274504, 0.0074409167, 0.0308066, -0.024976877, + -0.0084504215, -0.04816205, -0.034978326, -0.026795322, 0.022302695, + 0.021473696, 0.026380824, 0.012668946, 0.030164795, -0.02254337, + -0.037144415, -0.010863872, -0.025337892, 0.01309013, 0.004419089, + -0.022356177, -0.004235239, -0.019133786, -0.003247462, 0.057762373, + -0.041744012, 0.0194948, 0.011398708, -0.006063712, -0.009486668, + -0.0020958916, -0.015296332, 0.012401528, 0.0038207402, 0.015911395, + 0.020577846, -0.028239382, 0.0024518925, -0.031475145, 0.009633748, + -0.02968344, -0.025538456, 0.027945222, -0.066854596, -0.010014819, + -0.010242124, 0.01961514, -0.012147481, 0.0024284932, 0.0013521343, + -0.02224921, -0.020430764, -0.028774219, 0.012033828, 0.046236638, + 0.010496172, 0.0006723066, 0.027102854, -0.01228119, 0.013190412, + 0.04540764, -0.015189365, 0.022784047, 0.0034797818, 0.0067155445, + 0.013852273, 0.014734753, -0.03417607, -0.018705916, 0.007153442, + -0.005655899, 0.047038894, -0.019227382, 0.014280142, -0.005726096, + 0.02559194, 0.005953402, 0.017502533, -0.061452746, -0.007848729, + 0.010603139, 0.038133863, 0.0049572685, 0.015831169, -0.024187993, + -0.0009952978, 0.017783323, -0.031154243, 0.015189365, 0.019722106, + -0.0050943205, 0.040246468, 0.0031672365, 0.017114777, -0.0077952463, + 0.02255674, -0.013638338, 0.0082431715, 0.0037538856, 0.026688356, + 0.019695364, -0.014199916, 0.014293513, 0.09648456, -0.0027794798, + 0.0014331954, -0.05744147, 0.032250658, -0.026969144, 0.009139024, + -0.00021477039, -0.023853721, 0.03171582, -0.006558436, -0.013437774, + 0.03660958, -0.0069328216, -0.05744147, -0.024254847, 0.004482601, + 0.00037626602, -0.013484572, 0.0049940385, -0.012047199, 0.0022697137, + 0.013772047, -0.040674336, -0.009647119, 0.03388191, 0.040326692, + -0.041824237, -0.013023276, -0.0011139647, 0.013564798, -0.031608853, + -0.013397661, -0.0061472803, 0.0476807, -0.04551461, 0.016646795, + 0.025698908, -0.004405718, 0.0075278278, 0.021126052, -0.009352958, + -0.00041324497, -0.053884804, -0.027544094, 0.0046965354, 0.032491334, + -0.0044525163, -0.008303341, -0.021206278, -0.005518847, 0.033079654, + -0.025311152, -0.032464594, -0.005060893, -0.038882636, -0.032170434, + -0.056478765, -0.017556017, 0.033106398, 0.033641234, 0.03709093, + 0.0050943205, -0.0116126435, -0.00060210924, 0.0009735701, + -0.013377605, 0.007614739, 0.022008533, -0.0083501395, 0.027597578, + 0.015135881, 0.05861811, -0.0003760571, -0.04027321, 0.01359154, + -0.029736925, 0.0015944821, -0.041075464, 0.00032257344, 0.024789685, + -0.03414933, 0.020176718, -0.030860083, 0.017502533, -0.012147481, + 0.052761648, -0.007848729, 0.034363262, -0.006966249, 0.0009476639, + 0.012568665, 0.028587027, -0.047600474, -0.0066654035, 0.009399757, + 0.019588398, -0.036154967, -0.00439569, 0.021928309, -0.04420426, + -0.017101405, 0.001215918, 0.005716068, -0.0014047823, -0.030699631, + 0.010990896, -0.029790409, -0.032063466, 0.0048970995, 0.007835359, + 0.037866443, 0.02180797, -0.017101405, 0.0005615786, 0.0032708612, + -0.01587128, 0.022275953, 0.003260833, -0.021072568, -0.006852596, + -0.009145709, -0.018090853, 0.004803503, 0.020470878, -0.04460539, + 0.0023114977, -0.034951583, -0.019320978, -0.017034551, 0.029870635, + 0.009767457, -0.020551104, 0.02283753, 0.033052914, -0.006488239, + 0.007333949, 0.020577846, -0.0011114577, -0.022971239, 0.01337092, + 0.01715489, -0.026073294, -0.017101405, 0.01859895, -0.00064723607, + 0.020898746, 0.025351264, 0.03476439, 0.0149888005, -0.016967697, + -0.0055088187, -0.021714373, -0.010422632, 0.0010964154, 0.042118397, + -0.009072169, -0.0007608889, 0.022717193, 0.015911395, -0.029309055, + -0.0116527565, 0.018064111, -0.0075211423, 0.014427222, 0.01874603, + 0.0071935547, 0.017703097, -0.048803855, 0.012301246, 0.028132414, + -0.006397985, 0.0065383795, -0.011732982, 0.0049906955, -0.04070108, + 0.02397406, -0.015149252, 0.011244943, -0.07022407, 0.00016839002, + -0.00016348038, 0.009947964, -0.022048647, -0.0046865074, + 0.0036669746, -0.04928521, -0.026514534, 0.02298461, 0.020992344, + 0.0025271038, 0.02762432, -0.009299475, 0.0058297208, 0.019575026, + -0.014895204, 0.0005126912, -0.006484896, 0.046557542, -0.0047466764, + 0.003187293, -0.009078855, -0.0037973411, 0.02109931, 0.009727344, + -0.0041717268, 0.025043732, -0.0051544895, 0.023586303, -0.012849454, + -0.0016354306, 0.02500362, -0.010148528, -0.0049906955, 0.008744582, + -0.023171803, 0.007928955, 0.00183683, -0.02020346, 0.023720011, + 0.012635519, -0.01889311, 0.00762811, 0.03385517, -0.0024218077, + 0.018090853, -0.03230414, 0.025110587, -0.008544018, -0.0038575104, + 0.010496172, 0.008991944, -0.0035031808, -0.012401528, -0.00945324, + -0.002876419, -0.034069102, 0.05166523, 0.024388557, 0.030218279, + 0.07097284, 0.018130967, 0.019668622, 0.0016462945, -0.010155213, + -0.000043638294, 0.059206434, 0.0028229353, 0.0083768815, + -0.019949412, -0.0011992044, 0.043562457, 0.041637044, -0.015229478, + 0.012956421, -0.023238659, 0.0150556555, -0.019294238, 0.010255495, + -0.006237534, 0.017101405, -0.009359644, -0.0007011376, 0.017983887, + -0.027022628, -0.013177041, -0.0020307084, -0.0029499591, + 0.0027811513, -0.014173174, 0.0020724926, 0.027651062, 0.0051645176, + 0.015175994, -0.018144337, -0.027517352, -0.018064111, -0.017034551, + -0.026140148, -0.005555617, -0.012655576, -0.028025446, 0.0046330234, + 0.019387834, -0.014641157, 0.008249857, 0.028961413, -0.015964879, + 0.012769228, 0.014226658, 0.059848234, 0.013217154, -0.007140071, + 0.010382519, -0.003210692, -0.040005792, -0.0067122015, -0.005445307, + -0.021727744, 0.02124639, 0.007066531, -0.014948688, -0.00734732, + 0.013304065, 0.014159804, -0.018398385, 0.006461497, 0.014066207, + 0.019147158, -0.009914537, 0.018438498, 0.039176796, -0.0050776065, + -0.020577846, -0.017395567, -0.017047923, 0.0064146984, -0.006585178, + -0.011164717, -0.019561656, -0.006521666, -0.047466766, -0.0057996362, + 0.013865643, -0.005207973, 0.02618026, -0.01801063, -0.022570113, + 0.025912842, 0.0041650413, 0.015617234, 0.0003679092, -0.003874224, + 0.02426822, 0.012368101, 0.012100683, 0.008697784, -0.026634872, + 0.00047550333, -0.039310504, 0.009225935, 0.009112282, -0.020711554, + 0.06311074, -0.013150299, -0.009346273, -0.016499715, 0.007715021, + -0.038508248, 0.0083835665, -0.002807893, 0.009780828, 0.022596855, + -0.017783323, -0.014667898, 0.011385338, 0.013417718, -0.05468706, + -0.005318283, -0.003274204, 0.01948143, -0.031689078, 0.00828997, + -0.016566569, 0.0023465964, -0.02442867, 0.017368825, -0.0057327817, + -0.002192831, -0.01127837, 0.039577924, 0.011111234, -0.00581635, + -0.010957468, 0.015376558, -0.00061213743, 0.004516028, -0.016366005, + -0.00046171458, 0.0022145587, 0.008818122, -0.011732982, 0.020564474, + -0.004442488, 0.0053249686, 0.046156414, 0.01264889, 0.019815702, + 0.019254124, 0.0357271, -0.0068726526, -0.00089919433, 0.018278047, + 0.04941892, 0.008784695, -0.015470154, -0.011519047, 0.024254847, + -0.006016914, 0.019307608, -0.0084370505, -0.024214735, 0.009840997, + 0.018839626, -0.0065651215, 0.031876273, 0.027731286, 0.00064096844, + -0.0035600073, -0.02601981, -0.015456783, -0.018665804, 0.018291418, + 0.006458154, -0.004850301, 0.021781228, -0.014119691, 0.0017933745, + 0.02196842, -0.015523638, 0.01700781, 0.005940031, 0.027731286, + 0.0021092626, 0.0056960117, -0.024602491, -0.017756581, 0.016446231, + 0.031394918, 0.028827703, 0.012582036, 0.025164071, -0.01905356, + -0.022449775, -0.017181631, -0.0052915416, 0.03203672, 0.006672089, + 0.01469464, -0.016312521, 0.01213411, 0.0083367685, 0.0066888025, + -0.0016830645, -0.0012994863, -0.015630605, 0.01657994, 0.042653233, + 0.007460973, -0.012040513, -0.018786142, 0.012602092, 0.023145061, + -0.008657671, 0.016098587, 0.027891738, -0.03358775, 0.024602491, + 0.011432136, 0.027062742, 0.025859358, -0.02412114, 0.0025254325, + 0.026394196, 0.022650337, -0.025859358, -0.010482801, 0.01874603, + -0.02866725, -0.010790332, -0.0345772, -0.009560208, 0.028346349, + -0.015269591, -0.027597578, -0.032785494, 0.008597502, -0.0007416682, + -0.020831892, -0.0047199344, 0.016700279, -0.008931775, -0.0019404547, + 0.016446231, -0.015563751, -0.021754486, 0.0056960117, 0.00012347208, + -0.021032456, -0.010362463, -0.0048737, -0.03535271, -0.0050809495, + 0.013999353, -0.0034797818, 0.008176317, -0.03564687, 0.042679977, + -0.024655975, -0.023465963, 0.005766209, -0.054045256, -0.006200764, + -0.0021326616, -0.0039644777, -0.035834063, 0.01484172, 0.044765837, + 0.0172886, 0.011031008, 0.033935394, -0.006080426, 0.016927585, + 0.021286504, -0.005589044, -0.0055957297, -0.01992267, -0.0023833665, + -0.019882558, -0.022784047, -0.0040480457, -0.025805874, 0.0021059199, + 0.015162623, 0.010903985, -0.025845988, -0.0010170256, -0.020256944, + -0.01410632, -0.030218279, 0.014667898, 0.0019588398, -0.006137252, + 0.0022997982, 0.004950583, 0.021286504, 0.026073294, -0.004312122, + 0.045568094, -0.039765116, 0.030913565, -0.0046731364, -0.010803703, + 0.016352635, -0.01440048, 0.019280866, -0.018064111, -0.030057827, + -0.027463868, 0.015389929, 0.005268142, -0.0035666928, 0.009533466, + -0.007815302, -0.02093886, -0.016566569, 0.012842768, -0.0061840503, + 0.004509343, -0.014453964, 0.003315988, -0.0046363664, -0.00026532917, + 0.015617234, -0.005502133, 0.011719611, -0.017195003, -0.039658148, + -0.030405471, 0.006387957, 0.010623195, 0.011860006, 0.04738654, + -0.0043789763, 0.02559194, 0.030218279, 0.030592663, 0.010014819, + -0.015804427, -0.0012919651, -0.010609824, -0.011759724, 0.03735835, + 0.04112895, 0.018251305, -0.0389896, 0.003058598, 0.0035700356, + -0.03912331, -0.037545543, 0.0056692697, 0.003068626, 0.021152794, + -0.018625692, -0.010583083, 0.0015694117, -0.012876196, -0.002958316, + 0.015670719, -0.057548437, -0.02136673, -0.032758754, -0.0042051543, + 0.002339911, -0.005602415, -0.007949011, -0.002766109, -0.0059834863, + -0.021687632, -0.012561979, -0.019361092, -0.015603863, -0.0023165117, + 0.016539827, 0.013009905, 0.012836083, 0.020083122, 0.015938137, + -0.010736848, 0.0068024555, 0.022209097, 0.016593311, -0.023332255, + -0.0016103601, -0.0070932726, 0.0118265785, -0.042091656, + -0.00030836684, -0.0017465764, -0.002321526, 0.015242849, + -0.0024853197, -0.025244296, -0.007661537, 0.006939507, 0.0075478842, + -0.014266771, -0.008076035, -0.024682717, -0.014320255, -0.035967775, + -0.014868462, 0.0064314124, -0.019013448, 0.006153966, -0.0232654, + 0.0075278278, 0.019748848, -0.0085105905, -0.0064648394, 0.01789029, + -0.002985058, 0.014935317, -0.0065885205, -0.011064436, 0.017609501, + 0.008958516, 0.0005795458, 0.0154033, -0.017342083, 0.01039589, + 0.011338539, 0.021259762, 0.003371143, 0.023185175, 0.0044525163, + -0.02470946, -0.0150155425, -0.0002939513, 0.026554646, 0.0028680624, + -0.012715745, 0.005294884, -0.011004266, -0.00013809653, 0.024522267, + 0.0011666127, 0.0036903739, 0.01206057, -0.017903661, -0.024401927, + -0.012441641, 0.027209822, -0.00982094, -0.011117919, 0.004312122, + -0.017636243, -0.0026557988, -0.00056701055, -0.008430365, + -0.0062208204, 0.018478611, -0.017328711, -0.028560285, 0.020992344, + 0.020524362, -0.01105775, -0.01615207, -0.023867091, 0.0034730963, + -0.03794667, 0.003787313, -0.005739467, -0.012100683, -0.010389204, + -0.011024323, 0.011873377, 0.0013479558, 0.02254337, 0.005161175, + 0.0017365481, -0.0052647996, 0.0115725305, 0.025658794, 0.022944499, + -0.004445831, -0.003700402, -0.0072804657, 0.010623195, 0.024589121, + 0.0033861853, 0.0052012876, -0.021580664, -0.005652556, -0.024468783, + -0.007333949, -0.026220374, 0.00031526122, 0.02006975, 0.00013705193, + -0.0038441394, 0.031582113, 0.0083434535, -0.012381472, 0.025498344, + -0.020885376, -0.0056726127, -0.00068776665, 0.026407566, + 0.0051043485, -0.020310426, -0.021259762, -0.008176317, -0.0075946823, + -0.027945222, 0.014935317, -0.013484572, 0.002486991, -0.0009852697, + -0.017756581, 0.022730563, 0.01615207, -0.007761819, -0.014948688, + -0.014560931, 0.02166089, 0.024482153, -0.007079902, -0.014199916, + 0.019227382, 0.028827703, -0.0038474821, 0.020002896, -0.008824807, + 0.006992991, -0.00052940485, -0.032758754, 0.012107368, 0.00031150065, + -0.00012493452, -0.004258638, 0.023171803, 0.042546265, -0.0045026573, + -0.0028195926, 0.00023315541, -0.005652556, -0.006387957, 0.029068379, + 0.020604586, -0.0070732166, 0.026862178, -0.012080626, -0.002560531, + 0.008838178, 0.009586949, -0.022997981, 0.0149754295, -0.027597578, + -0.0062442194, 0.0011657771, 0.012989848, -0.0057996362, -0.005552274, + 0.0154033, -0.03430978, 0.0003353176, 0.019788962, -0.026487792, + 0.031047275, 0.0020775066, -0.019681994, 0.03037873, -0.0016571584, + -0.005665927, -0.016660165, -0.018278047, -0.009032056, 0.01934772, + 0.0017022851, 0.011592587, 0.0071267, -0.01294305, -0.007908899, + -0.0033210022, -0.009700602, -0.010870557, 0.006448126, 0.03800015, + -0.0038407966, -0.016620053, -0.024040913, -0.017221745, 0.005184574, + -0.018184451, 0.03519226, -0.005970116, 0.0037973411, -0.020617958, + -0.025805874, 0.028346349, -0.032785494, 0.0065049524, 0.001495036, + -0.0064915814, -0.010476115, 0.009312846, 0.016620053, -0.005458678, + -0.02080515, 0.008971887, 0.006919451, -0.039176796, -0.004044703, + -0.0004968132, 0.010389204, -0.002303141, 0.01469464, 0.022035275, + 0.012080626, 0.004034675, -0.015242849, 0.0076080533, -0.02369327, + 0.011445507, -0.03174256, 0.007333949, -0.0054887626, -0.0031304664, + 0.0021092626, 0.0041616987, -0.042679977, -0.0020724926, 0.017088035, + 0.00087830226, 0.010924041, -0.010964154, -0.051852427, -0.013136929, + 0.008042608, 0.011619329, -0.0023750095, -0.002560531, 0.009861053, + 0.0060135713, 0.035539903, -0.0150155425, 0.005024123, -0.013350863, + 0.015470154, 0.008664356, 0.0075813113, -0.0055589597, -0.012180908, + -0.027651062, -0.030753115, 0.014520818, -0.0034998383, -0.021126052, + 0.022316065, -0.02180797, 0.014240029, -0.013985982, 0.0071668127, + 0.013217154, 0.012769228, -0.027236562, 0.014092949, 0.008764638, + -0.030779857, 0.0036435756, 0.007962382, -0.005903261, 0.02428159, + 0.014748124, -0.006728915, 0.020230202, 0.0076347953, -0.014159804, + -0.031020533, 0.0025454888, 0.024976877, 0.000036038808, 0.0014231673, + 0.00334273, 0.006892709, 0.026808694, -0.018050741, 0.0038441394, + -0.004405718, 0.000800166, 0.022864273, 0.022784047, 0.024615863, + -0.016192183, -0.004927184, -0.0023549532, -0.021701002, + 0.00094348553, 0.013076759, 0.004813531, -0.013798789, 0.00348981, + 0.009827626, 0.013731934, 0.010235439, 0.011920175, 0.01469464, + 0.014092949, -0.033052914, -0.023399109, -0.029496249, -0.0062542474, + 0.0010888942, -0.0023833665, -0.00015428787, 0.0040380177, + -0.004743334, -0.017342083, -0.0062542474, 0.03631542, 0.0071333856, + -0.016740391, 0.00435892, 0.0005340011, 0.038481507, -0.0070464746, + 0.02240966, 0.0015008857, -0.010609824, -0.009145709, 0.027864996, + 0.0082565425, 0.004973982, -0.03487136, 0.017248487, -0.014774866, + 0.022382919, 0.011946917, 0.00065851776, -0.00034388332, 0.010074988, + -0.012007086, -0.005532218, 0.0004813531, 0.0116327, -0.01772984, + 0.005940031, 0.011191459, 0.002358296, -0.017983887, -0.0032859035, + 0.00027222355, -0.014199916, 0.027009258, 0.01889311, 0.015443412, + -0.008985258, 0.011719611, 0.002161075, -0.0018134309, 0.031341437, + 0.013578169, -0.002849677, -0.014320255, -0.025057103, 0.0008014195, + 0.008704469, -0.0055355607, 0.009433184, 0.022329435, 0.018050741, + 0.018117595, 0.0057327817, -0.029041637, 0.010736848, -0.021607406, + 0.00045377557, 0.020243572, 0.012114053, 0.015831169, 0.00012284532, + 0.0029733581, 0.038909376, -0.008657671, -0.019534914, 0.0073072077, + 0.004529399, -0.03460394, 0.001413139, 0.008991944, -0.0011515705, + 0.017208373, -0.0058263782, 0.0049840105, -0.017756581, 0.01976222, + -0.023907203, 0.016406119, 0.007969068, 0.01714152, 0.0046263384, + -0.0030335274, 0.006103825, 0.0058965757, -0.0030552552, 0.032678526, + 0.0116995545, 0.018291418, 0.0061573084, -0.028185898, 0.012047199, + 0.0013655052, 0.006759, -0.004007933, 0.011452192, -0.029549733, + -0.0016128671, 0.017088035, 0.029202089, -0.020671442, -0.01933435, + -0.012902937, -0.01700781, 0.010776961, 0.014721382, -0.0015802756, + -0.018064111, -0.011264999, 0.00085239613, -0.017783323, + 0.00029666728, 0.0059634303, -0.0150155425, -0.0116260145, + 0.027597578, -0.013932498, 0.0012393171, -0.0061840503, -0.03233088, + -0.004867015, -0.02893467, -0.02543149, -0.0036302046, -0.015456783, + 0.013083445, -0.0037639139, 0.01587128, 0.0083635105, 0.0024736202, + -0.0028914614, -0.016205555, -0.024401927, 0.0023683242, + -0.0007943162, 0.000023359937, -0.010930726, 0.017475791, 0.017609501, + -0.0074542877, 0.0151091395, -0.020778408, -0.007888842, 0.008637614, + -0.0070331036, -0.009640433, -0.023532819, 0.013304065, -0.018251305, + -0.015363187, -0.005060893, -0.0002413033, 0.0010696736, + -0.0046263384, 0.0083969375, -0.008096091, -0.013598225, -0.008891662, + 0.009861053, -0.0043890043, 0.017609501, 0.0027694518, -0.012475068, + 0.018278047, 0.014159804, 0.006484896, -0.014306884, -0.020978972, + -0.011819893, 0.0009635419, 0.018973336, -0.014240029, -0.0084236795, + 0.018772772, 0.024589121, -0.016686907, -0.014881833, -0.012301246, + 0.0013379276, -0.005358396, 0.0028246068, -0.02749061, 0.0006530858, + -0.015523638, 0.00334273, 0.0034363263, 0.009145709, -0.031689078, + 0.043883357, -0.015991619, -0.0007629781, 0.0028329636, -0.02426822, + 0.027758028, -0.0037070874, -0.012896252, -0.016954325, 0.005712725, + -0.0044658873, -0.022275953, -0.00024339253, -0.015443412, + -0.008610873, 0.035005067, 0.01512251, -0.016339263, -0.032090206, + 0.021848083, -0.012996534, -0.0058263782, -0.00064765394, 0.004348892, + 0.029469507, 0.025752392, 0.018037371, 0.016071845, 0.009332902, + 0.0029198746, -0.0008833164, 0.028560285, -0.013865643, -0.026073294, + -0.020925488, 0.01220765, -0.0028229353, 0.016874101, -0.02049762, + 0.02382698, 0.008771324, 0.02529778, 0.011238258, -0.02500362, + -0.027517352, 0.0063311304, -0.0048335874, 0.021714373, 0.009065484, + 0.021834712, 0.0030151424, 0.022717193, -0.010001448, 0.012668946, + 0.013518, 0.00027786443, 0.004245267, 0.006214135, -0.003100382, + -0.023252029, 0.028613769, 0.014507447, 0.0150690265, -0.005642528, + -0.0040246467, -0.024615863, 0.007113329, 0.023038095, -0.0005879026, + -0.0045962534, -0.0142534, -0.025097216, 0.007066531, 0.004743334, + -0.0117262965, 0.016098587, 0.0078621, 0.007387433, 0.014052836, + -0.010108415, -0.016085217, 0.003860853, 0.014708011, 0.0024736202, + -0.00037355005, 0.0150690265, -0.0116393855, 0.0050074095, 0.02224921, + 0.010342406, -0.008617558, 0.01758276, -0.017997257, -0.023185175, + -0.0022429717, -0.0038441394, -0.014012723, 0.0039243647, 0.012053884, + 0.0047934745, -0.016713649, 0.005990172, 0.011117919, -0.032812238, + -0.009279418, -0.0037338294, -0.0032073492, 0.015951507, -0.013919127, + 0.02718308, -0.027945222, -0.010903985, 0.0032591617, -0.0017866892, + -0.0015702473, -0.0024702775, 0.0065517505, 0.011458878, 0.0014390452, + 0.00960032, -0.0064648394, -0.0051645176, -0.0048335874, 0.004903785, + 0.012053884, -0.0061105103, -0.029870635, 0.021152794, -0.00762811, + 0.006411356, -0.0076815933, 0.00367366, 0.003274204, -0.017515905, + 0.011111234, 0.0021828027, 0.012481754, -0.021152794, -0.014280142, + -0.0031956497, 0.00770165, -0.01525622, -0.023479335, -0.010081673, + -0.012836083, 0.022489887, 0.009720659, 0.012167537, -0.0025889443, + -0.014280142, -0.017810065, -0.008965202, 0.023358997, 0.002395066, + 0.01294305, -0.0038207402, 0.022931127, -0.0068726526, -0.005615786, + 0.0036669746, 0.008477163, -0.017676355, 0.011465563, -0.014587673, + 0.019895928, -0.013023276, -0.0023465964, -0.012087312, -0.0077551333, + -0.023960687, 0.011305112, 0.001978896, 0.026715098, 0.014066207, + -0.0058497773, -0.007875471, -0.00828997, 0.021834712, 0.0060971393, + 0.006829197, -0.015617234, -0.008918404, 0.0039143367, 0.0010487816, + 0.014159804, -0.008724525, 0.005692669, -0.0019120415, -0.009339588, + -0.017542647, -0.0036703174, -0.0084504215, -0.013705192, 0.010315664, + 0.0021543894, -0.0009760771, 0.0016972711, -0.01771647, -0.004582883, + -0.012261134, -0.02283753, -0.004509343, 0.006508295, 0.0013613268, + 0.00010038636, 0.0077685043, -0.009433184, -0.022342807, -0.011458878, + 0.009326217, -0.01962851, -0.00031484338, 0.021594035, 0.019294238, + -0.010329035, 0.0029332454, -0.0011198145, 0.021420212, 0.0025154043, + -0.0006635319, -0.009205878, -0.008457107, -0.0017022851, + -0.0055489317, 0.025418118, 0.006525009, 0.0040848157, 0.003255819, + 0.00058330636, 0.0019237411, 0.010349092, -0.020230202, -0.013758676, + -0.0059233173, 0.0064915814, 0.010937412, -0.010208697, 0.0009886124, + -0.00057286036, 0.015349816, -0.009165766, -0.005268142, + -0.0014306884, -0.015510267, 0.011592587, -0.0081963735, -0.017074665, + -0.020096492, -0.023024723, 0.01601836, -0.023492705, 0.005070921, + -0.006040313, 0.016900843, -0.017355453, -0.008263228, + -0.000093335286, -0.01440048, -0.025498344, 0.0003679092, -0.009012, + -0.014881833, 0.012067255, 0.0023917232, 0.023572931, 0.009439869, + -0.017489163, -0.0022179012, 0.021607406, 0.01629915, -0.027116224, + -0.019695364, -0.029309055, 0.0008925089, -0.0078019314, -0.009713973, + -0.012755857, 0.0022245867, -0.014534189, 0.0022229154, 0.009065484, + 0.011405394, 0.006561779, -0.0013989324, 0.010783646, -0.026140148, + -0.010402575, -0.0048101884, 0.014855091, -0.0032073492, 0.013484572, + 0.01948143, -0.013230525, 0.022663709, -0.022623597, -0.018050741, + -0.000022654829, -0.0013830545, -0.005619129, -0.013203783, + -0.0005523861, -0.005940031, 0.0023315542, 0.022182355, -0.015416671, + 0.003934393, 0.020256944, 0.0005870669, 0.01845187, 0.002863048, + -0.015537009, -0.011291741, 0.0084103085, 0.0023064837, 0.005739467, + 0.0033444013, 0.014039465, 0.007848729, -0.014828349, 0.015523638, + 0.00472662, 0.007922269, 0.0065283515, 0.025351264, 0.015777685, + 0.0013354206, -0.018037371, 0.0016462945, -0.0037538856, -0.01206057, + -0.012020457, -0.013611596, 0.0059199748, -0.013658394, -0.01946806, + -0.009934593, 0.0020490934, 0.0117931515, 0.013584854, 0.029897377, + -0.027971964, 0.0015234492, -0.015363187, 0.0035366083, -0.0001981612, + -0.025204184, -0.010001448, -0.031796046, -0.013718563, 0.008517276, + -0.019066932, -0.012689003, -0.009727344, 0.007140071, -0.01616544, + 0.00031755934, -0.016379377, 0.0117797805, 0.005976801, 0.01069005, + 0.01904019, -0.016847359, 0.0040112757, -0.015978249, 0.013558112, + 0.011532418, 0.01992267, 0.010977525, -0.0071066436, 0.0072537237, + -0.017636243, -0.0073272637, -0.030619405, -0.015684089, -0.018037371, + -0.015643977, 0.0052280296, -0.00360012, 0.007013047, -0.017047923, + -0.0065016095, -0.01976222, 0.00062968675, -0.0021945022, + -0.00011125023, -0.006374586, 0.014681269, 0.027089482, -0.035005067, + 0.00886492, 0.012889567, 0.022222469, -0.006401328, -0.00679577, + -0.04377639, 0.0025655453, 0.0026691698, 0.00995465, -0.016807245, + -0.011318483, 0.024094397, -0.005037494, -0.0116460705, 0.017342083, + -0.009092226, -0.013103501, 0.027838254, -0.017515905, 0.010970839, + 0.0067021735, 0.01992267, 0.009840997, 0.010349092, -0.005615786, + -0.00015731722, -0.006585178, 0.032464594, -0.0020724926, + 0.0014382096, 0.0081696315, 0.01264889, -0.0084236795, -0.005639185, + 0.013404347, 0.027303418, 0.018986706, 0.003623519, 0.024495525, + -0.006334473, 0.000105504914, 0.0016220597, -0.000747518, 0.017114777, + -0.00995465, -0.0068392255, 0.004850301, 0.009988077, -0.005248086, + -0.005615786, 0.02484317, -0.012628834, 0.0040380177, -0.007434231, + 0.00018802856, -0.0013220496, -0.011592587, 0.016406119, 0.004335521, + -0.0074810293, 0.012327988, -0.008757953, -0.008263228, -0.015898023, + -0.016753763, -0.004144985, 0.02255674, -0.017088035, 0.002715968, + -0.034817874, 0.0031889642, 0.037331607, 0.0037505429, -0.0014599373, + -0.010843816, 0.011298427, 0.036128223, 0.017662985, 0.002802879, + 0.0034430118, 0.0068726526, 0.023666527, -0.01555038, -0.00092259346, + -0.0015100782, 0.011164717, -0.00081646175, 0.0067489715, 0.008737896, + 0.0047099064, 0.015537009, -0.023345625, 0.0041951262, -0.011244943, + -0.005542246, -0.004903785, 0.00030168137, -0.0017783323, + -0.028426575, 0.0070331036, -0.006458154, -0.014534189, 0.0036469183, + -0.0040480457, -0.006224163, 0.00040196328, -0.000242139, 0.007688279, + 0.02691566, -0.009981391, 0.0029817151, 0.029870635, -0.006488239, + 0.028025446, -0.00982094, -0.016459603, 0.010442688, -0.0020223516, + 0.015430042, 0.020377282, 0.022824159, 0.0007374898, -0.025605312, + 0.00041157362, 0.00945324, 0.019936042, -0.0057461527, 0.011458878, + -0.0078019314, 0.0009627062, 0.022489887, -0.008771324, -0.015349816, + 0.005097663, -0.008818122, 0.005431936, 0.002500362, -0.00821643, + -0.02139347, 0.014306884, 0.033748202, -0.01629915, 0.00067648495, + 0.013919127, 0.016620053, 0.014146433, -0.0034296408, -0.0010947441, + 0.020457506, -0.014306884, 0.009326217, 0.024094397, -0.0053650816, + -0.001657994, -0.01076359, -0.010476115, -0.006635319, 0.001992267, + -0.013390976, -0.011244943, 0.00056408567, -0.00063386513, + -0.002229601, 0.015269591, 0.0022145587, -0.0007717528, -0.0019755533, + 0.023947317, 0.0017164918, 0.00028789262, 0.013551427, 0.0046731364, + 0.00020965184, 0.041155692, 0.010924041, -0.015643977, 0.025498344, + -0.012140795, 0.006595206, 0.0044257743, 0.0062943604, 0.013892385, + -0.030779857, 0.00020171286, -0.031635597, -0.010603139, 0.018946594, + 0.006508295, 0.015670719, 0.0015167637, 0.015577122, -0.008804751, + 0.00236331, -0.005903261, 0.0032357625, 0.008557389, 0.007715021, + 0.02500362, -0.027169708, 0.024468783, -0.006608577, -0.0035566646, + 0.004679822, -0.010863872, 0.0002584348, 0.014520818, -0.0057962937, + 0.016646795, -0.008798065, 0.008577445, -0.010476115, -0.008597502, + 0.0018886424, 0.026581388, 0.010349092, -0.016058475, -0.016459603, + -0.023613043, -0.00995465, 0.004850301, -0.010997581, 0.0052046305, + -0.0030368702, 0.016807245, 0.0037639139, -0.018973336, 0.003586749, + 0.008691098, -0.005726096, -0.0051143765, -0.016098587, 0.01890648, + -0.023158433, -0.01890648, 0.008657671, -0.012889567, 0.011465563, + 0.0047600474, 0.009132338, -0.010496172, -0.006545065, -0.005304912, + 0.020778408, 0.00009411874, 0.003068626, -0.01629915, -0.019534914, + -0.022182355, -0.010442688, -0.02398743, 0.0073673767, -0.03152863, + 0.008136204, -0.024348445, 0.000120024895, -0.006668746, + -0.0055255326, 0.016472973, 0.000026350113, -0.008858235, 0.017515905, + -0.009974706, 0.010141842, 0.01600499, 0.023372367, 0.0014549232, + 0.010489486, -0.020925488, -0.02195505, -0.016686907, 0.009092226, + -0.00664869, -0.020564474, -0.01344446, 0.011973659, 0.010623195, + -0.0020357226, 0.020002896, 0.01329738, -0.0033009457, -0.0015426698, + -0.019695364, 0.016887471, -0.01861232, 0.014026094, -0.014935317, + 0.0027142966, -0.017756581, -0.0036402328, -0.017061293, -0.01076359, + 0.00930616, 0.007186869, 0.006364558, -0.025123958, -0.013772047, + 0.008035922, 0.00345304, -0.014052836, 0.016767133, 0.009399757, + 0.007066531, 0.024936765, -0.010335721, -0.008978573, 0.015844539, + -0.0053617386, 0.0028530199, -0.025498344, 0.03896286, -0.029068379, + 0.011412079, -0.010081673, -0.004044703, 0.019936042, -0.00048678505, + 0.00668546, 0.008096091, -0.049231727, -0.020965602, -0.0034797818, + -0.002784494, -0.0057494952, 0.011077807, -0.012675632, -0.036074743, + -0.011031008, -0.0029599874, 0.0064915814, 0.0023549532, + -0.0028279496, 0.004144985, -0.006227506, -0.00023628921, + -0.0015226135, -0.0035165518, 0.008042608, -0.010703421, + -0.0033143167, -0.00035015095, 0.018425127, -0.027731286, + -0.000002350357, 0.01889311, 0.0058531202, 0.007748448, 0.024776313, + -0.0033878568, 0.011251628, 0.0014139748, 0.0075679403, -0.003045227, + 0.0053784526, -0.0067255725, -0.03663632, 0.007026418, 0.023867091, + 0.014681269, -0.009780828, 0.010449373, -0.010903985, 0.0027276676, + -0.015242849, -0.005789608, 0.0011732982, 0.007735077, 0.0026090008, + 0.0059099463, -0.018050741, -0.012996534, -0.01338429, -0.0228509, + -0.0024385215, 0.007333949, 0.0083100265, 0.005298227, -0.010890614, + 0.0083969375, 0.0015267918, -0.00010049082, -0.0084972195, + 0.009466611, 0.027704546, -0.010349092, -0.0150289135, 0.0033210022, + -0.0022429717, 0.0068726526, 0.0072938367, 0.0116661275, -0.021326616, + 0.027383644, 0.016713649, 0.0044491733, -0.022302695, -0.0056826407, + -0.024308331, -0.007962382, -0.0075946823, -0.0040981867, 0.010783646, + -0.007039789, -0.014801608, -0.006508295, 0.016339263, 0.0049472405, + -0.0081696315, -0.0084236795, -0.010576397, -0.0017833464, + -0.011051065, -0.011318483, -0.012227706, -0.0011682841, 0.018331531, + 0.007955697, 0.01571083, 0.0053617386, -0.023706641, -0.010522914, + 0.010148528, -0.0034831245, -0.027758028, 0.012735801, 0.023720011, + 0.0031471802, -0.0048436155, 0.0050207805, 0.009459926, -0.0023198545, + 0.01250181, -0.018518724, -0.025511714, 0.01228119, 0.023800237, + 0.004790132, -0.0057695517, -0.0068358826, 0.023800237, 0.0046731364, + -0.018090853, -0.0011741339, -0.005010752, -0.00443246, 0.017997257, + -0.0031287952, -0.013431089, 0.006337816, -0.010663308, -0.017997257, + 0.0012050541, 0.008617558, -0.0075545697, 0.0018050742, 0.019855816, + -0.001195026, 0.010342406, -0.012314617, 0.0047867894, -0.018050741, + 0.0036435756, 0.0017499191, 0.0061573084, 0.023185175, 0.0030418842, + 0.012227706, 0.021192908, -0.017836807, 0.0007074052, -0.0027828226, + -0.016352635, 0.0023432537, -0.007908899, 0.009012, -0.0026440993, + 0.016833987, -0.011358596, 0.012408214, -0.013491258, 0.011144661, + 0.004318807, -0.006548408, -0.016045103, 0.009988077, -0.0011691197, + -0.0008544853, -0.014855091, -0.018986706, -0.009546837, -0.018652434, + 0.0021059199, -0.010268866, 0.037171155, -0.010977525, -0.0024218077, + 0.009078855, -0.022289323, -0.008844864, -0.0069996766, -0.023318883, + -0.011198145, -0.011886748, -0.00096019916, 0.0081963735, + -0.0048970995, 0.015657347, 0.016646795, 0.0061472803, -0.0118399495, + 0.005294884, -0.009118967, -0.0010955797, 0.013678451, -0.008844864, + 0.009794199, -0.002849677, 0.0082097445, -0.004863672, -0.011037694, + -0.029442765, -0.0032959317, 0.026367454, 0.004445831, -0.017034551, + 0.010556341, -0.0011599272, 0.005515504, 0.009038742, 0.015670719, + 0.018291418, 0.0024970192, -0.02226258, 0.0011682841, 0.012735801, + -0.019000076, 0.013491258, 0.023867091, -0.0266215, -0.039390728, + -0.009092226, -0.008898347, -0.0081562605, -0.008015866, 0.020430764, + -0.0086443, -0.040192984, -0.013170356, -0.014026094, 0.009065484, + -0.009032056, 0.022636967, -0.010369148, -0.00019481848, 0.0043789763, + -0.004863672, -0.0024535637, 0.0008118655, -0.0052380576, + -0.00019983258, 0.010516228, -0.0041884407, -0.013364234, 0.015964879, + -0.0012719088, -0.011291741, 0.017462421, 0.0059066038, 0.0022379577, + 0.0064915814, 0.01076359, 0.0012109039, -0.018866368, -0.006939507, + -0.0010696736, -0.004813531, -0.012615463, 0.01440048, 0.023238659, + -0.030271761, -0.0142534, 0.025110587, 0.0068225116, 0.0039377357, + 0.004656423, -0.008430365, 0.0027727943, -0.016740391, 0.018665804, + 0.009225935, 0.006327788, -0.021928309, -0.014039465, -0.008035922, + -0.000056617486, 0.01815771, -0.015175994, 0.0058063217, 0.02270382, + 0.0045494554, 0.011238258, -0.02369327, 0.0057829227, 0.00683254, + 0.00011427958, 0.0043656053, -0.021286504, 0.0011407066, -0.03152863, + -0.0004721606, 0.00040593275, 0.020671442, 0.009125653, 0.020190088, + 0.008483849, 0.015924765, 0.0077083353, 0.018826256, 0.011478934, + -0.01366508, 0.001181655, 0.009627062, -0.023064837, 0.022824159, + -0.0081830025, 0.009185822, 0.009473297, 0.009419813, -0.0010446031, + 0.0012953079, 0.006521666, -0.00032988566, -0.0014273457, + -0.015884653, 0.028827703, -0.0011866691, 0.01730197, 0.0077818753, + -0.00439569, -0.006080426, 0.012956421, -0.02226258, -0.008557389, + -0.005662584, -0.016713649, 0.012902937, 0.004606282, -0.0013696835, + -0.007969068, -0.0010412603, 0.003155537, -0.0038775667, -0.016472973, + 0.010128472, -0.009640433, -0.018130967, 0.0054887626, 0.010783646, + 0.01756939, 0.023867091, 0.01091067, -0.0149888005, -0.023532819, + -0.01759613, -0.010268866, -0.0064815534, 0.0014047823, -0.04043366, + -0.0013471202, -0.021781228, -0.00806935, -0.015978249, -0.02079178, + -0.0032992745, 0.010609824, 0.018558836, -0.0014457307, 0.006026942, + 0.009573579, 0.0036435756, -0.010389204, 0.014921946, -0.026059922, + 0.011104548, 0.009767457, -0.011184774, -0.0061071673, 0.012461697, + -0.014453964, 0.0049906955, -0.006287675, -0.0010980868, -0.00435892, + -0.040032532, 0.010115101, -0.009861053, 0.0067723705, 0.0017967173, + 0.011966973, -0.005345025, 0.009326217, 0.003787313, 0.003150523, + 0.015924765, 0.004602939, -0.021580664, 0.012287875, -0.010990896, + 0.00006069144, -0.02659476, -0.00886492, 0.011191459, 0.0013337493, + -0.001713149, -0.008109462, -0.023412481, -0.001176641, -0.0062107923, + -0.008029237, -0.0083768815, 0.013578169, 0.0031856217, -0.0043288353, + -0.009132338, 0.0040480457, 0.0037806274, 0.009466611, -0.004516028, + -0.020765038, 0.007447602, 0.009660489, -0.0012359744, 0.03738509, + -0.025177442, -0.022423033, 0.008477163, 0.0014407166, 0.002010652, + -0.010415946, -0.008724525, 0.0049940385, 0.0056224717, -0.021286504, + -0.0024034227, -0.010008133, -0.019387834, -0.0011649413, + -0.0044391453, -0.0075946823, -0.022021905, 0.002605658, 0.0069127656, + -0.007748448, 0.0014206602, 0.023011353, 0.014921946, -0.008203059, + -0.0072670947, 0.004017961, 0.017195003, 0.01787692, -0.0015267918, + 0.0150556555, 0.011325168, 0.012455012, -0.009927908, 0.020765038, + 0.0016488015, 0.0054386216, 0.0021995162, -0.020109862, -0.030191537, + -0.0031170957, 0.014146433, 0.0025220898, 0.019681994, 0.008971887, + -0.0072804657, 0.004602939, -0.0022045304, -0.0017733182, + -0.029790409, -0.013076759, -0.024615863, 0.027263304, 0.006531694, + -0.031341437, 0.021901567, 0.0018919851, 0.0022413004, 0.009152395, + -0.0027677803, -0.011926861, 0.005111034, 0.0015535337, -0.0083969375, + -0.01585791, -0.011819893, -0.003837454, 0.00172652, -0.006040313, + 0.020925488, -0.0030385414, -0.016914213, -0.0055489317, -0.011331854, + 0.01787692, -0.024896652, -0.011077807, 0.004877043, -0.007908899, + 0.031180985, 0.018852996, -0.0016354306, -0.02500362, -0.017395567, + 0.019681994, 0.0041616987, 0.0047834464, 0.0047600474, -0.02021683, + 0.017034551, -0.013424403, 0.013758676, -0.016058475, -0.018652434, + -0.0007124193, 0.026059922, 0.0027861653, 0.015844539, 0.0020691499, + 0.011525732, 0.025351264, 0.006040313, -0.012983163, 0.0047199344, + 0.021714373, -0.015844539, -0.003407913, 0.0073406347, -0.015323074, + -0.01250181, 0.021500438, 0.01616544, -0.03562013, -0.015443412, + -0.012555294, -0.02837309, 0.0034129273, 0.005211316, -0.0053517106, + 0.016325893, -0.012408214, 0.0004408225, 0.015537009, -0.014119691, + -0.0063043884, -0.0006321938, 0.012628834, 0.026153518, -0.0063077314, + 0.03390865, 0.014346996, -0.007788561, 0.012929679, 0.0009560207, + -0.013217154, -0.007868786, -0.0035366083, -0.009359644, -0.022931127, + 0.01454756, 0.00446923, 0.0007387433, -0.0017967173, -0.025658794, + -0.005421908, -0.014186545, -0.016847359, 0.008517276, -0.01440048, + 0.032945946, 0.0062341914, 0.0082966555, -0.008577445, 0.004643052, + 0.0084370505, -0.018973336, -0.008109462, 0.0015953179, 0.018024, + -0.014868462, -0.026688356, 0.001584454, 0.021567293, 0.017756581, + -0.001618717, 0.015898023, -0.014734753, -0.014199916, 0.021607406, + -0.0023783522, -0.0063679004, -0.01628578, 0.013865643, 0.012695688, + 0.010449373, -0.0010320678, -0.002573902, 0.018959964, -0.01381216, + -0.00454277, -0.017917031, 0.0039410787, 0.009540151, 0.021767857, + -0.0048837285, -0.0044491733, 0.0055723307, -0.0031605512, + 0.0062107923, 0.0027076113, -0.018478611, 0.027383644, -0.004482601, + 0.001415646, -0.0073072077, -0.011412079, -0.009433184, -0.00800918, + 0.011900119, 0.010121786, 0.009473297, -0.012702374, -0.021741115, + -0.0049305265, -0.0030652834, -0.013658394, -0.006892709, 0.020591216, + -0.001108115, 0.0029198746, 0.00087913795, -0.001250181, 0.0031053962, + 0.03631542, 0.012020457, 0.01228119, -0.0061974213, 0.012541923, + -0.00013360473, -0.011833264, -0.0037037446, 0.005568988, 0.022677079, + 0.0068860236, 0.013919127, 0.005405194, -0.0061706793, -0.016085217, + 0.010001448, 0.011264999, 0.023104949, -0.0056859837, -0.028346349, + -0.018358273, 0.009439869, 0.034416746, 0.008664356, -0.009185822, + -0.010268866, -0.017542647, -0.019427946, 0.013631652, -0.0006154801, + -0.008918404, -0.00777519, 0.007287151, 0.0082097445, 0.02136673, + -0.009118967, 0.002308155, 0.0015593835, -0.037545543, -0.00014478198, + 0.008610873, 0.0005260621, 0.014199916, -0.013745305, -0.010449373, + -0.0028714049, 0.0032157062, -0.0005327476, 0.014413851, 0.024589121, + -0.023118319, 0.0060837683, -0.0042920653, 0.00023900518, 0.008403623, + -0.01046943, -0.008022551, 0.00038859234, -0.00727378, 0.005655899, + 0.0035165518, -0.02807893, -0.010603139, -0.020591216, -0.0021894882, + -0.008711154, -0.017783323, -0.012013772, 0.009994762, -0.03797341, + 0.006892709, 0.009947964, -0.012414899, -0.008637614, 0.0025889443, + 0.011452192, -0.0031806075, -0.025391376, 0.018478611, -0.003063612, + 0.03024502, 0.0030719687, -0.000847382, -0.007026418, 0.021594035, + 0.007989124, 0.022369549, -0.017676355, 0.021126052, 0.0030552552, + -0.0046865074, 0.016352635, 0.002881433, 0.0036937166, 0.009914537, + 0.007969068, -0.022356177, -0.01032235, 0.0007019733, 0.013838902, + 0.0199093, -0.007561255, 0.028480059, -0.0032708612, -0.010810388, + 0.0051444615, -0.0014574302, -0.012374786, -0.009219249, 0.006040313, + -0.0075278278, -0.0116861835, 0.0116260145, 0.010054931, + -0.0011139647, -0.023011353, -0.000012300201, 0.035887547, + -0.0038073694, 0.010810388, -0.0038474821, 0.011305112, 0.014266771, + -0.015430042, -0.009279418, 0.026875548, -0.005345025, -0.003118767, + -0.009981391, 0.030592663, -0.009332902, 0.004529399, 0.050836235, + -0.0031354807, -0.006992991, -0.025966326, -0.014801608, 0.016178813, + -0.022142243, -0.0052514286, 0.008998629, -0.0047567044, 0.0071066436, + 0.007661537, 0.018425127, 0.002568888, 0.0025204183, 0.025565198, + -0.0015485196, -0.015978249, -0.011458878, 0.01091067, 0.0011114577, + 0.00982094, -0.0015451768, 0.012976478, 0.019521542, 0.0049472405, + -0.0020440794, 0.016660165, -0.0026223715, 0.024615863, -0.014133062, + 0.022877643, 0.0016220597, -0.027570836, -0.0057728947, -0.0012008757, + 0.018866368, 0.0021192906, 0.0031104102, 0.0058865473, 0.00777519, + 0.0075077713, 0.021848083, 0.020337168, 0.0057929507, -0.020671442, + -0.010081673, -0.0049906955, 0.016031733, 0.020029638, 0.0071200146, + 0.0029666729, -0.0058464347, 0.008858235, -0.006769028, -0.022583483, + 0.0010563027, 0.030565921, 0.002894804, -0.014012723, -0.018732658, + 0.022757305, 0.012742487, 0.012528552, -0.017676355, 0.01469464, + 0.001868586, 0.0398186, 0.004492629, -0.0053283116, 0.01615207, + -0.010937412 + ] + } + ] + }, + { + "name": "online-marketing-v-koralkach-katlas", + "title": "Online Marketing v Korálkách Katlas", + "content": "Standa, zodpovědný za online aktivity v Korálkách Katlas, vedl prezentaci. Více informací o něm lze najít na stanislavvavrik.cz.\n\n---", + "keywords": [ + "Standa", + "Korálky Katlas", + "online aktivity", + "prezentace", + "stanislavvavrik.cz" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + -0.024927013, -0.041797765, -0.022209998, -0.026238129, 0.035700284, + -0.009470053, -0.012976895, 0.0453362, -0.003113902, -0.00042132454, + -0.0037990788, 0.032256626, -0.03964943, -0.017960718, 0.020693528, + 0.007096616, -0.009138324, 0.03658489, -0.04792684, -0.005287907, + 0.0066780066, -0.022462744, -0.06170146, 0.012139676, -0.009288392, + -0.02072512, -0.0044032983, -0.016136212, 0.013506081, 0.015006756, + 0.010402052, -0.0035917487, 0.013008488, -0.04236644, -0.02202044, + -0.017802753, 0.04321946, 0.010307272, -0.026253926, 0.033141237, + 0.0068162265, -0.01300059, -0.016586415, 0.00090040534, -0.03250937, + 0.026206534, -0.01878214, -0.02650667, 0.0072466834, -0.0066503626, + 0.020962069, 0.016902346, 0.014422283, 0.013664046, 0.014959366, + -0.007424395, 0.033014864, 0.061353937, 0.0059118723, -0.0038918837, + 0.06347068, 0.006508193, -0.014769807, 0.007894344, 0.018608378, + 0.016302075, -0.0039491463, 0.0029045972, -0.01165788, 0.062206954, + -0.0049877716, 0.0038800363, 0.013055878, 0.001667527, -0.011460423, + -0.024958605, 0.023205183, -0.009154121, -0.005746008, -0.028181108, + -0.010868051, 0.01938241, -0.03389947, -0.030661173, 0.010654797, + -0.079046115, 0.015148926, -0.073675275, 0.0073849037, -0.029539615, + -0.011997507, 0.0074480902, -0.029981919, -0.011144491, -0.008016767, + -0.012431913, -0.0054577203, -0.0068162265, -0.0010810788, + 0.016665397, -0.0082616145, -0.012258151, 0.019745732, -0.0015539891, + 0.040281292, 0.06527149, -0.009446358, 0.020740917, 0.007661344, + -0.023631692, 0.036332145, -0.0077679707, 0.01868736, 0.0070255315, + -0.0057539064, 0.011578897, -0.017550007, -0.07563405, -0.008301105, + -0.05816303, -0.036711264, -0.04852711, 0.014935671, -0.008711817, + -0.0029618598, -0.0038701633, -0.01952458, 0.017407836, 0.04081838, + 0.020804103, -0.060058616, -0.0240582, -0.006776735, -0.017818548, + 0.014564452, -0.007696886, -0.0020752763, -0.023031421, 0.008925071, + 0.03487886, -0.019319223, -0.02650667, -0.00864863, 0.0021740051, + -0.058194622, -0.03418381, 0.00851436, -0.032762118, -0.024263555, + 0.002770326, -0.011847439, 0.009896561, -0.035479132, 0.020804103, + 0.013577165, -0.015733398, -0.018450411, -0.0012380574, -0.03386788, + 0.015046247, -0.024816435, -0.01753421, -0.0051022973, 0.02530613, + 0.027707212, -0.028797176, 0.045399386, 0.004584959, -0.03803818, + 0.024816435, -0.008316902, 0.020709323, 0.012526692, -0.011262965, + -0.056583367, 0.028939344, 0.018624173, -0.044419996, -0.032383, + -0.02581162, -0.019208647, 0.017518412, -0.017423633, 0.0133323185, + -0.017060312, -0.005588042, 0.064892374, -0.014430181, 0.012013303, + -0.018655768, 0.05130731, -0.036521707, -0.0124240145, -0.038101364, + -0.016633805, 0.004399349, 0.003882011, -0.059900653, 0.019635156, + -0.02919209, -0.026174942, -0.010986526, 0.037121974, -0.022636507, + -0.03626896, 0.029223682, -0.007452039, 0.021783492, -0.0017001075, + -0.024547894, -0.028007345, 0.038891193, 0.03626896, 0.027091144, + -0.009920255, 0.005449822, -0.008253716, 0.008285309, -0.002553123, + 0.012968997, 0.049885616, 0.010251984, -0.009138324, -0.0071163615, + -0.014872485, -0.015836077, 0.018450411, -0.0161915, 0.0066306167, + 0.031719543, -0.056646556, -0.027059551, -0.004154502, 0.019919494, + -0.034626115, -0.021009458, 0.024816435, -0.021799289, 0.01600984, + -0.036426924, -0.013395504, 0.015622823, -0.027912566, -0.037027195, + 0.024737453, 0.011713168, 0.0005143763, 0.012992691, 0.0025412757, + 0.0013930614, -0.04103953, 0.009849171, -0.0103151705, 0.003224478, + -0.014975163, 0.060437735, 0.032635745, -0.020693528, 0.0060619395, + 0.0053945337, 0.0059987535, -0.056804523, -0.02410559, 0.020424984, + -0.02511657, 0.006851769, -0.00846697, -0.012471404, -0.012052795, + 0.021120034, 0.00020276399, 0.0023122253, 0.010654797, 0.031451, + -0.0027071398, 0.033330794, 0.032635745, 0.025211351, 0.002142412, + 0.031893305, 0.022478541, -0.022731286, -0.023031421, 0.015994042, + -0.017139295, 0.027786193, 0.020140646, -0.004099214, 0.0008352444, + 0.040534038, -0.019856308, -0.0023655386, -0.016349467, 0.0086249355, + -0.008380089, 0.014880383, -0.020756714, -0.01998268, 0.0029776564, + -0.028023142, 0.009778086, -0.028655006, -0.023805454, 0.007890394, + 0.02168871, 0.021356983, -0.0049482803, 0.012416116, -0.02585901, + -0.030155681, 0.0069189044, 0.009335782, 0.014319604, 0.001019867, + -0.015559637, 0.0033705963, 0.0015204214, 0.01957197, -0.018545192, + -0.04830596, 0.035542317, 0.012495099, -0.03233561, 0.010275679, + 0.032446187, 0.0044230437, 0.004010358, 0.00911463, -0.0145328585, + 0.03260415, -0.0017267643, -0.018166073, 0.01059161, 0.020804103, + -0.0064292103, -0.032414593, 0.028765582, 0.029492225, -0.0023378946, + 0.01563862, -0.0024445215, -0.040281292, 0.00966751, 0.0056038387, + -0.052823782, -0.03113507, 0.03154578, 0.027880974, 0.01253459, + 0.02211522, 0.0007695899, 0.027043754, 0.055572387, 0.025005994, + 0.033457167, -0.029760767, 0.016065128, 0.0036746806, 0.005370839, + 0.0070057856, 0.035542317, -0.032856897, -0.009612222, -0.0069189044, + 0.002132539, 0.003696401, -0.010939135, -0.028465446, -0.023979217, + 0.017360447, -0.0027782244, 0.00629099, -0.06906267, 0.0062475493, + -0.0319091, -0.0005003075, 0.0061014313, 0.014398588, -0.052160326, + -0.0036351893, 0.012992691, 0.008103648, -0.017913328, 0.011839541, + 0.006010601, 0.034784082, 0.044609558, 0.0016033534, 0.027754601, + 0.025511485, -0.014509164, -0.0052207713, -0.0015490527, 0.029602801, + 0.042808745, -0.004667891, 0.013308623, 0.024848029, 0.028086329, + -0.019303428, -0.029160496, -0.005907923, -0.02118322, -0.024326742, + -0.0008505474, -0.016128313, 0.024358334, -0.0073059206, + -0.0053866357, -0.025511485, 0.01832404, -0.05140209, -0.03775384, + 0.016823363, 0.02276288, 0.029429039, 0.014461774, -0.011302457, + 0.0014799426, 0.017392041, 0.017091906, -0.008545953, 0.011152389, + 0.03576347, -0.048748262, 0.01957197, 0.007858802, 0.011049711, + -0.019603562, -0.01188693, 0.004043926, 0.012321336, 0.0073809545, + 0.006989989, 0.014856689, -0.028244294, 0.014690825, 0.0057420586, + 0.012653065, 0.019619359, -0.019003293, -0.020819899, 0.022162609, + 0.023837047, -0.028007345, 0.016302075, -0.024942808, -0.005777601, + 0.053360865, 0.029318463, -0.010433645, -0.004146604, -0.03759587, + -0.05070704, -0.037248347, -0.055888318, -0.011002322, 0.07386483, + 0.008119445, 0.004174248, 0.010575814, -0.010670593, -0.0039017566, + -0.01591506, 0.0058447365, 0.011736863, -0.017739566, -0.00070541626, + 0.0075902594, 0.014374893, 0.035605505, -0.06369183, 0.08365872, + 0.007554717, 0.019919494, 0.017692175, -0.022336371, -0.010678492, + -0.015622823, -0.026253926, -0.015614925, -0.0013042056, -0.025701044, + 0.01517262, 0.004391451, -0.013845707, -0.0022273185, -0.022841863, + -0.01679177, 0.01674438, -0.019350817, 0.017913328, 0.0028888006, + 0.015907161, -0.023947623, -0.012945302, -0.011373541, -0.03968102, + -0.026822602, -0.0048692976, -0.031245645, 0.010228289, 0.0077126827, + 0.015843974, -0.018924309, -0.025969587, 0.02549569, 0.012858421, + -0.0048377044, 0.031577375, 0.021214815, 0.03519479, 0.03456293, + -0.035700284, 0.0020061664, -0.0009695154, -0.030566392, -0.020804103, + 0.0053313477, -0.01008612, -0.0058960756, 0.013142759, -0.020045867, + 0.01595455, 0.0092252055, -0.008775003, -0.018766344, 0.027770396, + -0.024484707, -0.037690654, 0.030139886, 0.0034870962, 0.0061093294, + 0.03525798, -0.014066859, -0.0057302113, 0.00400246, 0.034120623, + 0.0017820523, -0.0029934528, -0.037216756, 0.006448956, 0.029097311, + 0.028844565, 0.014438079, -0.003579901, 0.017423633, -0.01818187, + -0.008853986, -0.00233592, -0.0035443588, -0.011618389, -0.004158451, + 0.015235807, 0.026791008, 0.031466797, -0.01928763, -0.035479132, + -0.007349361, 0.016870754, -0.015464857, -0.0053945337, 0.021056848, + 0.018750547, 0.0014355147, -0.036837637, 0.010307272, 0.03737472, + -0.01568601, 0.013956283, 0.0026538263, -0.009912358, -0.026743619, + -0.003575952, 0.0045928573, 0.027454466, -0.028402261, -0.0029184192, + 0.013616657, 0.015812382, 0.032540966, -0.02257332, 0.0024682165, + -0.015733398, -0.029981919, 0.026332907, 0.023615895, -0.031024493, + -0.0048732464, -0.012266048, 0.019540377, -0.016159907, -0.010015035, + 0.03260415, -0.000788842, -0.0006284079, -0.0019380436, 0.020282816, + 0.0012262099, -0.01863997, 0.021767695, 0.021167424, -0.0062159565, + 0.018987495, -0.028876157, 0.010670593, 0.038101364, 0.011160288, + 0.0101887975, -0.012558285, 0.004213739, 0.03250937, -0.006737244, + 0.041639797, 0.007301972, 0.012021202, 0.04612603, 0.03983899, + -0.009485849, 0.03465771, -0.003386393, -0.0074362424, 0.024389928, + -0.048179585, 0.01887692, -0.01415374, -0.0059592617, -0.010165103, + 0.008174733, 0.017091906, -0.0094068665, -0.0036470366, -0.013845707, + -0.053076528, 0.032414593, 0.016665397, 0.021783492, 0.07803513, + 0.026364502, -0.015077841, -0.00081253686, 0.0051022973, 0.029334258, + 0.051654834, -0.0068715145, 0.0073454124, -0.022178406, -0.011744761, + 0.03487886, 0.028986733, 0.0072150906, 0.016223093, -0.00023744242, + 0.013245437, -0.01813448, -0.0019696367, 0.0002910027, -0.012668861, + 0.0061172275, -0.004284824, -0.030645376, -0.025780028, -0.011428829, + 0.0024780894, 0.000950757, 0.02007746, -0.027027957, 0.0103704585, + -0.009951849, 0.027786193, 0.04043926, -0.021957254, -0.033046458, + -0.002655801, 0.009951849, -0.009493748, -0.021278001, -0.007835106, + -0.01873475, 0.024595283, 0.019793121, -0.022241592, 0.014469672, + 0.01170527, 0.01743943, 0.011365643, -0.019729935, 0.019682545, + 0.0014394638, -0.021783492, -0.016120415, 0.020267019, -0.044198845, + 0.0040873666, -0.013956283, -0.009888662, -0.010015035, -0.006144872, + -0.012447709, -0.008869783, -0.01207649, 0.0048416536, -0.015141027, + 0.01887692, -0.0024109539, 0.025669452, -0.007843005, -0.007151904, + -0.00550511, -0.034341775, 0.004541518, -0.028212702, 0.0024464962, + 0.004237434, 0.024279352, 0.0125503875, -0.011997507, -0.02118322, + -0.036237366, 0.032856897, 0.027596634, -0.012100184, -0.013174352, + -0.015946653, -0.021499153, 0.009414765, 0.038891193, -0.0038918837, + 0.016136212, -0.00342391, -0.008940867, -0.010409949, 0.060311362, + 0.016428448, -0.014461774, 0.015227908, -0.0068833623, 0.0061093294, + 0.0024168775, -0.017550007, 0.044988677, -0.034815673, -0.013671945, + -0.015614925, -0.004956179, -0.0582894, -0.00037442843, 0.022794472, + 0.005805245, 0.019856308, -0.029460631, 0.0034910454, -0.0069702435, + 0.017565804, -0.03759587, 0.0020298613, 0.0037121975, 0.0025550977, + -0.010812763, -0.010141408, 0.021546543, 0.0030467664, -0.034152217, + 0.07070552, -0.033646725, -0.009454256, -0.0078074625, -0.0013683792, + 0.02211522, -0.03904916, 0.0012795234, -0.005927669, -0.004663942, + -0.0029855547, -0.0072111413, -0.0046797385, -0.021041052, + 0.011357745, -0.014880383, -0.0041189594, -0.030171478, 0.0031632662, + -0.01813448, -0.004229536, -0.015701806, -0.0061883125, 0.047042232, + 0.028876157, 0.021499153, -0.011160288, 0.03696401, 0.011373541, + -0.0143433, -0.0053471443, -0.0021779542, -0.00078588014, 0.018940106, + 0.01327703, -0.0154964505, -0.0074599376, 0.007021582, 0.004201892, + 0.0009917293, 0.015054146, 0.012771539, -0.016491635, 0.0019479165, + -0.03104029, -0.0143511975, -0.0077521745, 0.021056848, -0.00155004, + 0.017170887, 0.0024346488, -0.013111166, 0.012313439, -0.012708353, + 0.017091906, -0.008190529, -0.013545572, 0.002572869, 0.026238129, + -0.014430181, -0.03806977, 0.028149515, 0.009043545, 0.01253459, + 0.01633367, 0.01008612, -0.014698722, -0.020093257, 0.049696058, + -0.043409016, 0.010473136, -0.0041939933, 0.0131506575, + -0.00008755998, -0.015204214, -0.0066227186, 0.0047903145, + 0.027059551, -0.0118790325, 0.0036924519, -0.021988846, 0.036332145, + 0.011863235, -0.028560227, -0.036616486, 0.039396685, -0.022747083, + 0.039902173, 0.019003293, 0.0035641047, -0.031166663, 0.034152217, + -0.009454256, 0.006520041, 0.01952458, 0.030613782, 0.0028256143, + -0.005528805, 0.0031375969, -0.008917172, -0.03535276, 0.016112518, + -0.01577289, 0.003196834, -0.045999657, 0.01119188, 0.024200369, + -0.002049607, -0.029539615, -0.028828768, 0.019966884, 0.0006057003, + -0.011428829, -0.03686923, 0.0067135487, 0.006721447, 0.0033548, + 0.0094068665, 0.010330967, -0.0073730564, -0.019966884, 0.011294559, + -0.03566869, -0.013308623, -0.0066582607, 0.033709913, 0.0047903145, + 0.020756714, 0.0041150106, -0.013916791, -0.08220543, 0.03399425, + 0.004956179, -0.022968234, 0.01452496, -0.056267437, 0.024832232, + 0.01660221, 0.012518794, -0.0018679462, 0.016112518, 0.04713701, + 0.02780199, 0.02974497, 0.034310184, 0.019398207, 0.0025471994, + 0.017518412, -0.0071361074, -0.028607616, -0.020298611, -0.022131016, + 0.014919875, 0.0016931965, -0.03326761, 0.02803894, -0.00018659717, + 0.01914546, 0.037722245, -0.015520145, 0.016159907, 0.0018570861, + 0.016760178, -0.00142268, -0.0094068665, 0.014438079, -0.014746113, + -0.0013209895, 0.011326152, 0.017360447, -0.0013841758, 0.0010761424, + 0.022209998, -0.03307805, 0.018718954, -0.0023754116, -0.023584303, + 0.022367965, -0.023063015, 0.036332145, 0.0010603458, -0.0003159317, + -0.029318463, -0.012621472, -0.028986733, -0.012273947, 0.014106351, + 0.021783492, -0.03418381, -0.014145842, 0.000893988, 0.0036332146, + 0.015235807, -0.0065516336, 0.008767105, 0.021546543, 0.012826827, + 0.007341463, -0.0019380436, 0.022889253, -0.026569856, -0.030724358, + -0.01128666, 0.010891746, -0.018339835, 0.015362179, 0.02281027, + -0.007933835, 0.03456293, 0.017423633, -0.006050092, 0.002304327, + -0.017060312, -0.014556553, -0.0012301591, -0.023552708, 0.0026794958, + 0.02391603, -0.004502027, -0.011681575, 0.017565804, 0.020314408, + -0.030155681, -0.03298327, 0.031514186, 0.003613469, 0.009872866, + -0.035321165, 0.016057229, -0.005039111, -0.013253335, -0.0016586415, + 0.039807394, -0.035036825, -0.031766932, -0.0043282644, -0.009248901, + -0.00007707006, 0.0031790629, -0.013490284, 0.0018807809, 0.008411681, + 0.006504244, -0.011160288, 0.027328093, -0.034531336, 0.023631692, + 0.016728584, 0.036995605, 0.033615135, -0.037121974, 0.017218279, + -0.027043754, 0.024990197, 0.017913328, 0.010165103, 0.021941457, + -0.013363911, -0.04154502, -0.0024050302, -0.01614411, 0.022083627, + -0.0014197181, 0.007258531, 0.019256037, -0.00020918135, 0.004833755, + -0.015046247, 0.009422664, 0.025937993, 0.003882011, 0.0071953447, + -0.04890623, -0.014564452, -0.04745294, -0.020614544, 0.026806805, + 0.00074737595, 0.0020041917, -0.03386788, 0.018197667, -0.002280632, + -0.046283994, -0.0153147895, 0.019903697, 0.024500504, -0.0075112763, + -0.01753421, -0.0049008904, 0.013679843, -0.00025027714, -0.008332699, + 0.023094608, 0.013071675, 0.010165103, 0.001267676, 0.014556553, + -0.016349467, 0.0007903229, -0.0300767, -0.011807947, -0.0019034885, + -0.0064568543, 0.029065717, 0.033457167, 0.044577964, -0.007961479, + 0.021072645, 0.022968234, 0.029018328, 0.0030447918, 0.0032264525, + 0.0056038387, -0.009880764, -0.007839056, -0.0107337795, 0.011255067, + -0.0059987535, -0.032383, 0.013395504, -0.004182146, -0.0026103857, + 0.002794021, 0.016412653, 0.0015065994, 0.0067569893, -0.018797936, + -0.03737472, 0.020740917, 0.00846697, 0.012929505, -0.0086249355, + -0.014469672, -0.013434996, -0.014288012, 0.0025847163, 0.019161258, + -0.007921987, -0.02017224, -0.01119188, 0.025132367, 0.0057341605, + 0.038227737, 0.009051443, 0.02017224, 0.008751309, 0.016302075, + 0.0038938583, 0.0072427345, -0.028876157, 0.022273185, -0.014303808, + 0.00400246, -0.0032856897, -0.0018886792, 0.012471404, 0.00911463, + 0.012155472, -0.034310184, 0.005047009, 0.005860533, -0.027043754, + -0.019619359, -0.009580629, 0.017613193, 0.024184572, 0.01614411, + -0.002142412, -0.004446739, -0.019129666, -0.005568296, -0.022873456, + 0.014935671, 0.011120796, -0.018260853, -0.008814495, -0.0025689197, + -0.0148961805, -0.018276649, 0.025685247, -0.016696991, -0.0042808745, + 0.006642464, -0.009872866, 0.02595379, 0.0101335095, 0.012787336, + -0.038006585, 0.0013673919, 0.021499153, 0.015946653, -0.00024262568, + -0.0120606925, 0.017107703, 0.007835106, -0.015520145, 0.018008107, + -0.02142017, 0.014232723, -0.0007843992, -0.034910455, -0.03137202, + 0.0109707285, 0.024389928, -0.02585901, 0.02401081, 0.025701044, + 0.014185334, -0.0011640108, -0.048779856, -0.013608758, -0.017076109, + 0.01947719, 0.016823363, -0.0052207713, 0.022747083, -0.0076810895, + 0.006042194, -0.0039392733, -0.009809679, -0.019066479, 0.0019676622, + -0.014564452, 0.016570618, -0.0055209068, 0.030866528, -0.0014394638, + -0.0152674, 0.024295148, -0.0011057609, 0.001140316, 0.0008357381, + -0.02803894, 0.021072645, 0.00513389, -0.015851874, 0.01615201, + -0.021641321, 0.0013387606, -0.013956283, -0.0080246655, -0.006824125, + 0.018466208, -0.002290505, -0.017834345, -0.004746874, 0.015757093, + -0.018655768, -0.00012149796, 0.0017800777, 0.0015036375, 0.010820661, + 0.030961307, 0.006820176, -0.013371809, -0.00011902974, -0.024358334, + 0.0031060036, 0.009912358, 0.009138324, 0.001977535, -0.0027683517, + 0.0011590745, -0.017012922, 0.031340424, -0.028828768, 0.010512628, + 0.01105761, -0.014690825, 0.00943846, 0.01522001, 0.02336315, + -0.027280703, 0.023015626, -0.0048850942, 0.016997125, -0.015836077, + -0.02396342, -0.018482005, -0.013577165, -0.0023793606, 0.03759587, + 0.0011185957, 0.02626972, 0.040407665, -0.010291476, -0.0066464134, + -0.024500504, 0.0022055982, -0.02415298, 0.016317872, -0.003283715, + 0.0010475111, -0.00656743, 0.0076731914, -0.028228499, 0.0042176885, + -0.004853501, 0.0051180935, 0.033646725, 0.0045375694, 0.000053498592, + -0.024184572, -0.023852844, -0.0009176829, 0.016902346, 0.009612222, + -0.009612222, 0.016207296, -0.0044072475, 0.001711955, -0.0023201234, + -0.011420932, -0.004229536, 0.008640733, 0.0074480902, -0.030519003, + -0.0040913154, 0.008245817, -0.027833583, -0.0019261962, -0.006472651, + -0.017581599, 0.035100013, -0.021546543, 0.0057420586, 0.008751309, + -0.013284928, 0.009193612, 0.018418819, -0.018545192, 0.0161915, + -0.00911463, -0.018671565, -0.010125611, 0.015512247, -0.019082274, + -0.008909274, -0.014161639, 0.014880383, 0.016712787, -0.028165312, + 0.005797347, -0.03178273, -0.016412653, -0.0038938583, 0.008166835, + 0.025132367, -0.019177055, 0.013466589, 0.0374695, -0.02377386, + 0.0053273984, 0.015630722, -0.0050351615, 0.019856308, 0.004237434, + 0.0047824164, -0.01008612, -0.012202863, -0.022968234, 0.010583712, + 0.0018333911, 0.018513598, -0.005362941, -0.023173591, 0.007424395, + -0.010433645, 0.017155092, 0.031751137, -0.00453362, -0.0016724635, + -0.003252122, -0.020251222, 0.026538264, -0.016412653, -0.022367965, + 0.013995775, 0.006172516, -0.00027224427, 0.0141142495, -0.035036825, + -0.00091718923, 0.0101887975, 0.01753421, 0.024579488, 0.0008135241, + 0.017723769, -0.0074007004, 0.031514186, -0.011081304, -0.012968997, + 0.020124849, 0.0007049226, 0.010875949, 0.026980568, -0.0044427896, + -0.002821665, -0.03127724, 0.006954447, -0.023789657, 0.023110405, + 0.0063265325, 0.024453115, 0.006792532, -0.026080163, -0.025685247, + -0.00911463, 0.0029164446, 0.026727822, 0.00010705889, -0.005493263, + -0.01832404, 0.004379603, -0.016775973, 0.022494338, -0.0034436556, + -0.019319223, 0.016025636, 0.008577546, 0.023236778, -0.018418819, + 0.0072940732, -0.013308623, 0.002549174, 0.022510134, 0.015488552, + 0.01230554, -0.031845916, -0.01813448, -0.0069860397, -0.007977275, + -0.0017425609, -0.0058170925, 0.048369143, -0.008380089, 0.009841273, + 0.009612222, -0.02734389, -0.003931375, 0.010259883, 0.0017178786, + -0.0001630257, -0.0041110613, 0.00864863, -0.006294939, 0.0097149, + 0.011397236, 0.010101916, -0.005777601, 0.013356013, 0.000101999045, + -0.026822602, -0.0110418135, 0.0044704336, -0.016475838, 0.021704508, + -0.00730987, 0.0004368743, -0.019966884, 0.0076218527, -0.02576423, + -0.004778467, 0.011049711, 0.0083248, 0.00072022557, 0.011760558, + 0.019082274, 0.0023477676, 0.017139295, 0.043251052, 0.00018869515, + 0.017423633, -0.00097198365, -0.03020307, 0.0047666198, -0.02410559, + -0.023236778, -0.017818548, 0.005449822, -0.0016379084, -0.0037003502, + 0.011736863, 0.021388577, -0.026127553, 0.029160496, 0.001956802, + -0.002355666, -0.0043203663, -0.009904459, -0.015480653, -0.012352929, + 0.008166835, -0.003396266, -0.006425261, -0.0034179862, 0.01577289, + -0.029729174, -0.010567916, 0.0076218527, 0.011318253, -0.009177816, + -0.0036016214, 0.013774622, -0.028623413, 0.007104514, -0.011136592, + 0.0031612916, -0.028054737, 0.008696021, 0.014651333, 0.025511485, + -0.0025590467, 0.0024464962, -0.018797936, -0.01008612, -0.01813448, + 0.014422283, -0.009628018, 0.010623204, -0.0076850387, -0.0069741923, + 0.0152674, -0.006914955, -0.012345032, 0.037911806, 0.0017326879, + 0.020093257, -0.015212112, -0.00934368, -0.010212492, -0.0023438183, + -0.019034885, 0.0023754116, -0.0067530405, 0.004644196, -0.008309004, + 0.0002964328, 0.01818187, -0.007199294, -0.012313439, -0.011855338, + -0.006314685, -0.013237539, 0.014374893, 0.005256314, -0.011713168, + -0.0036095197, -0.010820661, 0.015243705, -0.020140646, 0.0065990235, + -0.0024682165, 0.005390585, 0.0015085739, -0.027880974, -0.018055497, + 0.024358334, 0.007365158, -0.0374695, -0.016633805, -0.03165636, + -0.012005405, 0.012850522, -0.0047152806, -0.007854852, -0.013427097, + -0.0061685666, -0.017660582, 0.0072506326, 0.009051443, -0.024073996, + 0.017107703, -0.005390585, 0.003617418, 0.022336371, -0.008609139, + 0.0049877716, -0.021704508, -0.027312296, -0.017360447, 0.007254582, + 0.0037418162, -0.018213462, -0.013379708, -0.014430181, -0.0075665643, + 0.007846953, 0.00018338849, -0.013671945, -0.014596045, 0.007388853, + -0.040692005, -0.0019410055, 0.011942219, -0.014872485, 0.0022352168, + 0.017739566, 0.016997125, 0.006772786, -0.0023813352, -0.0147145195, + 0.009319985, 0.0151331285, 0.0048179585, -0.03127724, -0.015717603, + 0.0040952647, 0.014611841, 0.013687741, -0.013261233, 0.006527939, + -0.0043045697, 0.007574463, 0.0036707316, -0.01863997, -0.030708563, + -0.017076109, 0.0011037864, -0.021514948, -0.007850903, 0.022209998, + 0.007925937, 0.046884265, -0.03044002, 0.018497802, -0.007933835, + -0.0053273984, 0.014027368, -0.0057618045, 0.0023398693, -0.015227908, + -0.0021029203, 0.03399425, 0.024974402, 0.01452496, -0.026759416, + -0.015401671, -0.008996155, 0.005607788, -0.008214224, 0.017107703, + -0.0022411405, -0.008846087, 0.018908512, -0.02160973, -0.011294559, + 0.034310184, 0.0009892612, -0.010244085, 0.00957273, 0.00038578224, + 0.020693528, -0.020061664, 0.006520041, 0.0118790325, 0.0074401917, + -0.01596245, -0.027707212, 0.021388577, 0.006097482, 0.013948385, + -0.0009828438, -0.002525479, -0.028828768, -0.018513598, 0.0047824164, + -0.012597777, 0.00916202, -0.0007399713, 0.020061664, 0.01753421, + -0.0068399217, -0.008553851, 0.0020654036, -0.011270864, 0.014398588, + -0.0030507157, -0.0013733157, 0.02077251, 0.0082616145, 0.019161258, + -0.0075428695, -0.02040919, 0.007933835, -0.03326761, -0.012637269, + 0.0058210418, 0.00032234908, 0.006267295, 0.0070887175, -0.020346003, + -0.007286175, -0.016057229, -0.012992691, -0.0012331209, -0.009477952, + -0.0065358374, 0.011744761, 0.02295244, -0.006480549, 0.0100071365, + -0.0012963073, -0.002580767, 0.016349467, -0.018071294, -0.006555583, + -0.0023398693, 0.0008742423, -0.029587004, -0.001392074, -0.013040082, + 0.023299964, 0.0000945944, 0.0062238546, -0.011760558, 0.007764022, + 0.01808709, 0.013727233, 0.016381059, 0.0053195003, -0.0041189594, + 0.00036850473, 0.007499429, 0.02123061, 0.0063067866, 0.0043598576, + 0.010180899, 0.0036490113, -0.0014414384, -0.0030961307, + -0.0038602906, 0.01119188, 0.008917172, 0.009738595, -0.009122528, + 0.02720172, -0.00874341, -0.016775973, -0.017186685, 0.0022885303, + -0.0037260195, -0.014145842, 0.0034021896, 0.030961307, -0.004032078, + -0.014872485, -0.010583712, -0.021862473, 0.025479892, 0.016712787, + 0.022920845, -0.036237366, -0.0024800638, -0.0014730316, -0.015212112, + -0.0063857697, -0.006583227, 0.017502617, -0.010244085, -0.017597396, + 0.0011738838, 0.0008772041, -0.00071084633, 0.0035601554, + -0.0033784946, -0.0025610214, -0.007688988, 0.016712787, 0.005611737, + 0.014366995, -0.021041052, -0.014335401, -0.019271834, 0.011847439, + -0.009477952, 0.021262204, 0.016491635, 0.0014493368, -0.020930476, + -0.004889043, 0.008190529, -0.00045193045, 0.015393772, -0.003104029, + 0.010125611, 0.015243705, -0.008443275, -0.02803894, -0.010346764, + -0.0050904495, -0.0066582607, -0.018482005, -0.006022448, + -0.007092667, -0.010441543, 0.01767638, 0.009943951, 0.0024030556, + 0.011492016, 0.0032501474, -0.026127553, 0.02295244, -0.017281465, + -0.017281465, -0.00985707, -0.0006659248, -0.009233104, -0.0066464134, + -0.00060273847, 0.018908512, -0.010425746, 0.022494338, 0.02765982, + 0.0012192989, 0.015425365, 0.013506081, -0.005765754, -0.0057697026, + -0.025242943, -0.016570618, -0.018276649, -0.007969378, -0.011104999, + 0.01938241, -0.00901985, 0.00043070377, 0.0021858525, 0.019003293, + -0.01327703, -0.019113868, -0.0094068665, 0.004419095, -0.015164722, + 0.013158556, -0.016065128, 0.016033534, 0.01595455, -0.029855546, + 0.02142017, 0.019256037, 0.00966751, -0.027091144, -0.02132539, + -0.014493367, 0.009090935, 0.0040873666, -0.001639883, -0.023284167, + 0.0035305368, -0.011902727, 0.0092252055, 0.0067411927, -0.008956664, + 0.016396856, 0.011871134, -0.00012735996, -0.024911216, 0.003806977, + 0.0020812, -0.0076455474, 0.009699103, 0.0043006204, 0.020977866, + 0.007835106, 0.001436502, -0.009691205, 0.0035680537, -0.00003362945, + -0.013016386, 0.0058723805, -0.016033534, 0.010646898, -0.01813448, + 0.0013298751, 0.017155092, 0.004269027, 0.010054527, 0.018655768, + -0.013845707, 0.028512837, 0.0049482803, -0.010536322, 0.009967646, + 0.0035463334, -0.0026834449, 0.0031731392, -0.00088806427, + 0.008782902, -0.0009680345, 0.005078602, 0.018513598, 0.0060856347, + -0.0038267227, 0.018213462, -0.014501265, 0.0024425471, 0.018150277, + -0.011871134, 0.031672154, 0.0061488207, -0.008435377, -0.026633043, + 0.008538054, 0.0072703785, -0.0007014671, -0.009501646, -0.010796966, + 0.01674438, 0.018055497, -0.008403784, 0.0076929373, -0.003984689, + 0.012147574, -0.0029480378, -0.005935567, 0.012155472, -0.013032183, + 0.024200369, -0.019729935, -0.017581599, 0.015156824, -0.01868736, + -0.016349467, -0.008182631, -0.004450688, -0.0074125477, 0.009683307, + -0.025669452, -0.0074007004, -0.0036332146, 0.013498182, 0.015346383, + 0.0060540414, -0.013916791, -0.024626877, -0.0029875292, + -0.0062159565, 0.0029539615, -0.005232619, -0.008340597, 0.035036825, + 0.0017238024, -0.010346764, -0.011373541, 0.005370839, -0.015535941, + -0.0063739223, 0.0076850387, 0.021846678, -0.0036806045, 0.0056512286, + -0.0017129423, -0.017960718, -0.006859667, 0.0032501474, -0.011397236, + -0.0015026502, 0.01128666, 0.039554648, -0.009825476, -0.010947034, + 0.007815361, 0.019682545, 0.0063581257, 0.008474868, -0.021041052, + -0.012313439, -0.0072150906, 0.03459452, -0.014911977, 0.0013555444, + 0.013142759, -0.0044032983, -0.0035937233, 0.014596045, -0.022288982, + -0.0031158766, 0.03519479, -0.007132158, -0.01901909, 0.014453876, + 0.0015046247, 0.015417468, -0.00045217725, -0.02988714, -0.0037497145, + -0.006417363, 0.010931237, 0.0021819035, 0.008403784, 0.027880974, + -0.0024247759, -0.011026016, 0.0054379744, 0.009722798, 0.02516396, + 0.00809575, 0.0030546647, 0.031340424, 0.0060895835, -0.006046143, + 0.00094384595, -0.016554821, 0.009130427, -0.015077841, -0.002572869, + -0.003765511, 0.011341948, 0.003414037, -0.016444245, 0.0161915, + -0.0025926146, 0.0067096, -0.0153147895, -0.0047824164, 0.012226557, + -0.015346383, 0.002016039, 0.0061409227, -0.021941457, -0.006559532, + 0.0030605884, 0.008925071, -0.011049711, -0.0049601276, 0.0037398415, + 0.010765373, -0.028970938, -0.018213462, -0.010425746, -0.019492986, + 0.026980568, 0.020061664, -0.009193612, -0.02137278, 0.0008174733, + 0.023505319, -0.0069583957, -0.0032777914, -0.003779333, -0.013837809, + 0.004020231, 0.0013545572, 0.008909274, -0.007068972, -0.011247169, + 0.005813143, 0.021593932, -0.007179548, 0.016681194, 0.0129532, + -0.013592962, 0.00837219, 0.0036095197, -0.0036016214, 0.019587766, + 0.008601241, -0.008932969, 0.0039906125, -0.00085647113, 0.009288392, + -0.009612222, -0.016854957, 0.009730697, -0.00030260335, 0.008490665, + -0.009904459, -0.0034732742, 0.037216756, 0.0035917487, -0.0059197703, + 0.003400215, -0.015709704, 0.009896561, -0.0017001075, -0.015456959, + 0.012882115, -0.007937784, 0.008569648, 0.004636298, 0.03118246, + 0.0022450897, -0.01933502, 0.013735131, 0.015251603, 0.0021601832, + -0.011326152, -0.0051022973, 0.008380089, 0.015377976, 0.016365262, + -0.010567916, -0.00518128, -0.0145328585, -0.0088065965, 0.023710676, + -0.01596245, -0.0118237445, 0.024263555, 0.0053747883, 0.016065128, + -0.005323449, 0.0029184192, 0.006263346, 0.011768457, -0.016823363, + 0.00851436, 0.009936052, 0.02415298, -0.0031751136, -0.002863131, + 0.009628018, -0.019082274, -0.0028394363, -0.029855546, 0.0052168225, + -0.0031790629, -0.0063265325, 0.006764888, -0.00846697, -0.0057539064, + -0.0054656183, -0.0053826864, 0.0050549074, -0.009011952, 0.007791666, + 0.0077600726, 0.0018778191, -0.00053066656, -0.004099214, + 0.0013999724, 0.025669452, 0.01068639, 0.01748682, 0.0042808745, + 0.016618008, 0.030455817, -0.008782902, -0.002780199, 0.0066740573, + -0.013142759, -0.016365262, -0.04407247, 0.002318149, -0.011318253, + -0.00024879622, 0.020977866, 0.004561264, -0.018797936, -0.019350817, + -0.00029445824, -0.024642672, -0.014864587, -0.009241003, + 0.0051101954, 0.023141997, 0.0009413778, -0.008735511, -0.0030191224, + 0.019935291, -0.01563862, -0.011026016, 0.0030566393, 0.0002143646, + -0.004213739, -0.0059197703, -0.0057420586, -0.0033291304, + 0.004004434, 0.0033923166, -0.008538054, -0.00064469816, -0.015843974, + 0.026617246, 0.010378357, -0.0062712445, 0.0116973715, -0.0029421141, + 0.010402052, -0.009722798, -0.012266048, 0.017723769, 0.0044230437, + 0.0088618845, 0.016823363, -0.0059237196, 0.0075152256, -0.0035956977, + 0.02281027, -0.015330587, -0.033014864, 0.015946653, 0.013158556, + -0.0145881465, 0.0072150906, -0.0053866357, 0.01161049, 0.016775973, + -0.004426993, -0.0070255315, -0.0034811725, -0.0124240145, + -0.0012133752, -0.0016882601, -0.008735511, -0.022004643, + -0.015986145, -0.020946272, 0.00096359174, 0.004269027, 0.010046628, + -0.024026606, 0.013071675, -0.015575433, 0.0070808195, -0.0019232343, + 0.0038050024, 0.009083036, 0.008309004, 0.0053313477, 0.0012923582, + -0.013790419, 0.021135831, 0.001570773, 0.011958015, -0.0062712445, + -0.008427478, 0.004079468, -0.008940867, -0.010575814, 0.002770326, + -0.005607788, -0.018892717, -0.012471404, -0.013498182, 0.0035423844, + -0.016223093, -0.0077047846, -0.007254582, -0.006042194, 0.0026952925, + -0.0074954797, 0.0031731392, -0.00629099, 0.010860153, -0.008617037, + -0.008135241, -0.027880974, -0.012495099, -0.017313058, 0.001684311, + 0.008798698, -0.0009724773, 0.01837143, -0.031751137, -0.01748682, + 0.02660145, 0.028212702, 0.007337514, 0.010710085, -0.0011501889, + -0.009351579, -0.0054221777, -0.007843005, -0.004513874, 0.0013496208, + 0.0018057472, -0.0010534348, -0.03367832, 0.015030451, -0.021767695, + 0.0015421417, -0.002835487, -0.0036095197, -0.003358749, -0.00883819, + -0.02017224, -0.00707687, -0.038196143, -0.0058170925, -0.005343195, + -0.0044664843, -0.011760558, -0.014651333, 0.0024484708, -0.021293797, + 0.006907057, -0.0074362424, 0.007068972, -0.00458101, -0.0008757232, + -0.013300725, -0.005145738, 0.007606056, 0.0029184192, 0.015472756, + 0.001705044, 0.006133024, -0.043882914, -0.0014414384, 0.017060312, + -0.007558666, -0.000001770174, 0.012139676, -0.012321336, 0.011112898, + 0.026806805, 0.0024109539, 0.007132158, -0.01563862, -0.0007414522, + -0.03077175, -0.0057854992, -0.0071755988, -0.029855546, 0.01309537, + 0.0065753288, -0.00045933507, -0.0023398693, 0.0009023799, + 0.008474868, -0.0033409777, -0.028260091, 0.027549244, 0.001026778, + 0.0034298336, -0.00976229, 0.0060619395, -0.0044625355, -0.018102886, + 0.002507708, -0.02211522, -0.004584959, 0.0072782766, -0.0013259259, + 0.0014068834, 0.00025965637, -0.0062159565, 0.010836458, 0.0034712995, + -0.0006126113, 0.011002322, 0.011997507, -0.009549036, -0.017076109, + 0.009035647, -0.007290124, -0.0062514986, 0.0029085462, 0.0051299413, + -0.0070571247, 0.011365643, -0.0033133337, -0.0069307517, + -0.0007932848, 0.0067332946, 0.000047019523, -0.0038662143, + -0.008901376, -0.018102886, 0.007471785, -0.00052918564, 0.011231372, + 0.005240517, 0.008909274, -0.019034885, -0.015109434, -0.012637269, + 0.010465238, -0.013450792, -0.017202482, -0.009604324, 0.0014355147, + -0.01938241, 0.004458586, -0.003617418, 0.015780788, -0.020472374, + -0.023536913, -0.0064963456, -0.0052800085, 0.019129666, -0.005852635, + -0.0030289954, 0.028718192, 0.012818929, -0.0037773584, -0.014264316, + -0.010670593, -0.013087471, -0.0061488207, -0.0052049747, + -0.0031237747, 0.009809679, 0.014540757, 0.015069943, -0.001147227, + -0.0008737486, 0.015907161, 0.008593342, -0.010417848, 0.0046836874, + -0.0154964505, -0.020235427, 0.018150277, -0.015109434, + 0.000048191927, 0.025716841, 0.008696021, 0.0057065166, 0.011231372, + 0.0161915, -0.0027584787, 0.0049759243, 0.011460423, 0.008364292, + -0.002873004, -0.013237539, 0.010678492, 0.021451764, 0.001533256, + -0.0012716252, 0.017392041, 0.013111166, 0.009454256, 0.015922958, + 0.020756714, -0.011073407, -0.000023062397, 0.0020170265, + -0.027170127, 0.005694669, -0.009904459, -0.009880764, 0.007313819, + -0.0049877716, -0.0007765009, -0.0011245194, 0.018292446, 0.009549036, + -0.012084388, -0.014864587, 0.00043613385, -0.010947034, -0.007554717, + -0.0037753838, -0.007234836, -0.015227908, 0.0037181212, -0.035321165, + 0.0050825514, -0.015054146, 0.028528633, 0.009193612, 0.013182251, + 0.020598747, 0.0012084388, -0.0042453324, -0.0051141446, 0.0014868536, + 0.005805245, -0.023189388, -0.0074836323, 0.0008609139, 0.012044896, + -0.0039886376, -0.017218279, 0.00846697, -0.017929124, 0.009596426, + 0.01138144, 0.00043662748, 0.002645928, -0.0031336476, -0.0075428695, + 0.0064963456, -0.022794472, -0.014785604, -0.02156234, -0.04423044, + -0.011681575, 0.01503835, -0.0076455474, -0.0021601832, 0.027501855, + -0.014540757, -0.028275888, 0.013513979, 0.01674438, 0.008893478, + -0.031672154, -0.005260263, 0.0010188797, -0.0013269131, + -0.0056828214, 0.03320442, -0.0016497559, -0.025432503, -0.019366615, + -0.0071874466, -0.02132539, -0.0010085133, 0.007945683, 0.004881145, + -0.0021068696, -0.026222331, -0.008036513, 0.0014651333, + -0.0072387853, 0.006448956, 0.013956283, 0.0037181212, -0.0017159041, + -0.0072506326, -0.031466797, 0.013261233, -0.000046834408, + -0.022794472, -0.008830291, 0.0011106974, -0.0024050302, -0.016965533, + 0.012226557, -0.017613193, -0.021672916, 0.0053826864, -0.0051891785, + -0.0061211768, -0.008846087, 0.002511657, 0.013766725, -0.012179167, + -0.0059513636, 0.012044896, -0.0057578553, -0.02410559, -0.010125611, + 0.019161258, -0.051338904, 0.0071677007, 0.018118683, -0.00069702434, + 0.009849171, 0.012779438, -0.010409949, 0.0043401117, -0.030124089, + 0.0060816854, -0.007396751, -0.0013802267, -0.004584959, + -0.0059474143, 0.006820176, 0.005212873, 0.011081304, -0.013071675, + 0.017518412, 0.028781379, 0.004541518, 0.011183983, -0.007795615, + 0.003968892, 0.003358749, -0.0048969416, 0.004541518, -0.012653065, + 0.017550007, -0.009785985, 0.0027031908, 0.008356393, 0.0020693527, + -0.022146814, 0.004419095, 0.01729726, 0.01947719, -0.014169537, + 0.012171269, 0.031940695, -0.011144491, 0.008609139, -0.009454256, + 0.0029954275, 0.028323278, 0.011420932, 0.000049210066, 0.0026479026, + 0.0092252055, -0.000521781, -0.004991721, 0.01813448, 0.017929124, + 0.020440781, -0.013822013, 0.03459452, 0.0022549625, -0.015891366, + -0.017976515, 0.009659612, 0.013245437, -0.009896561, -0.03164056, + -0.01698133, -0.0116973715, -0.002373437, -0.003937299, 0.0074283443, + -0.0013683792, -0.011547304, 0.0043480103, -0.035637096, 0.0037813077, + 0.0068083284, 0.0026242076, -0.011270864, 0.012013303, 0.024326742, + -0.0152674, 0.02345793, 0.015393772, 0.0065160915, -0.018339835, + -0.008909274, 0.013158556, -0.01165788, 0.0062238546, -0.010931237, + 0.017281465, -0.013664046, -0.0015944678, -0.010939135, -0.006907057, + -0.025669452, -0.016854957, 0.02123061, 0.017044516, 0.004569162, + -0.002104895, 0.0074480902, -0.00934368, 0.021514948, -0.010165103, + -0.020456579, -0.0009611235, -0.017076109, -0.010583712, 0.007799564, + 0.009643815, -0.0065516336, 0.008474868, 0.0036529603, 0.0018373403, + -0.0034337828, -0.000078427576, 0.0015283197, -0.0032856897, + 0.0013377733, 0.0032027578, 0.0012508921, -0.017170887, 0.007151904, + 0.00939107, -0.0019202724, 0.023047218, -0.013395504, -0.022841863, + 0.013237539, -0.014043164, 0.0018876919, -0.022826066, -0.018497802, + 0.020630341, 0.01952458, 0.013474488, -0.003937299, 0.001378252, + 0.01863997, -0.0012331209, -0.018008107, -0.014975163, 0.008190529, + 0.00966751, -0.009098833, 0.008348496, -0.004051824, 0.0018580734, + 0.009430561, -0.01628628, -0.0044230437, 0.0021483356, 0.0023655386, + -0.00874341, 0.0070887175, -0.007013684, -0.015733398, 0.016396856, + 0.003613469, -0.022920845, -0.014193232, -0.0039629685, -0.0017287388, + -0.0071834973, -0.033804692, 0.0027880974, -0.01138144, -0.0009946913, + 0.011073407, -0.0021917762, 0.0021522848, 0.012668861, 0.0019607511, + 0.0041505527, -0.0066938032, 0.004711332, 0.0060342955, 0.025100775, + 0.0029440885, -0.0011995533, 0.002470191, 0.013893097, 0.012123879, + 0.008277411, 0.0032758168, 0.02401081, 0.0023339456, -0.0118790325, + 0.020614544, -0.0059118723, 0.0010820661, 0.017913328, -0.0025235044, + -0.0058407877, -0.0018886792, 0.015433264, -0.0049759243, 0.028923549, + -0.00010156711, -0.009707002, -0.0041308072, 0.007827208, 0.014098452, + -0.011973811, -0.0072782766, -0.0049206363, 0.021278001, 0.0005963211, + -0.02614335, -0.004478332, 0.009264697, 0.0044309422, 0.008269512, + 0.0025649706, -0.011089203, -0.0011906676, 0.0226681, 0.012471404, + -0.0033014864, 0.0011511762, -0.010670593, -0.010196696, -0.012818929, + 0.007854852, 0.0062396512, -0.023047218, -0.00601455, -0.0133876065, + 0.015764993, -0.019872105, -0.0007064036, 0.0009611235, -0.0015737348, + 0.026490875, 0.011792151, -0.016807567, -0.029539615, -0.010054527, + 0.02040919, -0.00499567, 0.006824125, 0.0018521496, -0.010947034, + 0.023789657, 0.0050667548, 0.007037379, -0.010433645, -0.009004054, + -0.0026084112, 0.026285518, -0.00458101, -0.004371705, 0.0049759243, + 0.0026775212, -0.000063926804, -0.010504729, -0.004663942, + 0.0026794958, -0.0051299413, -0.009138324, -0.016949736, -0.002073302, + -0.0066740573, -0.014098452, -0.0028433853, 0.004213739, -0.02156234, + -0.005291856, -0.04381973, -0.011563101, 0.010338865, -0.01794492, + 0.0053352965, 0.003617418, -0.009075139, 0.010212492, -0.012913709, + -0.009541137, -0.024453115, 0.0036904772, 0.019745732, 0.036806043, + 0.02192566, -0.007420446, 0.015283196, -0.00043045694, 0.01355347, + -0.0058091944, -0.00730987, -0.0030921816, -0.020251222, + -0.0077403267, -0.0072940732, -0.0036648079, -0.0043282644, + -0.004458586, 0.0034594522, -0.03816455, -0.0065437355, -0.002600513, + 0.011065508, 0.017692175, 0.0035542317, -0.0031770882, -0.0061132787, + 0.0063067866, 0.006982091, -0.017139295, 0.022352168, -0.01068639, + -0.014682926, -0.015575433, 0.0024682165, -0.00085992663, + -0.032698933, 0.0043322137, 0.0073849037, 0.01327703, -0.011926422, + 0.013434996, 0.0062238546, -0.014983062, 0.017407836, 0.0101887975, + 0.011065508, -0.025179757, 0.0034890708, -0.0029756818, 0.01036256, + 0.019887902, -0.0072980225, -0.011918523, -0.029823953, -0.01161049, + -0.0062514986, 0.0061764647, -0.00026878878, 0.011270864, + -0.008846087, -0.010836458, -0.013837809, -0.021799289, 0.013087471, + 0.022036236, -0.04081838, 0.017328855, -0.0012015278, 0.017091906, + -0.033741508, 0.0009862992, -0.0021266153, 0.0034495792, 0.039586242, + 0.03803818, 0.018908512, -0.0018521496, -0.021767695, 0.019129666, + -0.021815084, -0.019556172, -0.010417848, -0.0012301591, + -0.0074836323, -0.003506842, 0.0070808195, 0.014651333, -0.009185715, + 0.048495516, 0.009193612, -0.0024306995, 0.0052247206, 0.022857659, + -0.005501161, 0.012866319, -0.014840892, -0.010607407, -0.004458586, + 0.0010484983, -0.0077798185, 0.002882877, 0.013016386, -0.011018118, + 0.0048653483, 0.0033705963, 0.00194298, -0.021830881, -0.010765373, + -0.013719334, -0.000696037, 0.025795825, 0.0026794958, 0.0040636715, + -0.017076109, -0.025574671, 0.011555202, 0.007839056, -0.027217517, + -0.019603562, -0.015101535, -0.016057229, 0.0054103304, 0.020282816, + 0.009951849, -0.026174942, 0.013103267, -0.011855338, -0.01952458, + -0.018545192, 0.0020298613, 0.04246122, -0.006010601, 0.0054853642, + -0.01554384, -0.0029124955, -0.0055169575, -0.0034179862, + 0.0056867707, -0.011918523, 0.028165312, -0.007613954, -0.004991721, + 0.015654417, -0.007519175, -0.016104618, -0.015417468, 0.015069943, + 0.004968026, 0.026332907, -0.011452524, -0.002002217, -0.026948975, + -0.007337514, 0.00035690411, -0.005145738, -0.02168871, 0.023394743, + -0.013221742, -0.017249871, 0.0017109676, 0.00149179, -0.0049008904, + -0.016570618, 0.008301105, -0.006606922, -0.051275715, 0.0019518656, + -0.0055524996, 0.019635156, -0.000107922766, -0.01369564, + -0.0028749786, -0.008072055, 0.021278001, 0.025969587, -0.022731286, + 0.030503206, -0.012755742, -0.0036865282, 0.0134192, -0.0009734646, + 0.0028117923, -0.0026360552, 0.0014345275, -0.026822602, -0.038132958, + 0.007341463, 0.011247169, 0.017865937, 0.00018881857, 0.019098071, + 0.020661933, -0.0071282093, -0.013774622, -0.012408217, -0.017044516, + -0.0016112517, -0.0009408841, -0.037848618, -0.016136212, + -0.0060619395, 0.011073407, -0.014888282, 0.010141408, 0.025179757, + 0.010465238, -0.016712787, -0.004541518, 0.004233485, 0.0069031077, + 0.0021700559, -0.0010781169, 0.0003021097, 0.017960718, -0.015630722, + 0.0026992415, -0.0079535805, 0.023789657, -0.0003499931, + -0.0023063014, 0.03077175, 0.0039945617, -0.019192852, -0.01475401, + -0.010646898, 0.0025590467, -0.0013200021, 0.015101535, 0.00013167935, + 0.022367965, -0.009912358, 0.012115981, -0.012779438, -0.0003043311, + -0.014959366, 0.004695535, -0.01679177, -0.011634185, -0.011942219, + 0.009193612, 0.004833755, 0.024374131, -0.0043361625, 0.006227804, + 0.0029460632, 0.0007503378, 0.021072645, -0.0004228055, 0.01837143, + 0.0057341605, -0.009083036, 0.023521116, 0.014785604, -0.010338865, + -0.0020575053, -0.0050035683, 0.0006032321, 0.012747845, -0.00633838, + 0.017613193, -0.008028614, -0.0055248556, 0.00014994416, -0.009146223, + 0.009296291, -0.023931827, -0.0060342955, -0.021246407, 0.016949736, + 0.0055209068, 0.0057539064, -0.008293208, 0.018055497, -0.010947034, + -0.008064156, -0.0070018363, 0.011349847, 0.022146814, 0.016412653, + 0.0068083284, -0.0068438705, 0.0040281294, -0.0029756818, 0.01059161, + -0.021625526, -0.00080068945, -0.013284928, 0.025432503, 0.00212859, + -0.02696477, -0.017407836, -0.010402052 + ] + } + ] + } ] diff --git a/src/scrapers/website/examples/www-pavolhejny-com.knowledge.json b/src/scrapers/website/examples/www-pavolhejny-com.knowledge.json index fbf2cc2fac..008410e5da 100644 --- a/src/scrapers/website/examples/www-pavolhejny-com.knowledge.json +++ b/src/scrapers/website/examples/www-pavolhejny-com.knowledge.json @@ -1,63 +1,1926 @@ [ - { - "name": "untitled", - "title": "Untitled", - "content": "---\n\n**Generative AI** is emerging as a transformative technology, comparable to the rise of personal computers and the early days of the internet.", - "keywords": [ - "Generative AI", - "transformative technology", - "personal computers", - "early days of the internet" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.026481831, 0.04921357, -0.002366284, -0.021460302, -0.027380828, -0.0036216667, 0.012605198, -0.006658986, -0.06981341, 0.021896956, -0.03893934, 0.025441566, -0.013189545, -0.0004924407, 0.0061902246, 0.013844527, -0.015963588, 0.04998414, 0.0066975146, -0.0013460849, 0.0354461, -0.0061163786, -0.060566597, 0.044179197, 0.012682254, 0.00824507, -0.030694267, 0.0064470805, -0.026327718, 0.0017578568, -0.003660195, -0.001222473, 0.045232303, 0.001574847, 0.009728413, -0.019392611, 0.0015339106, -0.020548463, 0.029872328, 0.034881018, 0.011256705, 0.0020708998, 0.018943114, 0.010145804, 0.028896276, -0.02853668, 0.0003991298, -0.036550578, 0.021190602, 0.023232607, 0.016002117, -0.014949007, -0.021293346, 0.041456524, 0.011834631, -0.004276649, 0.011937373, 0.019135756, -0.007423132, -0.050857447, 0.00016474891, -0.014332553, -0.024452671, 0.009426608, 0.012438241, 0.0215502, 0.017299237, -0.002600665, 0.04433331, 0.020317294, 0.029666845, 0.06642291, 0.0055320314, 0.029332932, 0.054299314, -0.008938582, -0.03149052, 0.01201443, -0.025955278, 0.013651885, 0.02559568, -0.028100023, -0.020188866, 0.0044949763, -0.0021094282, 0.0000010096138, 0.011256705, -0.073666245, 0.0030164502, 0.0036056133, 0.010672358, -0.013728942, -0.011584196, -0.020908061, -0.044487424, -0.027791796, 0.045566216, -0.010299917, -0.005217383, 0.024298558, 0.011956637, -0.002674511, -0.0404291, 0.01731208, 0.0005526413, 0.016438771, -0.0006758518, -0.006209489, -0.016143387, 0.013112488, 0.025955278, -0.003226751, -0.0071662767, -0.0046651433, -0.027201029, 0.0028061494, -0.009728413, -0.017800106, 0.044127826, 0.015115963, -0.0037211983, 0.014743523, -0.03226109, 0.006071429, -0.021601573, 0.010254968, 0.020432878, -0.017453352, 0.01961094, -0.032158345, 0.02591675, 0.011353026, -0.0068516284, 0.011333762, 0.009336708, -0.04225278, -0.0065401904, 0.031798746, 0.028100023, 0.048263203, -0.06390572, -0.007159855, 0.019790739, -0.00056187203, -0.0013733758, -0.025467252, -0.016952483, 0.01606633, 0.012733626, -0.026327718, -0.024195816, -0.013780314, -0.027509255, 0.00446608, 0.018095491, 0.020047594, 0.0060842717, -0.024414143, 0.05676513, -0.046901867, -0.018801844, -0.013587671, 0.028228452, -0.036807436, -0.011166805, -0.0064246054, -0.020201707, -0.00026046782, 0.0016093621, 0.0034482891, 0.042278465, -0.0054389215, -0.04572033, -0.07489915, 0.0060329004, -0.025261767, -0.014204125, -0.014846264, -0.005731095, 0.024375614, 0.008501926, -0.0031464836, 0.028022967, -0.009779784, -0.020908061, -0.022128126, 0.006890157, 0.023964645, 0.038554054, -0.021575887, 0.00079223956, 0.051448215, 0.013883055, 0.0060040043, 0.02905039, 0.0014680914, -0.011725467, -0.033365566, 0.015937902, 0.026032334, -0.054196574, 0.015385662, -0.02205107, -0.014782051, 0.0024593943, 0.015937902, -0.009535771, -0.0095743, 0.020869533, 0.030642897, -0.01035771, -0.017607465, -0.017787263, 0.042612378, -0.014936164, 0.02817708, -0.017183652, -0.007352497, -0.017389137, 0.0459515, 0.026404776, 0.037115663, 0.007513032, -0.026404776, -0.018557832, -0.009304601, -0.06303241, 0.0059751077, 0.004613772, -0.026918488, 0.032158345, 0.01790285, 0.0072625973, 0.056919243, -0.0379376, 0.06133716, 0.024555413, 0.0073268116, -0.01717081, 0.034315933, 0.027406514, 0.02899902, -0.00980547, -0.02463247, -0.007904737, -0.028074337, 0.023733474, 0.0059141046, 0.0075708246, 0.028716477, 0.00035177203, 0.0030084234, 0.051936243, 0.00994674, -0.011930952, 0.009709149, -0.005750359, 0.006633301, -0.008906474, 0.0073139686, 0.0019874217, 0.010370553, -0.017157966, 0.017479036, 0.005731095, -0.035112187, 0.0005357851, 0.0032652793, 0.0019537094, -0.029153133, -0.001703275, -0.004225278, 0.007288283, 0.017466193, 0.028613735, 0.010903528, 0.042612378, 0.02301428, -0.03125935, -0.04623404, -0.021113547, 0.009054166, 0.0051242732, -0.025518622, 0.0045559797, 0.01243182, -0.007943265, 0.027098287, 0.0022137759, -0.046799127, -0.011423661, 0.0044436054, 0.030899752, 0.025711264, 0.030129185, -0.0028848117, 0.0030662161, -0.0033487573, -0.046901867, 0.019508198, 0.018146861, 0.02661026, -0.007866208, -0.015231549, 0.0122648645, -0.015128806, 0.005050427, -0.00395879, -0.026160762, -0.05404246, -0.009221123, -0.053374633, 0.0028334404, 0.025698422, -0.011995166, -0.00033009984, 0.017222181, -0.014139911, 0.038502682, -0.017440509, -0.020009067, 0.0029875538, 0.018429402, 0.019906323, -0.012348342, 0.023399564, -0.000100685465, -0.012727205, 0.01592506, 0.01201443, -0.03457279, 0.004626615, -0.017812949, -0.05106293, 0.011051221, -0.012759311, 0.018763315, -0.008296442, 0.0010033429, 0.019970538, -0.004976581, 0.0057664127, 0.009908212, 0.037192717, 0.026507517, -0.02149883, -0.009818313, -0.0053971824, -0.013163859, 0.01740198, 0.000666621, 0.003351968, 0.05501851, -0.0039620004, 0.017363451, -0.006283335, -0.02499207, -0.02283448, 0.01606633, -0.039581478, -0.024439828, 0.0067809927, -0.009458714, -0.0022812006, -0.019482512, 0.019495355, 0.00073284167, 0.009400922, 0.030617211, 0.0062191207, 0.013240916, 0.014011484, -0.0122648645, 0.011956637, 0.013497772, 0.006960792, -0.017363451, -0.008604669, -0.012386871, -0.00061725656, 0.032440886, 0.025531465, -0.019944852, 0.00086769095, -0.0011991955, -0.0030260824, -0.0017899638, -0.017838635, -0.017337766, 0.011089749, -0.008084536, -0.013382187, -0.00469725, 0.0030517678, -0.0038464156, 0.017517565, -0.044281937, -0.0033583895, 0.028896276, 0.0041642743, 0.023232607, -0.03056584, 0.002181669, 0.031336408, -0.020753948, -0.0052976506, -0.0019970539, -0.057124726, 0.012662991, -0.014011484, -0.024786584, -0.0030180556, 0.003393707, -0.012322657, -0.02899902, -0.05008688, -0.016541513, 0.03714135, 0.015488404, 0.034059078, -0.01449951, -0.037911914, 0.0063314955, 0.0039555794, -0.015411347, 0.0030276875, 0.012740048, 0.025698422, -0.0381174, -0.021357559, -0.031464834, -0.009477979, 0.0027917014, 0.010306339, -0.015000379, 0.050985876, -0.03688449, 0.0045591905, 0.03978696, 0.013356501, -0.02946136, 0.005962265, 0.032338142, -0.018095491, -0.013574828, 0.022487724, 0.02684143, 0.01873763, 0.024337087, 0.056919243, -0.014165597, 0.03434162, 0.016567199, 0.011622725, 0.007416711, 0.0011245467, -0.042920604, 0.012496035, -0.007988215, 0.031798746, 0.023399564, -0.023181235, 0.014345396, -0.0352663, -0.013972955, 0.02513334, 0.0461313, 0.00971557, 0.000971236, -0.017877163, -0.0072561763, -0.025582837, -0.039581478, 0.02541588, -0.045925815, -0.0031063498, 0.004482134, 0.012996903, 0.022962907, -0.015848001, -0.040814385, 0.022718895, -0.044924077, 0.015950745, -0.0050632698, 0.026147919, -0.025929593, -0.027663369, 0.0106274085, -0.015167334, 0.052295838, 0.0053522326, 0.008765204, -0.020227393, 0.010807208, 0.0070378487, -0.03724409, -0.055223994, 0.02370779, 0.033005968, -0.0033969178, 0.03084838, 0.037629373, -0.0029153132, 0.003204276, -0.0039427364, -0.02030445, -0.03521493, 0.013472087, 0.016323186, 0.061491277, 0.0035125029, -0.028279822, -0.020009067, 0.016913954, -0.015475562, -0.012386871, -0.010210018, 0.029409988, -0.009895369, -0.009580721, 0.011622725, -0.0025525044, 0.006543401, -0.019790739, 0.012579513, 0.0033038077, -0.0009977242, 0.029872328, -0.021241974, -0.033262827, 0.0057696234, 0.0056090886, 0.028022967, 0.06405983, -0.02513334, -0.02126766, -0.019238498, -0.0017867531, 0.042715117, -0.03351968, -0.057535697, 0.012412556, 0.03190149, 0.01850646, 0.006154907, -0.0047807284, -0.012207071, 0.0028703634, -0.03167032, 0.053888347, -0.014949007, 0.005888419, 0.0030581893, -0.014653623, -0.007121327, -0.014692151, -0.02007328, -0.037038606, -0.013330815, 0.02140893, 0.011243862, 0.021845585, -0.004456448, 0.0062865457, 0.015475562, 0.016002117, -0.011282391, 0.005410025, 0.0017562514, 0.011218177, 0.012451084, 0.026327718, 0.0064663445, 0.019906323, 0.011719045, -0.018660573, -0.015398505, 0.012367606, 0.011423661, -0.008360656, 0.019649468, 0.031387776, 0.025390195, 0.0005686948, 0.0066782506, -0.00814875, -0.030077813, -0.0034033393, 0.044769965, 0.036627635, 0.023450933, -0.0029265506, 0.015115963, 0.0010563195, 0.009060588, -0.027534941, 0.015681047, -0.028665107, -0.029435674, 0.0068387855, -0.0041450104, 0.015732417, -0.020458564, -0.024388457, 0.008463399, 0.023643576, 0.0153086055, 0.018904585, 0.010511823, 0.0024465513, 0.020831006, 0.0017129071, -0.008026743, -0.032286774, -0.006315442, 0.03403339, 0.029076075, 0.010800786, -0.022359297, -0.014294025, -0.024414143, -0.0074552395, -0.009766942, 0.025377352, -0.021421773, 0.012451084, -0.016323186, -0.0031175872, 0.0018814687, -0.001079597, 0.009914634, -0.015526933, 0.006459923, -0.01473068, -0.019687997, 0.0061805923, -0.008418448, -0.02080532, -0.012784997, 0.0026488253, 0.058306266, 0.016926797, 0.03624235, -0.004639458, 0.00016755828, -0.021100704, 0.024902169, 0.007191962, 0.004899524, 0.055994563, -0.028048653, 0.009477979, -0.0011646805, -0.00091825944, 0.01800559, 0.02776611, -0.030257612, 0.028048653, -0.003241199, -0.016836897, 0.014897636, 0.0004274241, 0.006877314, 0.012412556, 0.020908061, -0.03737252, 0.038091715, 0.017928535, -0.0034771853, -0.014332553, 0.017915692, -0.004116114, 0.011449347, -0.010132961, 0.0067938357, 0.0032797274, -0.0069351066, -0.010486138, 0.0020548464, 0.010486138, -0.0164773, -0.02946136, -0.005731095, 0.013818841, -0.021190602, 0.044025082, 0.03673038, -0.005506346, -0.01758178, 0.019366927, 0.033493996, -0.0018959168, -0.007827681, 0.031053865, 0.004337652, -0.0011686939, 0.031233665, 0.01150714, -0.014846264, 0.0020773213, 0.023373878, 0.01740198, -0.039966762, 0.010736573, 0.009766942, -0.0018333083, -0.024619628, -0.05224447, -0.008643198, 0.006703936, -0.004626615, 0.030694267, -0.011076906, 0.053888347, -0.035112187, 0.040583216, -0.035137873, 0.008495505, 0.039966762, 0.02375916, 0.008707412, -0.053785603, -0.012117172, -0.03236383, 0.008309285, -0.029230189, -0.025788322, 0.01473068, 0.0060650073, -0.038399942, 0.009599986, 0.026250662, -0.009266073, 0.026738688, -0.001855783, -0.013818841, -0.02499207, 0.022962907, -0.007172698, 0.0067424644, -0.032081287, 0.04374254, 0.006652565, 0.0006770558, 0.017055225, -0.0025187922, -0.003660195, -0.059179574, 0.0028510992, -0.038656797, -0.019816425, 0.038785223, 0.016271815, 0.0037500947, 0.00022956486, 0.012142858, 0.0070378487, 0.01588653, 0.0358057, 0.023656419, 0.010858579, 0.00202595, 0.0025492937, 0.0077827303, 0.0022137759, 0.004816046, -0.010113697, -0.0027130393, 0.0065947725, 0.03323714, -0.0023261502, -0.005262333, -0.0018188601, 0.022397825, -0.00060922984, 0.010967743, -0.011988744, 0.019662311, -0.03398202, 0.034649845, -0.006071429, 0.002319729, -0.0023405985, -0.007577246, -0.019392611, 0.009978848, -0.002969895, 0.0044725016, -0.013818841, -0.008444134, -0.004597719, -0.017684521, 0.010537509, -0.0093945, -0.016541513, 0.0115199825, -0.012194228, 0.044410367, 0.006601194, 0.012309814, 0.019598097, -0.010017375, 0.010909949, -0.030334668, 0.0030533732, 0.007179119, 0.013985798, -0.016708469, 0.018211076, 0.040994186, -0.022269398, 0.020009067, -0.0081102215, -0.036422152, -0.027534941, 0.016567199, -0.0024096284, 0.00019906324, -0.02003475, 0.034778275, -0.021421773, 0.0432802, -0.010794365, 0.020908061, 0.02306565, 0.023720633, -0.026481831, 0.00043344416, 0.018955957, -0.010730151, -0.0024609996, -0.013703257, -0.009734835, -0.00971557, 0.011564932, 0.0070185843, 0.0067681503, 0.02928156, -0.009323865, -0.0076157744, -0.015462719, 0.027971596, -0.0072947047, 0.010210018, 0.04199592, 0.0072112265, -0.054813027, 0.015038907, -0.04148221, -0.012984061, 0.017890006, -0.01510312, 0.019726524, -0.025467252, 0.02666163, 0.024683842, 0.0083542345, 0.0038110977, 0.008450556, 0.007943265, -0.012573091, 0.03102818, 0.0044500264, 0.004931631, 0.004074375, 0.0023839427, 0.012001587, -0.018724788, -0.0052655437, 0.015090277, 0.0019248131, -0.013960112, -0.024645314, -0.007493768, 0.014692151, -0.009291758, -0.01937977, -0.012252022, 0.013446401, -0.025390195, -0.004116114, -0.010556773, -0.033262827, 0.024285715, 0.03084838, -0.019970538, -0.012194228, 0.012149279, -0.021794215, 0.0012915031, 0.004543137, -0.0024064176, -0.006090693, 0.012457506, 0.025801165, -0.018917428, -0.020086123, -0.015629675, 0.027689055, 0.020381507, -0.008668883, 0.0035574527, -0.017235024, -0.0040165824, 0.00022976553, 0.023656419, 0.0075066104, -0.0037372517, -0.0025974542, -0.00833497, 0.012374028, 0.0052591222, -0.03369948, -0.00865604, 0.001333242, -0.02840825, -0.0136262, -0.00015060177, 0.031002494, 0.03231246, 0.020060437, -0.02817708, 0.0046651433, -0.00045351102, -0.0055866134, 0.014396767, -0.023502305, -0.01597643, -0.0005907683, 0.015603989, 0.0076992526, 0.0073717614, 0.006068218, 0.01864773, -0.021370402, -0.038887966, 0.0018943115, -0.01072373, 0.028973334, -0.008540455, 0.016567199, -0.023001436, -0.01836519, -0.0067103575, 0.044949763, -0.01868626, 0.025081968, 0.038476996, -0.0025043439, 0.010909949, 0.013202388, -0.010800786, 0.011410818, 0.047030296, 0.008277178, -0.025030596, 0.03218403, -0.01946967, 0.019135756, 0.004957317, 0.0035478207, -0.030899752, -0.008636776, 0.013651885, -0.008822996, -0.018390875, -0.06477903, 0.000050116978, -0.0009913028, 0.024157288, -0.0140500115, -0.015655361, -0.015205863, 0.00051050086, 0.03601118, -0.007724938, 0.0053522326, 0.024773741, 0.02559568, 0.015642518, 0.021627259, 0.007564403, 0.042021606, 0.02858805, -0.017568937, -0.022115285, -0.016053487, -0.004209224, 0.0015001984, 0.022256555, -0.025608523, -0.0017000643, -0.0069543705, 0.052655436, -0.010614566, 0.0023405985, 0.0005803336, -0.0041321674, 0.014704994, 0.035291985, -0.003554242, -0.018609202, -0.00071518286, -0.01634887, 0.02905039, -0.044179197, 0.0028832064, -0.014627937, -0.0056861453, 0.021370402, 0.0010282259, 0.032517944, -0.025801165, -0.008360656, -0.025069125, -0.0075001894, -0.016657097, -0.018519303, -0.00478715, 0.00285431, 0.018095491, 0.0045367153, -0.012733626, -0.0042284885, 0.010832893, 0.014627937, 0.010351288, 0.02324545, 0.02393896, 0.021807056, 0.00014438105, -0.0044949763, 0.014062854, -0.034290247, -0.007930422, -0.008135907, 0.05589182, 0.0023887588, 0.017697364, 0.012746469, -0.035985496, 0.021896956, 0.0022009332, 0.010473295, -0.014011484, -0.008084536, 0.003798255, 0.018172547, 0.01772305, 0.00055344397, 0.0025075546, -0.00028073534, 0.0032973862, -0.033622425, -0.008501926, 0.02858805, 0.02545441, 0.0018509671, -0.017568937, 0.005644406, -0.012528141, 0.006684672, -0.0027290927, 0.02066405, -0.00060481514, -0.023990331, 0.017607465, 0.0037372517, 0.009580721, 0.0019681575, -0.009709149, 0.035189245, -0.020766791, 0.019816425, 0.016104858, -0.01505175, -0.01431971, -0.0036762485, -0.0329546, -0.0017369873, 0.018211076, -0.018532146, 0.031875804, 0.015770946, -0.03816877, 0.012560248, 0.006251228, -0.013003324, 0.015501247, -0.016284658, -0.014576566, 0.0028575207, -0.009182595, 0.05332326, -0.024837956, 0.0155783035, 0.049316313, -0.013549143, -0.014704994, -0.016913954, -0.0056989877, 0.0055577173, -0.053374633, 0.0032973862, 0.004032636, -0.03940168, -0.0002867554, -0.034906704, -0.012964796, -0.008938582, 0.032517944, -0.021473145, 0.014846264, 0.013330815, 0.021241974, -0.017979905, -0.0019392612, -0.0050632698, -0.020959433, -0.011012692, -0.014692151, -0.029384302, 0.0020291607, 0.023104178, -0.023669261, -0.012849211, -0.0074295537, 0.018339504, -0.0155783035, -0.0077313595, 0.019777896, -0.016246129, -0.03770643, 0.0075322962, 0.0030854803, 0.01615623, -0.024593942, 0.028742163, -0.021768529, 0.015038907, 0.0031962493, -0.013138174, -0.020291608, 0.02158873, -0.01026781, 0.034007706, -0.0040390575, 0.0032829382, -0.013998641, -0.028433936, -0.005721463, 0.00814875, -0.0011510351, -0.016888268, -0.008784468, -0.0067360434, -0.01054393, -0.03411045, 0.022295082, -0.021935485, -0.001362941, -0.008168014, 0.0105182445, -0.0005365878, -0.0017626729, 0.021396087, 0.00015551815, 0.0028302297, 0.03246657, 0.0010458847, 0.0007224069, -0.018172547, -0.029332932, 0.008437713, -0.026687317, 0.00598474, -0.0076157744, -0.00059116964, 0.011744731, -0.008945003, -0.01164841, -0.020895218, 0.013151017, -0.013048274, -0.026353404, -0.008296442, -0.019199971, -0.031978548, -0.0028607314, 0.010132961, -0.01675984, 0.037834857, 0.023605047, 0.0048642065, -0.01449951, -0.000033310986, -0.0012296971, 0.012829946, 0.026481831, -0.017915692, 0.031233665, -0.011372291, -0.009728413, 0.014615094, -0.017967062, 0.0014416032, -0.011038378, -0.002725882, -0.0219098, 0.013163859, 0.012271285, 0.014473824, -0.02518471, 0.012842789, -0.008893631, -0.038143083, 0.014127068, 0.03246657, -0.0035799276, 0.0042702276, 0.0012642121, -0.0350865, 0.024131602, 0.0062769135, 0.016554356, -0.019816425, 0.010710887, -0.007269019, 0.005853101, -0.010100854, 0.013844527, -0.012817104, -0.0036955127, -0.0057888874, -0.031362094, -0.025403038, 0.029589787, 0.0043665483, 0.02292438, 0.020792477, -0.006228753, 0.028254136, -0.0032235403, -0.0017385926, 0.020086123, -0.035780013, -0.032029916, 0.015398505, -0.037860543, 0.0049348418, 0.0086110905, 0.011385133, -0.019122913, -0.02541588, -0.011622725, -0.029101761, 0.011808945, -0.020753948, 0.000802273, 0.019559568, -0.004616983, -0.005695777, 0.029076075, 0.021485986, 0.008168014, -0.013189545, -0.0007500992, 0.011320919, -0.0008291626, -0.0012401319, 0.025325982, -0.015270077, -0.015180177, 0.018198233, 0.019546725, -0.019803582, -0.007737781, 0.03310871, 0.0123997135, -0.0016583252, 0.008078115, -0.0016647466, -0.0054710284, -0.002546083, -0.00216401, 0.025980964, -0.008283599, 0.024439828, 0.00644387, 0.03195286, 0.011494297, 0.012862054, 0.003083875, -0.02545441, -0.00893216, 0.023797689, -0.04723578, -0.005422868, -0.001841335, 0.025801165, -0.02587822, -0.0097476775, -0.019996224, -0.008033165, -0.010601723, -0.019456826, -0.0005947817, -0.009182595, 0.003157721, 0.020895218, 0.0045463475, -0.022256555, -0.006036111, 0.004379391, -0.0069543705, 0.018082649, -0.004783939, -0.020497093, 0.0016181915, -0.050035506, -0.0018172547, -0.013523458, 0.0016872215, 0.01449951, -0.009831156, 0.02035582, 0.0033327038, -0.00019735756, 0.02205107, -0.0037725694, 0.0074102897, -0.020728262, -0.0037308303, -0.012303392, -0.02518471, 0.023258291, -0.02481227, 0.04538642, 0.0064213946, 0.014255497, 0.02853668, -0.00010665535, 0.016104858, 0.01173831, 0.004915578, -0.00045190565, -0.019122913, 0.01288774, -0.027278084, -0.008501926, 0.027534941, -0.023874747, 0.0032540418, -0.031362094, 0.0115199825, -0.005833837, 0.013112488, 0.015783789, 0.012720783, 0.0072625973, 0.02789454, 0.023052808, -0.0028880222, -0.0057664127, -0.015629675, -0.00547745, -0.017658835, -0.010807208, -0.011044799, 0.019443983, 0.015347133, 0.015038907, 0.021139232, 0.0032700952, -0.0034258142, -0.0051949085, -0.021884114, -0.0034129713, -0.0071470123, -0.0029650792, -0.0012634094, 0.012027272, 0.004658722, 0.013587671, 0.032286774, 0.025017753, 0.0057439376, 0.01279784, 0.0076029315, 0.0023213343, 0.0012834762, 0.011083327, 0.0011735099, -0.010768679, 0.0066268793, 0.0047261463, -0.0077056736, 0.004061532, 0.0025974542, -0.0019023382, -0.0061805923, 0.001878258, -0.011275969, 0.04949611, -0.0046940395, 0.0021014016, -0.007423132, -0.013960112, 0.013728942, 0.004581665, -0.015526933, 0.028279822, -0.035112187, -0.008142329, -0.023810532, -0.010280653, 0.031798746, 0.007236912, -0.016271815, 0.009028481, -0.01072373, -0.019803582, 0.009978848, -0.00018702312, -0.036807436, -0.016772684, -0.02066405, -0.007757045, -0.0012987271, -0.008816575, 0.011828209, -0.015899373, -0.004485344, 0.009535771, -0.00024240765, -0.003586349, -0.0023373878, -0.013080381, -0.0056604594, -0.014294025, -0.010640251, -0.0023839427, 0.026353404, -0.011712624, 0.0014335763, -0.0077827303, -0.033648107, -0.011719045, -0.0059911613, -0.027046915, 0.0020548464, -0.012618041, -0.009567878, 0.023861904, 0.012425399, 0.01666994, 0.02434993, 0.006344338, -0.017157966, 0.005288019, -0.010967743, 0.011796102, 0.015539776, 0.0049284203, -0.02016318, -0.00012391285, 0.015372819, -0.008822996, 0.0064984513, -0.0065530334, 0.0005153169, 0.0048513636, 0.022808794, 0.013960112, 0.025467252, 0.032158345, 0.014024327, -0.019212814, -0.014460981, 0.0033969178, 0.016258972, 0.0043858127, 0.0030662161, -0.003522135, 0.02408023, -0.006658986, 0.0034129713, 0.0012481586, 0.026918488, -0.02794591, 0.022436354, 0.02237214, -0.012772154, -0.019944852, -0.0022763845, 0.016734155, 0.015822317, 0.017055225, 0.028870592, -0.0021190604, 0.01473068, -0.013780314, -0.014512353, 0.0013284261, 0.013215231, 0.012528141, -0.034726903, -0.0023807322, -0.011866737, 0.011886002, 0.028485307, -0.007757045, 0.020586992, -0.03269774, 0.02173, 0.020753948, -0.001878258, 0.018416561, 0.0023887588, 0.021832742, 0.045823075, 0.029692529, -0.01763315, 0.014101383, -0.021370402, 0.0040904284, 0.007204805, -0.0015828739, -0.028921962, -0.0025942435, 0.017979905, -0.02748357, -0.010858579, -0.0037147768, 0.011430083, -0.011385133, 0.0054581854, -0.0244912, -0.008668883, -0.008758782, 0.0032620686, 0.016284658, -0.0020692945, -0.011051221, 0.018275289, 0.0015347133, -0.007879051, -0.021010803, -0.0072754403, -0.042201407, -0.007885473, 0.004254174, -0.0069094207, -0.0030967176, 0.019418297, -0.007795573, 0.034906704, 0.000084631974, 0.009433029, -0.00446608, 0.02481227, 0.010210018, -0.0062865457, -0.0019039436, -0.011173227, -0.0022812006, -0.011622725, 0.00704427, -0.014525195, 0.0077827303, 0.007821259, 0.0076221954, 0.003907419, -0.0242215, -0.007288283, -0.0145637235, -0.0011422056, 0.012040115, 0.013099645, -0.024735212, -0.0011767206, -0.02269321, -0.004135378, 0.014371081, 0.013870213, -0.013016167, -0.001703275, 0.02518471, -0.0021624046, -0.0053811288, 0.0071662767, 0.0016230076, 0.0006746478, -0.011038378, -0.013433558, 0.02375916, -0.014717837, -0.009638513, -0.025428724, -0.00584668, 0.01588653, -0.033802222, 0.0022908326, -0.009587143, -0.0016422718, 0.009683464, -0.004539926, -0.002513976, -0.0036698272, -0.00855972, -0.019366927, -0.0048642065, -0.005994372, -0.02223087, 0.016143387, -0.0007814035, -0.013433558, -0.008521191, 0.006652565, -0.012059379, -0.010813629, -0.012354763, 0.039324623, -0.022590468, 0.011597039, -0.007140591, -0.014717837, 0.0066654077, 0.01003664, -0.0013252153, 0.013497772, 0.021049332, 0.0014761181, -0.0072433334, -0.005088955, 0.019790739, -0.011918109, 0.007814838, 0.0047807284, 0.041816123, -0.0072754403, -0.013041853, 0.03560021, 0.025197553, -0.009336708, 0.0014464192, -0.0005991964, -0.00024702304, -0.0116548315, 0.0018525724, -0.004437184, 0.0035413993, 0.010145804, -0.007994637, -0.027278084, -0.0058402587, -0.010563195, 0.00013474896, -0.00088454713, 0.016284658, 0.0065209265, 0.008964268, -0.0103833955, 0.009240387, 0.020291608, 0.012008009, -0.016400242, -0.0127914185, -0.024439828, 0.009176173, -0.0039555794, 0.0022539096, -0.011468611, -0.0060168467, 0.0060842717, -0.020728262, 0.0143068675, 0.011301654, -0.0064470805, 0.009214701, -0.009099117, 0.039941076, 0.004722936, 0.0064759767, 0.0021431404, -0.01449951, -0.0075451387, -0.0040390575, -0.015488404, 0.0099916905, -0.0032941755, -0.00019805989, -0.009696306, -0.00038347766, 0.011571353, 0.0046779863, -0.01649014, 0.005419657, -0.025300296, 0.004010161, -0.013844527, 0.007692831, -0.0009856841, -0.0058434694, -0.019559568, -0.018121175, 0.010486138, 0.0014600647, 0.0013220047, 0.00095598516, -0.004957317, 0.006556244, -0.0012706335, 0.016708469, 0.015398505, 0.023489462, 0.013279445, -0.0001430767, 0.016130544, 0.0035349778, -0.022436354, -0.011346605, 0.026995543, 0.005496714, 0.004411498, -0.009811891, 0.008758782, -0.008970688, -0.026533203, -0.012085065, -0.010858579, -0.0001264212, -0.015732417, -0.0062640705, -0.013253759, 0.0017787263, -0.00575357, -0.003917051, 0.006081061, -0.007005742, 0.0060200575, 0.015950745, 0.00382073, -0.028151395, 0.027149657, -0.008906474, 0.007679988, 0.0033583895, 0.012226336, 0.004170696, -0.008842261, 0.0023951803, 0.0048353104, -0.0008765204, -0.032903228, -0.029358618, 0.014576566, 0.0054581854, 0.0143068675, 0.0026311665, 0.0047582537, 0.0013838105, 0.004437184, 0.012695097, 0.01464078, 0.029332932, -0.016258972, 0.019187128, 0.008534034, 0.01464078, 0.030488782, -0.007224069, 0.016926797, 0.0056604594, -0.004828889, 0.0049637384, 0.008765204, -0.023515148, 0.01440961, -0.03369948, -0.014062854, -0.018557832, -0.008142329, -0.030411726, 0.012926267, -0.012457506, 0.0070506916, -0.005262333, 0.025711264, -0.0008395974, 0.00066341035, -0.014345396, -0.013587671, 0.012515298, -0.017183652, -0.010409081, 0.03863111, 0.019456826, 0.03701292, -0.00023919696, -0.012727205, 0.015719574, -0.012733626, -0.025287453, -0.0035895596, -0.000015940612, 0.006440659, -0.0045367153, 0.024452671, 0.00043665484, 0.009831156, 0.009555035, -0.017877163, 0.0111411195, 0.007814838, 0.023181235, 0.006684672, -0.01141724, 0.010556773, -0.0057439376, -0.00029177213, -0.046259727, -0.00013384596, 0.014242654, -0.004915578, 0.014139911, -0.0013693625, 0.010254968, -0.027586313, 0.010935635, -0.000024381234, 0.0048674173, 0.0008516375, -0.009818313, -0.020908061, 0.013703257, 0.0039106295, 0.008578983, -0.02454257, 0.010338445, -0.0054838713, -0.0071277483, 0.005593035, 0.015411347, -0.00013163859, -0.0010306338, -0.003364811, -0.016310344, 0.030617211, 0.0008267546, 0.013960112, 0.0068516284, 0.0017947798, -0.006748886, 0.0009961189, -0.008778047, -0.0033198611, 0.006357181, 0.0009616039, 0.004254174, -0.011160384, -0.027046915, -0.0011285602, -0.012136436, 0.009587143, 0.011308076, -0.02375916, 0.017466193, -0.031875804, 0.0040294253, -0.0069672135, -0.010742993, -0.015758103, -0.0012385265, -0.0070506916, -0.0070250058, -0.003586349, -0.023219764, -0.00589484, 0.008309285, -0.001671168, -0.0031705636, 0.022282239, -0.028896276, 0.017812949, -0.026507517, 0.008450556, 0.017761579, 0.009831156, -0.017299237, -0.008482663, -0.024439828, -0.009696306, 0.006270492, -0.018583516, -0.0127785755, -0.00028595273, -0.0065369797, -0.0029843433, 0.0051371157, 0.0061452747, -0.0055962456, -0.028690793, -0.013998641, 0.008052429, 0.0162076, -0.0019039436, 0.0016270209, -0.008964268, -0.026815744, 0.009644935, -0.030000757, -0.0063475487, -0.0029201293, 0.010287074, -0.00014347804, 0.012444663, 0.0059718974, 0.007879051, 0.013484929, -0.008277178, -0.0037789908, -0.021845585, -0.015706731, 0.013472087, -0.010158647, 0.00031605302, -0.01542419, 0.025120497, 0.010595301, 0.008970688, -0.0067809927, 0.006257649, 0.039735593, -0.0028478885, -0.0040390575, 0.031130921, -0.032492258, -0.0009118381, 0.025248924, -0.011539247, 0.022885852, 0.007667145, -0.01040266, 0.0024786585, 0.004986213, 0.009002795, -0.02053562, -0.021896956, -0.0066268793, 0.00014387938, 0.009233966, -0.015706731, 0.00072280824, -0.022577625, -0.007159855, 0.004498187, 0.01772305, 0.03195286, -0.011937373, 0.021858428, -0.012219914, -0.021922642, 0.004713304, -0.007140591, 0.0076864096, 0.0014175229, 0.02357936, 0.013125331, 0.023283977, -0.0003373239, 0.011198913, 0.030180555, 0.008514769, 0.00069351064, -0.0034001286, 0.011153962, -0.019649468, -0.016554356, -0.020599835, 0.021575887, -0.021485986, -0.00589484, 0.014062854, 0.00007720724, -0.019161442, 0.002610297, -0.005326547, -0.0006112365, 0.00076133665, -0.016708469, 0.0022169866, -0.008367077, -0.024786584, -0.011340183, 0.010030218, 0.005763202, -0.013960112, -0.0067553073, 0.0026761163, -0.013960112, -0.0027066178, 0.0055127675, -0.0052527008, 0.004860996, 0.015154492, 0.0148077365, 0.00003150497, 0.002297254, 0.0067553073, 0.011385133, 0.0038945759, 0.020407192, -0.02509481, 0.015732417, 0.009433029, -0.006055375, 0.040994186, 0.0023100968, 0.00244013, -0.020098966, -0.00855972, 0.01615623, 0.026404776, -0.0014857502, -0.0016101648, 0.0021158496, 0.010871422, -0.012720783, -0.006819521, -0.009933898, -0.023052808, -0.018108333, 0.0015917033, 0.0031785904, 0.013844527, 0.019854954, 0.028254136, 0.023656419, 0.015963588, -0.01882753, -0.012091487, 0.007891894, 0.011873159, 0.0073589184, -0.019906323, -0.02016318, 0.004337652, 0.00034193927, 0.006052165, 0.015668204, 0.002435314, 0.008399185, 0.010306339, 0.0010266205, -0.015437033, 0.015205863, -0.0013525062, 0.038682483, 0.00063772476, -0.013613357, 0.026263505, 0.015437033, -0.016130544, 0.007236912, 0.010980585, 0.012945532, 0.00870099, -0.008187278, 0.0048963134, 0.014846264, 0.00009371223, 0.013908741, 0.012425399, -0.005069691, 0.006800257, 0.018673416, -0.0005787282, 0.026327718, -0.020612678, -0.024889326, -0.0035991918, 0.00019655489, -0.020009067, 0.017247867, -0.022012541, 0.023682104, 0.0069029992, 0.023027122, -0.012130015, 0.00515638, -0.0032556471, -0.0038656797, 0.013266602, -0.0029056813, -0.00764146, -0.012123593, 0.011532825, -0.011218177, -0.008232228, -0.019880638, 0.023463776, -0.00070153736, 0.009927477, -0.013780314, -0.0299237, 0.023258291, -0.0045784544, -0.0044628694, 0.005943001, -0.011590618, -0.004915578, -0.010010955, -0.009677042, 0.0016302316, -0.016952483, 0.008212964, 0.03190149, -0.001942472, -0.007352497, -0.021832742, 0.0034611318, -0.01542419, 0.020060437, -0.00354461, -0.017003853, 0.015000379, 0.0107622575, -0.0011036772, 0.0069543705, 0.00025886248, -0.0019986592, -0.0008460188, -0.02881922, 0.009047746, -0.005981529, -0.013960112, 0.0029538416, 0.0071020625, 0.022539096, -0.0022346454, 0.011083327, 0.019302713, -0.02301428, 0.022539096, -0.0038271514, -0.0019167863, -0.018814687, -0.023283977, 0.01675984, 0.0004017385, 0.029204503, -0.003692302, 0.0013059512, 0.005647617, -0.011622725, -0.01786432, 0.008585405, -0.019867796, 0.0038817332, 0.01330513, -0.0017113017, 0.016644256, 0.018609202, 0.0017899638, 0.0047614644, 0.019765053, 0.019803582, -0.020599835, -0.006469555, 0.020766791, 0.020240236, -0.011333762, 0.0098696835, 0.0020741106, -0.0051082196, -0.00418996, 0.016002117, 0.000589163, 0.009876105, 0.0019216024, 0.0006646144, 0.0029265506, -0.037321147, 0.015013221, -0.04397371, -0.012919846, -0.027201029, 0.006524137, 0.019598097, 0.012990482, 0.013343658, 0.016965326, -0.015912216, -0.0012192624, 0.024465514, -0.010184332, -0.003393707, 0.00034033394, -0.01542419, -0.004138589, 0.020381507, 0.0002448157, -0.0015892952, -0.01473068, 0.011391554, -0.026790058, 0.0024979226, -0.014435296, 0.015270077, 0.008456977, 0.0067553073, 0.0148077365, -0.0043023345, -0.021164916, -0.0035638742, 0.025492938, -0.014666466, 0.0034290247, -0.0013444795, -0.0136262, 0.0101265395, -0.01196948, 0.005464607, -0.0047582537, -0.0033230719, -0.023309663, -0.01606633, 0.04068596, 0.005853101, 0.009612828, -0.018378032, 0.009375236, 0.0071983836, 0.0012738442, 0.02067689, -0.027611997, -0.011532825, -0.0127914185, 0.021049332, 0.0039523686, 0.028202767, -0.012046536, -0.003480396, -0.019443983, 0.014204125, 0.007583667, 0.0020468195, 0.0034418677, -0.023733474, 0.02187127, -0.011834631, -0.0043472843, -0.0034482891, 0.0020933747, 0.016014958, -0.01178968, 0.009105538, -0.011430083, -0.0010563195, -0.017003853, -0.0036955127, -0.0072947047, 0.025261767, 0.009047746, -0.0000065593545, -0.004902735, 0.001979395, -0.033853594, -0.004860996, -0.0010466874, 0.034958072, 0.02158873, -0.031233665, 0.025467252, -0.019765053, -0.011686939, 0.031053865, -0.015437033, -0.011545667, -0.0046490896, 0.021216288, 0.007468082, -0.008822996, -0.0016109674, 0.018018434, 0.0021030067, 0.0075194533, -0.017825792, 0.022179497, 0.009368815, 0.00319946, -0.016811213, 0.044769965, 0.0038817332, 0.015603989, 0.007435975, 0.008322128, 0.0150645925, 0.01155851, -0.0010121723, -0.0043440736, 0.01758178, -0.00056187203, 0.005474239, 0.0032668847, -0.009330287, 0.0092468085, -0.0070378487, 0.009657778, -0.0021656153, 0.01684974, -0.016323186, 0.01952104, 0.005474239, -0.0023999964, -0.0065626656, 0.021742843, -0.0046940395, -0.0005245477, -0.017055225, 0.0011967875, -0.0029570523, 0.0003947151, 0.013754628, 0.00065377826, -0.0018028066, 0.01408854, -0.0062897564, 0.013106067, 0.017196495, -0.004912367, 0.021254817, 0.00010986605, -0.0061934353, -0.016978167, 0.0026985912, -0.013420715, -0.045412105, -0.00053177174, 0.009407343, -0.016169071, -0.00051772495, -0.008302864, 0.026083706, 0.0028494939, -0.00621591, 0.014859107, -0.02269321, 0.02361789, 0.018275289, -0.000034916335, 0.0053426004, 0.0116548315, -0.03310871, 0.023322506, -0.0008917712, -0.012457506, 0.010794365, -0.004254174, -0.012457506, 0.008925739, -0.029307246, -0.0070185843, 0.014435296, 0.003968422, 0.003711566, 0.007063534, -0.010068747, 0.0070506916, 0.0111411195, -0.0052366476, 0.013343658, -0.009426608, -0.014949007, 0.0093559725, -0.008675304, -0.0064502913, 0.016785527, 0.023630733, 0.0028783903, -0.006832364, -0.016130544, 0.00354461, 0.0021479565, -0.0043344414, 0.0011751152, 0.006832364, 0.0075451387, -0.015488404, -0.015642518, -0.0053104935, -0.0076350383, -0.014139911, -0.020497093, 0.011776838, 0.0062897564, 0.011109013, -0.010531087, 0.009028481, 0.012181386, 0.030052127, 0.009786205, 0.00759651, 0.008341392, -0.01708091, 0.000027265844, 0.020098966, -0.005349022, 0.013093224, 0.003250831, -0.02794591, 0.023425248, 0.0053104935, -0.030694267, 0.008174435, -0.00064575154, -0.0052462798, 0.009144066, 0.026456147, 0.00040213985, 0.00068026653, 0.0035638742, -0.012207071, -0.0095743, 0.016888268, -0.001979395, -0.0030036075, -0.0012730415, 0.02509481, 0.006414973, -0.0009455504, 0.026212133, -0.006716779, -0.016913954, -0.010646673, -0.015180177, 0.018146861, 0.03177306, 0.001321202, -0.0024931065, -0.03038604, 0.014011484, -0.012907004, -0.0016872215, 0.015501247, 0.030000757, -0.03359674, -0.02002191, -0.006652565, 0.0081102215, 0.022218026, 0.006620458, -0.017915692, 0.0053233365, 0.005159591, 0.0006810692, 0.0137931565, -0.020111809, -0.0056283525, 0.022757424, -0.019983381, -0.02545441, 0.005689356, 0.030976808, 0.0358057, 0.0145508805, -0.0162076, 0.02219234, 0.016053487, 0.0069543705, 0.005644406, -0.00009918045, -0.010710887, 0.022076756, -0.0016398637, -0.000030702293, 0.004382602, 0.0028623368, 0.014846264, 0.014486667, 0.0011221387, -0.020522777, -0.006992899, -0.003978054, -0.027098287, -0.007435975, 0.04107124, 0.002536451, 0.012130015, 0.0067360434, -0.016079172, 0.0061902246, -0.014627937, -0.012091487, -0.0048192567, -0.0060425326, -0.004655511, 0.03020624, -0.011173227, -0.027637683, 0.009131223, 0.016464457, -0.014152754, -0.01708091, -0.008810153, 0.006980056, 0.0068580494, -0.0102613885, 0.0036248774, -0.009799048, 0.000051346076, -0.017761579, 0.008553298, 0.021884114, 0.028562363, -0.01864773, -0.00018270875, -0.004658722, 0.029101761, -0.0026568521, -0.019803582, -0.013446401, 0.032672055, -0.013317972, -0.003024477, 0.013228074, -0.0023518358, 0.023361035, -0.013690414, -0.0015186599, -0.0068580494, 0.017658835, 0.0043119667, 0.018390875, 0.016040644, -0.0050568483, -0.0041032713, -0.004084007, -0.0132922875, 0.0014624727, 0.0065658763, 0.018442245, -0.0041995924, 0.004816046, -0.010203596, -0.021293346, -0.007744202, 0.00621591, 0.0030726374, 0.0022442774, -0.012412556, 0.013202388, 0.0033327038, -0.013131753, 0.021100704, -0.002896049, -0.017222181, 0.023078494, -0.00488026, -0.007821259, -0.011391554, -0.020098966, -0.0025252134, 0.0012425398, -0.012643727, 0.026250662, -0.012393292, 0.005451764, 0.007269019, 0.010742993, 0.009792627, -0.020227393, -0.013574828, 0.0045046085, 0.00980547, -0.017196495, 0.0014424059, -0.0070121633, -0.013754628, 0.0072818617, 0.0036505628, -0.00188789, 0.0050568483, 0.024504043, 0.0033391253, 0.0043729697, 0.0014175229, 0.0021286923, 0.0066268793, 0.011391554, 0.0023502305, 0.010768679, -0.01510312, -0.004774307, 0.00529444, -0.00446608, -0.0021880902, 0.0033070184, -0.022641838, -0.00024060164, -0.0072433334, 0.009966005, 0.0039844755, -0.0071855406, 0.00036381214, 0.011950215, 0.026186448, 0.0044403947, -0.0008893632, 0.021601573, 0.013407872, 0.010935635, 0.002385548, -0.009272494, -0.0122777065, -0.009728413, -0.00042622007, 0.00704427, 0.0065498226, -0.008887211, 0.016862582, -0.0067745717, 0.015989274, 0.013151017, -0.015501247, -0.011076906, -0.012547405, -0.010113697, 0.005763202, -0.015681047, 0.0036730377, -0.0021431404, 0.0002586618, -0.0034771853, -0.0140500115, -0.006556244, -0.0041450104, -0.005612299, -0.028279822, -0.019174285, -0.005538453, 0.008636776, 0.0136262, -0.022320768, 0.009856841, 0.0020981908, 0.017620306, 0.0076543028, 0.0067103575, -0.003933104, -0.0038496263, 0.00870099, 0.035934124, -0.0137931565, 0.00043705618, 0.0036377201, -0.0065755085, -0.01649014, -0.0010370553, -0.022115285, 0.0018429403, 0.012136436, 0.00056387874, -0.005349022, 0.0011863527, -0.0030501625, 0.0015170545, -0.012829946, 0.01408854, 0.0016310343, 0.022205183, -0.017183652, 0.02016318, 0.003310229, 0.008135907, -0.004369759, 0.01026781, 0.008270756, -0.008649618, 0.011635567, 0.031747375, -0.013549143, 0.0191486, 0.021357559, 0.0074038683, 0.03665332, -0.00976052, -0.016580041, -0.012894161, -0.013240916, -0.018596359, -0.0014351817, 0.0082193855, -0.0053650755, 0.008546877, -0.0015981246, 0.009856841, 0.005650827, 0.0070250058, -0.0028783903, -0.0081937, 0.022474881, -0.006100325, -0.007525875, 0.007468082, 0.010479717, -0.0060457434, 0.017389137, 0.015693888, 0.0093559725, -0.001772305, 0.0072754403, 0.015796632, 0.012380449, -0.005666881, 0.014037169, -0.0077506234, -0.023219764, 0.009041324, 0.013106067, 0.04084007, 0.006418184, 0.009182595, 0.021563044, 0.009458714, -0.010768679, 0.0021351138, 0.009362394, -0.0031224033, -0.021665785, 0.015899373, 0.009009217, 0.0010587275, 0.0037436732, 0.014679309, 0.0025123707, -0.005782466, 0.016721312, -0.01082005, -0.0031159818, 0.011404397, 0.008675304, -0.009144066, -0.0038014657, -0.0032379883, 0.0040679537, -0.004523873, -0.01961094, -0.0005590627, -0.014974693, -0.0055641388, -0.026635945, -0.006703936, -0.0011759179, -0.008412027, 0.030540153, -0.0023550466, -0.0067938357, -0.0011052826, 0.0007472898, 0.0070956415, -0.024915012, -0.029076075, 0.0044275518, 0.026289191, 0.004568822, 0.001528292, -0.026083706, -0.00708922, -0.0025717686, 0.018544989, 0.008026743, -0.0024240764, -0.0009038113, -0.011340183, -0.033365566, -0.00055665465, 0.025839692, 0.02748357, -0.009933898, 0.008604669, -0.009471557, 0.014627937, -0.009477979, 0.0064053414, 0.0010571221, -0.003618456, 0.014846264, 0.022423511, -0.013818841, 0.0059044724, -0.014974693, 0.019212814, 0.018211076, 0.0023775215, -0.018416561, -0.023271134, 0.011905266, -0.021190602, 0.0028992598, -0.0032973862, 0.004084007, -0.019508198, -0.001153443, 0.00041377862, 0.011731888, -0.011609882, -0.009150487, 0.015115963, 0.008386342, 0.027509255, -0.011757574, -0.006411763, 0.019919166, -0.005085745, 0.0041000606, 0.021062175, 0.016284658, -0.009458714, 0.008142329, 0.0012634094, 0.024118759, 0.002284411, -0.0056925663, -0.021563044, 0.023399564, -0.0043986556, 0.008200121, 0.004568822, 0.015475562, -0.00055665465, -0.0018204654, -0.0012682255, 0.020638363, -0.0022860165, -0.015693888, -0.018557832, -0.013960112, 0.0028238085, 0.009426608, -0.015732417, -0.007493768, -0.015372819, -0.02591675, -0.0047454108, -0.011564932, 0.002412839, 0.017106596, -0.0061805923, -0.013382187, 0.0162975, -0.0093816575, -0.021845585, -0.00038668836, -0.0027852799, -0.024324244, -0.025236081, -0.036756065, -0.010698044, 0.00003102838, 0.0014014695, -0.00071839354, -0.0140500115, 0.044769965, -0.018442245, -0.023643576, 0.0358057, 0.009966005, -0.0071662767, -0.003949158, 0.0013725731, -0.010415503, -0.0020548464, -0.025724107, -0.036910176, -0.039838333, -0.011211756, -0.0043729697, 0.003945947, 0.02702123, 0.0037083556, 0.012977639, 0.002859126, -0.012065801, -0.010190753, 0.0020933747, -0.0097476775, 0.007757045, 0.021357559, -0.0061324323, 0.0030373198, 0.010370553, 0.008347813, 0.031644635, -0.025608523, -0.02108786, 0.011629146, 0.0081102215, -0.008168014, 0.0014424059, 0.020432878, -0.0059879506, 0.0005377918, -0.015270077, 0.0143068675, -0.007583667, -0.005589824, -0.01173831, -0.010004533, 0.022115285, -0.003907419, -0.00093110226, -0.0013525062, -0.013690414, 0.0017674889, 0.017273553, -0.016310344, -0.00515638, 0.021357559, 0.02661026, 0.0034258142, 0.023232607, 0.026687317, 0.017761579, 0.023836218, -0.009921055, 0.00069110264, -0.0065658763, -0.010685201, 0.006440659, 0.001574847, 0.011975901, 0.00037906296, -0.003711566, 0.0037372517, 0.013356501, 0.005888419, 0.016836897, -0.0022956487, -0.006819521, 0.023258291, -0.002827019, -0.017684521, 0.011841052, -0.018018434, 0.011173227, 0.009580721, 0.021614416, -0.004469291, 0.01758178, -0.02062552, -0.026867116, -0.016400242, 0.0022812006, 0.0026167184, -0.0036890914, 0.0059751077, 0.016592884, 0.010511823, 0.0051114303, 0.018878901, 0.0015395294, -0.0076029315, -0.015128806, 0.0024369194, 0.019212814, -0.025287453, -0.007988215, -0.0031384567, -0.005082534, 0.016258972, 0.022616152, -0.0055448744, 0.010325603, 0.020548463, 0.0024850797, 0.029641159, -0.00092387816, -0.005952633, 0.022847323, 0.0047935713, -0.002550899, 0.010441188, -0.016220443, -0.009137644, -0.015026064, -0.0075387177, -0.016284658, 0.000010064784, 0.00032026708, 0.006559455, -0.022577625, 0.0017498301, -0.0059590545, -0.023091335, -0.011545667, 0.012329078, -0.0025541098, -0.013446401, 0.029692529, 0.025569994, 0.023797689, 0.004138589, -0.008052429, 0.010088012, -0.003300597, -0.008071694, -0.0041995924, -0.010100854, -0.0056411955, -0.01496185, 0.008534034, -0.0018670205, -0.0155783035, 0.008347813, 0.0027001966, -0.0087202545, 0.028716477, -0.002127087, -0.013677571, 0.0031641424, -0.032851856, -0.0003864877, 0.001675984, -0.003329493, -0.004751832, -0.015488404, 0.022140969, -0.012123593, -0.006501662, 0.0069543705, 0.013459244, 0.010017375, -0.008437713, -0.010152225, -0.007936844, 0.005377918, 0.024902169, 0.0060939034, 0.019187128, 0.0040711644, 0.016181914, 0.0011076906, 0.005878787, -0.03899071, 0.007879051, 0.01058888, 0.024889326, 0.0021977224, 0.00082434656, -0.004751832, -0.012091487, 0.016567199, 0.019970538, 0.011012692, 0.0046908287, 0.0014536432, -0.010332025, 0.010120118, -0.0358057, -0.008161593, -0.00049725675, 0.016772684, -0.017055225, 0.013459244, 0.037732117, -0.014371081, 0.0140500115, -0.009522929, -0.0024882904 - ] - } - ] - }, - { - "name": "untitled", - "title": "Untitled", - "content": "Technological advances, particularly in generative AI, are expected to impact almost every sector of human activity, creating both opportunities and challenges.", - "keywords": [ - "Technological advances", - "generative AI", - "impact", - "sector", - "human activity", - "opportunities", - "challenges" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.026481831, 0.04921357, -0.002366284, -0.021460302, -0.027380828, -0.0036216667, 0.012605198, -0.006658986, -0.06981341, 0.021896956, -0.03893934, 0.025441566, -0.013189545, -0.0004924407, 0.0061902246, 0.013844527, -0.015963588, 0.04998414, 0.0066975146, -0.0013460849, 0.0354461, -0.0061163786, -0.060566597, 0.044179197, 0.012682254, 0.00824507, -0.030694267, 0.0064470805, -0.026327718, 0.0017578568, -0.003660195, -0.001222473, 0.045232303, 0.001574847, 0.009728413, -0.019392611, 0.0015339106, -0.020548463, 0.029872328, 0.034881018, 0.011256705, 0.0020708998, 0.018943114, 0.010145804, 0.028896276, -0.02853668, 0.0003991298, -0.036550578, 0.021190602, 0.023232607, 0.016002117, -0.014949007, -0.021293346, 0.041456524, 0.011834631, -0.004276649, 0.011937373, 0.019135756, -0.007423132, -0.050857447, 0.00016474891, -0.014332553, -0.024452671, 0.009426608, 0.012438241, 0.0215502, 0.017299237, -0.002600665, 0.04433331, 0.020317294, 0.029666845, 0.06642291, 0.0055320314, 0.029332932, 0.054299314, -0.008938582, -0.03149052, 0.01201443, -0.025955278, 0.013651885, 0.02559568, -0.028100023, -0.020188866, 0.0044949763, -0.0021094282, 0.0000010096138, 0.011256705, -0.073666245, 0.0030164502, 0.0036056133, 0.010672358, -0.013728942, -0.011584196, -0.020908061, -0.044487424, -0.027791796, 0.045566216, -0.010299917, -0.005217383, 0.024298558, 0.011956637, -0.002674511, -0.0404291, 0.01731208, 0.0005526413, 0.016438771, -0.0006758518, -0.006209489, -0.016143387, 0.013112488, 0.025955278, -0.003226751, -0.0071662767, -0.0046651433, -0.027201029, 0.0028061494, -0.009728413, -0.017800106, 0.044127826, 0.015115963, -0.0037211983, 0.014743523, -0.03226109, 0.006071429, -0.021601573, 0.010254968, 0.020432878, -0.017453352, 0.01961094, -0.032158345, 0.02591675, 0.011353026, -0.0068516284, 0.011333762, 0.009336708, -0.04225278, -0.0065401904, 0.031798746, 0.028100023, 0.048263203, -0.06390572, -0.007159855, 0.019790739, -0.00056187203, -0.0013733758, -0.025467252, -0.016952483, 0.01606633, 0.012733626, -0.026327718, -0.024195816, -0.013780314, -0.027509255, 0.00446608, 0.018095491, 0.020047594, 0.0060842717, -0.024414143, 0.05676513, -0.046901867, -0.018801844, -0.013587671, 0.028228452, -0.036807436, -0.011166805, -0.0064246054, -0.020201707, -0.00026046782, 0.0016093621, 0.0034482891, 0.042278465, -0.0054389215, -0.04572033, -0.07489915, 0.0060329004, -0.025261767, -0.014204125, -0.014846264, -0.005731095, 0.024375614, 0.008501926, -0.0031464836, 0.028022967, -0.009779784, -0.020908061, -0.022128126, 0.006890157, 0.023964645, 0.038554054, -0.021575887, 0.00079223956, 0.051448215, 0.013883055, 0.0060040043, 0.02905039, 0.0014680914, -0.011725467, -0.033365566, 0.015937902, 0.026032334, -0.054196574, 0.015385662, -0.02205107, -0.014782051, 0.0024593943, 0.015937902, -0.009535771, -0.0095743, 0.020869533, 0.030642897, -0.01035771, -0.017607465, -0.017787263, 0.042612378, -0.014936164, 0.02817708, -0.017183652, -0.007352497, -0.017389137, 0.0459515, 0.026404776, 0.037115663, 0.007513032, -0.026404776, -0.018557832, -0.009304601, -0.06303241, 0.0059751077, 0.004613772, -0.026918488, 0.032158345, 0.01790285, 0.0072625973, 0.056919243, -0.0379376, 0.06133716, 0.024555413, 0.0073268116, -0.01717081, 0.034315933, 0.027406514, 0.02899902, -0.00980547, -0.02463247, -0.007904737, -0.028074337, 0.023733474, 0.0059141046, 0.0075708246, 0.028716477, 0.00035177203, 0.0030084234, 0.051936243, 0.00994674, -0.011930952, 0.009709149, -0.005750359, 0.006633301, -0.008906474, 0.0073139686, 0.0019874217, 0.010370553, -0.017157966, 0.017479036, 0.005731095, -0.035112187, 0.0005357851, 0.0032652793, 0.0019537094, -0.029153133, -0.001703275, -0.004225278, 0.007288283, 0.017466193, 0.028613735, 0.010903528, 0.042612378, 0.02301428, -0.03125935, -0.04623404, -0.021113547, 0.009054166, 0.0051242732, -0.025518622, 0.0045559797, 0.01243182, -0.007943265, 0.027098287, 0.0022137759, -0.046799127, -0.011423661, 0.0044436054, 0.030899752, 0.025711264, 0.030129185, -0.0028848117, 0.0030662161, -0.0033487573, -0.046901867, 0.019508198, 0.018146861, 0.02661026, -0.007866208, -0.015231549, 0.0122648645, -0.015128806, 0.005050427, -0.00395879, -0.026160762, -0.05404246, -0.009221123, -0.053374633, 0.0028334404, 0.025698422, -0.011995166, -0.00033009984, 0.017222181, -0.014139911, 0.038502682, -0.017440509, -0.020009067, 0.0029875538, 0.018429402, 0.019906323, -0.012348342, 0.023399564, -0.000100685465, -0.012727205, 0.01592506, 0.01201443, -0.03457279, 0.004626615, -0.017812949, -0.05106293, 0.011051221, -0.012759311, 0.018763315, -0.008296442, 0.0010033429, 0.019970538, -0.004976581, 0.0057664127, 0.009908212, 0.037192717, 0.026507517, -0.02149883, -0.009818313, -0.0053971824, -0.013163859, 0.01740198, 0.000666621, 0.003351968, 0.05501851, -0.0039620004, 0.017363451, -0.006283335, -0.02499207, -0.02283448, 0.01606633, -0.039581478, -0.024439828, 0.0067809927, -0.009458714, -0.0022812006, -0.019482512, 0.019495355, 0.00073284167, 0.009400922, 0.030617211, 0.0062191207, 0.013240916, 0.014011484, -0.0122648645, 0.011956637, 0.013497772, 0.006960792, -0.017363451, -0.008604669, -0.012386871, -0.00061725656, 0.032440886, 0.025531465, -0.019944852, 0.00086769095, -0.0011991955, -0.0030260824, -0.0017899638, -0.017838635, -0.017337766, 0.011089749, -0.008084536, -0.013382187, -0.00469725, 0.0030517678, -0.0038464156, 0.017517565, -0.044281937, -0.0033583895, 0.028896276, 0.0041642743, 0.023232607, -0.03056584, 0.002181669, 0.031336408, -0.020753948, -0.0052976506, -0.0019970539, -0.057124726, 0.012662991, -0.014011484, -0.024786584, -0.0030180556, 0.003393707, -0.012322657, -0.02899902, -0.05008688, -0.016541513, 0.03714135, 0.015488404, 0.034059078, -0.01449951, -0.037911914, 0.0063314955, 0.0039555794, -0.015411347, 0.0030276875, 0.012740048, 0.025698422, -0.0381174, -0.021357559, -0.031464834, -0.009477979, 0.0027917014, 0.010306339, -0.015000379, 0.050985876, -0.03688449, 0.0045591905, 0.03978696, 0.013356501, -0.02946136, 0.005962265, 0.032338142, -0.018095491, -0.013574828, 0.022487724, 0.02684143, 0.01873763, 0.024337087, 0.056919243, -0.014165597, 0.03434162, 0.016567199, 0.011622725, 0.007416711, 0.0011245467, -0.042920604, 0.012496035, -0.007988215, 0.031798746, 0.023399564, -0.023181235, 0.014345396, -0.0352663, -0.013972955, 0.02513334, 0.0461313, 0.00971557, 0.000971236, -0.017877163, -0.0072561763, -0.025582837, -0.039581478, 0.02541588, -0.045925815, -0.0031063498, 0.004482134, 0.012996903, 0.022962907, -0.015848001, -0.040814385, 0.022718895, -0.044924077, 0.015950745, -0.0050632698, 0.026147919, -0.025929593, -0.027663369, 0.0106274085, -0.015167334, 0.052295838, 0.0053522326, 0.008765204, -0.020227393, 0.010807208, 0.0070378487, -0.03724409, -0.055223994, 0.02370779, 0.033005968, -0.0033969178, 0.03084838, 0.037629373, -0.0029153132, 0.003204276, -0.0039427364, -0.02030445, -0.03521493, 0.013472087, 0.016323186, 0.061491277, 0.0035125029, -0.028279822, -0.020009067, 0.016913954, -0.015475562, -0.012386871, -0.010210018, 0.029409988, -0.009895369, -0.009580721, 0.011622725, -0.0025525044, 0.006543401, -0.019790739, 0.012579513, 0.0033038077, -0.0009977242, 0.029872328, -0.021241974, -0.033262827, 0.0057696234, 0.0056090886, 0.028022967, 0.06405983, -0.02513334, -0.02126766, -0.019238498, -0.0017867531, 0.042715117, -0.03351968, -0.057535697, 0.012412556, 0.03190149, 0.01850646, 0.006154907, -0.0047807284, -0.012207071, 0.0028703634, -0.03167032, 0.053888347, -0.014949007, 0.005888419, 0.0030581893, -0.014653623, -0.007121327, -0.014692151, -0.02007328, -0.037038606, -0.013330815, 0.02140893, 0.011243862, 0.021845585, -0.004456448, 0.0062865457, 0.015475562, 0.016002117, -0.011282391, 0.005410025, 0.0017562514, 0.011218177, 0.012451084, 0.026327718, 0.0064663445, 0.019906323, 0.011719045, -0.018660573, -0.015398505, 0.012367606, 0.011423661, -0.008360656, 0.019649468, 0.031387776, 0.025390195, 0.0005686948, 0.0066782506, -0.00814875, -0.030077813, -0.0034033393, 0.044769965, 0.036627635, 0.023450933, -0.0029265506, 0.015115963, 0.0010563195, 0.009060588, -0.027534941, 0.015681047, -0.028665107, -0.029435674, 0.0068387855, -0.0041450104, 0.015732417, -0.020458564, -0.024388457, 0.008463399, 0.023643576, 0.0153086055, 0.018904585, 0.010511823, 0.0024465513, 0.020831006, 0.0017129071, -0.008026743, -0.032286774, -0.006315442, 0.03403339, 0.029076075, 0.010800786, -0.022359297, -0.014294025, -0.024414143, -0.0074552395, -0.009766942, 0.025377352, -0.021421773, 0.012451084, -0.016323186, -0.0031175872, 0.0018814687, -0.001079597, 0.009914634, -0.015526933, 0.006459923, -0.01473068, -0.019687997, 0.0061805923, -0.008418448, -0.02080532, -0.012784997, 0.0026488253, 0.058306266, 0.016926797, 0.03624235, -0.004639458, 0.00016755828, -0.021100704, 0.024902169, 0.007191962, 0.004899524, 0.055994563, -0.028048653, 0.009477979, -0.0011646805, -0.00091825944, 0.01800559, 0.02776611, -0.030257612, 0.028048653, -0.003241199, -0.016836897, 0.014897636, 0.0004274241, 0.006877314, 0.012412556, 0.020908061, -0.03737252, 0.038091715, 0.017928535, -0.0034771853, -0.014332553, 0.017915692, -0.004116114, 0.011449347, -0.010132961, 0.0067938357, 0.0032797274, -0.0069351066, -0.010486138, 0.0020548464, 0.010486138, -0.0164773, -0.02946136, -0.005731095, 0.013818841, -0.021190602, 0.044025082, 0.03673038, -0.005506346, -0.01758178, 0.019366927, 0.033493996, -0.0018959168, -0.007827681, 0.031053865, 0.004337652, -0.0011686939, 0.031233665, 0.01150714, -0.014846264, 0.0020773213, 0.023373878, 0.01740198, -0.039966762, 0.010736573, 0.009766942, -0.0018333083, -0.024619628, -0.05224447, -0.008643198, 0.006703936, -0.004626615, 0.030694267, -0.011076906, 0.053888347, -0.035112187, 0.040583216, -0.035137873, 0.008495505, 0.039966762, 0.02375916, 0.008707412, -0.053785603, -0.012117172, -0.03236383, 0.008309285, -0.029230189, -0.025788322, 0.01473068, 0.0060650073, -0.038399942, 0.009599986, 0.026250662, -0.009266073, 0.026738688, -0.001855783, -0.013818841, -0.02499207, 0.022962907, -0.007172698, 0.0067424644, -0.032081287, 0.04374254, 0.006652565, 0.0006770558, 0.017055225, -0.0025187922, -0.003660195, -0.059179574, 0.0028510992, -0.038656797, -0.019816425, 0.038785223, 0.016271815, 0.0037500947, 0.00022956486, 0.012142858, 0.0070378487, 0.01588653, 0.0358057, 0.023656419, 0.010858579, 0.00202595, 0.0025492937, 0.0077827303, 0.0022137759, 0.004816046, -0.010113697, -0.0027130393, 0.0065947725, 0.03323714, -0.0023261502, -0.005262333, -0.0018188601, 0.022397825, -0.00060922984, 0.010967743, -0.011988744, 0.019662311, -0.03398202, 0.034649845, -0.006071429, 0.002319729, -0.0023405985, -0.007577246, -0.019392611, 0.009978848, -0.002969895, 0.0044725016, -0.013818841, -0.008444134, -0.004597719, -0.017684521, 0.010537509, -0.0093945, -0.016541513, 0.0115199825, -0.012194228, 0.044410367, 0.006601194, 0.012309814, 0.019598097, -0.010017375, 0.010909949, -0.030334668, 0.0030533732, 0.007179119, 0.013985798, -0.016708469, 0.018211076, 0.040994186, -0.022269398, 0.020009067, -0.0081102215, -0.036422152, -0.027534941, 0.016567199, -0.0024096284, 0.00019906324, -0.02003475, 0.034778275, -0.021421773, 0.0432802, -0.010794365, 0.020908061, 0.02306565, 0.023720633, -0.026481831, 0.00043344416, 0.018955957, -0.010730151, -0.0024609996, -0.013703257, -0.009734835, -0.00971557, 0.011564932, 0.0070185843, 0.0067681503, 0.02928156, -0.009323865, -0.0076157744, -0.015462719, 0.027971596, -0.0072947047, 0.010210018, 0.04199592, 0.0072112265, -0.054813027, 0.015038907, -0.04148221, -0.012984061, 0.017890006, -0.01510312, 0.019726524, -0.025467252, 0.02666163, 0.024683842, 0.0083542345, 0.0038110977, 0.008450556, 0.007943265, -0.012573091, 0.03102818, 0.0044500264, 0.004931631, 0.004074375, 0.0023839427, 0.012001587, -0.018724788, -0.0052655437, 0.015090277, 0.0019248131, -0.013960112, -0.024645314, -0.007493768, 0.014692151, -0.009291758, -0.01937977, -0.012252022, 0.013446401, -0.025390195, -0.004116114, -0.010556773, -0.033262827, 0.024285715, 0.03084838, -0.019970538, -0.012194228, 0.012149279, -0.021794215, 0.0012915031, 0.004543137, -0.0024064176, -0.006090693, 0.012457506, 0.025801165, -0.018917428, -0.020086123, -0.015629675, 0.027689055, 0.020381507, -0.008668883, 0.0035574527, -0.017235024, -0.0040165824, 0.00022976553, 0.023656419, 0.0075066104, -0.0037372517, -0.0025974542, -0.00833497, 0.012374028, 0.0052591222, -0.03369948, -0.00865604, 0.001333242, -0.02840825, -0.0136262, -0.00015060177, 0.031002494, 0.03231246, 0.020060437, -0.02817708, 0.0046651433, -0.00045351102, -0.0055866134, 0.014396767, -0.023502305, -0.01597643, -0.0005907683, 0.015603989, 0.0076992526, 0.0073717614, 0.006068218, 0.01864773, -0.021370402, -0.038887966, 0.0018943115, -0.01072373, 0.028973334, -0.008540455, 0.016567199, -0.023001436, -0.01836519, -0.0067103575, 0.044949763, -0.01868626, 0.025081968, 0.038476996, -0.0025043439, 0.010909949, 0.013202388, -0.010800786, 0.011410818, 0.047030296, 0.008277178, -0.025030596, 0.03218403, -0.01946967, 0.019135756, 0.004957317, 0.0035478207, -0.030899752, -0.008636776, 0.013651885, -0.008822996, -0.018390875, -0.06477903, 0.000050116978, -0.0009913028, 0.024157288, -0.0140500115, -0.015655361, -0.015205863, 0.00051050086, 0.03601118, -0.007724938, 0.0053522326, 0.024773741, 0.02559568, 0.015642518, 0.021627259, 0.007564403, 0.042021606, 0.02858805, -0.017568937, -0.022115285, -0.016053487, -0.004209224, 0.0015001984, 0.022256555, -0.025608523, -0.0017000643, -0.0069543705, 0.052655436, -0.010614566, 0.0023405985, 0.0005803336, -0.0041321674, 0.014704994, 0.035291985, -0.003554242, -0.018609202, -0.00071518286, -0.01634887, 0.02905039, -0.044179197, 0.0028832064, -0.014627937, -0.0056861453, 0.021370402, 0.0010282259, 0.032517944, -0.025801165, -0.008360656, -0.025069125, -0.0075001894, -0.016657097, -0.018519303, -0.00478715, 0.00285431, 0.018095491, 0.0045367153, -0.012733626, -0.0042284885, 0.010832893, 0.014627937, 0.010351288, 0.02324545, 0.02393896, 0.021807056, 0.00014438105, -0.0044949763, 0.014062854, -0.034290247, -0.007930422, -0.008135907, 0.05589182, 0.0023887588, 0.017697364, 0.012746469, -0.035985496, 0.021896956, 0.0022009332, 0.010473295, -0.014011484, -0.008084536, 0.003798255, 0.018172547, 0.01772305, 0.00055344397, 0.0025075546, -0.00028073534, 0.0032973862, -0.033622425, -0.008501926, 0.02858805, 0.02545441, 0.0018509671, -0.017568937, 0.005644406, -0.012528141, 0.006684672, -0.0027290927, 0.02066405, -0.00060481514, -0.023990331, 0.017607465, 0.0037372517, 0.009580721, 0.0019681575, -0.009709149, 0.035189245, -0.020766791, 0.019816425, 0.016104858, -0.01505175, -0.01431971, -0.0036762485, -0.0329546, -0.0017369873, 0.018211076, -0.018532146, 0.031875804, 0.015770946, -0.03816877, 0.012560248, 0.006251228, -0.013003324, 0.015501247, -0.016284658, -0.014576566, 0.0028575207, -0.009182595, 0.05332326, -0.024837956, 0.0155783035, 0.049316313, -0.013549143, -0.014704994, -0.016913954, -0.0056989877, 0.0055577173, -0.053374633, 0.0032973862, 0.004032636, -0.03940168, -0.0002867554, -0.034906704, -0.012964796, -0.008938582, 0.032517944, -0.021473145, 0.014846264, 0.013330815, 0.021241974, -0.017979905, -0.0019392612, -0.0050632698, -0.020959433, -0.011012692, -0.014692151, -0.029384302, 0.0020291607, 0.023104178, -0.023669261, -0.012849211, -0.0074295537, 0.018339504, -0.0155783035, -0.0077313595, 0.019777896, -0.016246129, -0.03770643, 0.0075322962, 0.0030854803, 0.01615623, -0.024593942, 0.028742163, -0.021768529, 0.015038907, 0.0031962493, -0.013138174, -0.020291608, 0.02158873, -0.01026781, 0.034007706, -0.0040390575, 0.0032829382, -0.013998641, -0.028433936, -0.005721463, 0.00814875, -0.0011510351, -0.016888268, -0.008784468, -0.0067360434, -0.01054393, -0.03411045, 0.022295082, -0.021935485, -0.001362941, -0.008168014, 0.0105182445, -0.0005365878, -0.0017626729, 0.021396087, 0.00015551815, 0.0028302297, 0.03246657, 0.0010458847, 0.0007224069, -0.018172547, -0.029332932, 0.008437713, -0.026687317, 0.00598474, -0.0076157744, -0.00059116964, 0.011744731, -0.008945003, -0.01164841, -0.020895218, 0.013151017, -0.013048274, -0.026353404, -0.008296442, -0.019199971, -0.031978548, -0.0028607314, 0.010132961, -0.01675984, 0.037834857, 0.023605047, 0.0048642065, -0.01449951, -0.000033310986, -0.0012296971, 0.012829946, 0.026481831, -0.017915692, 0.031233665, -0.011372291, -0.009728413, 0.014615094, -0.017967062, 0.0014416032, -0.011038378, -0.002725882, -0.0219098, 0.013163859, 0.012271285, 0.014473824, -0.02518471, 0.012842789, -0.008893631, -0.038143083, 0.014127068, 0.03246657, -0.0035799276, 0.0042702276, 0.0012642121, -0.0350865, 0.024131602, 0.0062769135, 0.016554356, -0.019816425, 0.010710887, -0.007269019, 0.005853101, -0.010100854, 0.013844527, -0.012817104, -0.0036955127, -0.0057888874, -0.031362094, -0.025403038, 0.029589787, 0.0043665483, 0.02292438, 0.020792477, -0.006228753, 0.028254136, -0.0032235403, -0.0017385926, 0.020086123, -0.035780013, -0.032029916, 0.015398505, -0.037860543, 0.0049348418, 0.0086110905, 0.011385133, -0.019122913, -0.02541588, -0.011622725, -0.029101761, 0.011808945, -0.020753948, 0.000802273, 0.019559568, -0.004616983, -0.005695777, 0.029076075, 0.021485986, 0.008168014, -0.013189545, -0.0007500992, 0.011320919, -0.0008291626, -0.0012401319, 0.025325982, -0.015270077, -0.015180177, 0.018198233, 0.019546725, -0.019803582, -0.007737781, 0.03310871, 0.0123997135, -0.0016583252, 0.008078115, -0.0016647466, -0.0054710284, -0.002546083, -0.00216401, 0.025980964, -0.008283599, 0.024439828, 0.00644387, 0.03195286, 0.011494297, 0.012862054, 0.003083875, -0.02545441, -0.00893216, 0.023797689, -0.04723578, -0.005422868, -0.001841335, 0.025801165, -0.02587822, -0.0097476775, -0.019996224, -0.008033165, -0.010601723, -0.019456826, -0.0005947817, -0.009182595, 0.003157721, 0.020895218, 0.0045463475, -0.022256555, -0.006036111, 0.004379391, -0.0069543705, 0.018082649, -0.004783939, -0.020497093, 0.0016181915, -0.050035506, -0.0018172547, -0.013523458, 0.0016872215, 0.01449951, -0.009831156, 0.02035582, 0.0033327038, -0.00019735756, 0.02205107, -0.0037725694, 0.0074102897, -0.020728262, -0.0037308303, -0.012303392, -0.02518471, 0.023258291, -0.02481227, 0.04538642, 0.0064213946, 0.014255497, 0.02853668, -0.00010665535, 0.016104858, 0.01173831, 0.004915578, -0.00045190565, -0.019122913, 0.01288774, -0.027278084, -0.008501926, 0.027534941, -0.023874747, 0.0032540418, -0.031362094, 0.0115199825, -0.005833837, 0.013112488, 0.015783789, 0.012720783, 0.0072625973, 0.02789454, 0.023052808, -0.0028880222, -0.0057664127, -0.015629675, -0.00547745, -0.017658835, -0.010807208, -0.011044799, 0.019443983, 0.015347133, 0.015038907, 0.021139232, 0.0032700952, -0.0034258142, -0.0051949085, -0.021884114, -0.0034129713, -0.0071470123, -0.0029650792, -0.0012634094, 0.012027272, 0.004658722, 0.013587671, 0.032286774, 0.025017753, 0.0057439376, 0.01279784, 0.0076029315, 0.0023213343, 0.0012834762, 0.011083327, 0.0011735099, -0.010768679, 0.0066268793, 0.0047261463, -0.0077056736, 0.004061532, 0.0025974542, -0.0019023382, -0.0061805923, 0.001878258, -0.011275969, 0.04949611, -0.0046940395, 0.0021014016, -0.007423132, -0.013960112, 0.013728942, 0.004581665, -0.015526933, 0.028279822, -0.035112187, -0.008142329, -0.023810532, -0.010280653, 0.031798746, 0.007236912, -0.016271815, 0.009028481, -0.01072373, -0.019803582, 0.009978848, -0.00018702312, -0.036807436, -0.016772684, -0.02066405, -0.007757045, -0.0012987271, -0.008816575, 0.011828209, -0.015899373, -0.004485344, 0.009535771, -0.00024240765, -0.003586349, -0.0023373878, -0.013080381, -0.0056604594, -0.014294025, -0.010640251, -0.0023839427, 0.026353404, -0.011712624, 0.0014335763, -0.0077827303, -0.033648107, -0.011719045, -0.0059911613, -0.027046915, 0.0020548464, -0.012618041, -0.009567878, 0.023861904, 0.012425399, 0.01666994, 0.02434993, 0.006344338, -0.017157966, 0.005288019, -0.010967743, 0.011796102, 0.015539776, 0.0049284203, -0.02016318, -0.00012391285, 0.015372819, -0.008822996, 0.0064984513, -0.0065530334, 0.0005153169, 0.0048513636, 0.022808794, 0.013960112, 0.025467252, 0.032158345, 0.014024327, -0.019212814, -0.014460981, 0.0033969178, 0.016258972, 0.0043858127, 0.0030662161, -0.003522135, 0.02408023, -0.006658986, 0.0034129713, 0.0012481586, 0.026918488, -0.02794591, 0.022436354, 0.02237214, -0.012772154, -0.019944852, -0.0022763845, 0.016734155, 0.015822317, 0.017055225, 0.028870592, -0.0021190604, 0.01473068, -0.013780314, -0.014512353, 0.0013284261, 0.013215231, 0.012528141, -0.034726903, -0.0023807322, -0.011866737, 0.011886002, 0.028485307, -0.007757045, 0.020586992, -0.03269774, 0.02173, 0.020753948, -0.001878258, 0.018416561, 0.0023887588, 0.021832742, 0.045823075, 0.029692529, -0.01763315, 0.014101383, -0.021370402, 0.0040904284, 0.007204805, -0.0015828739, -0.028921962, -0.0025942435, 0.017979905, -0.02748357, -0.010858579, -0.0037147768, 0.011430083, -0.011385133, 0.0054581854, -0.0244912, -0.008668883, -0.008758782, 0.0032620686, 0.016284658, -0.0020692945, -0.011051221, 0.018275289, 0.0015347133, -0.007879051, -0.021010803, -0.0072754403, -0.042201407, -0.007885473, 0.004254174, -0.0069094207, -0.0030967176, 0.019418297, -0.007795573, 0.034906704, 0.000084631974, 0.009433029, -0.00446608, 0.02481227, 0.010210018, -0.0062865457, -0.0019039436, -0.011173227, -0.0022812006, -0.011622725, 0.00704427, -0.014525195, 0.0077827303, 0.007821259, 0.0076221954, 0.003907419, -0.0242215, -0.007288283, -0.0145637235, -0.0011422056, 0.012040115, 0.013099645, -0.024735212, -0.0011767206, -0.02269321, -0.004135378, 0.014371081, 0.013870213, -0.013016167, -0.001703275, 0.02518471, -0.0021624046, -0.0053811288, 0.0071662767, 0.0016230076, 0.0006746478, -0.011038378, -0.013433558, 0.02375916, -0.014717837, -0.009638513, -0.025428724, -0.00584668, 0.01588653, -0.033802222, 0.0022908326, -0.009587143, -0.0016422718, 0.009683464, -0.004539926, -0.002513976, -0.0036698272, -0.00855972, -0.019366927, -0.0048642065, -0.005994372, -0.02223087, 0.016143387, -0.0007814035, -0.013433558, -0.008521191, 0.006652565, -0.012059379, -0.010813629, -0.012354763, 0.039324623, -0.022590468, 0.011597039, -0.007140591, -0.014717837, 0.0066654077, 0.01003664, -0.0013252153, 0.013497772, 0.021049332, 0.0014761181, -0.0072433334, -0.005088955, 0.019790739, -0.011918109, 0.007814838, 0.0047807284, 0.041816123, -0.0072754403, -0.013041853, 0.03560021, 0.025197553, -0.009336708, 0.0014464192, -0.0005991964, -0.00024702304, -0.0116548315, 0.0018525724, -0.004437184, 0.0035413993, 0.010145804, -0.007994637, -0.027278084, -0.0058402587, -0.010563195, 0.00013474896, -0.00088454713, 0.016284658, 0.0065209265, 0.008964268, -0.0103833955, 0.009240387, 0.020291608, 0.012008009, -0.016400242, -0.0127914185, -0.024439828, 0.009176173, -0.0039555794, 0.0022539096, -0.011468611, -0.0060168467, 0.0060842717, -0.020728262, 0.0143068675, 0.011301654, -0.0064470805, 0.009214701, -0.009099117, 0.039941076, 0.004722936, 0.0064759767, 0.0021431404, -0.01449951, -0.0075451387, -0.0040390575, -0.015488404, 0.0099916905, -0.0032941755, -0.00019805989, -0.009696306, -0.00038347766, 0.011571353, 0.0046779863, -0.01649014, 0.005419657, -0.025300296, 0.004010161, -0.013844527, 0.007692831, -0.0009856841, -0.0058434694, -0.019559568, -0.018121175, 0.010486138, 0.0014600647, 0.0013220047, 0.00095598516, -0.004957317, 0.006556244, -0.0012706335, 0.016708469, 0.015398505, 0.023489462, 0.013279445, -0.0001430767, 0.016130544, 0.0035349778, -0.022436354, -0.011346605, 0.026995543, 0.005496714, 0.004411498, -0.009811891, 0.008758782, -0.008970688, -0.026533203, -0.012085065, -0.010858579, -0.0001264212, -0.015732417, -0.0062640705, -0.013253759, 0.0017787263, -0.00575357, -0.003917051, 0.006081061, -0.007005742, 0.0060200575, 0.015950745, 0.00382073, -0.028151395, 0.027149657, -0.008906474, 0.007679988, 0.0033583895, 0.012226336, 0.004170696, -0.008842261, 0.0023951803, 0.0048353104, -0.0008765204, -0.032903228, -0.029358618, 0.014576566, 0.0054581854, 0.0143068675, 0.0026311665, 0.0047582537, 0.0013838105, 0.004437184, 0.012695097, 0.01464078, 0.029332932, -0.016258972, 0.019187128, 0.008534034, 0.01464078, 0.030488782, -0.007224069, 0.016926797, 0.0056604594, -0.004828889, 0.0049637384, 0.008765204, -0.023515148, 0.01440961, -0.03369948, -0.014062854, -0.018557832, -0.008142329, -0.030411726, 0.012926267, -0.012457506, 0.0070506916, -0.005262333, 0.025711264, -0.0008395974, 0.00066341035, -0.014345396, -0.013587671, 0.012515298, -0.017183652, -0.010409081, 0.03863111, 0.019456826, 0.03701292, -0.00023919696, -0.012727205, 0.015719574, -0.012733626, -0.025287453, -0.0035895596, -0.000015940612, 0.006440659, -0.0045367153, 0.024452671, 0.00043665484, 0.009831156, 0.009555035, -0.017877163, 0.0111411195, 0.007814838, 0.023181235, 0.006684672, -0.01141724, 0.010556773, -0.0057439376, -0.00029177213, -0.046259727, -0.00013384596, 0.014242654, -0.004915578, 0.014139911, -0.0013693625, 0.010254968, -0.027586313, 0.010935635, -0.000024381234, 0.0048674173, 0.0008516375, -0.009818313, -0.020908061, 0.013703257, 0.0039106295, 0.008578983, -0.02454257, 0.010338445, -0.0054838713, -0.0071277483, 0.005593035, 0.015411347, -0.00013163859, -0.0010306338, -0.003364811, -0.016310344, 0.030617211, 0.0008267546, 0.013960112, 0.0068516284, 0.0017947798, -0.006748886, 0.0009961189, -0.008778047, -0.0033198611, 0.006357181, 0.0009616039, 0.004254174, -0.011160384, -0.027046915, -0.0011285602, -0.012136436, 0.009587143, 0.011308076, -0.02375916, 0.017466193, -0.031875804, 0.0040294253, -0.0069672135, -0.010742993, -0.015758103, -0.0012385265, -0.0070506916, -0.0070250058, -0.003586349, -0.023219764, -0.00589484, 0.008309285, -0.001671168, -0.0031705636, 0.022282239, -0.028896276, 0.017812949, -0.026507517, 0.008450556, 0.017761579, 0.009831156, -0.017299237, -0.008482663, -0.024439828, -0.009696306, 0.006270492, -0.018583516, -0.0127785755, -0.00028595273, -0.0065369797, -0.0029843433, 0.0051371157, 0.0061452747, -0.0055962456, -0.028690793, -0.013998641, 0.008052429, 0.0162076, -0.0019039436, 0.0016270209, -0.008964268, -0.026815744, 0.009644935, -0.030000757, -0.0063475487, -0.0029201293, 0.010287074, -0.00014347804, 0.012444663, 0.0059718974, 0.007879051, 0.013484929, -0.008277178, -0.0037789908, -0.021845585, -0.015706731, 0.013472087, -0.010158647, 0.00031605302, -0.01542419, 0.025120497, 0.010595301, 0.008970688, -0.0067809927, 0.006257649, 0.039735593, -0.0028478885, -0.0040390575, 0.031130921, -0.032492258, -0.0009118381, 0.025248924, -0.011539247, 0.022885852, 0.007667145, -0.01040266, 0.0024786585, 0.004986213, 0.009002795, -0.02053562, -0.021896956, -0.0066268793, 0.00014387938, 0.009233966, -0.015706731, 0.00072280824, -0.022577625, -0.007159855, 0.004498187, 0.01772305, 0.03195286, -0.011937373, 0.021858428, -0.012219914, -0.021922642, 0.004713304, -0.007140591, 0.0076864096, 0.0014175229, 0.02357936, 0.013125331, 0.023283977, -0.0003373239, 0.011198913, 0.030180555, 0.008514769, 0.00069351064, -0.0034001286, 0.011153962, -0.019649468, -0.016554356, -0.020599835, 0.021575887, -0.021485986, -0.00589484, 0.014062854, 0.00007720724, -0.019161442, 0.002610297, -0.005326547, -0.0006112365, 0.00076133665, -0.016708469, 0.0022169866, -0.008367077, -0.024786584, -0.011340183, 0.010030218, 0.005763202, -0.013960112, -0.0067553073, 0.0026761163, -0.013960112, -0.0027066178, 0.0055127675, -0.0052527008, 0.004860996, 0.015154492, 0.0148077365, 0.00003150497, 0.002297254, 0.0067553073, 0.011385133, 0.0038945759, 0.020407192, -0.02509481, 0.015732417, 0.009433029, -0.006055375, 0.040994186, 0.0023100968, 0.00244013, -0.020098966, -0.00855972, 0.01615623, 0.026404776, -0.0014857502, -0.0016101648, 0.0021158496, 0.010871422, -0.012720783, -0.006819521, -0.009933898, -0.023052808, -0.018108333, 0.0015917033, 0.0031785904, 0.013844527, 0.019854954, 0.028254136, 0.023656419, 0.015963588, -0.01882753, -0.012091487, 0.007891894, 0.011873159, 0.0073589184, -0.019906323, -0.02016318, 0.004337652, 0.00034193927, 0.006052165, 0.015668204, 0.002435314, 0.008399185, 0.010306339, 0.0010266205, -0.015437033, 0.015205863, -0.0013525062, 0.038682483, 0.00063772476, -0.013613357, 0.026263505, 0.015437033, -0.016130544, 0.007236912, 0.010980585, 0.012945532, 0.00870099, -0.008187278, 0.0048963134, 0.014846264, 0.00009371223, 0.013908741, 0.012425399, -0.005069691, 0.006800257, 0.018673416, -0.0005787282, 0.026327718, -0.020612678, -0.024889326, -0.0035991918, 0.00019655489, -0.020009067, 0.017247867, -0.022012541, 0.023682104, 0.0069029992, 0.023027122, -0.012130015, 0.00515638, -0.0032556471, -0.0038656797, 0.013266602, -0.0029056813, -0.00764146, -0.012123593, 0.011532825, -0.011218177, -0.008232228, -0.019880638, 0.023463776, -0.00070153736, 0.009927477, -0.013780314, -0.0299237, 0.023258291, -0.0045784544, -0.0044628694, 0.005943001, -0.011590618, -0.004915578, -0.010010955, -0.009677042, 0.0016302316, -0.016952483, 0.008212964, 0.03190149, -0.001942472, -0.007352497, -0.021832742, 0.0034611318, -0.01542419, 0.020060437, -0.00354461, -0.017003853, 0.015000379, 0.0107622575, -0.0011036772, 0.0069543705, 0.00025886248, -0.0019986592, -0.0008460188, -0.02881922, 0.009047746, -0.005981529, -0.013960112, 0.0029538416, 0.0071020625, 0.022539096, -0.0022346454, 0.011083327, 0.019302713, -0.02301428, 0.022539096, -0.0038271514, -0.0019167863, -0.018814687, -0.023283977, 0.01675984, 0.0004017385, 0.029204503, -0.003692302, 0.0013059512, 0.005647617, -0.011622725, -0.01786432, 0.008585405, -0.019867796, 0.0038817332, 0.01330513, -0.0017113017, 0.016644256, 0.018609202, 0.0017899638, 0.0047614644, 0.019765053, 0.019803582, -0.020599835, -0.006469555, 0.020766791, 0.020240236, -0.011333762, 0.0098696835, 0.0020741106, -0.0051082196, -0.00418996, 0.016002117, 0.000589163, 0.009876105, 0.0019216024, 0.0006646144, 0.0029265506, -0.037321147, 0.015013221, -0.04397371, -0.012919846, -0.027201029, 0.006524137, 0.019598097, 0.012990482, 0.013343658, 0.016965326, -0.015912216, -0.0012192624, 0.024465514, -0.010184332, -0.003393707, 0.00034033394, -0.01542419, -0.004138589, 0.020381507, 0.0002448157, -0.0015892952, -0.01473068, 0.011391554, -0.026790058, 0.0024979226, -0.014435296, 0.015270077, 0.008456977, 0.0067553073, 0.0148077365, -0.0043023345, -0.021164916, -0.0035638742, 0.025492938, -0.014666466, 0.0034290247, -0.0013444795, -0.0136262, 0.0101265395, -0.01196948, 0.005464607, -0.0047582537, -0.0033230719, -0.023309663, -0.01606633, 0.04068596, 0.005853101, 0.009612828, -0.018378032, 0.009375236, 0.0071983836, 0.0012738442, 0.02067689, -0.027611997, -0.011532825, -0.0127914185, 0.021049332, 0.0039523686, 0.028202767, -0.012046536, -0.003480396, -0.019443983, 0.014204125, 0.007583667, 0.0020468195, 0.0034418677, -0.023733474, 0.02187127, -0.011834631, -0.0043472843, -0.0034482891, 0.0020933747, 0.016014958, -0.01178968, 0.009105538, -0.011430083, -0.0010563195, -0.017003853, -0.0036955127, -0.0072947047, 0.025261767, 0.009047746, -0.0000065593545, -0.004902735, 0.001979395, -0.033853594, -0.004860996, -0.0010466874, 0.034958072, 0.02158873, -0.031233665, 0.025467252, -0.019765053, -0.011686939, 0.031053865, -0.015437033, -0.011545667, -0.0046490896, 0.021216288, 0.007468082, -0.008822996, -0.0016109674, 0.018018434, 0.0021030067, 0.0075194533, -0.017825792, 0.022179497, 0.009368815, 0.00319946, -0.016811213, 0.044769965, 0.0038817332, 0.015603989, 0.007435975, 0.008322128, 0.0150645925, 0.01155851, -0.0010121723, -0.0043440736, 0.01758178, -0.00056187203, 0.005474239, 0.0032668847, -0.009330287, 0.0092468085, -0.0070378487, 0.009657778, -0.0021656153, 0.01684974, -0.016323186, 0.01952104, 0.005474239, -0.0023999964, -0.0065626656, 0.021742843, -0.0046940395, -0.0005245477, -0.017055225, 0.0011967875, -0.0029570523, 0.0003947151, 0.013754628, 0.00065377826, -0.0018028066, 0.01408854, -0.0062897564, 0.013106067, 0.017196495, -0.004912367, 0.021254817, 0.00010986605, -0.0061934353, -0.016978167, 0.0026985912, -0.013420715, -0.045412105, -0.00053177174, 0.009407343, -0.016169071, -0.00051772495, -0.008302864, 0.026083706, 0.0028494939, -0.00621591, 0.014859107, -0.02269321, 0.02361789, 0.018275289, -0.000034916335, 0.0053426004, 0.0116548315, -0.03310871, 0.023322506, -0.0008917712, -0.012457506, 0.010794365, -0.004254174, -0.012457506, 0.008925739, -0.029307246, -0.0070185843, 0.014435296, 0.003968422, 0.003711566, 0.007063534, -0.010068747, 0.0070506916, 0.0111411195, -0.0052366476, 0.013343658, -0.009426608, -0.014949007, 0.0093559725, -0.008675304, -0.0064502913, 0.016785527, 0.023630733, 0.0028783903, -0.006832364, -0.016130544, 0.00354461, 0.0021479565, -0.0043344414, 0.0011751152, 0.006832364, 0.0075451387, -0.015488404, -0.015642518, -0.0053104935, -0.0076350383, -0.014139911, -0.020497093, 0.011776838, 0.0062897564, 0.011109013, -0.010531087, 0.009028481, 0.012181386, 0.030052127, 0.009786205, 0.00759651, 0.008341392, -0.01708091, 0.000027265844, 0.020098966, -0.005349022, 0.013093224, 0.003250831, -0.02794591, 0.023425248, 0.0053104935, -0.030694267, 0.008174435, -0.00064575154, -0.0052462798, 0.009144066, 0.026456147, 0.00040213985, 0.00068026653, 0.0035638742, -0.012207071, -0.0095743, 0.016888268, -0.001979395, -0.0030036075, -0.0012730415, 0.02509481, 0.006414973, -0.0009455504, 0.026212133, -0.006716779, -0.016913954, -0.010646673, -0.015180177, 0.018146861, 0.03177306, 0.001321202, -0.0024931065, -0.03038604, 0.014011484, -0.012907004, -0.0016872215, 0.015501247, 0.030000757, -0.03359674, -0.02002191, -0.006652565, 0.0081102215, 0.022218026, 0.006620458, -0.017915692, 0.0053233365, 0.005159591, 0.0006810692, 0.0137931565, -0.020111809, -0.0056283525, 0.022757424, -0.019983381, -0.02545441, 0.005689356, 0.030976808, 0.0358057, 0.0145508805, -0.0162076, 0.02219234, 0.016053487, 0.0069543705, 0.005644406, -0.00009918045, -0.010710887, 0.022076756, -0.0016398637, -0.000030702293, 0.004382602, 0.0028623368, 0.014846264, 0.014486667, 0.0011221387, -0.020522777, -0.006992899, -0.003978054, -0.027098287, -0.007435975, 0.04107124, 0.002536451, 0.012130015, 0.0067360434, -0.016079172, 0.0061902246, -0.014627937, -0.012091487, -0.0048192567, -0.0060425326, -0.004655511, 0.03020624, -0.011173227, -0.027637683, 0.009131223, 0.016464457, -0.014152754, -0.01708091, -0.008810153, 0.006980056, 0.0068580494, -0.0102613885, 0.0036248774, -0.009799048, 0.000051346076, -0.017761579, 0.008553298, 0.021884114, 0.028562363, -0.01864773, -0.00018270875, -0.004658722, 0.029101761, -0.0026568521, -0.019803582, -0.013446401, 0.032672055, -0.013317972, -0.003024477, 0.013228074, -0.0023518358, 0.023361035, -0.013690414, -0.0015186599, -0.0068580494, 0.017658835, 0.0043119667, 0.018390875, 0.016040644, -0.0050568483, -0.0041032713, -0.004084007, -0.0132922875, 0.0014624727, 0.0065658763, 0.018442245, -0.0041995924, 0.004816046, -0.010203596, -0.021293346, -0.007744202, 0.00621591, 0.0030726374, 0.0022442774, -0.012412556, 0.013202388, 0.0033327038, -0.013131753, 0.021100704, -0.002896049, -0.017222181, 0.023078494, -0.00488026, -0.007821259, -0.011391554, -0.020098966, -0.0025252134, 0.0012425398, -0.012643727, 0.026250662, -0.012393292, 0.005451764, 0.007269019, 0.010742993, 0.009792627, -0.020227393, -0.013574828, 0.0045046085, 0.00980547, -0.017196495, 0.0014424059, -0.0070121633, -0.013754628, 0.0072818617, 0.0036505628, -0.00188789, 0.0050568483, 0.024504043, 0.0033391253, 0.0043729697, 0.0014175229, 0.0021286923, 0.0066268793, 0.011391554, 0.0023502305, 0.010768679, -0.01510312, -0.004774307, 0.00529444, -0.00446608, -0.0021880902, 0.0033070184, -0.022641838, -0.00024060164, -0.0072433334, 0.009966005, 0.0039844755, -0.0071855406, 0.00036381214, 0.011950215, 0.026186448, 0.0044403947, -0.0008893632, 0.021601573, 0.013407872, 0.010935635, 0.002385548, -0.009272494, -0.0122777065, -0.009728413, -0.00042622007, 0.00704427, 0.0065498226, -0.008887211, 0.016862582, -0.0067745717, 0.015989274, 0.013151017, -0.015501247, -0.011076906, -0.012547405, -0.010113697, 0.005763202, -0.015681047, 0.0036730377, -0.0021431404, 0.0002586618, -0.0034771853, -0.0140500115, -0.006556244, -0.0041450104, -0.005612299, -0.028279822, -0.019174285, -0.005538453, 0.008636776, 0.0136262, -0.022320768, 0.009856841, 0.0020981908, 0.017620306, 0.0076543028, 0.0067103575, -0.003933104, -0.0038496263, 0.00870099, 0.035934124, -0.0137931565, 0.00043705618, 0.0036377201, -0.0065755085, -0.01649014, -0.0010370553, -0.022115285, 0.0018429403, 0.012136436, 0.00056387874, -0.005349022, 0.0011863527, -0.0030501625, 0.0015170545, -0.012829946, 0.01408854, 0.0016310343, 0.022205183, -0.017183652, 0.02016318, 0.003310229, 0.008135907, -0.004369759, 0.01026781, 0.008270756, -0.008649618, 0.011635567, 0.031747375, -0.013549143, 0.0191486, 0.021357559, 0.0074038683, 0.03665332, -0.00976052, -0.016580041, -0.012894161, -0.013240916, -0.018596359, -0.0014351817, 0.0082193855, -0.0053650755, 0.008546877, -0.0015981246, 0.009856841, 0.005650827, 0.0070250058, -0.0028783903, -0.0081937, 0.022474881, -0.006100325, -0.007525875, 0.007468082, 0.010479717, -0.0060457434, 0.017389137, 0.015693888, 0.0093559725, -0.001772305, 0.0072754403, 0.015796632, 0.012380449, -0.005666881, 0.014037169, -0.0077506234, -0.023219764, 0.009041324, 0.013106067, 0.04084007, 0.006418184, 0.009182595, 0.021563044, 0.009458714, -0.010768679, 0.0021351138, 0.009362394, -0.0031224033, -0.021665785, 0.015899373, 0.009009217, 0.0010587275, 0.0037436732, 0.014679309, 0.0025123707, -0.005782466, 0.016721312, -0.01082005, -0.0031159818, 0.011404397, 0.008675304, -0.009144066, -0.0038014657, -0.0032379883, 0.0040679537, -0.004523873, -0.01961094, -0.0005590627, -0.014974693, -0.0055641388, -0.026635945, -0.006703936, -0.0011759179, -0.008412027, 0.030540153, -0.0023550466, -0.0067938357, -0.0011052826, 0.0007472898, 0.0070956415, -0.024915012, -0.029076075, 0.0044275518, 0.026289191, 0.004568822, 0.001528292, -0.026083706, -0.00708922, -0.0025717686, 0.018544989, 0.008026743, -0.0024240764, -0.0009038113, -0.011340183, -0.033365566, -0.00055665465, 0.025839692, 0.02748357, -0.009933898, 0.008604669, -0.009471557, 0.014627937, -0.009477979, 0.0064053414, 0.0010571221, -0.003618456, 0.014846264, 0.022423511, -0.013818841, 0.0059044724, -0.014974693, 0.019212814, 0.018211076, 0.0023775215, -0.018416561, -0.023271134, 0.011905266, -0.021190602, 0.0028992598, -0.0032973862, 0.004084007, -0.019508198, -0.001153443, 0.00041377862, 0.011731888, -0.011609882, -0.009150487, 0.015115963, 0.008386342, 0.027509255, -0.011757574, -0.006411763, 0.019919166, -0.005085745, 0.0041000606, 0.021062175, 0.016284658, -0.009458714, 0.008142329, 0.0012634094, 0.024118759, 0.002284411, -0.0056925663, -0.021563044, 0.023399564, -0.0043986556, 0.008200121, 0.004568822, 0.015475562, -0.00055665465, -0.0018204654, -0.0012682255, 0.020638363, -0.0022860165, -0.015693888, -0.018557832, -0.013960112, 0.0028238085, 0.009426608, -0.015732417, -0.007493768, -0.015372819, -0.02591675, -0.0047454108, -0.011564932, 0.002412839, 0.017106596, -0.0061805923, -0.013382187, 0.0162975, -0.0093816575, -0.021845585, -0.00038668836, -0.0027852799, -0.024324244, -0.025236081, -0.036756065, -0.010698044, 0.00003102838, 0.0014014695, -0.00071839354, -0.0140500115, 0.044769965, -0.018442245, -0.023643576, 0.0358057, 0.009966005, -0.0071662767, -0.003949158, 0.0013725731, -0.010415503, -0.0020548464, -0.025724107, -0.036910176, -0.039838333, -0.011211756, -0.0043729697, 0.003945947, 0.02702123, 0.0037083556, 0.012977639, 0.002859126, -0.012065801, -0.010190753, 0.0020933747, -0.0097476775, 0.007757045, 0.021357559, -0.0061324323, 0.0030373198, 0.010370553, 0.008347813, 0.031644635, -0.025608523, -0.02108786, 0.011629146, 0.0081102215, -0.008168014, 0.0014424059, 0.020432878, -0.0059879506, 0.0005377918, -0.015270077, 0.0143068675, -0.007583667, -0.005589824, -0.01173831, -0.010004533, 0.022115285, -0.003907419, -0.00093110226, -0.0013525062, -0.013690414, 0.0017674889, 0.017273553, -0.016310344, -0.00515638, 0.021357559, 0.02661026, 0.0034258142, 0.023232607, 0.026687317, 0.017761579, 0.023836218, -0.009921055, 0.00069110264, -0.0065658763, -0.010685201, 0.006440659, 0.001574847, 0.011975901, 0.00037906296, -0.003711566, 0.0037372517, 0.013356501, 0.005888419, 0.016836897, -0.0022956487, -0.006819521, 0.023258291, -0.002827019, -0.017684521, 0.011841052, -0.018018434, 0.011173227, 0.009580721, 0.021614416, -0.004469291, 0.01758178, -0.02062552, -0.026867116, -0.016400242, 0.0022812006, 0.0026167184, -0.0036890914, 0.0059751077, 0.016592884, 0.010511823, 0.0051114303, 0.018878901, 0.0015395294, -0.0076029315, -0.015128806, 0.0024369194, 0.019212814, -0.025287453, -0.007988215, -0.0031384567, -0.005082534, 0.016258972, 0.022616152, -0.0055448744, 0.010325603, 0.020548463, 0.0024850797, 0.029641159, -0.00092387816, -0.005952633, 0.022847323, 0.0047935713, -0.002550899, 0.010441188, -0.016220443, -0.009137644, -0.015026064, -0.0075387177, -0.016284658, 0.000010064784, 0.00032026708, 0.006559455, -0.022577625, 0.0017498301, -0.0059590545, -0.023091335, -0.011545667, 0.012329078, -0.0025541098, -0.013446401, 0.029692529, 0.025569994, 0.023797689, 0.004138589, -0.008052429, 0.010088012, -0.003300597, -0.008071694, -0.0041995924, -0.010100854, -0.0056411955, -0.01496185, 0.008534034, -0.0018670205, -0.0155783035, 0.008347813, 0.0027001966, -0.0087202545, 0.028716477, -0.002127087, -0.013677571, 0.0031641424, -0.032851856, -0.0003864877, 0.001675984, -0.003329493, -0.004751832, -0.015488404, 0.022140969, -0.012123593, -0.006501662, 0.0069543705, 0.013459244, 0.010017375, -0.008437713, -0.010152225, -0.007936844, 0.005377918, 0.024902169, 0.0060939034, 0.019187128, 0.0040711644, 0.016181914, 0.0011076906, 0.005878787, -0.03899071, 0.007879051, 0.01058888, 0.024889326, 0.0021977224, 0.00082434656, -0.004751832, -0.012091487, 0.016567199, 0.019970538, 0.011012692, 0.0046908287, 0.0014536432, -0.010332025, 0.010120118, -0.0358057, -0.008161593, -0.00049725675, 0.016772684, -0.017055225, 0.013459244, 0.037732117, -0.014371081, 0.0140500115, -0.009522929, -0.0024882904 - ] - } - ] - }, - { - "name": "untitled", - "title": "Untitled", - "content": "Staying up-to-date and discerning key technological trends amidst constant advancements can be overwhelming. Assistance in navigating these changes can be beneficial.", - "keywords": [ - "technological trends", - "advancements", - "up-to-date", - "navigating changes", - "beneficial" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - 0.026481831, 0.04921357, -0.002366284, -0.021460302, -0.027380828, -0.0036216667, 0.012605198, -0.006658986, -0.06981341, 0.021896956, -0.03893934, 0.025441566, -0.013189545, -0.0004924407, 0.0061902246, 0.013844527, -0.015963588, 0.04998414, 0.0066975146, -0.0013460849, 0.0354461, -0.0061163786, -0.060566597, 0.044179197, 0.012682254, 0.00824507, -0.030694267, 0.0064470805, -0.026327718, 0.0017578568, -0.003660195, -0.001222473, 0.045232303, 0.001574847, 0.009728413, -0.019392611, 0.0015339106, -0.020548463, 0.029872328, 0.034881018, 0.011256705, 0.0020708998, 0.018943114, 0.010145804, 0.028896276, -0.02853668, 0.0003991298, -0.036550578, 0.021190602, 0.023232607, 0.016002117, -0.014949007, -0.021293346, 0.041456524, 0.011834631, -0.004276649, 0.011937373, 0.019135756, -0.007423132, -0.050857447, 0.00016474891, -0.014332553, -0.024452671, 0.009426608, 0.012438241, 0.0215502, 0.017299237, -0.002600665, 0.04433331, 0.020317294, 0.029666845, 0.06642291, 0.0055320314, 0.029332932, 0.054299314, -0.008938582, -0.03149052, 0.01201443, -0.025955278, 0.013651885, 0.02559568, -0.028100023, -0.020188866, 0.0044949763, -0.0021094282, 0.0000010096138, 0.011256705, -0.073666245, 0.0030164502, 0.0036056133, 0.010672358, -0.013728942, -0.011584196, -0.020908061, -0.044487424, -0.027791796, 0.045566216, -0.010299917, -0.005217383, 0.024298558, 0.011956637, -0.002674511, -0.0404291, 0.01731208, 0.0005526413, 0.016438771, -0.0006758518, -0.006209489, -0.016143387, 0.013112488, 0.025955278, -0.003226751, -0.0071662767, -0.0046651433, -0.027201029, 0.0028061494, -0.009728413, -0.017800106, 0.044127826, 0.015115963, -0.0037211983, 0.014743523, -0.03226109, 0.006071429, -0.021601573, 0.010254968, 0.020432878, -0.017453352, 0.01961094, -0.032158345, 0.02591675, 0.011353026, -0.0068516284, 0.011333762, 0.009336708, -0.04225278, -0.0065401904, 0.031798746, 0.028100023, 0.048263203, -0.06390572, -0.007159855, 0.019790739, -0.00056187203, -0.0013733758, -0.025467252, -0.016952483, 0.01606633, 0.012733626, -0.026327718, -0.024195816, -0.013780314, -0.027509255, 0.00446608, 0.018095491, 0.020047594, 0.0060842717, -0.024414143, 0.05676513, -0.046901867, -0.018801844, -0.013587671, 0.028228452, -0.036807436, -0.011166805, -0.0064246054, -0.020201707, -0.00026046782, 0.0016093621, 0.0034482891, 0.042278465, -0.0054389215, -0.04572033, -0.07489915, 0.0060329004, -0.025261767, -0.014204125, -0.014846264, -0.005731095, 0.024375614, 0.008501926, -0.0031464836, 0.028022967, -0.009779784, -0.020908061, -0.022128126, 0.006890157, 0.023964645, 0.038554054, -0.021575887, 0.00079223956, 0.051448215, 0.013883055, 0.0060040043, 0.02905039, 0.0014680914, -0.011725467, -0.033365566, 0.015937902, 0.026032334, -0.054196574, 0.015385662, -0.02205107, -0.014782051, 0.0024593943, 0.015937902, -0.009535771, -0.0095743, 0.020869533, 0.030642897, -0.01035771, -0.017607465, -0.017787263, 0.042612378, -0.014936164, 0.02817708, -0.017183652, -0.007352497, -0.017389137, 0.0459515, 0.026404776, 0.037115663, 0.007513032, -0.026404776, -0.018557832, -0.009304601, -0.06303241, 0.0059751077, 0.004613772, -0.026918488, 0.032158345, 0.01790285, 0.0072625973, 0.056919243, -0.0379376, 0.06133716, 0.024555413, 0.0073268116, -0.01717081, 0.034315933, 0.027406514, 0.02899902, -0.00980547, -0.02463247, -0.007904737, -0.028074337, 0.023733474, 0.0059141046, 0.0075708246, 0.028716477, 0.00035177203, 0.0030084234, 0.051936243, 0.00994674, -0.011930952, 0.009709149, -0.005750359, 0.006633301, -0.008906474, 0.0073139686, 0.0019874217, 0.010370553, -0.017157966, 0.017479036, 0.005731095, -0.035112187, 0.0005357851, 0.0032652793, 0.0019537094, -0.029153133, -0.001703275, -0.004225278, 0.007288283, 0.017466193, 0.028613735, 0.010903528, 0.042612378, 0.02301428, -0.03125935, -0.04623404, -0.021113547, 0.009054166, 0.0051242732, -0.025518622, 0.0045559797, 0.01243182, -0.007943265, 0.027098287, 0.0022137759, -0.046799127, -0.011423661, 0.0044436054, 0.030899752, 0.025711264, 0.030129185, -0.0028848117, 0.0030662161, -0.0033487573, -0.046901867, 0.019508198, 0.018146861, 0.02661026, -0.007866208, -0.015231549, 0.0122648645, -0.015128806, 0.005050427, -0.00395879, -0.026160762, -0.05404246, -0.009221123, -0.053374633, 0.0028334404, 0.025698422, -0.011995166, -0.00033009984, 0.017222181, -0.014139911, 0.038502682, -0.017440509, -0.020009067, 0.0029875538, 0.018429402, 0.019906323, -0.012348342, 0.023399564, -0.000100685465, -0.012727205, 0.01592506, 0.01201443, -0.03457279, 0.004626615, -0.017812949, -0.05106293, 0.011051221, -0.012759311, 0.018763315, -0.008296442, 0.0010033429, 0.019970538, -0.004976581, 0.0057664127, 0.009908212, 0.037192717, 0.026507517, -0.02149883, -0.009818313, -0.0053971824, -0.013163859, 0.01740198, 0.000666621, 0.003351968, 0.05501851, -0.0039620004, 0.017363451, -0.006283335, -0.02499207, -0.02283448, 0.01606633, -0.039581478, -0.024439828, 0.0067809927, -0.009458714, -0.0022812006, -0.019482512, 0.019495355, 0.00073284167, 0.009400922, 0.030617211, 0.0062191207, 0.013240916, 0.014011484, -0.0122648645, 0.011956637, 0.013497772, 0.006960792, -0.017363451, -0.008604669, -0.012386871, -0.00061725656, 0.032440886, 0.025531465, -0.019944852, 0.00086769095, -0.0011991955, -0.0030260824, -0.0017899638, -0.017838635, -0.017337766, 0.011089749, -0.008084536, -0.013382187, -0.00469725, 0.0030517678, -0.0038464156, 0.017517565, -0.044281937, -0.0033583895, 0.028896276, 0.0041642743, 0.023232607, -0.03056584, 0.002181669, 0.031336408, -0.020753948, -0.0052976506, -0.0019970539, -0.057124726, 0.012662991, -0.014011484, -0.024786584, -0.0030180556, 0.003393707, -0.012322657, -0.02899902, -0.05008688, -0.016541513, 0.03714135, 0.015488404, 0.034059078, -0.01449951, -0.037911914, 0.0063314955, 0.0039555794, -0.015411347, 0.0030276875, 0.012740048, 0.025698422, -0.0381174, -0.021357559, -0.031464834, -0.009477979, 0.0027917014, 0.010306339, -0.015000379, 0.050985876, -0.03688449, 0.0045591905, 0.03978696, 0.013356501, -0.02946136, 0.005962265, 0.032338142, -0.018095491, -0.013574828, 0.022487724, 0.02684143, 0.01873763, 0.024337087, 0.056919243, -0.014165597, 0.03434162, 0.016567199, 0.011622725, 0.007416711, 0.0011245467, -0.042920604, 0.012496035, -0.007988215, 0.031798746, 0.023399564, -0.023181235, 0.014345396, -0.0352663, -0.013972955, 0.02513334, 0.0461313, 0.00971557, 0.000971236, -0.017877163, -0.0072561763, -0.025582837, -0.039581478, 0.02541588, -0.045925815, -0.0031063498, 0.004482134, 0.012996903, 0.022962907, -0.015848001, -0.040814385, 0.022718895, -0.044924077, 0.015950745, -0.0050632698, 0.026147919, -0.025929593, -0.027663369, 0.0106274085, -0.015167334, 0.052295838, 0.0053522326, 0.008765204, -0.020227393, 0.010807208, 0.0070378487, -0.03724409, -0.055223994, 0.02370779, 0.033005968, -0.0033969178, 0.03084838, 0.037629373, -0.0029153132, 0.003204276, -0.0039427364, -0.02030445, -0.03521493, 0.013472087, 0.016323186, 0.061491277, 0.0035125029, -0.028279822, -0.020009067, 0.016913954, -0.015475562, -0.012386871, -0.010210018, 0.029409988, -0.009895369, -0.009580721, 0.011622725, -0.0025525044, 0.006543401, -0.019790739, 0.012579513, 0.0033038077, -0.0009977242, 0.029872328, -0.021241974, -0.033262827, 0.0057696234, 0.0056090886, 0.028022967, 0.06405983, -0.02513334, -0.02126766, -0.019238498, -0.0017867531, 0.042715117, -0.03351968, -0.057535697, 0.012412556, 0.03190149, 0.01850646, 0.006154907, -0.0047807284, -0.012207071, 0.0028703634, -0.03167032, 0.053888347, -0.014949007, 0.005888419, 0.0030581893, -0.014653623, -0.007121327, -0.014692151, -0.02007328, -0.037038606, -0.013330815, 0.02140893, 0.011243862, 0.021845585, -0.004456448, 0.0062865457, 0.015475562, 0.016002117, -0.011282391, 0.005410025, 0.0017562514, 0.011218177, 0.012451084, 0.026327718, 0.0064663445, 0.019906323, 0.011719045, -0.018660573, -0.015398505, 0.012367606, 0.011423661, -0.008360656, 0.019649468, 0.031387776, 0.025390195, 0.0005686948, 0.0066782506, -0.00814875, -0.030077813, -0.0034033393, 0.044769965, 0.036627635, 0.023450933, -0.0029265506, 0.015115963, 0.0010563195, 0.009060588, -0.027534941, 0.015681047, -0.028665107, -0.029435674, 0.0068387855, -0.0041450104, 0.015732417, -0.020458564, -0.024388457, 0.008463399, 0.023643576, 0.0153086055, 0.018904585, 0.010511823, 0.0024465513, 0.020831006, 0.0017129071, -0.008026743, -0.032286774, -0.006315442, 0.03403339, 0.029076075, 0.010800786, -0.022359297, -0.014294025, -0.024414143, -0.0074552395, -0.009766942, 0.025377352, -0.021421773, 0.012451084, -0.016323186, -0.0031175872, 0.0018814687, -0.001079597, 0.009914634, -0.015526933, 0.006459923, -0.01473068, -0.019687997, 0.0061805923, -0.008418448, -0.02080532, -0.012784997, 0.0026488253, 0.058306266, 0.016926797, 0.03624235, -0.004639458, 0.00016755828, -0.021100704, 0.024902169, 0.007191962, 0.004899524, 0.055994563, -0.028048653, 0.009477979, -0.0011646805, -0.00091825944, 0.01800559, 0.02776611, -0.030257612, 0.028048653, -0.003241199, -0.016836897, 0.014897636, 0.0004274241, 0.006877314, 0.012412556, 0.020908061, -0.03737252, 0.038091715, 0.017928535, -0.0034771853, -0.014332553, 0.017915692, -0.004116114, 0.011449347, -0.010132961, 0.0067938357, 0.0032797274, -0.0069351066, -0.010486138, 0.0020548464, 0.010486138, -0.0164773, -0.02946136, -0.005731095, 0.013818841, -0.021190602, 0.044025082, 0.03673038, -0.005506346, -0.01758178, 0.019366927, 0.033493996, -0.0018959168, -0.007827681, 0.031053865, 0.004337652, -0.0011686939, 0.031233665, 0.01150714, -0.014846264, 0.0020773213, 0.023373878, 0.01740198, -0.039966762, 0.010736573, 0.009766942, -0.0018333083, -0.024619628, -0.05224447, -0.008643198, 0.006703936, -0.004626615, 0.030694267, -0.011076906, 0.053888347, -0.035112187, 0.040583216, -0.035137873, 0.008495505, 0.039966762, 0.02375916, 0.008707412, -0.053785603, -0.012117172, -0.03236383, 0.008309285, -0.029230189, -0.025788322, 0.01473068, 0.0060650073, -0.038399942, 0.009599986, 0.026250662, -0.009266073, 0.026738688, -0.001855783, -0.013818841, -0.02499207, 0.022962907, -0.007172698, 0.0067424644, -0.032081287, 0.04374254, 0.006652565, 0.0006770558, 0.017055225, -0.0025187922, -0.003660195, -0.059179574, 0.0028510992, -0.038656797, -0.019816425, 0.038785223, 0.016271815, 0.0037500947, 0.00022956486, 0.012142858, 0.0070378487, 0.01588653, 0.0358057, 0.023656419, 0.010858579, 0.00202595, 0.0025492937, 0.0077827303, 0.0022137759, 0.004816046, -0.010113697, -0.0027130393, 0.0065947725, 0.03323714, -0.0023261502, -0.005262333, -0.0018188601, 0.022397825, -0.00060922984, 0.010967743, -0.011988744, 0.019662311, -0.03398202, 0.034649845, -0.006071429, 0.002319729, -0.0023405985, -0.007577246, -0.019392611, 0.009978848, -0.002969895, 0.0044725016, -0.013818841, -0.008444134, -0.004597719, -0.017684521, 0.010537509, -0.0093945, -0.016541513, 0.0115199825, -0.012194228, 0.044410367, 0.006601194, 0.012309814, 0.019598097, -0.010017375, 0.010909949, -0.030334668, 0.0030533732, 0.007179119, 0.013985798, -0.016708469, 0.018211076, 0.040994186, -0.022269398, 0.020009067, -0.0081102215, -0.036422152, -0.027534941, 0.016567199, -0.0024096284, 0.00019906324, -0.02003475, 0.034778275, -0.021421773, 0.0432802, -0.010794365, 0.020908061, 0.02306565, 0.023720633, -0.026481831, 0.00043344416, 0.018955957, -0.010730151, -0.0024609996, -0.013703257, -0.009734835, -0.00971557, 0.011564932, 0.0070185843, 0.0067681503, 0.02928156, -0.009323865, -0.0076157744, -0.015462719, 0.027971596, -0.0072947047, 0.010210018, 0.04199592, 0.0072112265, -0.054813027, 0.015038907, -0.04148221, -0.012984061, 0.017890006, -0.01510312, 0.019726524, -0.025467252, 0.02666163, 0.024683842, 0.0083542345, 0.0038110977, 0.008450556, 0.007943265, -0.012573091, 0.03102818, 0.0044500264, 0.004931631, 0.004074375, 0.0023839427, 0.012001587, -0.018724788, -0.0052655437, 0.015090277, 0.0019248131, -0.013960112, -0.024645314, -0.007493768, 0.014692151, -0.009291758, -0.01937977, -0.012252022, 0.013446401, -0.025390195, -0.004116114, -0.010556773, -0.033262827, 0.024285715, 0.03084838, -0.019970538, -0.012194228, 0.012149279, -0.021794215, 0.0012915031, 0.004543137, -0.0024064176, -0.006090693, 0.012457506, 0.025801165, -0.018917428, -0.020086123, -0.015629675, 0.027689055, 0.020381507, -0.008668883, 0.0035574527, -0.017235024, -0.0040165824, 0.00022976553, 0.023656419, 0.0075066104, -0.0037372517, -0.0025974542, -0.00833497, 0.012374028, 0.0052591222, -0.03369948, -0.00865604, 0.001333242, -0.02840825, -0.0136262, -0.00015060177, 0.031002494, 0.03231246, 0.020060437, -0.02817708, 0.0046651433, -0.00045351102, -0.0055866134, 0.014396767, -0.023502305, -0.01597643, -0.0005907683, 0.015603989, 0.0076992526, 0.0073717614, 0.006068218, 0.01864773, -0.021370402, -0.038887966, 0.0018943115, -0.01072373, 0.028973334, -0.008540455, 0.016567199, -0.023001436, -0.01836519, -0.0067103575, 0.044949763, -0.01868626, 0.025081968, 0.038476996, -0.0025043439, 0.010909949, 0.013202388, -0.010800786, 0.011410818, 0.047030296, 0.008277178, -0.025030596, 0.03218403, -0.01946967, 0.019135756, 0.004957317, 0.0035478207, -0.030899752, -0.008636776, 0.013651885, -0.008822996, -0.018390875, -0.06477903, 0.000050116978, -0.0009913028, 0.024157288, -0.0140500115, -0.015655361, -0.015205863, 0.00051050086, 0.03601118, -0.007724938, 0.0053522326, 0.024773741, 0.02559568, 0.015642518, 0.021627259, 0.007564403, 0.042021606, 0.02858805, -0.017568937, -0.022115285, -0.016053487, -0.004209224, 0.0015001984, 0.022256555, -0.025608523, -0.0017000643, -0.0069543705, 0.052655436, -0.010614566, 0.0023405985, 0.0005803336, -0.0041321674, 0.014704994, 0.035291985, -0.003554242, -0.018609202, -0.00071518286, -0.01634887, 0.02905039, -0.044179197, 0.0028832064, -0.014627937, -0.0056861453, 0.021370402, 0.0010282259, 0.032517944, -0.025801165, -0.008360656, -0.025069125, -0.0075001894, -0.016657097, -0.018519303, -0.00478715, 0.00285431, 0.018095491, 0.0045367153, -0.012733626, -0.0042284885, 0.010832893, 0.014627937, 0.010351288, 0.02324545, 0.02393896, 0.021807056, 0.00014438105, -0.0044949763, 0.014062854, -0.034290247, -0.007930422, -0.008135907, 0.05589182, 0.0023887588, 0.017697364, 0.012746469, -0.035985496, 0.021896956, 0.0022009332, 0.010473295, -0.014011484, -0.008084536, 0.003798255, 0.018172547, 0.01772305, 0.00055344397, 0.0025075546, -0.00028073534, 0.0032973862, -0.033622425, -0.008501926, 0.02858805, 0.02545441, 0.0018509671, -0.017568937, 0.005644406, -0.012528141, 0.006684672, -0.0027290927, 0.02066405, -0.00060481514, -0.023990331, 0.017607465, 0.0037372517, 0.009580721, 0.0019681575, -0.009709149, 0.035189245, -0.020766791, 0.019816425, 0.016104858, -0.01505175, -0.01431971, -0.0036762485, -0.0329546, -0.0017369873, 0.018211076, -0.018532146, 0.031875804, 0.015770946, -0.03816877, 0.012560248, 0.006251228, -0.013003324, 0.015501247, -0.016284658, -0.014576566, 0.0028575207, -0.009182595, 0.05332326, -0.024837956, 0.0155783035, 0.049316313, -0.013549143, -0.014704994, -0.016913954, -0.0056989877, 0.0055577173, -0.053374633, 0.0032973862, 0.004032636, -0.03940168, -0.0002867554, -0.034906704, -0.012964796, -0.008938582, 0.032517944, -0.021473145, 0.014846264, 0.013330815, 0.021241974, -0.017979905, -0.0019392612, -0.0050632698, -0.020959433, -0.011012692, -0.014692151, -0.029384302, 0.0020291607, 0.023104178, -0.023669261, -0.012849211, -0.0074295537, 0.018339504, -0.0155783035, -0.0077313595, 0.019777896, -0.016246129, -0.03770643, 0.0075322962, 0.0030854803, 0.01615623, -0.024593942, 0.028742163, -0.021768529, 0.015038907, 0.0031962493, -0.013138174, -0.020291608, 0.02158873, -0.01026781, 0.034007706, -0.0040390575, 0.0032829382, -0.013998641, -0.028433936, -0.005721463, 0.00814875, -0.0011510351, -0.016888268, -0.008784468, -0.0067360434, -0.01054393, -0.03411045, 0.022295082, -0.021935485, -0.001362941, -0.008168014, 0.0105182445, -0.0005365878, -0.0017626729, 0.021396087, 0.00015551815, 0.0028302297, 0.03246657, 0.0010458847, 0.0007224069, -0.018172547, -0.029332932, 0.008437713, -0.026687317, 0.00598474, -0.0076157744, -0.00059116964, 0.011744731, -0.008945003, -0.01164841, -0.020895218, 0.013151017, -0.013048274, -0.026353404, -0.008296442, -0.019199971, -0.031978548, -0.0028607314, 0.010132961, -0.01675984, 0.037834857, 0.023605047, 0.0048642065, -0.01449951, -0.000033310986, -0.0012296971, 0.012829946, 0.026481831, -0.017915692, 0.031233665, -0.011372291, -0.009728413, 0.014615094, -0.017967062, 0.0014416032, -0.011038378, -0.002725882, -0.0219098, 0.013163859, 0.012271285, 0.014473824, -0.02518471, 0.012842789, -0.008893631, -0.038143083, 0.014127068, 0.03246657, -0.0035799276, 0.0042702276, 0.0012642121, -0.0350865, 0.024131602, 0.0062769135, 0.016554356, -0.019816425, 0.010710887, -0.007269019, 0.005853101, -0.010100854, 0.013844527, -0.012817104, -0.0036955127, -0.0057888874, -0.031362094, -0.025403038, 0.029589787, 0.0043665483, 0.02292438, 0.020792477, -0.006228753, 0.028254136, -0.0032235403, -0.0017385926, 0.020086123, -0.035780013, -0.032029916, 0.015398505, -0.037860543, 0.0049348418, 0.0086110905, 0.011385133, -0.019122913, -0.02541588, -0.011622725, -0.029101761, 0.011808945, -0.020753948, 0.000802273, 0.019559568, -0.004616983, -0.005695777, 0.029076075, 0.021485986, 0.008168014, -0.013189545, -0.0007500992, 0.011320919, -0.0008291626, -0.0012401319, 0.025325982, -0.015270077, -0.015180177, 0.018198233, 0.019546725, -0.019803582, -0.007737781, 0.03310871, 0.0123997135, -0.0016583252, 0.008078115, -0.0016647466, -0.0054710284, -0.002546083, -0.00216401, 0.025980964, -0.008283599, 0.024439828, 0.00644387, 0.03195286, 0.011494297, 0.012862054, 0.003083875, -0.02545441, -0.00893216, 0.023797689, -0.04723578, -0.005422868, -0.001841335, 0.025801165, -0.02587822, -0.0097476775, -0.019996224, -0.008033165, -0.010601723, -0.019456826, -0.0005947817, -0.009182595, 0.003157721, 0.020895218, 0.0045463475, -0.022256555, -0.006036111, 0.004379391, -0.0069543705, 0.018082649, -0.004783939, -0.020497093, 0.0016181915, -0.050035506, -0.0018172547, -0.013523458, 0.0016872215, 0.01449951, -0.009831156, 0.02035582, 0.0033327038, -0.00019735756, 0.02205107, -0.0037725694, 0.0074102897, -0.020728262, -0.0037308303, -0.012303392, -0.02518471, 0.023258291, -0.02481227, 0.04538642, 0.0064213946, 0.014255497, 0.02853668, -0.00010665535, 0.016104858, 0.01173831, 0.004915578, -0.00045190565, -0.019122913, 0.01288774, -0.027278084, -0.008501926, 0.027534941, -0.023874747, 0.0032540418, -0.031362094, 0.0115199825, -0.005833837, 0.013112488, 0.015783789, 0.012720783, 0.0072625973, 0.02789454, 0.023052808, -0.0028880222, -0.0057664127, -0.015629675, -0.00547745, -0.017658835, -0.010807208, -0.011044799, 0.019443983, 0.015347133, 0.015038907, 0.021139232, 0.0032700952, -0.0034258142, -0.0051949085, -0.021884114, -0.0034129713, -0.0071470123, -0.0029650792, -0.0012634094, 0.012027272, 0.004658722, 0.013587671, 0.032286774, 0.025017753, 0.0057439376, 0.01279784, 0.0076029315, 0.0023213343, 0.0012834762, 0.011083327, 0.0011735099, -0.010768679, 0.0066268793, 0.0047261463, -0.0077056736, 0.004061532, 0.0025974542, -0.0019023382, -0.0061805923, 0.001878258, -0.011275969, 0.04949611, -0.0046940395, 0.0021014016, -0.007423132, -0.013960112, 0.013728942, 0.004581665, -0.015526933, 0.028279822, -0.035112187, -0.008142329, -0.023810532, -0.010280653, 0.031798746, 0.007236912, -0.016271815, 0.009028481, -0.01072373, -0.019803582, 0.009978848, -0.00018702312, -0.036807436, -0.016772684, -0.02066405, -0.007757045, -0.0012987271, -0.008816575, 0.011828209, -0.015899373, -0.004485344, 0.009535771, -0.00024240765, -0.003586349, -0.0023373878, -0.013080381, -0.0056604594, -0.014294025, -0.010640251, -0.0023839427, 0.026353404, -0.011712624, 0.0014335763, -0.0077827303, -0.033648107, -0.011719045, -0.0059911613, -0.027046915, 0.0020548464, -0.012618041, -0.009567878, 0.023861904, 0.012425399, 0.01666994, 0.02434993, 0.006344338, -0.017157966, 0.005288019, -0.010967743, 0.011796102, 0.015539776, 0.0049284203, -0.02016318, -0.00012391285, 0.015372819, -0.008822996, 0.0064984513, -0.0065530334, 0.0005153169, 0.0048513636, 0.022808794, 0.013960112, 0.025467252, 0.032158345, 0.014024327, -0.019212814, -0.014460981, 0.0033969178, 0.016258972, 0.0043858127, 0.0030662161, -0.003522135, 0.02408023, -0.006658986, 0.0034129713, 0.0012481586, 0.026918488, -0.02794591, 0.022436354, 0.02237214, -0.012772154, -0.019944852, -0.0022763845, 0.016734155, 0.015822317, 0.017055225, 0.028870592, -0.0021190604, 0.01473068, -0.013780314, -0.014512353, 0.0013284261, 0.013215231, 0.012528141, -0.034726903, -0.0023807322, -0.011866737, 0.011886002, 0.028485307, -0.007757045, 0.020586992, -0.03269774, 0.02173, 0.020753948, -0.001878258, 0.018416561, 0.0023887588, 0.021832742, 0.045823075, 0.029692529, -0.01763315, 0.014101383, -0.021370402, 0.0040904284, 0.007204805, -0.0015828739, -0.028921962, -0.0025942435, 0.017979905, -0.02748357, -0.010858579, -0.0037147768, 0.011430083, -0.011385133, 0.0054581854, -0.0244912, -0.008668883, -0.008758782, 0.0032620686, 0.016284658, -0.0020692945, -0.011051221, 0.018275289, 0.0015347133, -0.007879051, -0.021010803, -0.0072754403, -0.042201407, -0.007885473, 0.004254174, -0.0069094207, -0.0030967176, 0.019418297, -0.007795573, 0.034906704, 0.000084631974, 0.009433029, -0.00446608, 0.02481227, 0.010210018, -0.0062865457, -0.0019039436, -0.011173227, -0.0022812006, -0.011622725, 0.00704427, -0.014525195, 0.0077827303, 0.007821259, 0.0076221954, 0.003907419, -0.0242215, -0.007288283, -0.0145637235, -0.0011422056, 0.012040115, 0.013099645, -0.024735212, -0.0011767206, -0.02269321, -0.004135378, 0.014371081, 0.013870213, -0.013016167, -0.001703275, 0.02518471, -0.0021624046, -0.0053811288, 0.0071662767, 0.0016230076, 0.0006746478, -0.011038378, -0.013433558, 0.02375916, -0.014717837, -0.009638513, -0.025428724, -0.00584668, 0.01588653, -0.033802222, 0.0022908326, -0.009587143, -0.0016422718, 0.009683464, -0.004539926, -0.002513976, -0.0036698272, -0.00855972, -0.019366927, -0.0048642065, -0.005994372, -0.02223087, 0.016143387, -0.0007814035, -0.013433558, -0.008521191, 0.006652565, -0.012059379, -0.010813629, -0.012354763, 0.039324623, -0.022590468, 0.011597039, -0.007140591, -0.014717837, 0.0066654077, 0.01003664, -0.0013252153, 0.013497772, 0.021049332, 0.0014761181, -0.0072433334, -0.005088955, 0.019790739, -0.011918109, 0.007814838, 0.0047807284, 0.041816123, -0.0072754403, -0.013041853, 0.03560021, 0.025197553, -0.009336708, 0.0014464192, -0.0005991964, -0.00024702304, -0.0116548315, 0.0018525724, -0.004437184, 0.0035413993, 0.010145804, -0.007994637, -0.027278084, -0.0058402587, -0.010563195, 0.00013474896, -0.00088454713, 0.016284658, 0.0065209265, 0.008964268, -0.0103833955, 0.009240387, 0.020291608, 0.012008009, -0.016400242, -0.0127914185, -0.024439828, 0.009176173, -0.0039555794, 0.0022539096, -0.011468611, -0.0060168467, 0.0060842717, -0.020728262, 0.0143068675, 0.011301654, -0.0064470805, 0.009214701, -0.009099117, 0.039941076, 0.004722936, 0.0064759767, 0.0021431404, -0.01449951, -0.0075451387, -0.0040390575, -0.015488404, 0.0099916905, -0.0032941755, -0.00019805989, -0.009696306, -0.00038347766, 0.011571353, 0.0046779863, -0.01649014, 0.005419657, -0.025300296, 0.004010161, -0.013844527, 0.007692831, -0.0009856841, -0.0058434694, -0.019559568, -0.018121175, 0.010486138, 0.0014600647, 0.0013220047, 0.00095598516, -0.004957317, 0.006556244, -0.0012706335, 0.016708469, 0.015398505, 0.023489462, 0.013279445, -0.0001430767, 0.016130544, 0.0035349778, -0.022436354, -0.011346605, 0.026995543, 0.005496714, 0.004411498, -0.009811891, 0.008758782, -0.008970688, -0.026533203, -0.012085065, -0.010858579, -0.0001264212, -0.015732417, -0.0062640705, -0.013253759, 0.0017787263, -0.00575357, -0.003917051, 0.006081061, -0.007005742, 0.0060200575, 0.015950745, 0.00382073, -0.028151395, 0.027149657, -0.008906474, 0.007679988, 0.0033583895, 0.012226336, 0.004170696, -0.008842261, 0.0023951803, 0.0048353104, -0.0008765204, -0.032903228, -0.029358618, 0.014576566, 0.0054581854, 0.0143068675, 0.0026311665, 0.0047582537, 0.0013838105, 0.004437184, 0.012695097, 0.01464078, 0.029332932, -0.016258972, 0.019187128, 0.008534034, 0.01464078, 0.030488782, -0.007224069, 0.016926797, 0.0056604594, -0.004828889, 0.0049637384, 0.008765204, -0.023515148, 0.01440961, -0.03369948, -0.014062854, -0.018557832, -0.008142329, -0.030411726, 0.012926267, -0.012457506, 0.0070506916, -0.005262333, 0.025711264, -0.0008395974, 0.00066341035, -0.014345396, -0.013587671, 0.012515298, -0.017183652, -0.010409081, 0.03863111, 0.019456826, 0.03701292, -0.00023919696, -0.012727205, 0.015719574, -0.012733626, -0.025287453, -0.0035895596, -0.000015940612, 0.006440659, -0.0045367153, 0.024452671, 0.00043665484, 0.009831156, 0.009555035, -0.017877163, 0.0111411195, 0.007814838, 0.023181235, 0.006684672, -0.01141724, 0.010556773, -0.0057439376, -0.00029177213, -0.046259727, -0.00013384596, 0.014242654, -0.004915578, 0.014139911, -0.0013693625, 0.010254968, -0.027586313, 0.010935635, -0.000024381234, 0.0048674173, 0.0008516375, -0.009818313, -0.020908061, 0.013703257, 0.0039106295, 0.008578983, -0.02454257, 0.010338445, -0.0054838713, -0.0071277483, 0.005593035, 0.015411347, -0.00013163859, -0.0010306338, -0.003364811, -0.016310344, 0.030617211, 0.0008267546, 0.013960112, 0.0068516284, 0.0017947798, -0.006748886, 0.0009961189, -0.008778047, -0.0033198611, 0.006357181, 0.0009616039, 0.004254174, -0.011160384, -0.027046915, -0.0011285602, -0.012136436, 0.009587143, 0.011308076, -0.02375916, 0.017466193, -0.031875804, 0.0040294253, -0.0069672135, -0.010742993, -0.015758103, -0.0012385265, -0.0070506916, -0.0070250058, -0.003586349, -0.023219764, -0.00589484, 0.008309285, -0.001671168, -0.0031705636, 0.022282239, -0.028896276, 0.017812949, -0.026507517, 0.008450556, 0.017761579, 0.009831156, -0.017299237, -0.008482663, -0.024439828, -0.009696306, 0.006270492, -0.018583516, -0.0127785755, -0.00028595273, -0.0065369797, -0.0029843433, 0.0051371157, 0.0061452747, -0.0055962456, -0.028690793, -0.013998641, 0.008052429, 0.0162076, -0.0019039436, 0.0016270209, -0.008964268, -0.026815744, 0.009644935, -0.030000757, -0.0063475487, -0.0029201293, 0.010287074, -0.00014347804, 0.012444663, 0.0059718974, 0.007879051, 0.013484929, -0.008277178, -0.0037789908, -0.021845585, -0.015706731, 0.013472087, -0.010158647, 0.00031605302, -0.01542419, 0.025120497, 0.010595301, 0.008970688, -0.0067809927, 0.006257649, 0.039735593, -0.0028478885, -0.0040390575, 0.031130921, -0.032492258, -0.0009118381, 0.025248924, -0.011539247, 0.022885852, 0.007667145, -0.01040266, 0.0024786585, 0.004986213, 0.009002795, -0.02053562, -0.021896956, -0.0066268793, 0.00014387938, 0.009233966, -0.015706731, 0.00072280824, -0.022577625, -0.007159855, 0.004498187, 0.01772305, 0.03195286, -0.011937373, 0.021858428, -0.012219914, -0.021922642, 0.004713304, -0.007140591, 0.0076864096, 0.0014175229, 0.02357936, 0.013125331, 0.023283977, -0.0003373239, 0.011198913, 0.030180555, 0.008514769, 0.00069351064, -0.0034001286, 0.011153962, -0.019649468, -0.016554356, -0.020599835, 0.021575887, -0.021485986, -0.00589484, 0.014062854, 0.00007720724, -0.019161442, 0.002610297, -0.005326547, -0.0006112365, 0.00076133665, -0.016708469, 0.0022169866, -0.008367077, -0.024786584, -0.011340183, 0.010030218, 0.005763202, -0.013960112, -0.0067553073, 0.0026761163, -0.013960112, -0.0027066178, 0.0055127675, -0.0052527008, 0.004860996, 0.015154492, 0.0148077365, 0.00003150497, 0.002297254, 0.0067553073, 0.011385133, 0.0038945759, 0.020407192, -0.02509481, 0.015732417, 0.009433029, -0.006055375, 0.040994186, 0.0023100968, 0.00244013, -0.020098966, -0.00855972, 0.01615623, 0.026404776, -0.0014857502, -0.0016101648, 0.0021158496, 0.010871422, -0.012720783, -0.006819521, -0.009933898, -0.023052808, -0.018108333, 0.0015917033, 0.0031785904, 0.013844527, 0.019854954, 0.028254136, 0.023656419, 0.015963588, -0.01882753, -0.012091487, 0.007891894, 0.011873159, 0.0073589184, -0.019906323, -0.02016318, 0.004337652, 0.00034193927, 0.006052165, 0.015668204, 0.002435314, 0.008399185, 0.010306339, 0.0010266205, -0.015437033, 0.015205863, -0.0013525062, 0.038682483, 0.00063772476, -0.013613357, 0.026263505, 0.015437033, -0.016130544, 0.007236912, 0.010980585, 0.012945532, 0.00870099, -0.008187278, 0.0048963134, 0.014846264, 0.00009371223, 0.013908741, 0.012425399, -0.005069691, 0.006800257, 0.018673416, -0.0005787282, 0.026327718, -0.020612678, -0.024889326, -0.0035991918, 0.00019655489, -0.020009067, 0.017247867, -0.022012541, 0.023682104, 0.0069029992, 0.023027122, -0.012130015, 0.00515638, -0.0032556471, -0.0038656797, 0.013266602, -0.0029056813, -0.00764146, -0.012123593, 0.011532825, -0.011218177, -0.008232228, -0.019880638, 0.023463776, -0.00070153736, 0.009927477, -0.013780314, -0.0299237, 0.023258291, -0.0045784544, -0.0044628694, 0.005943001, -0.011590618, -0.004915578, -0.010010955, -0.009677042, 0.0016302316, -0.016952483, 0.008212964, 0.03190149, -0.001942472, -0.007352497, -0.021832742, 0.0034611318, -0.01542419, 0.020060437, -0.00354461, -0.017003853, 0.015000379, 0.0107622575, -0.0011036772, 0.0069543705, 0.00025886248, -0.0019986592, -0.0008460188, -0.02881922, 0.009047746, -0.005981529, -0.013960112, 0.0029538416, 0.0071020625, 0.022539096, -0.0022346454, 0.011083327, 0.019302713, -0.02301428, 0.022539096, -0.0038271514, -0.0019167863, -0.018814687, -0.023283977, 0.01675984, 0.0004017385, 0.029204503, -0.003692302, 0.0013059512, 0.005647617, -0.011622725, -0.01786432, 0.008585405, -0.019867796, 0.0038817332, 0.01330513, -0.0017113017, 0.016644256, 0.018609202, 0.0017899638, 0.0047614644, 0.019765053, 0.019803582, -0.020599835, -0.006469555, 0.020766791, 0.020240236, -0.011333762, 0.0098696835, 0.0020741106, -0.0051082196, -0.00418996, 0.016002117, 0.000589163, 0.009876105, 0.0019216024, 0.0006646144, 0.0029265506, -0.037321147, 0.015013221, -0.04397371, -0.012919846, -0.027201029, 0.006524137, 0.019598097, 0.012990482, 0.013343658, 0.016965326, -0.015912216, -0.0012192624, 0.024465514, -0.010184332, -0.003393707, 0.00034033394, -0.01542419, -0.004138589, 0.020381507, 0.0002448157, -0.0015892952, -0.01473068, 0.011391554, -0.026790058, 0.0024979226, -0.014435296, 0.015270077, 0.008456977, 0.0067553073, 0.0148077365, -0.0043023345, -0.021164916, -0.0035638742, 0.025492938, -0.014666466, 0.0034290247, -0.0013444795, -0.0136262, 0.0101265395, -0.01196948, 0.005464607, -0.0047582537, -0.0033230719, -0.023309663, -0.01606633, 0.04068596, 0.005853101, 0.009612828, -0.018378032, 0.009375236, 0.0071983836, 0.0012738442, 0.02067689, -0.027611997, -0.011532825, -0.0127914185, 0.021049332, 0.0039523686, 0.028202767, -0.012046536, -0.003480396, -0.019443983, 0.014204125, 0.007583667, 0.0020468195, 0.0034418677, -0.023733474, 0.02187127, -0.011834631, -0.0043472843, -0.0034482891, 0.0020933747, 0.016014958, -0.01178968, 0.009105538, -0.011430083, -0.0010563195, -0.017003853, -0.0036955127, -0.0072947047, 0.025261767, 0.009047746, -0.0000065593545, -0.004902735, 0.001979395, -0.033853594, -0.004860996, -0.0010466874, 0.034958072, 0.02158873, -0.031233665, 0.025467252, -0.019765053, -0.011686939, 0.031053865, -0.015437033, -0.011545667, -0.0046490896, 0.021216288, 0.007468082, -0.008822996, -0.0016109674, 0.018018434, 0.0021030067, 0.0075194533, -0.017825792, 0.022179497, 0.009368815, 0.00319946, -0.016811213, 0.044769965, 0.0038817332, 0.015603989, 0.007435975, 0.008322128, 0.0150645925, 0.01155851, -0.0010121723, -0.0043440736, 0.01758178, -0.00056187203, 0.005474239, 0.0032668847, -0.009330287, 0.0092468085, -0.0070378487, 0.009657778, -0.0021656153, 0.01684974, -0.016323186, 0.01952104, 0.005474239, -0.0023999964, -0.0065626656, 0.021742843, -0.0046940395, -0.0005245477, -0.017055225, 0.0011967875, -0.0029570523, 0.0003947151, 0.013754628, 0.00065377826, -0.0018028066, 0.01408854, -0.0062897564, 0.013106067, 0.017196495, -0.004912367, 0.021254817, 0.00010986605, -0.0061934353, -0.016978167, 0.0026985912, -0.013420715, -0.045412105, -0.00053177174, 0.009407343, -0.016169071, -0.00051772495, -0.008302864, 0.026083706, 0.0028494939, -0.00621591, 0.014859107, -0.02269321, 0.02361789, 0.018275289, -0.000034916335, 0.0053426004, 0.0116548315, -0.03310871, 0.023322506, -0.0008917712, -0.012457506, 0.010794365, -0.004254174, -0.012457506, 0.008925739, -0.029307246, -0.0070185843, 0.014435296, 0.003968422, 0.003711566, 0.007063534, -0.010068747, 0.0070506916, 0.0111411195, -0.0052366476, 0.013343658, -0.009426608, -0.014949007, 0.0093559725, -0.008675304, -0.0064502913, 0.016785527, 0.023630733, 0.0028783903, -0.006832364, -0.016130544, 0.00354461, 0.0021479565, -0.0043344414, 0.0011751152, 0.006832364, 0.0075451387, -0.015488404, -0.015642518, -0.0053104935, -0.0076350383, -0.014139911, -0.020497093, 0.011776838, 0.0062897564, 0.011109013, -0.010531087, 0.009028481, 0.012181386, 0.030052127, 0.009786205, 0.00759651, 0.008341392, -0.01708091, 0.000027265844, 0.020098966, -0.005349022, 0.013093224, 0.003250831, -0.02794591, 0.023425248, 0.0053104935, -0.030694267, 0.008174435, -0.00064575154, -0.0052462798, 0.009144066, 0.026456147, 0.00040213985, 0.00068026653, 0.0035638742, -0.012207071, -0.0095743, 0.016888268, -0.001979395, -0.0030036075, -0.0012730415, 0.02509481, 0.006414973, -0.0009455504, 0.026212133, -0.006716779, -0.016913954, -0.010646673, -0.015180177, 0.018146861, 0.03177306, 0.001321202, -0.0024931065, -0.03038604, 0.014011484, -0.012907004, -0.0016872215, 0.015501247, 0.030000757, -0.03359674, -0.02002191, -0.006652565, 0.0081102215, 0.022218026, 0.006620458, -0.017915692, 0.0053233365, 0.005159591, 0.0006810692, 0.0137931565, -0.020111809, -0.0056283525, 0.022757424, -0.019983381, -0.02545441, 0.005689356, 0.030976808, 0.0358057, 0.0145508805, -0.0162076, 0.02219234, 0.016053487, 0.0069543705, 0.005644406, -0.00009918045, -0.010710887, 0.022076756, -0.0016398637, -0.000030702293, 0.004382602, 0.0028623368, 0.014846264, 0.014486667, 0.0011221387, -0.020522777, -0.006992899, -0.003978054, -0.027098287, -0.007435975, 0.04107124, 0.002536451, 0.012130015, 0.0067360434, -0.016079172, 0.0061902246, -0.014627937, -0.012091487, -0.0048192567, -0.0060425326, -0.004655511, 0.03020624, -0.011173227, -0.027637683, 0.009131223, 0.016464457, -0.014152754, -0.01708091, -0.008810153, 0.006980056, 0.0068580494, -0.0102613885, 0.0036248774, -0.009799048, 0.000051346076, -0.017761579, 0.008553298, 0.021884114, 0.028562363, -0.01864773, -0.00018270875, -0.004658722, 0.029101761, -0.0026568521, -0.019803582, -0.013446401, 0.032672055, -0.013317972, -0.003024477, 0.013228074, -0.0023518358, 0.023361035, -0.013690414, -0.0015186599, -0.0068580494, 0.017658835, 0.0043119667, 0.018390875, 0.016040644, -0.0050568483, -0.0041032713, -0.004084007, -0.0132922875, 0.0014624727, 0.0065658763, 0.018442245, -0.0041995924, 0.004816046, -0.010203596, -0.021293346, -0.007744202, 0.00621591, 0.0030726374, 0.0022442774, -0.012412556, 0.013202388, 0.0033327038, -0.013131753, 0.021100704, -0.002896049, -0.017222181, 0.023078494, -0.00488026, -0.007821259, -0.011391554, -0.020098966, -0.0025252134, 0.0012425398, -0.012643727, 0.026250662, -0.012393292, 0.005451764, 0.007269019, 0.010742993, 0.009792627, -0.020227393, -0.013574828, 0.0045046085, 0.00980547, -0.017196495, 0.0014424059, -0.0070121633, -0.013754628, 0.0072818617, 0.0036505628, -0.00188789, 0.0050568483, 0.024504043, 0.0033391253, 0.0043729697, 0.0014175229, 0.0021286923, 0.0066268793, 0.011391554, 0.0023502305, 0.010768679, -0.01510312, -0.004774307, 0.00529444, -0.00446608, -0.0021880902, 0.0033070184, -0.022641838, -0.00024060164, -0.0072433334, 0.009966005, 0.0039844755, -0.0071855406, 0.00036381214, 0.011950215, 0.026186448, 0.0044403947, -0.0008893632, 0.021601573, 0.013407872, 0.010935635, 0.002385548, -0.009272494, -0.0122777065, -0.009728413, -0.00042622007, 0.00704427, 0.0065498226, -0.008887211, 0.016862582, -0.0067745717, 0.015989274, 0.013151017, -0.015501247, -0.011076906, -0.012547405, -0.010113697, 0.005763202, -0.015681047, 0.0036730377, -0.0021431404, 0.0002586618, -0.0034771853, -0.0140500115, -0.006556244, -0.0041450104, -0.005612299, -0.028279822, -0.019174285, -0.005538453, 0.008636776, 0.0136262, -0.022320768, 0.009856841, 0.0020981908, 0.017620306, 0.0076543028, 0.0067103575, -0.003933104, -0.0038496263, 0.00870099, 0.035934124, -0.0137931565, 0.00043705618, 0.0036377201, -0.0065755085, -0.01649014, -0.0010370553, -0.022115285, 0.0018429403, 0.012136436, 0.00056387874, -0.005349022, 0.0011863527, -0.0030501625, 0.0015170545, -0.012829946, 0.01408854, 0.0016310343, 0.022205183, -0.017183652, 0.02016318, 0.003310229, 0.008135907, -0.004369759, 0.01026781, 0.008270756, -0.008649618, 0.011635567, 0.031747375, -0.013549143, 0.0191486, 0.021357559, 0.0074038683, 0.03665332, -0.00976052, -0.016580041, -0.012894161, -0.013240916, -0.018596359, -0.0014351817, 0.0082193855, -0.0053650755, 0.008546877, -0.0015981246, 0.009856841, 0.005650827, 0.0070250058, -0.0028783903, -0.0081937, 0.022474881, -0.006100325, -0.007525875, 0.007468082, 0.010479717, -0.0060457434, 0.017389137, 0.015693888, 0.0093559725, -0.001772305, 0.0072754403, 0.015796632, 0.012380449, -0.005666881, 0.014037169, -0.0077506234, -0.023219764, 0.009041324, 0.013106067, 0.04084007, 0.006418184, 0.009182595, 0.021563044, 0.009458714, -0.010768679, 0.0021351138, 0.009362394, -0.0031224033, -0.021665785, 0.015899373, 0.009009217, 0.0010587275, 0.0037436732, 0.014679309, 0.0025123707, -0.005782466, 0.016721312, -0.01082005, -0.0031159818, 0.011404397, 0.008675304, -0.009144066, -0.0038014657, -0.0032379883, 0.0040679537, -0.004523873, -0.01961094, -0.0005590627, -0.014974693, -0.0055641388, -0.026635945, -0.006703936, -0.0011759179, -0.008412027, 0.030540153, -0.0023550466, -0.0067938357, -0.0011052826, 0.0007472898, 0.0070956415, -0.024915012, -0.029076075, 0.0044275518, 0.026289191, 0.004568822, 0.001528292, -0.026083706, -0.00708922, -0.0025717686, 0.018544989, 0.008026743, -0.0024240764, -0.0009038113, -0.011340183, -0.033365566, -0.00055665465, 0.025839692, 0.02748357, -0.009933898, 0.008604669, -0.009471557, 0.014627937, -0.009477979, 0.0064053414, 0.0010571221, -0.003618456, 0.014846264, 0.022423511, -0.013818841, 0.0059044724, -0.014974693, 0.019212814, 0.018211076, 0.0023775215, -0.018416561, -0.023271134, 0.011905266, -0.021190602, 0.0028992598, -0.0032973862, 0.004084007, -0.019508198, -0.001153443, 0.00041377862, 0.011731888, -0.011609882, -0.009150487, 0.015115963, 0.008386342, 0.027509255, -0.011757574, -0.006411763, 0.019919166, -0.005085745, 0.0041000606, 0.021062175, 0.016284658, -0.009458714, 0.008142329, 0.0012634094, 0.024118759, 0.002284411, -0.0056925663, -0.021563044, 0.023399564, -0.0043986556, 0.008200121, 0.004568822, 0.015475562, -0.00055665465, -0.0018204654, -0.0012682255, 0.020638363, -0.0022860165, -0.015693888, -0.018557832, -0.013960112, 0.0028238085, 0.009426608, -0.015732417, -0.007493768, -0.015372819, -0.02591675, -0.0047454108, -0.011564932, 0.002412839, 0.017106596, -0.0061805923, -0.013382187, 0.0162975, -0.0093816575, -0.021845585, -0.00038668836, -0.0027852799, -0.024324244, -0.025236081, -0.036756065, -0.010698044, 0.00003102838, 0.0014014695, -0.00071839354, -0.0140500115, 0.044769965, -0.018442245, -0.023643576, 0.0358057, 0.009966005, -0.0071662767, -0.003949158, 0.0013725731, -0.010415503, -0.0020548464, -0.025724107, -0.036910176, -0.039838333, -0.011211756, -0.0043729697, 0.003945947, 0.02702123, 0.0037083556, 0.012977639, 0.002859126, -0.012065801, -0.010190753, 0.0020933747, -0.0097476775, 0.007757045, 0.021357559, -0.0061324323, 0.0030373198, 0.010370553, 0.008347813, 0.031644635, -0.025608523, -0.02108786, 0.011629146, 0.0081102215, -0.008168014, 0.0014424059, 0.020432878, -0.0059879506, 0.0005377918, -0.015270077, 0.0143068675, -0.007583667, -0.005589824, -0.01173831, -0.010004533, 0.022115285, -0.003907419, -0.00093110226, -0.0013525062, -0.013690414, 0.0017674889, 0.017273553, -0.016310344, -0.00515638, 0.021357559, 0.02661026, 0.0034258142, 0.023232607, 0.026687317, 0.017761579, 0.023836218, -0.009921055, 0.00069110264, -0.0065658763, -0.010685201, 0.006440659, 0.001574847, 0.011975901, 0.00037906296, -0.003711566, 0.0037372517, 0.013356501, 0.005888419, 0.016836897, -0.0022956487, -0.006819521, 0.023258291, -0.002827019, -0.017684521, 0.011841052, -0.018018434, 0.011173227, 0.009580721, 0.021614416, -0.004469291, 0.01758178, -0.02062552, -0.026867116, -0.016400242, 0.0022812006, 0.0026167184, -0.0036890914, 0.0059751077, 0.016592884, 0.010511823, 0.0051114303, 0.018878901, 0.0015395294, -0.0076029315, -0.015128806, 0.0024369194, 0.019212814, -0.025287453, -0.007988215, -0.0031384567, -0.005082534, 0.016258972, 0.022616152, -0.0055448744, 0.010325603, 0.020548463, 0.0024850797, 0.029641159, -0.00092387816, -0.005952633, 0.022847323, 0.0047935713, -0.002550899, 0.010441188, -0.016220443, -0.009137644, -0.015026064, -0.0075387177, -0.016284658, 0.000010064784, 0.00032026708, 0.006559455, -0.022577625, 0.0017498301, -0.0059590545, -0.023091335, -0.011545667, 0.012329078, -0.0025541098, -0.013446401, 0.029692529, 0.025569994, 0.023797689, 0.004138589, -0.008052429, 0.010088012, -0.003300597, -0.008071694, -0.0041995924, -0.010100854, -0.0056411955, -0.01496185, 0.008534034, -0.0018670205, -0.0155783035, 0.008347813, 0.0027001966, -0.0087202545, 0.028716477, -0.002127087, -0.013677571, 0.0031641424, -0.032851856, -0.0003864877, 0.001675984, -0.003329493, -0.004751832, -0.015488404, 0.022140969, -0.012123593, -0.006501662, 0.0069543705, 0.013459244, 0.010017375, -0.008437713, -0.010152225, -0.007936844, 0.005377918, 0.024902169, 0.0060939034, 0.019187128, 0.0040711644, 0.016181914, 0.0011076906, 0.005878787, -0.03899071, 0.007879051, 0.01058888, 0.024889326, 0.0021977224, 0.00082434656, -0.004751832, -0.012091487, 0.016567199, 0.019970538, 0.011012692, 0.0046908287, 0.0014536432, -0.010332025, 0.010120118, -0.0358057, -0.008161593, -0.00049725675, 0.016772684, -0.017055225, 0.013459244, 0.037732117, -0.014371081, 0.0140500115, -0.009522929, -0.0024882904 - ] - } - ] - } + { + "name": "untitled", + "title": "Untitled", + "content": "---\n\n**Generative AI** is emerging as a transformative technology, comparable to the rise of personal computers and the early days of the internet.", + "keywords": [ + "Generative AI", + "transformative technology", + "personal computers", + "early days of the internet" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.026481831, 0.04921357, -0.002366284, -0.021460302, -0.027380828, + -0.0036216667, 0.012605198, -0.006658986, -0.06981341, 0.021896956, + -0.03893934, 0.025441566, -0.013189545, -0.0004924407, 0.0061902246, + 0.013844527, -0.015963588, 0.04998414, 0.0066975146, -0.0013460849, + 0.0354461, -0.0061163786, -0.060566597, 0.044179197, 0.012682254, + 0.00824507, -0.030694267, 0.0064470805, -0.026327718, 0.0017578568, + -0.003660195, -0.001222473, 0.045232303, 0.001574847, 0.009728413, + -0.019392611, 0.0015339106, -0.020548463, 0.029872328, 0.034881018, + 0.011256705, 0.0020708998, 0.018943114, 0.010145804, 0.028896276, + -0.02853668, 0.0003991298, -0.036550578, 0.021190602, 0.023232607, + 0.016002117, -0.014949007, -0.021293346, 0.041456524, 0.011834631, + -0.004276649, 0.011937373, 0.019135756, -0.007423132, -0.050857447, + 0.00016474891, -0.014332553, -0.024452671, 0.009426608, 0.012438241, + 0.0215502, 0.017299237, -0.002600665, 0.04433331, 0.020317294, + 0.029666845, 0.06642291, 0.0055320314, 0.029332932, 0.054299314, + -0.008938582, -0.03149052, 0.01201443, -0.025955278, 0.013651885, + 0.02559568, -0.028100023, -0.020188866, 0.0044949763, -0.0021094282, + 0.0000010096138, 0.011256705, -0.073666245, 0.0030164502, + 0.0036056133, 0.010672358, -0.013728942, -0.011584196, -0.020908061, + -0.044487424, -0.027791796, 0.045566216, -0.010299917, -0.005217383, + 0.024298558, 0.011956637, -0.002674511, -0.0404291, 0.01731208, + 0.0005526413, 0.016438771, -0.0006758518, -0.006209489, -0.016143387, + 0.013112488, 0.025955278, -0.003226751, -0.0071662767, -0.0046651433, + -0.027201029, 0.0028061494, -0.009728413, -0.017800106, 0.044127826, + 0.015115963, -0.0037211983, 0.014743523, -0.03226109, 0.006071429, + -0.021601573, 0.010254968, 0.020432878, -0.017453352, 0.01961094, + -0.032158345, 0.02591675, 0.011353026, -0.0068516284, 0.011333762, + 0.009336708, -0.04225278, -0.0065401904, 0.031798746, 0.028100023, + 0.048263203, -0.06390572, -0.007159855, 0.019790739, -0.00056187203, + -0.0013733758, -0.025467252, -0.016952483, 0.01606633, 0.012733626, + -0.026327718, -0.024195816, -0.013780314, -0.027509255, 0.00446608, + 0.018095491, 0.020047594, 0.0060842717, -0.024414143, 0.05676513, + -0.046901867, -0.018801844, -0.013587671, 0.028228452, -0.036807436, + -0.011166805, -0.0064246054, -0.020201707, -0.00026046782, + 0.0016093621, 0.0034482891, 0.042278465, -0.0054389215, -0.04572033, + -0.07489915, 0.0060329004, -0.025261767, -0.014204125, -0.014846264, + -0.005731095, 0.024375614, 0.008501926, -0.0031464836, 0.028022967, + -0.009779784, -0.020908061, -0.022128126, 0.006890157, 0.023964645, + 0.038554054, -0.021575887, 0.00079223956, 0.051448215, 0.013883055, + 0.0060040043, 0.02905039, 0.0014680914, -0.011725467, -0.033365566, + 0.015937902, 0.026032334, -0.054196574, 0.015385662, -0.02205107, + -0.014782051, 0.0024593943, 0.015937902, -0.009535771, -0.0095743, + 0.020869533, 0.030642897, -0.01035771, -0.017607465, -0.017787263, + 0.042612378, -0.014936164, 0.02817708, -0.017183652, -0.007352497, + -0.017389137, 0.0459515, 0.026404776, 0.037115663, 0.007513032, + -0.026404776, -0.018557832, -0.009304601, -0.06303241, 0.0059751077, + 0.004613772, -0.026918488, 0.032158345, 0.01790285, 0.0072625973, + 0.056919243, -0.0379376, 0.06133716, 0.024555413, 0.0073268116, + -0.01717081, 0.034315933, 0.027406514, 0.02899902, -0.00980547, + -0.02463247, -0.007904737, -0.028074337, 0.023733474, 0.0059141046, + 0.0075708246, 0.028716477, 0.00035177203, 0.0030084234, 0.051936243, + 0.00994674, -0.011930952, 0.009709149, -0.005750359, 0.006633301, + -0.008906474, 0.0073139686, 0.0019874217, 0.010370553, -0.017157966, + 0.017479036, 0.005731095, -0.035112187, 0.0005357851, 0.0032652793, + 0.0019537094, -0.029153133, -0.001703275, -0.004225278, 0.007288283, + 0.017466193, 0.028613735, 0.010903528, 0.042612378, 0.02301428, + -0.03125935, -0.04623404, -0.021113547, 0.009054166, 0.0051242732, + -0.025518622, 0.0045559797, 0.01243182, -0.007943265, 0.027098287, + 0.0022137759, -0.046799127, -0.011423661, 0.0044436054, 0.030899752, + 0.025711264, 0.030129185, -0.0028848117, 0.0030662161, -0.0033487573, + -0.046901867, 0.019508198, 0.018146861, 0.02661026, -0.007866208, + -0.015231549, 0.0122648645, -0.015128806, 0.005050427, -0.00395879, + -0.026160762, -0.05404246, -0.009221123, -0.053374633, 0.0028334404, + 0.025698422, -0.011995166, -0.00033009984, 0.017222181, -0.014139911, + 0.038502682, -0.017440509, -0.020009067, 0.0029875538, 0.018429402, + 0.019906323, -0.012348342, 0.023399564, -0.000100685465, -0.012727205, + 0.01592506, 0.01201443, -0.03457279, 0.004626615, -0.017812949, + -0.05106293, 0.011051221, -0.012759311, 0.018763315, -0.008296442, + 0.0010033429, 0.019970538, -0.004976581, 0.0057664127, 0.009908212, + 0.037192717, 0.026507517, -0.02149883, -0.009818313, -0.0053971824, + -0.013163859, 0.01740198, 0.000666621, 0.003351968, 0.05501851, + -0.0039620004, 0.017363451, -0.006283335, -0.02499207, -0.02283448, + 0.01606633, -0.039581478, -0.024439828, 0.0067809927, -0.009458714, + -0.0022812006, -0.019482512, 0.019495355, 0.00073284167, 0.009400922, + 0.030617211, 0.0062191207, 0.013240916, 0.014011484, -0.0122648645, + 0.011956637, 0.013497772, 0.006960792, -0.017363451, -0.008604669, + -0.012386871, -0.00061725656, 0.032440886, 0.025531465, -0.019944852, + 0.00086769095, -0.0011991955, -0.0030260824, -0.0017899638, + -0.017838635, -0.017337766, 0.011089749, -0.008084536, -0.013382187, + -0.00469725, 0.0030517678, -0.0038464156, 0.017517565, -0.044281937, + -0.0033583895, 0.028896276, 0.0041642743, 0.023232607, -0.03056584, + 0.002181669, 0.031336408, -0.020753948, -0.0052976506, -0.0019970539, + -0.057124726, 0.012662991, -0.014011484, -0.024786584, -0.0030180556, + 0.003393707, -0.012322657, -0.02899902, -0.05008688, -0.016541513, + 0.03714135, 0.015488404, 0.034059078, -0.01449951, -0.037911914, + 0.0063314955, 0.0039555794, -0.015411347, 0.0030276875, 0.012740048, + 0.025698422, -0.0381174, -0.021357559, -0.031464834, -0.009477979, + 0.0027917014, 0.010306339, -0.015000379, 0.050985876, -0.03688449, + 0.0045591905, 0.03978696, 0.013356501, -0.02946136, 0.005962265, + 0.032338142, -0.018095491, -0.013574828, 0.022487724, 0.02684143, + 0.01873763, 0.024337087, 0.056919243, -0.014165597, 0.03434162, + 0.016567199, 0.011622725, 0.007416711, 0.0011245467, -0.042920604, + 0.012496035, -0.007988215, 0.031798746, 0.023399564, -0.023181235, + 0.014345396, -0.0352663, -0.013972955, 0.02513334, 0.0461313, + 0.00971557, 0.000971236, -0.017877163, -0.0072561763, -0.025582837, + -0.039581478, 0.02541588, -0.045925815, -0.0031063498, 0.004482134, + 0.012996903, 0.022962907, -0.015848001, -0.040814385, 0.022718895, + -0.044924077, 0.015950745, -0.0050632698, 0.026147919, -0.025929593, + -0.027663369, 0.0106274085, -0.015167334, 0.052295838, 0.0053522326, + 0.008765204, -0.020227393, 0.010807208, 0.0070378487, -0.03724409, + -0.055223994, 0.02370779, 0.033005968, -0.0033969178, 0.03084838, + 0.037629373, -0.0029153132, 0.003204276, -0.0039427364, -0.02030445, + -0.03521493, 0.013472087, 0.016323186, 0.061491277, 0.0035125029, + -0.028279822, -0.020009067, 0.016913954, -0.015475562, -0.012386871, + -0.010210018, 0.029409988, -0.009895369, -0.009580721, 0.011622725, + -0.0025525044, 0.006543401, -0.019790739, 0.012579513, 0.0033038077, + -0.0009977242, 0.029872328, -0.021241974, -0.033262827, 0.0057696234, + 0.0056090886, 0.028022967, 0.06405983, -0.02513334, -0.02126766, + -0.019238498, -0.0017867531, 0.042715117, -0.03351968, -0.057535697, + 0.012412556, 0.03190149, 0.01850646, 0.006154907, -0.0047807284, + -0.012207071, 0.0028703634, -0.03167032, 0.053888347, -0.014949007, + 0.005888419, 0.0030581893, -0.014653623, -0.007121327, -0.014692151, + -0.02007328, -0.037038606, -0.013330815, 0.02140893, 0.011243862, + 0.021845585, -0.004456448, 0.0062865457, 0.015475562, 0.016002117, + -0.011282391, 0.005410025, 0.0017562514, 0.011218177, 0.012451084, + 0.026327718, 0.0064663445, 0.019906323, 0.011719045, -0.018660573, + -0.015398505, 0.012367606, 0.011423661, -0.008360656, 0.019649468, + 0.031387776, 0.025390195, 0.0005686948, 0.0066782506, -0.00814875, + -0.030077813, -0.0034033393, 0.044769965, 0.036627635, 0.023450933, + -0.0029265506, 0.015115963, 0.0010563195, 0.009060588, -0.027534941, + 0.015681047, -0.028665107, -0.029435674, 0.0068387855, -0.0041450104, + 0.015732417, -0.020458564, -0.024388457, 0.008463399, 0.023643576, + 0.0153086055, 0.018904585, 0.010511823, 0.0024465513, 0.020831006, + 0.0017129071, -0.008026743, -0.032286774, -0.006315442, 0.03403339, + 0.029076075, 0.010800786, -0.022359297, -0.014294025, -0.024414143, + -0.0074552395, -0.009766942, 0.025377352, -0.021421773, 0.012451084, + -0.016323186, -0.0031175872, 0.0018814687, -0.001079597, 0.009914634, + -0.015526933, 0.006459923, -0.01473068, -0.019687997, 0.0061805923, + -0.008418448, -0.02080532, -0.012784997, 0.0026488253, 0.058306266, + 0.016926797, 0.03624235, -0.004639458, 0.00016755828, -0.021100704, + 0.024902169, 0.007191962, 0.004899524, 0.055994563, -0.028048653, + 0.009477979, -0.0011646805, -0.00091825944, 0.01800559, 0.02776611, + -0.030257612, 0.028048653, -0.003241199, -0.016836897, 0.014897636, + 0.0004274241, 0.006877314, 0.012412556, 0.020908061, -0.03737252, + 0.038091715, 0.017928535, -0.0034771853, -0.014332553, 0.017915692, + -0.004116114, 0.011449347, -0.010132961, 0.0067938357, 0.0032797274, + -0.0069351066, -0.010486138, 0.0020548464, 0.010486138, -0.0164773, + -0.02946136, -0.005731095, 0.013818841, -0.021190602, 0.044025082, + 0.03673038, -0.005506346, -0.01758178, 0.019366927, 0.033493996, + -0.0018959168, -0.007827681, 0.031053865, 0.004337652, -0.0011686939, + 0.031233665, 0.01150714, -0.014846264, 0.0020773213, 0.023373878, + 0.01740198, -0.039966762, 0.010736573, 0.009766942, -0.0018333083, + -0.024619628, -0.05224447, -0.008643198, 0.006703936, -0.004626615, + 0.030694267, -0.011076906, 0.053888347, -0.035112187, 0.040583216, + -0.035137873, 0.008495505, 0.039966762, 0.02375916, 0.008707412, + -0.053785603, -0.012117172, -0.03236383, 0.008309285, -0.029230189, + -0.025788322, 0.01473068, 0.0060650073, -0.038399942, 0.009599986, + 0.026250662, -0.009266073, 0.026738688, -0.001855783, -0.013818841, + -0.02499207, 0.022962907, -0.007172698, 0.0067424644, -0.032081287, + 0.04374254, 0.006652565, 0.0006770558, 0.017055225, -0.0025187922, + -0.003660195, -0.059179574, 0.0028510992, -0.038656797, -0.019816425, + 0.038785223, 0.016271815, 0.0037500947, 0.00022956486, 0.012142858, + 0.0070378487, 0.01588653, 0.0358057, 0.023656419, 0.010858579, + 0.00202595, 0.0025492937, 0.0077827303, 0.0022137759, 0.004816046, + -0.010113697, -0.0027130393, 0.0065947725, 0.03323714, -0.0023261502, + -0.005262333, -0.0018188601, 0.022397825, -0.00060922984, 0.010967743, + -0.011988744, 0.019662311, -0.03398202, 0.034649845, -0.006071429, + 0.002319729, -0.0023405985, -0.007577246, -0.019392611, 0.009978848, + -0.002969895, 0.0044725016, -0.013818841, -0.008444134, -0.004597719, + -0.017684521, 0.010537509, -0.0093945, -0.016541513, 0.0115199825, + -0.012194228, 0.044410367, 0.006601194, 0.012309814, 0.019598097, + -0.010017375, 0.010909949, -0.030334668, 0.0030533732, 0.007179119, + 0.013985798, -0.016708469, 0.018211076, 0.040994186, -0.022269398, + 0.020009067, -0.0081102215, -0.036422152, -0.027534941, 0.016567199, + -0.0024096284, 0.00019906324, -0.02003475, 0.034778275, -0.021421773, + 0.0432802, -0.010794365, 0.020908061, 0.02306565, 0.023720633, + -0.026481831, 0.00043344416, 0.018955957, -0.010730151, -0.0024609996, + -0.013703257, -0.009734835, -0.00971557, 0.011564932, 0.0070185843, + 0.0067681503, 0.02928156, -0.009323865, -0.0076157744, -0.015462719, + 0.027971596, -0.0072947047, 0.010210018, 0.04199592, 0.0072112265, + -0.054813027, 0.015038907, -0.04148221, -0.012984061, 0.017890006, + -0.01510312, 0.019726524, -0.025467252, 0.02666163, 0.024683842, + 0.0083542345, 0.0038110977, 0.008450556, 0.007943265, -0.012573091, + 0.03102818, 0.0044500264, 0.004931631, 0.004074375, 0.0023839427, + 0.012001587, -0.018724788, -0.0052655437, 0.015090277, 0.0019248131, + -0.013960112, -0.024645314, -0.007493768, 0.014692151, -0.009291758, + -0.01937977, -0.012252022, 0.013446401, -0.025390195, -0.004116114, + -0.010556773, -0.033262827, 0.024285715, 0.03084838, -0.019970538, + -0.012194228, 0.012149279, -0.021794215, 0.0012915031, 0.004543137, + -0.0024064176, -0.006090693, 0.012457506, 0.025801165, -0.018917428, + -0.020086123, -0.015629675, 0.027689055, 0.020381507, -0.008668883, + 0.0035574527, -0.017235024, -0.0040165824, 0.00022976553, 0.023656419, + 0.0075066104, -0.0037372517, -0.0025974542, -0.00833497, 0.012374028, + 0.0052591222, -0.03369948, -0.00865604, 0.001333242, -0.02840825, + -0.0136262, -0.00015060177, 0.031002494, 0.03231246, 0.020060437, + -0.02817708, 0.0046651433, -0.00045351102, -0.0055866134, 0.014396767, + -0.023502305, -0.01597643, -0.0005907683, 0.015603989, 0.0076992526, + 0.0073717614, 0.006068218, 0.01864773, -0.021370402, -0.038887966, + 0.0018943115, -0.01072373, 0.028973334, -0.008540455, 0.016567199, + -0.023001436, -0.01836519, -0.0067103575, 0.044949763, -0.01868626, + 0.025081968, 0.038476996, -0.0025043439, 0.010909949, 0.013202388, + -0.010800786, 0.011410818, 0.047030296, 0.008277178, -0.025030596, + 0.03218403, -0.01946967, 0.019135756, 0.004957317, 0.0035478207, + -0.030899752, -0.008636776, 0.013651885, -0.008822996, -0.018390875, + -0.06477903, 0.000050116978, -0.0009913028, 0.024157288, + -0.0140500115, -0.015655361, -0.015205863, 0.00051050086, 0.03601118, + -0.007724938, 0.0053522326, 0.024773741, 0.02559568, 0.015642518, + 0.021627259, 0.007564403, 0.042021606, 0.02858805, -0.017568937, + -0.022115285, -0.016053487, -0.004209224, 0.0015001984, 0.022256555, + -0.025608523, -0.0017000643, -0.0069543705, 0.052655436, -0.010614566, + 0.0023405985, 0.0005803336, -0.0041321674, 0.014704994, 0.035291985, + -0.003554242, -0.018609202, -0.00071518286, -0.01634887, 0.02905039, + -0.044179197, 0.0028832064, -0.014627937, -0.0056861453, 0.021370402, + 0.0010282259, 0.032517944, -0.025801165, -0.008360656, -0.025069125, + -0.0075001894, -0.016657097, -0.018519303, -0.00478715, 0.00285431, + 0.018095491, 0.0045367153, -0.012733626, -0.0042284885, 0.010832893, + 0.014627937, 0.010351288, 0.02324545, 0.02393896, 0.021807056, + 0.00014438105, -0.0044949763, 0.014062854, -0.034290247, -0.007930422, + -0.008135907, 0.05589182, 0.0023887588, 0.017697364, 0.012746469, + -0.035985496, 0.021896956, 0.0022009332, 0.010473295, -0.014011484, + -0.008084536, 0.003798255, 0.018172547, 0.01772305, 0.00055344397, + 0.0025075546, -0.00028073534, 0.0032973862, -0.033622425, + -0.008501926, 0.02858805, 0.02545441, 0.0018509671, -0.017568937, + 0.005644406, -0.012528141, 0.006684672, -0.0027290927, 0.02066405, + -0.00060481514, -0.023990331, 0.017607465, 0.0037372517, 0.009580721, + 0.0019681575, -0.009709149, 0.035189245, -0.020766791, 0.019816425, + 0.016104858, -0.01505175, -0.01431971, -0.0036762485, -0.0329546, + -0.0017369873, 0.018211076, -0.018532146, 0.031875804, 0.015770946, + -0.03816877, 0.012560248, 0.006251228, -0.013003324, 0.015501247, + -0.016284658, -0.014576566, 0.0028575207, -0.009182595, 0.05332326, + -0.024837956, 0.0155783035, 0.049316313, -0.013549143, -0.014704994, + -0.016913954, -0.0056989877, 0.0055577173, -0.053374633, 0.0032973862, + 0.004032636, -0.03940168, -0.0002867554, -0.034906704, -0.012964796, + -0.008938582, 0.032517944, -0.021473145, 0.014846264, 0.013330815, + 0.021241974, -0.017979905, -0.0019392612, -0.0050632698, -0.020959433, + -0.011012692, -0.014692151, -0.029384302, 0.0020291607, 0.023104178, + -0.023669261, -0.012849211, -0.0074295537, 0.018339504, -0.0155783035, + -0.0077313595, 0.019777896, -0.016246129, -0.03770643, 0.0075322962, + 0.0030854803, 0.01615623, -0.024593942, 0.028742163, -0.021768529, + 0.015038907, 0.0031962493, -0.013138174, -0.020291608, 0.02158873, + -0.01026781, 0.034007706, -0.0040390575, 0.0032829382, -0.013998641, + -0.028433936, -0.005721463, 0.00814875, -0.0011510351, -0.016888268, + -0.008784468, -0.0067360434, -0.01054393, -0.03411045, 0.022295082, + -0.021935485, -0.001362941, -0.008168014, 0.0105182445, -0.0005365878, + -0.0017626729, 0.021396087, 0.00015551815, 0.0028302297, 0.03246657, + 0.0010458847, 0.0007224069, -0.018172547, -0.029332932, 0.008437713, + -0.026687317, 0.00598474, -0.0076157744, -0.00059116964, 0.011744731, + -0.008945003, -0.01164841, -0.020895218, 0.013151017, -0.013048274, + -0.026353404, -0.008296442, -0.019199971, -0.031978548, -0.0028607314, + 0.010132961, -0.01675984, 0.037834857, 0.023605047, 0.0048642065, + -0.01449951, -0.000033310986, -0.0012296971, 0.012829946, 0.026481831, + -0.017915692, 0.031233665, -0.011372291, -0.009728413, 0.014615094, + -0.017967062, 0.0014416032, -0.011038378, -0.002725882, -0.0219098, + 0.013163859, 0.012271285, 0.014473824, -0.02518471, 0.012842789, + -0.008893631, -0.038143083, 0.014127068, 0.03246657, -0.0035799276, + 0.0042702276, 0.0012642121, -0.0350865, 0.024131602, 0.0062769135, + 0.016554356, -0.019816425, 0.010710887, -0.007269019, 0.005853101, + -0.010100854, 0.013844527, -0.012817104, -0.0036955127, -0.0057888874, + -0.031362094, -0.025403038, 0.029589787, 0.0043665483, 0.02292438, + 0.020792477, -0.006228753, 0.028254136, -0.0032235403, -0.0017385926, + 0.020086123, -0.035780013, -0.032029916, 0.015398505, -0.037860543, + 0.0049348418, 0.0086110905, 0.011385133, -0.019122913, -0.02541588, + -0.011622725, -0.029101761, 0.011808945, -0.020753948, 0.000802273, + 0.019559568, -0.004616983, -0.005695777, 0.029076075, 0.021485986, + 0.008168014, -0.013189545, -0.0007500992, 0.011320919, -0.0008291626, + -0.0012401319, 0.025325982, -0.015270077, -0.015180177, 0.018198233, + 0.019546725, -0.019803582, -0.007737781, 0.03310871, 0.0123997135, + -0.0016583252, 0.008078115, -0.0016647466, -0.0054710284, + -0.002546083, -0.00216401, 0.025980964, -0.008283599, 0.024439828, + 0.00644387, 0.03195286, 0.011494297, 0.012862054, 0.003083875, + -0.02545441, -0.00893216, 0.023797689, -0.04723578, -0.005422868, + -0.001841335, 0.025801165, -0.02587822, -0.0097476775, -0.019996224, + -0.008033165, -0.010601723, -0.019456826, -0.0005947817, -0.009182595, + 0.003157721, 0.020895218, 0.0045463475, -0.022256555, -0.006036111, + 0.004379391, -0.0069543705, 0.018082649, -0.004783939, -0.020497093, + 0.0016181915, -0.050035506, -0.0018172547, -0.013523458, 0.0016872215, + 0.01449951, -0.009831156, 0.02035582, 0.0033327038, -0.00019735756, + 0.02205107, -0.0037725694, 0.0074102897, -0.020728262, -0.0037308303, + -0.012303392, -0.02518471, 0.023258291, -0.02481227, 0.04538642, + 0.0064213946, 0.014255497, 0.02853668, -0.00010665535, 0.016104858, + 0.01173831, 0.004915578, -0.00045190565, -0.019122913, 0.01288774, + -0.027278084, -0.008501926, 0.027534941, -0.023874747, 0.0032540418, + -0.031362094, 0.0115199825, -0.005833837, 0.013112488, 0.015783789, + 0.012720783, 0.0072625973, 0.02789454, 0.023052808, -0.0028880222, + -0.0057664127, -0.015629675, -0.00547745, -0.017658835, -0.010807208, + -0.011044799, 0.019443983, 0.015347133, 0.015038907, 0.021139232, + 0.0032700952, -0.0034258142, -0.0051949085, -0.021884114, + -0.0034129713, -0.0071470123, -0.0029650792, -0.0012634094, + 0.012027272, 0.004658722, 0.013587671, 0.032286774, 0.025017753, + 0.0057439376, 0.01279784, 0.0076029315, 0.0023213343, 0.0012834762, + 0.011083327, 0.0011735099, -0.010768679, 0.0066268793, 0.0047261463, + -0.0077056736, 0.004061532, 0.0025974542, -0.0019023382, + -0.0061805923, 0.001878258, -0.011275969, 0.04949611, -0.0046940395, + 0.0021014016, -0.007423132, -0.013960112, 0.013728942, 0.004581665, + -0.015526933, 0.028279822, -0.035112187, -0.008142329, -0.023810532, + -0.010280653, 0.031798746, 0.007236912, -0.016271815, 0.009028481, + -0.01072373, -0.019803582, 0.009978848, -0.00018702312, -0.036807436, + -0.016772684, -0.02066405, -0.007757045, -0.0012987271, -0.008816575, + 0.011828209, -0.015899373, -0.004485344, 0.009535771, -0.00024240765, + -0.003586349, -0.0023373878, -0.013080381, -0.0056604594, + -0.014294025, -0.010640251, -0.0023839427, 0.026353404, -0.011712624, + 0.0014335763, -0.0077827303, -0.033648107, -0.011719045, + -0.0059911613, -0.027046915, 0.0020548464, -0.012618041, -0.009567878, + 0.023861904, 0.012425399, 0.01666994, 0.02434993, 0.006344338, + -0.017157966, 0.005288019, -0.010967743, 0.011796102, 0.015539776, + 0.0049284203, -0.02016318, -0.00012391285, 0.015372819, -0.008822996, + 0.0064984513, -0.0065530334, 0.0005153169, 0.0048513636, 0.022808794, + 0.013960112, 0.025467252, 0.032158345, 0.014024327, -0.019212814, + -0.014460981, 0.0033969178, 0.016258972, 0.0043858127, 0.0030662161, + -0.003522135, 0.02408023, -0.006658986, 0.0034129713, 0.0012481586, + 0.026918488, -0.02794591, 0.022436354, 0.02237214, -0.012772154, + -0.019944852, -0.0022763845, 0.016734155, 0.015822317, 0.017055225, + 0.028870592, -0.0021190604, 0.01473068, -0.013780314, -0.014512353, + 0.0013284261, 0.013215231, 0.012528141, -0.034726903, -0.0023807322, + -0.011866737, 0.011886002, 0.028485307, -0.007757045, 0.020586992, + -0.03269774, 0.02173, 0.020753948, -0.001878258, 0.018416561, + 0.0023887588, 0.021832742, 0.045823075, 0.029692529, -0.01763315, + 0.014101383, -0.021370402, 0.0040904284, 0.007204805, -0.0015828739, + -0.028921962, -0.0025942435, 0.017979905, -0.02748357, -0.010858579, + -0.0037147768, 0.011430083, -0.011385133, 0.0054581854, -0.0244912, + -0.008668883, -0.008758782, 0.0032620686, 0.016284658, -0.0020692945, + -0.011051221, 0.018275289, 0.0015347133, -0.007879051, -0.021010803, + -0.0072754403, -0.042201407, -0.007885473, 0.004254174, -0.0069094207, + -0.0030967176, 0.019418297, -0.007795573, 0.034906704, 0.000084631974, + 0.009433029, -0.00446608, 0.02481227, 0.010210018, -0.0062865457, + -0.0019039436, -0.011173227, -0.0022812006, -0.011622725, 0.00704427, + -0.014525195, 0.0077827303, 0.007821259, 0.0076221954, 0.003907419, + -0.0242215, -0.007288283, -0.0145637235, -0.0011422056, 0.012040115, + 0.013099645, -0.024735212, -0.0011767206, -0.02269321, -0.004135378, + 0.014371081, 0.013870213, -0.013016167, -0.001703275, 0.02518471, + -0.0021624046, -0.0053811288, 0.0071662767, 0.0016230076, + 0.0006746478, -0.011038378, -0.013433558, 0.02375916, -0.014717837, + -0.009638513, -0.025428724, -0.00584668, 0.01588653, -0.033802222, + 0.0022908326, -0.009587143, -0.0016422718, 0.009683464, -0.004539926, + -0.002513976, -0.0036698272, -0.00855972, -0.019366927, -0.0048642065, + -0.005994372, -0.02223087, 0.016143387, -0.0007814035, -0.013433558, + -0.008521191, 0.006652565, -0.012059379, -0.010813629, -0.012354763, + 0.039324623, -0.022590468, 0.011597039, -0.007140591, -0.014717837, + 0.0066654077, 0.01003664, -0.0013252153, 0.013497772, 0.021049332, + 0.0014761181, -0.0072433334, -0.005088955, 0.019790739, -0.011918109, + 0.007814838, 0.0047807284, 0.041816123, -0.0072754403, -0.013041853, + 0.03560021, 0.025197553, -0.009336708, 0.0014464192, -0.0005991964, + -0.00024702304, -0.0116548315, 0.0018525724, -0.004437184, + 0.0035413993, 0.010145804, -0.007994637, -0.027278084, -0.0058402587, + -0.010563195, 0.00013474896, -0.00088454713, 0.016284658, + 0.0065209265, 0.008964268, -0.0103833955, 0.009240387, 0.020291608, + 0.012008009, -0.016400242, -0.0127914185, -0.024439828, 0.009176173, + -0.0039555794, 0.0022539096, -0.011468611, -0.0060168467, + 0.0060842717, -0.020728262, 0.0143068675, 0.011301654, -0.0064470805, + 0.009214701, -0.009099117, 0.039941076, 0.004722936, 0.0064759767, + 0.0021431404, -0.01449951, -0.0075451387, -0.0040390575, -0.015488404, + 0.0099916905, -0.0032941755, -0.00019805989, -0.009696306, + -0.00038347766, 0.011571353, 0.0046779863, -0.01649014, 0.005419657, + -0.025300296, 0.004010161, -0.013844527, 0.007692831, -0.0009856841, + -0.0058434694, -0.019559568, -0.018121175, 0.010486138, 0.0014600647, + 0.0013220047, 0.00095598516, -0.004957317, 0.006556244, -0.0012706335, + 0.016708469, 0.015398505, 0.023489462, 0.013279445, -0.0001430767, + 0.016130544, 0.0035349778, -0.022436354, -0.011346605, 0.026995543, + 0.005496714, 0.004411498, -0.009811891, 0.008758782, -0.008970688, + -0.026533203, -0.012085065, -0.010858579, -0.0001264212, -0.015732417, + -0.0062640705, -0.013253759, 0.0017787263, -0.00575357, -0.003917051, + 0.006081061, -0.007005742, 0.0060200575, 0.015950745, 0.00382073, + -0.028151395, 0.027149657, -0.008906474, 0.007679988, 0.0033583895, + 0.012226336, 0.004170696, -0.008842261, 0.0023951803, 0.0048353104, + -0.0008765204, -0.032903228, -0.029358618, 0.014576566, 0.0054581854, + 0.0143068675, 0.0026311665, 0.0047582537, 0.0013838105, 0.004437184, + 0.012695097, 0.01464078, 0.029332932, -0.016258972, 0.019187128, + 0.008534034, 0.01464078, 0.030488782, -0.007224069, 0.016926797, + 0.0056604594, -0.004828889, 0.0049637384, 0.008765204, -0.023515148, + 0.01440961, -0.03369948, -0.014062854, -0.018557832, -0.008142329, + -0.030411726, 0.012926267, -0.012457506, 0.0070506916, -0.005262333, + 0.025711264, -0.0008395974, 0.00066341035, -0.014345396, -0.013587671, + 0.012515298, -0.017183652, -0.010409081, 0.03863111, 0.019456826, + 0.03701292, -0.00023919696, -0.012727205, 0.015719574, -0.012733626, + -0.025287453, -0.0035895596, -0.000015940612, 0.006440659, + -0.0045367153, 0.024452671, 0.00043665484, 0.009831156, 0.009555035, + -0.017877163, 0.0111411195, 0.007814838, 0.023181235, 0.006684672, + -0.01141724, 0.010556773, -0.0057439376, -0.00029177213, -0.046259727, + -0.00013384596, 0.014242654, -0.004915578, 0.014139911, -0.0013693625, + 0.010254968, -0.027586313, 0.010935635, -0.000024381234, 0.0048674173, + 0.0008516375, -0.009818313, -0.020908061, 0.013703257, 0.0039106295, + 0.008578983, -0.02454257, 0.010338445, -0.0054838713, -0.0071277483, + 0.005593035, 0.015411347, -0.00013163859, -0.0010306338, -0.003364811, + -0.016310344, 0.030617211, 0.0008267546, 0.013960112, 0.0068516284, + 0.0017947798, -0.006748886, 0.0009961189, -0.008778047, -0.0033198611, + 0.006357181, 0.0009616039, 0.004254174, -0.011160384, -0.027046915, + -0.0011285602, -0.012136436, 0.009587143, 0.011308076, -0.02375916, + 0.017466193, -0.031875804, 0.0040294253, -0.0069672135, -0.010742993, + -0.015758103, -0.0012385265, -0.0070506916, -0.0070250058, + -0.003586349, -0.023219764, -0.00589484, 0.008309285, -0.001671168, + -0.0031705636, 0.022282239, -0.028896276, 0.017812949, -0.026507517, + 0.008450556, 0.017761579, 0.009831156, -0.017299237, -0.008482663, + -0.024439828, -0.009696306, 0.006270492, -0.018583516, -0.0127785755, + -0.00028595273, -0.0065369797, -0.0029843433, 0.0051371157, + 0.0061452747, -0.0055962456, -0.028690793, -0.013998641, 0.008052429, + 0.0162076, -0.0019039436, 0.0016270209, -0.008964268, -0.026815744, + 0.009644935, -0.030000757, -0.0063475487, -0.0029201293, 0.010287074, + -0.00014347804, 0.012444663, 0.0059718974, 0.007879051, 0.013484929, + -0.008277178, -0.0037789908, -0.021845585, -0.015706731, 0.013472087, + -0.010158647, 0.00031605302, -0.01542419, 0.025120497, 0.010595301, + 0.008970688, -0.0067809927, 0.006257649, 0.039735593, -0.0028478885, + -0.0040390575, 0.031130921, -0.032492258, -0.0009118381, 0.025248924, + -0.011539247, 0.022885852, 0.007667145, -0.01040266, 0.0024786585, + 0.004986213, 0.009002795, -0.02053562, -0.021896956, -0.0066268793, + 0.00014387938, 0.009233966, -0.015706731, 0.00072280824, -0.022577625, + -0.007159855, 0.004498187, 0.01772305, 0.03195286, -0.011937373, + 0.021858428, -0.012219914, -0.021922642, 0.004713304, -0.007140591, + 0.0076864096, 0.0014175229, 0.02357936, 0.013125331, 0.023283977, + -0.0003373239, 0.011198913, 0.030180555, 0.008514769, 0.00069351064, + -0.0034001286, 0.011153962, -0.019649468, -0.016554356, -0.020599835, + 0.021575887, -0.021485986, -0.00589484, 0.014062854, 0.00007720724, + -0.019161442, 0.002610297, -0.005326547, -0.0006112365, 0.00076133665, + -0.016708469, 0.0022169866, -0.008367077, -0.024786584, -0.011340183, + 0.010030218, 0.005763202, -0.013960112, -0.0067553073, 0.0026761163, + -0.013960112, -0.0027066178, 0.0055127675, -0.0052527008, 0.004860996, + 0.015154492, 0.0148077365, 0.00003150497, 0.002297254, 0.0067553073, + 0.011385133, 0.0038945759, 0.020407192, -0.02509481, 0.015732417, + 0.009433029, -0.006055375, 0.040994186, 0.0023100968, 0.00244013, + -0.020098966, -0.00855972, 0.01615623, 0.026404776, -0.0014857502, + -0.0016101648, 0.0021158496, 0.010871422, -0.012720783, -0.006819521, + -0.009933898, -0.023052808, -0.018108333, 0.0015917033, 0.0031785904, + 0.013844527, 0.019854954, 0.028254136, 0.023656419, 0.015963588, + -0.01882753, -0.012091487, 0.007891894, 0.011873159, 0.0073589184, + -0.019906323, -0.02016318, 0.004337652, 0.00034193927, 0.006052165, + 0.015668204, 0.002435314, 0.008399185, 0.010306339, 0.0010266205, + -0.015437033, 0.015205863, -0.0013525062, 0.038682483, 0.00063772476, + -0.013613357, 0.026263505, 0.015437033, -0.016130544, 0.007236912, + 0.010980585, 0.012945532, 0.00870099, -0.008187278, 0.0048963134, + 0.014846264, 0.00009371223, 0.013908741, 0.012425399, -0.005069691, + 0.006800257, 0.018673416, -0.0005787282, 0.026327718, -0.020612678, + -0.024889326, -0.0035991918, 0.00019655489, -0.020009067, 0.017247867, + -0.022012541, 0.023682104, 0.0069029992, 0.023027122, -0.012130015, + 0.00515638, -0.0032556471, -0.0038656797, 0.013266602, -0.0029056813, + -0.00764146, -0.012123593, 0.011532825, -0.011218177, -0.008232228, + -0.019880638, 0.023463776, -0.00070153736, 0.009927477, -0.013780314, + -0.0299237, 0.023258291, -0.0045784544, -0.0044628694, 0.005943001, + -0.011590618, -0.004915578, -0.010010955, -0.009677042, 0.0016302316, + -0.016952483, 0.008212964, 0.03190149, -0.001942472, -0.007352497, + -0.021832742, 0.0034611318, -0.01542419, 0.020060437, -0.00354461, + -0.017003853, 0.015000379, 0.0107622575, -0.0011036772, 0.0069543705, + 0.00025886248, -0.0019986592, -0.0008460188, -0.02881922, 0.009047746, + -0.005981529, -0.013960112, 0.0029538416, 0.0071020625, 0.022539096, + -0.0022346454, 0.011083327, 0.019302713, -0.02301428, 0.022539096, + -0.0038271514, -0.0019167863, -0.018814687, -0.023283977, 0.01675984, + 0.0004017385, 0.029204503, -0.003692302, 0.0013059512, 0.005647617, + -0.011622725, -0.01786432, 0.008585405, -0.019867796, 0.0038817332, + 0.01330513, -0.0017113017, 0.016644256, 0.018609202, 0.0017899638, + 0.0047614644, 0.019765053, 0.019803582, -0.020599835, -0.006469555, + 0.020766791, 0.020240236, -0.011333762, 0.0098696835, 0.0020741106, + -0.0051082196, -0.00418996, 0.016002117, 0.000589163, 0.009876105, + 0.0019216024, 0.0006646144, 0.0029265506, -0.037321147, 0.015013221, + -0.04397371, -0.012919846, -0.027201029, 0.006524137, 0.019598097, + 0.012990482, 0.013343658, 0.016965326, -0.015912216, -0.0012192624, + 0.024465514, -0.010184332, -0.003393707, 0.00034033394, -0.01542419, + -0.004138589, 0.020381507, 0.0002448157, -0.0015892952, -0.01473068, + 0.011391554, -0.026790058, 0.0024979226, -0.014435296, 0.015270077, + 0.008456977, 0.0067553073, 0.0148077365, -0.0043023345, -0.021164916, + -0.0035638742, 0.025492938, -0.014666466, 0.0034290247, -0.0013444795, + -0.0136262, 0.0101265395, -0.01196948, 0.005464607, -0.0047582537, + -0.0033230719, -0.023309663, -0.01606633, 0.04068596, 0.005853101, + 0.009612828, -0.018378032, 0.009375236, 0.0071983836, 0.0012738442, + 0.02067689, -0.027611997, -0.011532825, -0.0127914185, 0.021049332, + 0.0039523686, 0.028202767, -0.012046536, -0.003480396, -0.019443983, + 0.014204125, 0.007583667, 0.0020468195, 0.0034418677, -0.023733474, + 0.02187127, -0.011834631, -0.0043472843, -0.0034482891, 0.0020933747, + 0.016014958, -0.01178968, 0.009105538, -0.011430083, -0.0010563195, + -0.017003853, -0.0036955127, -0.0072947047, 0.025261767, 0.009047746, + -0.0000065593545, -0.004902735, 0.001979395, -0.033853594, + -0.004860996, -0.0010466874, 0.034958072, 0.02158873, -0.031233665, + 0.025467252, -0.019765053, -0.011686939, 0.031053865, -0.015437033, + -0.011545667, -0.0046490896, 0.021216288, 0.007468082, -0.008822996, + -0.0016109674, 0.018018434, 0.0021030067, 0.0075194533, -0.017825792, + 0.022179497, 0.009368815, 0.00319946, -0.016811213, 0.044769965, + 0.0038817332, 0.015603989, 0.007435975, 0.008322128, 0.0150645925, + 0.01155851, -0.0010121723, -0.0043440736, 0.01758178, -0.00056187203, + 0.005474239, 0.0032668847, -0.009330287, 0.0092468085, -0.0070378487, + 0.009657778, -0.0021656153, 0.01684974, -0.016323186, 0.01952104, + 0.005474239, -0.0023999964, -0.0065626656, 0.021742843, -0.0046940395, + -0.0005245477, -0.017055225, 0.0011967875, -0.0029570523, + 0.0003947151, 0.013754628, 0.00065377826, -0.0018028066, 0.01408854, + -0.0062897564, 0.013106067, 0.017196495, -0.004912367, 0.021254817, + 0.00010986605, -0.0061934353, -0.016978167, 0.0026985912, + -0.013420715, -0.045412105, -0.00053177174, 0.009407343, -0.016169071, + -0.00051772495, -0.008302864, 0.026083706, 0.0028494939, -0.00621591, + 0.014859107, -0.02269321, 0.02361789, 0.018275289, -0.000034916335, + 0.0053426004, 0.0116548315, -0.03310871, 0.023322506, -0.0008917712, + -0.012457506, 0.010794365, -0.004254174, -0.012457506, 0.008925739, + -0.029307246, -0.0070185843, 0.014435296, 0.003968422, 0.003711566, + 0.007063534, -0.010068747, 0.0070506916, 0.0111411195, -0.0052366476, + 0.013343658, -0.009426608, -0.014949007, 0.0093559725, -0.008675304, + -0.0064502913, 0.016785527, 0.023630733, 0.0028783903, -0.006832364, + -0.016130544, 0.00354461, 0.0021479565, -0.0043344414, 0.0011751152, + 0.006832364, 0.0075451387, -0.015488404, -0.015642518, -0.0053104935, + -0.0076350383, -0.014139911, -0.020497093, 0.011776838, 0.0062897564, + 0.011109013, -0.010531087, 0.009028481, 0.012181386, 0.030052127, + 0.009786205, 0.00759651, 0.008341392, -0.01708091, 0.000027265844, + 0.020098966, -0.005349022, 0.013093224, 0.003250831, -0.02794591, + 0.023425248, 0.0053104935, -0.030694267, 0.008174435, -0.00064575154, + -0.0052462798, 0.009144066, 0.026456147, 0.00040213985, 0.00068026653, + 0.0035638742, -0.012207071, -0.0095743, 0.016888268, -0.001979395, + -0.0030036075, -0.0012730415, 0.02509481, 0.006414973, -0.0009455504, + 0.026212133, -0.006716779, -0.016913954, -0.010646673, -0.015180177, + 0.018146861, 0.03177306, 0.001321202, -0.0024931065, -0.03038604, + 0.014011484, -0.012907004, -0.0016872215, 0.015501247, 0.030000757, + -0.03359674, -0.02002191, -0.006652565, 0.0081102215, 0.022218026, + 0.006620458, -0.017915692, 0.0053233365, 0.005159591, 0.0006810692, + 0.0137931565, -0.020111809, -0.0056283525, 0.022757424, -0.019983381, + -0.02545441, 0.005689356, 0.030976808, 0.0358057, 0.0145508805, + -0.0162076, 0.02219234, 0.016053487, 0.0069543705, 0.005644406, + -0.00009918045, -0.010710887, 0.022076756, -0.0016398637, + -0.000030702293, 0.004382602, 0.0028623368, 0.014846264, 0.014486667, + 0.0011221387, -0.020522777, -0.006992899, -0.003978054, -0.027098287, + -0.007435975, 0.04107124, 0.002536451, 0.012130015, 0.0067360434, + -0.016079172, 0.0061902246, -0.014627937, -0.012091487, -0.0048192567, + -0.0060425326, -0.004655511, 0.03020624, -0.011173227, -0.027637683, + 0.009131223, 0.016464457, -0.014152754, -0.01708091, -0.008810153, + 0.006980056, 0.0068580494, -0.0102613885, 0.0036248774, -0.009799048, + 0.000051346076, -0.017761579, 0.008553298, 0.021884114, 0.028562363, + -0.01864773, -0.00018270875, -0.004658722, 0.029101761, -0.0026568521, + -0.019803582, -0.013446401, 0.032672055, -0.013317972, -0.003024477, + 0.013228074, -0.0023518358, 0.023361035, -0.013690414, -0.0015186599, + -0.0068580494, 0.017658835, 0.0043119667, 0.018390875, 0.016040644, + -0.0050568483, -0.0041032713, -0.004084007, -0.0132922875, + 0.0014624727, 0.0065658763, 0.018442245, -0.0041995924, 0.004816046, + -0.010203596, -0.021293346, -0.007744202, 0.00621591, 0.0030726374, + 0.0022442774, -0.012412556, 0.013202388, 0.0033327038, -0.013131753, + 0.021100704, -0.002896049, -0.017222181, 0.023078494, -0.00488026, + -0.007821259, -0.011391554, -0.020098966, -0.0025252134, 0.0012425398, + -0.012643727, 0.026250662, -0.012393292, 0.005451764, 0.007269019, + 0.010742993, 0.009792627, -0.020227393, -0.013574828, 0.0045046085, + 0.00980547, -0.017196495, 0.0014424059, -0.0070121633, -0.013754628, + 0.0072818617, 0.0036505628, -0.00188789, 0.0050568483, 0.024504043, + 0.0033391253, 0.0043729697, 0.0014175229, 0.0021286923, 0.0066268793, + 0.011391554, 0.0023502305, 0.010768679, -0.01510312, -0.004774307, + 0.00529444, -0.00446608, -0.0021880902, 0.0033070184, -0.022641838, + -0.00024060164, -0.0072433334, 0.009966005, 0.0039844755, + -0.0071855406, 0.00036381214, 0.011950215, 0.026186448, 0.0044403947, + -0.0008893632, 0.021601573, 0.013407872, 0.010935635, 0.002385548, + -0.009272494, -0.0122777065, -0.009728413, -0.00042622007, 0.00704427, + 0.0065498226, -0.008887211, 0.016862582, -0.0067745717, 0.015989274, + 0.013151017, -0.015501247, -0.011076906, -0.012547405, -0.010113697, + 0.005763202, -0.015681047, 0.0036730377, -0.0021431404, 0.0002586618, + -0.0034771853, -0.0140500115, -0.006556244, -0.0041450104, + -0.005612299, -0.028279822, -0.019174285, -0.005538453, 0.008636776, + 0.0136262, -0.022320768, 0.009856841, 0.0020981908, 0.017620306, + 0.0076543028, 0.0067103575, -0.003933104, -0.0038496263, 0.00870099, + 0.035934124, -0.0137931565, 0.00043705618, 0.0036377201, + -0.0065755085, -0.01649014, -0.0010370553, -0.022115285, 0.0018429403, + 0.012136436, 0.00056387874, -0.005349022, 0.0011863527, -0.0030501625, + 0.0015170545, -0.012829946, 0.01408854, 0.0016310343, 0.022205183, + -0.017183652, 0.02016318, 0.003310229, 0.008135907, -0.004369759, + 0.01026781, 0.008270756, -0.008649618, 0.011635567, 0.031747375, + -0.013549143, 0.0191486, 0.021357559, 0.0074038683, 0.03665332, + -0.00976052, -0.016580041, -0.012894161, -0.013240916, -0.018596359, + -0.0014351817, 0.0082193855, -0.0053650755, 0.008546877, + -0.0015981246, 0.009856841, 0.005650827, 0.0070250058, -0.0028783903, + -0.0081937, 0.022474881, -0.006100325, -0.007525875, 0.007468082, + 0.010479717, -0.0060457434, 0.017389137, 0.015693888, 0.0093559725, + -0.001772305, 0.0072754403, 0.015796632, 0.012380449, -0.005666881, + 0.014037169, -0.0077506234, -0.023219764, 0.009041324, 0.013106067, + 0.04084007, 0.006418184, 0.009182595, 0.021563044, 0.009458714, + -0.010768679, 0.0021351138, 0.009362394, -0.0031224033, -0.021665785, + 0.015899373, 0.009009217, 0.0010587275, 0.0037436732, 0.014679309, + 0.0025123707, -0.005782466, 0.016721312, -0.01082005, -0.0031159818, + 0.011404397, 0.008675304, -0.009144066, -0.0038014657, -0.0032379883, + 0.0040679537, -0.004523873, -0.01961094, -0.0005590627, -0.014974693, + -0.0055641388, -0.026635945, -0.006703936, -0.0011759179, + -0.008412027, 0.030540153, -0.0023550466, -0.0067938357, + -0.0011052826, 0.0007472898, 0.0070956415, -0.024915012, -0.029076075, + 0.0044275518, 0.026289191, 0.004568822, 0.001528292, -0.026083706, + -0.00708922, -0.0025717686, 0.018544989, 0.008026743, -0.0024240764, + -0.0009038113, -0.011340183, -0.033365566, -0.00055665465, + 0.025839692, 0.02748357, -0.009933898, 0.008604669, -0.009471557, + 0.014627937, -0.009477979, 0.0064053414, 0.0010571221, -0.003618456, + 0.014846264, 0.022423511, -0.013818841, 0.0059044724, -0.014974693, + 0.019212814, 0.018211076, 0.0023775215, -0.018416561, -0.023271134, + 0.011905266, -0.021190602, 0.0028992598, -0.0032973862, 0.004084007, + -0.019508198, -0.001153443, 0.00041377862, 0.011731888, -0.011609882, + -0.009150487, 0.015115963, 0.008386342, 0.027509255, -0.011757574, + -0.006411763, 0.019919166, -0.005085745, 0.0041000606, 0.021062175, + 0.016284658, -0.009458714, 0.008142329, 0.0012634094, 0.024118759, + 0.002284411, -0.0056925663, -0.021563044, 0.023399564, -0.0043986556, + 0.008200121, 0.004568822, 0.015475562, -0.00055665465, -0.0018204654, + -0.0012682255, 0.020638363, -0.0022860165, -0.015693888, -0.018557832, + -0.013960112, 0.0028238085, 0.009426608, -0.015732417, -0.007493768, + -0.015372819, -0.02591675, -0.0047454108, -0.011564932, 0.002412839, + 0.017106596, -0.0061805923, -0.013382187, 0.0162975, -0.0093816575, + -0.021845585, -0.00038668836, -0.0027852799, -0.024324244, + -0.025236081, -0.036756065, -0.010698044, 0.00003102838, 0.0014014695, + -0.00071839354, -0.0140500115, 0.044769965, -0.018442245, + -0.023643576, 0.0358057, 0.009966005, -0.0071662767, -0.003949158, + 0.0013725731, -0.010415503, -0.0020548464, -0.025724107, -0.036910176, + -0.039838333, -0.011211756, -0.0043729697, 0.003945947, 0.02702123, + 0.0037083556, 0.012977639, 0.002859126, -0.012065801, -0.010190753, + 0.0020933747, -0.0097476775, 0.007757045, 0.021357559, -0.0061324323, + 0.0030373198, 0.010370553, 0.008347813, 0.031644635, -0.025608523, + -0.02108786, 0.011629146, 0.0081102215, -0.008168014, 0.0014424059, + 0.020432878, -0.0059879506, 0.0005377918, -0.015270077, 0.0143068675, + -0.007583667, -0.005589824, -0.01173831, -0.010004533, 0.022115285, + -0.003907419, -0.00093110226, -0.0013525062, -0.013690414, + 0.0017674889, 0.017273553, -0.016310344, -0.00515638, 0.021357559, + 0.02661026, 0.0034258142, 0.023232607, 0.026687317, 0.017761579, + 0.023836218, -0.009921055, 0.00069110264, -0.0065658763, -0.010685201, + 0.006440659, 0.001574847, 0.011975901, 0.00037906296, -0.003711566, + 0.0037372517, 0.013356501, 0.005888419, 0.016836897, -0.0022956487, + -0.006819521, 0.023258291, -0.002827019, -0.017684521, 0.011841052, + -0.018018434, 0.011173227, 0.009580721, 0.021614416, -0.004469291, + 0.01758178, -0.02062552, -0.026867116, -0.016400242, 0.0022812006, + 0.0026167184, -0.0036890914, 0.0059751077, 0.016592884, 0.010511823, + 0.0051114303, 0.018878901, 0.0015395294, -0.0076029315, -0.015128806, + 0.0024369194, 0.019212814, -0.025287453, -0.007988215, -0.0031384567, + -0.005082534, 0.016258972, 0.022616152, -0.0055448744, 0.010325603, + 0.020548463, 0.0024850797, 0.029641159, -0.00092387816, -0.005952633, + 0.022847323, 0.0047935713, -0.002550899, 0.010441188, -0.016220443, + -0.009137644, -0.015026064, -0.0075387177, -0.016284658, + 0.000010064784, 0.00032026708, 0.006559455, -0.022577625, + 0.0017498301, -0.0059590545, -0.023091335, -0.011545667, 0.012329078, + -0.0025541098, -0.013446401, 0.029692529, 0.025569994, 0.023797689, + 0.004138589, -0.008052429, 0.010088012, -0.003300597, -0.008071694, + -0.0041995924, -0.010100854, -0.0056411955, -0.01496185, 0.008534034, + -0.0018670205, -0.0155783035, 0.008347813, 0.0027001966, + -0.0087202545, 0.028716477, -0.002127087, -0.013677571, 0.0031641424, + -0.032851856, -0.0003864877, 0.001675984, -0.003329493, -0.004751832, + -0.015488404, 0.022140969, -0.012123593, -0.006501662, 0.0069543705, + 0.013459244, 0.010017375, -0.008437713, -0.010152225, -0.007936844, + 0.005377918, 0.024902169, 0.0060939034, 0.019187128, 0.0040711644, + 0.016181914, 0.0011076906, 0.005878787, -0.03899071, 0.007879051, + 0.01058888, 0.024889326, 0.0021977224, 0.00082434656, -0.004751832, + -0.012091487, 0.016567199, 0.019970538, 0.011012692, 0.0046908287, + 0.0014536432, -0.010332025, 0.010120118, -0.0358057, -0.008161593, + -0.00049725675, 0.016772684, -0.017055225, 0.013459244, 0.037732117, + -0.014371081, 0.0140500115, -0.009522929, -0.0024882904 + ] + } + ] + }, + { + "name": "untitled", + "title": "Untitled", + "content": "Technological advances, particularly in generative AI, are expected to impact almost every sector of human activity, creating both opportunities and challenges.", + "keywords": [ + "Technological advances", + "generative AI", + "impact", + "sector", + "human activity", + "opportunities", + "challenges" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.026481831, 0.04921357, -0.002366284, -0.021460302, -0.027380828, + -0.0036216667, 0.012605198, -0.006658986, -0.06981341, 0.021896956, + -0.03893934, 0.025441566, -0.013189545, -0.0004924407, 0.0061902246, + 0.013844527, -0.015963588, 0.04998414, 0.0066975146, -0.0013460849, + 0.0354461, -0.0061163786, -0.060566597, 0.044179197, 0.012682254, + 0.00824507, -0.030694267, 0.0064470805, -0.026327718, 0.0017578568, + -0.003660195, -0.001222473, 0.045232303, 0.001574847, 0.009728413, + -0.019392611, 0.0015339106, -0.020548463, 0.029872328, 0.034881018, + 0.011256705, 0.0020708998, 0.018943114, 0.010145804, 0.028896276, + -0.02853668, 0.0003991298, -0.036550578, 0.021190602, 0.023232607, + 0.016002117, -0.014949007, -0.021293346, 0.041456524, 0.011834631, + -0.004276649, 0.011937373, 0.019135756, -0.007423132, -0.050857447, + 0.00016474891, -0.014332553, -0.024452671, 0.009426608, 0.012438241, + 0.0215502, 0.017299237, -0.002600665, 0.04433331, 0.020317294, + 0.029666845, 0.06642291, 0.0055320314, 0.029332932, 0.054299314, + -0.008938582, -0.03149052, 0.01201443, -0.025955278, 0.013651885, + 0.02559568, -0.028100023, -0.020188866, 0.0044949763, -0.0021094282, + 0.0000010096138, 0.011256705, -0.073666245, 0.0030164502, + 0.0036056133, 0.010672358, -0.013728942, -0.011584196, -0.020908061, + -0.044487424, -0.027791796, 0.045566216, -0.010299917, -0.005217383, + 0.024298558, 0.011956637, -0.002674511, -0.0404291, 0.01731208, + 0.0005526413, 0.016438771, -0.0006758518, -0.006209489, -0.016143387, + 0.013112488, 0.025955278, -0.003226751, -0.0071662767, -0.0046651433, + -0.027201029, 0.0028061494, -0.009728413, -0.017800106, 0.044127826, + 0.015115963, -0.0037211983, 0.014743523, -0.03226109, 0.006071429, + -0.021601573, 0.010254968, 0.020432878, -0.017453352, 0.01961094, + -0.032158345, 0.02591675, 0.011353026, -0.0068516284, 0.011333762, + 0.009336708, -0.04225278, -0.0065401904, 0.031798746, 0.028100023, + 0.048263203, -0.06390572, -0.007159855, 0.019790739, -0.00056187203, + -0.0013733758, -0.025467252, -0.016952483, 0.01606633, 0.012733626, + -0.026327718, -0.024195816, -0.013780314, -0.027509255, 0.00446608, + 0.018095491, 0.020047594, 0.0060842717, -0.024414143, 0.05676513, + -0.046901867, -0.018801844, -0.013587671, 0.028228452, -0.036807436, + -0.011166805, -0.0064246054, -0.020201707, -0.00026046782, + 0.0016093621, 0.0034482891, 0.042278465, -0.0054389215, -0.04572033, + -0.07489915, 0.0060329004, -0.025261767, -0.014204125, -0.014846264, + -0.005731095, 0.024375614, 0.008501926, -0.0031464836, 0.028022967, + -0.009779784, -0.020908061, -0.022128126, 0.006890157, 0.023964645, + 0.038554054, -0.021575887, 0.00079223956, 0.051448215, 0.013883055, + 0.0060040043, 0.02905039, 0.0014680914, -0.011725467, -0.033365566, + 0.015937902, 0.026032334, -0.054196574, 0.015385662, -0.02205107, + -0.014782051, 0.0024593943, 0.015937902, -0.009535771, -0.0095743, + 0.020869533, 0.030642897, -0.01035771, -0.017607465, -0.017787263, + 0.042612378, -0.014936164, 0.02817708, -0.017183652, -0.007352497, + -0.017389137, 0.0459515, 0.026404776, 0.037115663, 0.007513032, + -0.026404776, -0.018557832, -0.009304601, -0.06303241, 0.0059751077, + 0.004613772, -0.026918488, 0.032158345, 0.01790285, 0.0072625973, + 0.056919243, -0.0379376, 0.06133716, 0.024555413, 0.0073268116, + -0.01717081, 0.034315933, 0.027406514, 0.02899902, -0.00980547, + -0.02463247, -0.007904737, -0.028074337, 0.023733474, 0.0059141046, + 0.0075708246, 0.028716477, 0.00035177203, 0.0030084234, 0.051936243, + 0.00994674, -0.011930952, 0.009709149, -0.005750359, 0.006633301, + -0.008906474, 0.0073139686, 0.0019874217, 0.010370553, -0.017157966, + 0.017479036, 0.005731095, -0.035112187, 0.0005357851, 0.0032652793, + 0.0019537094, -0.029153133, -0.001703275, -0.004225278, 0.007288283, + 0.017466193, 0.028613735, 0.010903528, 0.042612378, 0.02301428, + -0.03125935, -0.04623404, -0.021113547, 0.009054166, 0.0051242732, + -0.025518622, 0.0045559797, 0.01243182, -0.007943265, 0.027098287, + 0.0022137759, -0.046799127, -0.011423661, 0.0044436054, 0.030899752, + 0.025711264, 0.030129185, -0.0028848117, 0.0030662161, -0.0033487573, + -0.046901867, 0.019508198, 0.018146861, 0.02661026, -0.007866208, + -0.015231549, 0.0122648645, -0.015128806, 0.005050427, -0.00395879, + -0.026160762, -0.05404246, -0.009221123, -0.053374633, 0.0028334404, + 0.025698422, -0.011995166, -0.00033009984, 0.017222181, -0.014139911, + 0.038502682, -0.017440509, -0.020009067, 0.0029875538, 0.018429402, + 0.019906323, -0.012348342, 0.023399564, -0.000100685465, -0.012727205, + 0.01592506, 0.01201443, -0.03457279, 0.004626615, -0.017812949, + -0.05106293, 0.011051221, -0.012759311, 0.018763315, -0.008296442, + 0.0010033429, 0.019970538, -0.004976581, 0.0057664127, 0.009908212, + 0.037192717, 0.026507517, -0.02149883, -0.009818313, -0.0053971824, + -0.013163859, 0.01740198, 0.000666621, 0.003351968, 0.05501851, + -0.0039620004, 0.017363451, -0.006283335, -0.02499207, -0.02283448, + 0.01606633, -0.039581478, -0.024439828, 0.0067809927, -0.009458714, + -0.0022812006, -0.019482512, 0.019495355, 0.00073284167, 0.009400922, + 0.030617211, 0.0062191207, 0.013240916, 0.014011484, -0.0122648645, + 0.011956637, 0.013497772, 0.006960792, -0.017363451, -0.008604669, + -0.012386871, -0.00061725656, 0.032440886, 0.025531465, -0.019944852, + 0.00086769095, -0.0011991955, -0.0030260824, -0.0017899638, + -0.017838635, -0.017337766, 0.011089749, -0.008084536, -0.013382187, + -0.00469725, 0.0030517678, -0.0038464156, 0.017517565, -0.044281937, + -0.0033583895, 0.028896276, 0.0041642743, 0.023232607, -0.03056584, + 0.002181669, 0.031336408, -0.020753948, -0.0052976506, -0.0019970539, + -0.057124726, 0.012662991, -0.014011484, -0.024786584, -0.0030180556, + 0.003393707, -0.012322657, -0.02899902, -0.05008688, -0.016541513, + 0.03714135, 0.015488404, 0.034059078, -0.01449951, -0.037911914, + 0.0063314955, 0.0039555794, -0.015411347, 0.0030276875, 0.012740048, + 0.025698422, -0.0381174, -0.021357559, -0.031464834, -0.009477979, + 0.0027917014, 0.010306339, -0.015000379, 0.050985876, -0.03688449, + 0.0045591905, 0.03978696, 0.013356501, -0.02946136, 0.005962265, + 0.032338142, -0.018095491, -0.013574828, 0.022487724, 0.02684143, + 0.01873763, 0.024337087, 0.056919243, -0.014165597, 0.03434162, + 0.016567199, 0.011622725, 0.007416711, 0.0011245467, -0.042920604, + 0.012496035, -0.007988215, 0.031798746, 0.023399564, -0.023181235, + 0.014345396, -0.0352663, -0.013972955, 0.02513334, 0.0461313, + 0.00971557, 0.000971236, -0.017877163, -0.0072561763, -0.025582837, + -0.039581478, 0.02541588, -0.045925815, -0.0031063498, 0.004482134, + 0.012996903, 0.022962907, -0.015848001, -0.040814385, 0.022718895, + -0.044924077, 0.015950745, -0.0050632698, 0.026147919, -0.025929593, + -0.027663369, 0.0106274085, -0.015167334, 0.052295838, 0.0053522326, + 0.008765204, -0.020227393, 0.010807208, 0.0070378487, -0.03724409, + -0.055223994, 0.02370779, 0.033005968, -0.0033969178, 0.03084838, + 0.037629373, -0.0029153132, 0.003204276, -0.0039427364, -0.02030445, + -0.03521493, 0.013472087, 0.016323186, 0.061491277, 0.0035125029, + -0.028279822, -0.020009067, 0.016913954, -0.015475562, -0.012386871, + -0.010210018, 0.029409988, -0.009895369, -0.009580721, 0.011622725, + -0.0025525044, 0.006543401, -0.019790739, 0.012579513, 0.0033038077, + -0.0009977242, 0.029872328, -0.021241974, -0.033262827, 0.0057696234, + 0.0056090886, 0.028022967, 0.06405983, -0.02513334, -0.02126766, + -0.019238498, -0.0017867531, 0.042715117, -0.03351968, -0.057535697, + 0.012412556, 0.03190149, 0.01850646, 0.006154907, -0.0047807284, + -0.012207071, 0.0028703634, -0.03167032, 0.053888347, -0.014949007, + 0.005888419, 0.0030581893, -0.014653623, -0.007121327, -0.014692151, + -0.02007328, -0.037038606, -0.013330815, 0.02140893, 0.011243862, + 0.021845585, -0.004456448, 0.0062865457, 0.015475562, 0.016002117, + -0.011282391, 0.005410025, 0.0017562514, 0.011218177, 0.012451084, + 0.026327718, 0.0064663445, 0.019906323, 0.011719045, -0.018660573, + -0.015398505, 0.012367606, 0.011423661, -0.008360656, 0.019649468, + 0.031387776, 0.025390195, 0.0005686948, 0.0066782506, -0.00814875, + -0.030077813, -0.0034033393, 0.044769965, 0.036627635, 0.023450933, + -0.0029265506, 0.015115963, 0.0010563195, 0.009060588, -0.027534941, + 0.015681047, -0.028665107, -0.029435674, 0.0068387855, -0.0041450104, + 0.015732417, -0.020458564, -0.024388457, 0.008463399, 0.023643576, + 0.0153086055, 0.018904585, 0.010511823, 0.0024465513, 0.020831006, + 0.0017129071, -0.008026743, -0.032286774, -0.006315442, 0.03403339, + 0.029076075, 0.010800786, -0.022359297, -0.014294025, -0.024414143, + -0.0074552395, -0.009766942, 0.025377352, -0.021421773, 0.012451084, + -0.016323186, -0.0031175872, 0.0018814687, -0.001079597, 0.009914634, + -0.015526933, 0.006459923, -0.01473068, -0.019687997, 0.0061805923, + -0.008418448, -0.02080532, -0.012784997, 0.0026488253, 0.058306266, + 0.016926797, 0.03624235, -0.004639458, 0.00016755828, -0.021100704, + 0.024902169, 0.007191962, 0.004899524, 0.055994563, -0.028048653, + 0.009477979, -0.0011646805, -0.00091825944, 0.01800559, 0.02776611, + -0.030257612, 0.028048653, -0.003241199, -0.016836897, 0.014897636, + 0.0004274241, 0.006877314, 0.012412556, 0.020908061, -0.03737252, + 0.038091715, 0.017928535, -0.0034771853, -0.014332553, 0.017915692, + -0.004116114, 0.011449347, -0.010132961, 0.0067938357, 0.0032797274, + -0.0069351066, -0.010486138, 0.0020548464, 0.010486138, -0.0164773, + -0.02946136, -0.005731095, 0.013818841, -0.021190602, 0.044025082, + 0.03673038, -0.005506346, -0.01758178, 0.019366927, 0.033493996, + -0.0018959168, -0.007827681, 0.031053865, 0.004337652, -0.0011686939, + 0.031233665, 0.01150714, -0.014846264, 0.0020773213, 0.023373878, + 0.01740198, -0.039966762, 0.010736573, 0.009766942, -0.0018333083, + -0.024619628, -0.05224447, -0.008643198, 0.006703936, -0.004626615, + 0.030694267, -0.011076906, 0.053888347, -0.035112187, 0.040583216, + -0.035137873, 0.008495505, 0.039966762, 0.02375916, 0.008707412, + -0.053785603, -0.012117172, -0.03236383, 0.008309285, -0.029230189, + -0.025788322, 0.01473068, 0.0060650073, -0.038399942, 0.009599986, + 0.026250662, -0.009266073, 0.026738688, -0.001855783, -0.013818841, + -0.02499207, 0.022962907, -0.007172698, 0.0067424644, -0.032081287, + 0.04374254, 0.006652565, 0.0006770558, 0.017055225, -0.0025187922, + -0.003660195, -0.059179574, 0.0028510992, -0.038656797, -0.019816425, + 0.038785223, 0.016271815, 0.0037500947, 0.00022956486, 0.012142858, + 0.0070378487, 0.01588653, 0.0358057, 0.023656419, 0.010858579, + 0.00202595, 0.0025492937, 0.0077827303, 0.0022137759, 0.004816046, + -0.010113697, -0.0027130393, 0.0065947725, 0.03323714, -0.0023261502, + -0.005262333, -0.0018188601, 0.022397825, -0.00060922984, 0.010967743, + -0.011988744, 0.019662311, -0.03398202, 0.034649845, -0.006071429, + 0.002319729, -0.0023405985, -0.007577246, -0.019392611, 0.009978848, + -0.002969895, 0.0044725016, -0.013818841, -0.008444134, -0.004597719, + -0.017684521, 0.010537509, -0.0093945, -0.016541513, 0.0115199825, + -0.012194228, 0.044410367, 0.006601194, 0.012309814, 0.019598097, + -0.010017375, 0.010909949, -0.030334668, 0.0030533732, 0.007179119, + 0.013985798, -0.016708469, 0.018211076, 0.040994186, -0.022269398, + 0.020009067, -0.0081102215, -0.036422152, -0.027534941, 0.016567199, + -0.0024096284, 0.00019906324, -0.02003475, 0.034778275, -0.021421773, + 0.0432802, -0.010794365, 0.020908061, 0.02306565, 0.023720633, + -0.026481831, 0.00043344416, 0.018955957, -0.010730151, -0.0024609996, + -0.013703257, -0.009734835, -0.00971557, 0.011564932, 0.0070185843, + 0.0067681503, 0.02928156, -0.009323865, -0.0076157744, -0.015462719, + 0.027971596, -0.0072947047, 0.010210018, 0.04199592, 0.0072112265, + -0.054813027, 0.015038907, -0.04148221, -0.012984061, 0.017890006, + -0.01510312, 0.019726524, -0.025467252, 0.02666163, 0.024683842, + 0.0083542345, 0.0038110977, 0.008450556, 0.007943265, -0.012573091, + 0.03102818, 0.0044500264, 0.004931631, 0.004074375, 0.0023839427, + 0.012001587, -0.018724788, -0.0052655437, 0.015090277, 0.0019248131, + -0.013960112, -0.024645314, -0.007493768, 0.014692151, -0.009291758, + -0.01937977, -0.012252022, 0.013446401, -0.025390195, -0.004116114, + -0.010556773, -0.033262827, 0.024285715, 0.03084838, -0.019970538, + -0.012194228, 0.012149279, -0.021794215, 0.0012915031, 0.004543137, + -0.0024064176, -0.006090693, 0.012457506, 0.025801165, -0.018917428, + -0.020086123, -0.015629675, 0.027689055, 0.020381507, -0.008668883, + 0.0035574527, -0.017235024, -0.0040165824, 0.00022976553, 0.023656419, + 0.0075066104, -0.0037372517, -0.0025974542, -0.00833497, 0.012374028, + 0.0052591222, -0.03369948, -0.00865604, 0.001333242, -0.02840825, + -0.0136262, -0.00015060177, 0.031002494, 0.03231246, 0.020060437, + -0.02817708, 0.0046651433, -0.00045351102, -0.0055866134, 0.014396767, + -0.023502305, -0.01597643, -0.0005907683, 0.015603989, 0.0076992526, + 0.0073717614, 0.006068218, 0.01864773, -0.021370402, -0.038887966, + 0.0018943115, -0.01072373, 0.028973334, -0.008540455, 0.016567199, + -0.023001436, -0.01836519, -0.0067103575, 0.044949763, -0.01868626, + 0.025081968, 0.038476996, -0.0025043439, 0.010909949, 0.013202388, + -0.010800786, 0.011410818, 0.047030296, 0.008277178, -0.025030596, + 0.03218403, -0.01946967, 0.019135756, 0.004957317, 0.0035478207, + -0.030899752, -0.008636776, 0.013651885, -0.008822996, -0.018390875, + -0.06477903, 0.000050116978, -0.0009913028, 0.024157288, + -0.0140500115, -0.015655361, -0.015205863, 0.00051050086, 0.03601118, + -0.007724938, 0.0053522326, 0.024773741, 0.02559568, 0.015642518, + 0.021627259, 0.007564403, 0.042021606, 0.02858805, -0.017568937, + -0.022115285, -0.016053487, -0.004209224, 0.0015001984, 0.022256555, + -0.025608523, -0.0017000643, -0.0069543705, 0.052655436, -0.010614566, + 0.0023405985, 0.0005803336, -0.0041321674, 0.014704994, 0.035291985, + -0.003554242, -0.018609202, -0.00071518286, -0.01634887, 0.02905039, + -0.044179197, 0.0028832064, -0.014627937, -0.0056861453, 0.021370402, + 0.0010282259, 0.032517944, -0.025801165, -0.008360656, -0.025069125, + -0.0075001894, -0.016657097, -0.018519303, -0.00478715, 0.00285431, + 0.018095491, 0.0045367153, -0.012733626, -0.0042284885, 0.010832893, + 0.014627937, 0.010351288, 0.02324545, 0.02393896, 0.021807056, + 0.00014438105, -0.0044949763, 0.014062854, -0.034290247, -0.007930422, + -0.008135907, 0.05589182, 0.0023887588, 0.017697364, 0.012746469, + -0.035985496, 0.021896956, 0.0022009332, 0.010473295, -0.014011484, + -0.008084536, 0.003798255, 0.018172547, 0.01772305, 0.00055344397, + 0.0025075546, -0.00028073534, 0.0032973862, -0.033622425, + -0.008501926, 0.02858805, 0.02545441, 0.0018509671, -0.017568937, + 0.005644406, -0.012528141, 0.006684672, -0.0027290927, 0.02066405, + -0.00060481514, -0.023990331, 0.017607465, 0.0037372517, 0.009580721, + 0.0019681575, -0.009709149, 0.035189245, -0.020766791, 0.019816425, + 0.016104858, -0.01505175, -0.01431971, -0.0036762485, -0.0329546, + -0.0017369873, 0.018211076, -0.018532146, 0.031875804, 0.015770946, + -0.03816877, 0.012560248, 0.006251228, -0.013003324, 0.015501247, + -0.016284658, -0.014576566, 0.0028575207, -0.009182595, 0.05332326, + -0.024837956, 0.0155783035, 0.049316313, -0.013549143, -0.014704994, + -0.016913954, -0.0056989877, 0.0055577173, -0.053374633, 0.0032973862, + 0.004032636, -0.03940168, -0.0002867554, -0.034906704, -0.012964796, + -0.008938582, 0.032517944, -0.021473145, 0.014846264, 0.013330815, + 0.021241974, -0.017979905, -0.0019392612, -0.0050632698, -0.020959433, + -0.011012692, -0.014692151, -0.029384302, 0.0020291607, 0.023104178, + -0.023669261, -0.012849211, -0.0074295537, 0.018339504, -0.0155783035, + -0.0077313595, 0.019777896, -0.016246129, -0.03770643, 0.0075322962, + 0.0030854803, 0.01615623, -0.024593942, 0.028742163, -0.021768529, + 0.015038907, 0.0031962493, -0.013138174, -0.020291608, 0.02158873, + -0.01026781, 0.034007706, -0.0040390575, 0.0032829382, -0.013998641, + -0.028433936, -0.005721463, 0.00814875, -0.0011510351, -0.016888268, + -0.008784468, -0.0067360434, -0.01054393, -0.03411045, 0.022295082, + -0.021935485, -0.001362941, -0.008168014, 0.0105182445, -0.0005365878, + -0.0017626729, 0.021396087, 0.00015551815, 0.0028302297, 0.03246657, + 0.0010458847, 0.0007224069, -0.018172547, -0.029332932, 0.008437713, + -0.026687317, 0.00598474, -0.0076157744, -0.00059116964, 0.011744731, + -0.008945003, -0.01164841, -0.020895218, 0.013151017, -0.013048274, + -0.026353404, -0.008296442, -0.019199971, -0.031978548, -0.0028607314, + 0.010132961, -0.01675984, 0.037834857, 0.023605047, 0.0048642065, + -0.01449951, -0.000033310986, -0.0012296971, 0.012829946, 0.026481831, + -0.017915692, 0.031233665, -0.011372291, -0.009728413, 0.014615094, + -0.017967062, 0.0014416032, -0.011038378, -0.002725882, -0.0219098, + 0.013163859, 0.012271285, 0.014473824, -0.02518471, 0.012842789, + -0.008893631, -0.038143083, 0.014127068, 0.03246657, -0.0035799276, + 0.0042702276, 0.0012642121, -0.0350865, 0.024131602, 0.0062769135, + 0.016554356, -0.019816425, 0.010710887, -0.007269019, 0.005853101, + -0.010100854, 0.013844527, -0.012817104, -0.0036955127, -0.0057888874, + -0.031362094, -0.025403038, 0.029589787, 0.0043665483, 0.02292438, + 0.020792477, -0.006228753, 0.028254136, -0.0032235403, -0.0017385926, + 0.020086123, -0.035780013, -0.032029916, 0.015398505, -0.037860543, + 0.0049348418, 0.0086110905, 0.011385133, -0.019122913, -0.02541588, + -0.011622725, -0.029101761, 0.011808945, -0.020753948, 0.000802273, + 0.019559568, -0.004616983, -0.005695777, 0.029076075, 0.021485986, + 0.008168014, -0.013189545, -0.0007500992, 0.011320919, -0.0008291626, + -0.0012401319, 0.025325982, -0.015270077, -0.015180177, 0.018198233, + 0.019546725, -0.019803582, -0.007737781, 0.03310871, 0.0123997135, + -0.0016583252, 0.008078115, -0.0016647466, -0.0054710284, + -0.002546083, -0.00216401, 0.025980964, -0.008283599, 0.024439828, + 0.00644387, 0.03195286, 0.011494297, 0.012862054, 0.003083875, + -0.02545441, -0.00893216, 0.023797689, -0.04723578, -0.005422868, + -0.001841335, 0.025801165, -0.02587822, -0.0097476775, -0.019996224, + -0.008033165, -0.010601723, -0.019456826, -0.0005947817, -0.009182595, + 0.003157721, 0.020895218, 0.0045463475, -0.022256555, -0.006036111, + 0.004379391, -0.0069543705, 0.018082649, -0.004783939, -0.020497093, + 0.0016181915, -0.050035506, -0.0018172547, -0.013523458, 0.0016872215, + 0.01449951, -0.009831156, 0.02035582, 0.0033327038, -0.00019735756, + 0.02205107, -0.0037725694, 0.0074102897, -0.020728262, -0.0037308303, + -0.012303392, -0.02518471, 0.023258291, -0.02481227, 0.04538642, + 0.0064213946, 0.014255497, 0.02853668, -0.00010665535, 0.016104858, + 0.01173831, 0.004915578, -0.00045190565, -0.019122913, 0.01288774, + -0.027278084, -0.008501926, 0.027534941, -0.023874747, 0.0032540418, + -0.031362094, 0.0115199825, -0.005833837, 0.013112488, 0.015783789, + 0.012720783, 0.0072625973, 0.02789454, 0.023052808, -0.0028880222, + -0.0057664127, -0.015629675, -0.00547745, -0.017658835, -0.010807208, + -0.011044799, 0.019443983, 0.015347133, 0.015038907, 0.021139232, + 0.0032700952, -0.0034258142, -0.0051949085, -0.021884114, + -0.0034129713, -0.0071470123, -0.0029650792, -0.0012634094, + 0.012027272, 0.004658722, 0.013587671, 0.032286774, 0.025017753, + 0.0057439376, 0.01279784, 0.0076029315, 0.0023213343, 0.0012834762, + 0.011083327, 0.0011735099, -0.010768679, 0.0066268793, 0.0047261463, + -0.0077056736, 0.004061532, 0.0025974542, -0.0019023382, + -0.0061805923, 0.001878258, -0.011275969, 0.04949611, -0.0046940395, + 0.0021014016, -0.007423132, -0.013960112, 0.013728942, 0.004581665, + -0.015526933, 0.028279822, -0.035112187, -0.008142329, -0.023810532, + -0.010280653, 0.031798746, 0.007236912, -0.016271815, 0.009028481, + -0.01072373, -0.019803582, 0.009978848, -0.00018702312, -0.036807436, + -0.016772684, -0.02066405, -0.007757045, -0.0012987271, -0.008816575, + 0.011828209, -0.015899373, -0.004485344, 0.009535771, -0.00024240765, + -0.003586349, -0.0023373878, -0.013080381, -0.0056604594, + -0.014294025, -0.010640251, -0.0023839427, 0.026353404, -0.011712624, + 0.0014335763, -0.0077827303, -0.033648107, -0.011719045, + -0.0059911613, -0.027046915, 0.0020548464, -0.012618041, -0.009567878, + 0.023861904, 0.012425399, 0.01666994, 0.02434993, 0.006344338, + -0.017157966, 0.005288019, -0.010967743, 0.011796102, 0.015539776, + 0.0049284203, -0.02016318, -0.00012391285, 0.015372819, -0.008822996, + 0.0064984513, -0.0065530334, 0.0005153169, 0.0048513636, 0.022808794, + 0.013960112, 0.025467252, 0.032158345, 0.014024327, -0.019212814, + -0.014460981, 0.0033969178, 0.016258972, 0.0043858127, 0.0030662161, + -0.003522135, 0.02408023, -0.006658986, 0.0034129713, 0.0012481586, + 0.026918488, -0.02794591, 0.022436354, 0.02237214, -0.012772154, + -0.019944852, -0.0022763845, 0.016734155, 0.015822317, 0.017055225, + 0.028870592, -0.0021190604, 0.01473068, -0.013780314, -0.014512353, + 0.0013284261, 0.013215231, 0.012528141, -0.034726903, -0.0023807322, + -0.011866737, 0.011886002, 0.028485307, -0.007757045, 0.020586992, + -0.03269774, 0.02173, 0.020753948, -0.001878258, 0.018416561, + 0.0023887588, 0.021832742, 0.045823075, 0.029692529, -0.01763315, + 0.014101383, -0.021370402, 0.0040904284, 0.007204805, -0.0015828739, + -0.028921962, -0.0025942435, 0.017979905, -0.02748357, -0.010858579, + -0.0037147768, 0.011430083, -0.011385133, 0.0054581854, -0.0244912, + -0.008668883, -0.008758782, 0.0032620686, 0.016284658, -0.0020692945, + -0.011051221, 0.018275289, 0.0015347133, -0.007879051, -0.021010803, + -0.0072754403, -0.042201407, -0.007885473, 0.004254174, -0.0069094207, + -0.0030967176, 0.019418297, -0.007795573, 0.034906704, 0.000084631974, + 0.009433029, -0.00446608, 0.02481227, 0.010210018, -0.0062865457, + -0.0019039436, -0.011173227, -0.0022812006, -0.011622725, 0.00704427, + -0.014525195, 0.0077827303, 0.007821259, 0.0076221954, 0.003907419, + -0.0242215, -0.007288283, -0.0145637235, -0.0011422056, 0.012040115, + 0.013099645, -0.024735212, -0.0011767206, -0.02269321, -0.004135378, + 0.014371081, 0.013870213, -0.013016167, -0.001703275, 0.02518471, + -0.0021624046, -0.0053811288, 0.0071662767, 0.0016230076, + 0.0006746478, -0.011038378, -0.013433558, 0.02375916, -0.014717837, + -0.009638513, -0.025428724, -0.00584668, 0.01588653, -0.033802222, + 0.0022908326, -0.009587143, -0.0016422718, 0.009683464, -0.004539926, + -0.002513976, -0.0036698272, -0.00855972, -0.019366927, -0.0048642065, + -0.005994372, -0.02223087, 0.016143387, -0.0007814035, -0.013433558, + -0.008521191, 0.006652565, -0.012059379, -0.010813629, -0.012354763, + 0.039324623, -0.022590468, 0.011597039, -0.007140591, -0.014717837, + 0.0066654077, 0.01003664, -0.0013252153, 0.013497772, 0.021049332, + 0.0014761181, -0.0072433334, -0.005088955, 0.019790739, -0.011918109, + 0.007814838, 0.0047807284, 0.041816123, -0.0072754403, -0.013041853, + 0.03560021, 0.025197553, -0.009336708, 0.0014464192, -0.0005991964, + -0.00024702304, -0.0116548315, 0.0018525724, -0.004437184, + 0.0035413993, 0.010145804, -0.007994637, -0.027278084, -0.0058402587, + -0.010563195, 0.00013474896, -0.00088454713, 0.016284658, + 0.0065209265, 0.008964268, -0.0103833955, 0.009240387, 0.020291608, + 0.012008009, -0.016400242, -0.0127914185, -0.024439828, 0.009176173, + -0.0039555794, 0.0022539096, -0.011468611, -0.0060168467, + 0.0060842717, -0.020728262, 0.0143068675, 0.011301654, -0.0064470805, + 0.009214701, -0.009099117, 0.039941076, 0.004722936, 0.0064759767, + 0.0021431404, -0.01449951, -0.0075451387, -0.0040390575, -0.015488404, + 0.0099916905, -0.0032941755, -0.00019805989, -0.009696306, + -0.00038347766, 0.011571353, 0.0046779863, -0.01649014, 0.005419657, + -0.025300296, 0.004010161, -0.013844527, 0.007692831, -0.0009856841, + -0.0058434694, -0.019559568, -0.018121175, 0.010486138, 0.0014600647, + 0.0013220047, 0.00095598516, -0.004957317, 0.006556244, -0.0012706335, + 0.016708469, 0.015398505, 0.023489462, 0.013279445, -0.0001430767, + 0.016130544, 0.0035349778, -0.022436354, -0.011346605, 0.026995543, + 0.005496714, 0.004411498, -0.009811891, 0.008758782, -0.008970688, + -0.026533203, -0.012085065, -0.010858579, -0.0001264212, -0.015732417, + -0.0062640705, -0.013253759, 0.0017787263, -0.00575357, -0.003917051, + 0.006081061, -0.007005742, 0.0060200575, 0.015950745, 0.00382073, + -0.028151395, 0.027149657, -0.008906474, 0.007679988, 0.0033583895, + 0.012226336, 0.004170696, -0.008842261, 0.0023951803, 0.0048353104, + -0.0008765204, -0.032903228, -0.029358618, 0.014576566, 0.0054581854, + 0.0143068675, 0.0026311665, 0.0047582537, 0.0013838105, 0.004437184, + 0.012695097, 0.01464078, 0.029332932, -0.016258972, 0.019187128, + 0.008534034, 0.01464078, 0.030488782, -0.007224069, 0.016926797, + 0.0056604594, -0.004828889, 0.0049637384, 0.008765204, -0.023515148, + 0.01440961, -0.03369948, -0.014062854, -0.018557832, -0.008142329, + -0.030411726, 0.012926267, -0.012457506, 0.0070506916, -0.005262333, + 0.025711264, -0.0008395974, 0.00066341035, -0.014345396, -0.013587671, + 0.012515298, -0.017183652, -0.010409081, 0.03863111, 0.019456826, + 0.03701292, -0.00023919696, -0.012727205, 0.015719574, -0.012733626, + -0.025287453, -0.0035895596, -0.000015940612, 0.006440659, + -0.0045367153, 0.024452671, 0.00043665484, 0.009831156, 0.009555035, + -0.017877163, 0.0111411195, 0.007814838, 0.023181235, 0.006684672, + -0.01141724, 0.010556773, -0.0057439376, -0.00029177213, -0.046259727, + -0.00013384596, 0.014242654, -0.004915578, 0.014139911, -0.0013693625, + 0.010254968, -0.027586313, 0.010935635, -0.000024381234, 0.0048674173, + 0.0008516375, -0.009818313, -0.020908061, 0.013703257, 0.0039106295, + 0.008578983, -0.02454257, 0.010338445, -0.0054838713, -0.0071277483, + 0.005593035, 0.015411347, -0.00013163859, -0.0010306338, -0.003364811, + -0.016310344, 0.030617211, 0.0008267546, 0.013960112, 0.0068516284, + 0.0017947798, -0.006748886, 0.0009961189, -0.008778047, -0.0033198611, + 0.006357181, 0.0009616039, 0.004254174, -0.011160384, -0.027046915, + -0.0011285602, -0.012136436, 0.009587143, 0.011308076, -0.02375916, + 0.017466193, -0.031875804, 0.0040294253, -0.0069672135, -0.010742993, + -0.015758103, -0.0012385265, -0.0070506916, -0.0070250058, + -0.003586349, -0.023219764, -0.00589484, 0.008309285, -0.001671168, + -0.0031705636, 0.022282239, -0.028896276, 0.017812949, -0.026507517, + 0.008450556, 0.017761579, 0.009831156, -0.017299237, -0.008482663, + -0.024439828, -0.009696306, 0.006270492, -0.018583516, -0.0127785755, + -0.00028595273, -0.0065369797, -0.0029843433, 0.0051371157, + 0.0061452747, -0.0055962456, -0.028690793, -0.013998641, 0.008052429, + 0.0162076, -0.0019039436, 0.0016270209, -0.008964268, -0.026815744, + 0.009644935, -0.030000757, -0.0063475487, -0.0029201293, 0.010287074, + -0.00014347804, 0.012444663, 0.0059718974, 0.007879051, 0.013484929, + -0.008277178, -0.0037789908, -0.021845585, -0.015706731, 0.013472087, + -0.010158647, 0.00031605302, -0.01542419, 0.025120497, 0.010595301, + 0.008970688, -0.0067809927, 0.006257649, 0.039735593, -0.0028478885, + -0.0040390575, 0.031130921, -0.032492258, -0.0009118381, 0.025248924, + -0.011539247, 0.022885852, 0.007667145, -0.01040266, 0.0024786585, + 0.004986213, 0.009002795, -0.02053562, -0.021896956, -0.0066268793, + 0.00014387938, 0.009233966, -0.015706731, 0.00072280824, -0.022577625, + -0.007159855, 0.004498187, 0.01772305, 0.03195286, -0.011937373, + 0.021858428, -0.012219914, -0.021922642, 0.004713304, -0.007140591, + 0.0076864096, 0.0014175229, 0.02357936, 0.013125331, 0.023283977, + -0.0003373239, 0.011198913, 0.030180555, 0.008514769, 0.00069351064, + -0.0034001286, 0.011153962, -0.019649468, -0.016554356, -0.020599835, + 0.021575887, -0.021485986, -0.00589484, 0.014062854, 0.00007720724, + -0.019161442, 0.002610297, -0.005326547, -0.0006112365, 0.00076133665, + -0.016708469, 0.0022169866, -0.008367077, -0.024786584, -0.011340183, + 0.010030218, 0.005763202, -0.013960112, -0.0067553073, 0.0026761163, + -0.013960112, -0.0027066178, 0.0055127675, -0.0052527008, 0.004860996, + 0.015154492, 0.0148077365, 0.00003150497, 0.002297254, 0.0067553073, + 0.011385133, 0.0038945759, 0.020407192, -0.02509481, 0.015732417, + 0.009433029, -0.006055375, 0.040994186, 0.0023100968, 0.00244013, + -0.020098966, -0.00855972, 0.01615623, 0.026404776, -0.0014857502, + -0.0016101648, 0.0021158496, 0.010871422, -0.012720783, -0.006819521, + -0.009933898, -0.023052808, -0.018108333, 0.0015917033, 0.0031785904, + 0.013844527, 0.019854954, 0.028254136, 0.023656419, 0.015963588, + -0.01882753, -0.012091487, 0.007891894, 0.011873159, 0.0073589184, + -0.019906323, -0.02016318, 0.004337652, 0.00034193927, 0.006052165, + 0.015668204, 0.002435314, 0.008399185, 0.010306339, 0.0010266205, + -0.015437033, 0.015205863, -0.0013525062, 0.038682483, 0.00063772476, + -0.013613357, 0.026263505, 0.015437033, -0.016130544, 0.007236912, + 0.010980585, 0.012945532, 0.00870099, -0.008187278, 0.0048963134, + 0.014846264, 0.00009371223, 0.013908741, 0.012425399, -0.005069691, + 0.006800257, 0.018673416, -0.0005787282, 0.026327718, -0.020612678, + -0.024889326, -0.0035991918, 0.00019655489, -0.020009067, 0.017247867, + -0.022012541, 0.023682104, 0.0069029992, 0.023027122, -0.012130015, + 0.00515638, -0.0032556471, -0.0038656797, 0.013266602, -0.0029056813, + -0.00764146, -0.012123593, 0.011532825, -0.011218177, -0.008232228, + -0.019880638, 0.023463776, -0.00070153736, 0.009927477, -0.013780314, + -0.0299237, 0.023258291, -0.0045784544, -0.0044628694, 0.005943001, + -0.011590618, -0.004915578, -0.010010955, -0.009677042, 0.0016302316, + -0.016952483, 0.008212964, 0.03190149, -0.001942472, -0.007352497, + -0.021832742, 0.0034611318, -0.01542419, 0.020060437, -0.00354461, + -0.017003853, 0.015000379, 0.0107622575, -0.0011036772, 0.0069543705, + 0.00025886248, -0.0019986592, -0.0008460188, -0.02881922, 0.009047746, + -0.005981529, -0.013960112, 0.0029538416, 0.0071020625, 0.022539096, + -0.0022346454, 0.011083327, 0.019302713, -0.02301428, 0.022539096, + -0.0038271514, -0.0019167863, -0.018814687, -0.023283977, 0.01675984, + 0.0004017385, 0.029204503, -0.003692302, 0.0013059512, 0.005647617, + -0.011622725, -0.01786432, 0.008585405, -0.019867796, 0.0038817332, + 0.01330513, -0.0017113017, 0.016644256, 0.018609202, 0.0017899638, + 0.0047614644, 0.019765053, 0.019803582, -0.020599835, -0.006469555, + 0.020766791, 0.020240236, -0.011333762, 0.0098696835, 0.0020741106, + -0.0051082196, -0.00418996, 0.016002117, 0.000589163, 0.009876105, + 0.0019216024, 0.0006646144, 0.0029265506, -0.037321147, 0.015013221, + -0.04397371, -0.012919846, -0.027201029, 0.006524137, 0.019598097, + 0.012990482, 0.013343658, 0.016965326, -0.015912216, -0.0012192624, + 0.024465514, -0.010184332, -0.003393707, 0.00034033394, -0.01542419, + -0.004138589, 0.020381507, 0.0002448157, -0.0015892952, -0.01473068, + 0.011391554, -0.026790058, 0.0024979226, -0.014435296, 0.015270077, + 0.008456977, 0.0067553073, 0.0148077365, -0.0043023345, -0.021164916, + -0.0035638742, 0.025492938, -0.014666466, 0.0034290247, -0.0013444795, + -0.0136262, 0.0101265395, -0.01196948, 0.005464607, -0.0047582537, + -0.0033230719, -0.023309663, -0.01606633, 0.04068596, 0.005853101, + 0.009612828, -0.018378032, 0.009375236, 0.0071983836, 0.0012738442, + 0.02067689, -0.027611997, -0.011532825, -0.0127914185, 0.021049332, + 0.0039523686, 0.028202767, -0.012046536, -0.003480396, -0.019443983, + 0.014204125, 0.007583667, 0.0020468195, 0.0034418677, -0.023733474, + 0.02187127, -0.011834631, -0.0043472843, -0.0034482891, 0.0020933747, + 0.016014958, -0.01178968, 0.009105538, -0.011430083, -0.0010563195, + -0.017003853, -0.0036955127, -0.0072947047, 0.025261767, 0.009047746, + -0.0000065593545, -0.004902735, 0.001979395, -0.033853594, + -0.004860996, -0.0010466874, 0.034958072, 0.02158873, -0.031233665, + 0.025467252, -0.019765053, -0.011686939, 0.031053865, -0.015437033, + -0.011545667, -0.0046490896, 0.021216288, 0.007468082, -0.008822996, + -0.0016109674, 0.018018434, 0.0021030067, 0.0075194533, -0.017825792, + 0.022179497, 0.009368815, 0.00319946, -0.016811213, 0.044769965, + 0.0038817332, 0.015603989, 0.007435975, 0.008322128, 0.0150645925, + 0.01155851, -0.0010121723, -0.0043440736, 0.01758178, -0.00056187203, + 0.005474239, 0.0032668847, -0.009330287, 0.0092468085, -0.0070378487, + 0.009657778, -0.0021656153, 0.01684974, -0.016323186, 0.01952104, + 0.005474239, -0.0023999964, -0.0065626656, 0.021742843, -0.0046940395, + -0.0005245477, -0.017055225, 0.0011967875, -0.0029570523, + 0.0003947151, 0.013754628, 0.00065377826, -0.0018028066, 0.01408854, + -0.0062897564, 0.013106067, 0.017196495, -0.004912367, 0.021254817, + 0.00010986605, -0.0061934353, -0.016978167, 0.0026985912, + -0.013420715, -0.045412105, -0.00053177174, 0.009407343, -0.016169071, + -0.00051772495, -0.008302864, 0.026083706, 0.0028494939, -0.00621591, + 0.014859107, -0.02269321, 0.02361789, 0.018275289, -0.000034916335, + 0.0053426004, 0.0116548315, -0.03310871, 0.023322506, -0.0008917712, + -0.012457506, 0.010794365, -0.004254174, -0.012457506, 0.008925739, + -0.029307246, -0.0070185843, 0.014435296, 0.003968422, 0.003711566, + 0.007063534, -0.010068747, 0.0070506916, 0.0111411195, -0.0052366476, + 0.013343658, -0.009426608, -0.014949007, 0.0093559725, -0.008675304, + -0.0064502913, 0.016785527, 0.023630733, 0.0028783903, -0.006832364, + -0.016130544, 0.00354461, 0.0021479565, -0.0043344414, 0.0011751152, + 0.006832364, 0.0075451387, -0.015488404, -0.015642518, -0.0053104935, + -0.0076350383, -0.014139911, -0.020497093, 0.011776838, 0.0062897564, + 0.011109013, -0.010531087, 0.009028481, 0.012181386, 0.030052127, + 0.009786205, 0.00759651, 0.008341392, -0.01708091, 0.000027265844, + 0.020098966, -0.005349022, 0.013093224, 0.003250831, -0.02794591, + 0.023425248, 0.0053104935, -0.030694267, 0.008174435, -0.00064575154, + -0.0052462798, 0.009144066, 0.026456147, 0.00040213985, 0.00068026653, + 0.0035638742, -0.012207071, -0.0095743, 0.016888268, -0.001979395, + -0.0030036075, -0.0012730415, 0.02509481, 0.006414973, -0.0009455504, + 0.026212133, -0.006716779, -0.016913954, -0.010646673, -0.015180177, + 0.018146861, 0.03177306, 0.001321202, -0.0024931065, -0.03038604, + 0.014011484, -0.012907004, -0.0016872215, 0.015501247, 0.030000757, + -0.03359674, -0.02002191, -0.006652565, 0.0081102215, 0.022218026, + 0.006620458, -0.017915692, 0.0053233365, 0.005159591, 0.0006810692, + 0.0137931565, -0.020111809, -0.0056283525, 0.022757424, -0.019983381, + -0.02545441, 0.005689356, 0.030976808, 0.0358057, 0.0145508805, + -0.0162076, 0.02219234, 0.016053487, 0.0069543705, 0.005644406, + -0.00009918045, -0.010710887, 0.022076756, -0.0016398637, + -0.000030702293, 0.004382602, 0.0028623368, 0.014846264, 0.014486667, + 0.0011221387, -0.020522777, -0.006992899, -0.003978054, -0.027098287, + -0.007435975, 0.04107124, 0.002536451, 0.012130015, 0.0067360434, + -0.016079172, 0.0061902246, -0.014627937, -0.012091487, -0.0048192567, + -0.0060425326, -0.004655511, 0.03020624, -0.011173227, -0.027637683, + 0.009131223, 0.016464457, -0.014152754, -0.01708091, -0.008810153, + 0.006980056, 0.0068580494, -0.0102613885, 0.0036248774, -0.009799048, + 0.000051346076, -0.017761579, 0.008553298, 0.021884114, 0.028562363, + -0.01864773, -0.00018270875, -0.004658722, 0.029101761, -0.0026568521, + -0.019803582, -0.013446401, 0.032672055, -0.013317972, -0.003024477, + 0.013228074, -0.0023518358, 0.023361035, -0.013690414, -0.0015186599, + -0.0068580494, 0.017658835, 0.0043119667, 0.018390875, 0.016040644, + -0.0050568483, -0.0041032713, -0.004084007, -0.0132922875, + 0.0014624727, 0.0065658763, 0.018442245, -0.0041995924, 0.004816046, + -0.010203596, -0.021293346, -0.007744202, 0.00621591, 0.0030726374, + 0.0022442774, -0.012412556, 0.013202388, 0.0033327038, -0.013131753, + 0.021100704, -0.002896049, -0.017222181, 0.023078494, -0.00488026, + -0.007821259, -0.011391554, -0.020098966, -0.0025252134, 0.0012425398, + -0.012643727, 0.026250662, -0.012393292, 0.005451764, 0.007269019, + 0.010742993, 0.009792627, -0.020227393, -0.013574828, 0.0045046085, + 0.00980547, -0.017196495, 0.0014424059, -0.0070121633, -0.013754628, + 0.0072818617, 0.0036505628, -0.00188789, 0.0050568483, 0.024504043, + 0.0033391253, 0.0043729697, 0.0014175229, 0.0021286923, 0.0066268793, + 0.011391554, 0.0023502305, 0.010768679, -0.01510312, -0.004774307, + 0.00529444, -0.00446608, -0.0021880902, 0.0033070184, -0.022641838, + -0.00024060164, -0.0072433334, 0.009966005, 0.0039844755, + -0.0071855406, 0.00036381214, 0.011950215, 0.026186448, 0.0044403947, + -0.0008893632, 0.021601573, 0.013407872, 0.010935635, 0.002385548, + -0.009272494, -0.0122777065, -0.009728413, -0.00042622007, 0.00704427, + 0.0065498226, -0.008887211, 0.016862582, -0.0067745717, 0.015989274, + 0.013151017, -0.015501247, -0.011076906, -0.012547405, -0.010113697, + 0.005763202, -0.015681047, 0.0036730377, -0.0021431404, 0.0002586618, + -0.0034771853, -0.0140500115, -0.006556244, -0.0041450104, + -0.005612299, -0.028279822, -0.019174285, -0.005538453, 0.008636776, + 0.0136262, -0.022320768, 0.009856841, 0.0020981908, 0.017620306, + 0.0076543028, 0.0067103575, -0.003933104, -0.0038496263, 0.00870099, + 0.035934124, -0.0137931565, 0.00043705618, 0.0036377201, + -0.0065755085, -0.01649014, -0.0010370553, -0.022115285, 0.0018429403, + 0.012136436, 0.00056387874, -0.005349022, 0.0011863527, -0.0030501625, + 0.0015170545, -0.012829946, 0.01408854, 0.0016310343, 0.022205183, + -0.017183652, 0.02016318, 0.003310229, 0.008135907, -0.004369759, + 0.01026781, 0.008270756, -0.008649618, 0.011635567, 0.031747375, + -0.013549143, 0.0191486, 0.021357559, 0.0074038683, 0.03665332, + -0.00976052, -0.016580041, -0.012894161, -0.013240916, -0.018596359, + -0.0014351817, 0.0082193855, -0.0053650755, 0.008546877, + -0.0015981246, 0.009856841, 0.005650827, 0.0070250058, -0.0028783903, + -0.0081937, 0.022474881, -0.006100325, -0.007525875, 0.007468082, + 0.010479717, -0.0060457434, 0.017389137, 0.015693888, 0.0093559725, + -0.001772305, 0.0072754403, 0.015796632, 0.012380449, -0.005666881, + 0.014037169, -0.0077506234, -0.023219764, 0.009041324, 0.013106067, + 0.04084007, 0.006418184, 0.009182595, 0.021563044, 0.009458714, + -0.010768679, 0.0021351138, 0.009362394, -0.0031224033, -0.021665785, + 0.015899373, 0.009009217, 0.0010587275, 0.0037436732, 0.014679309, + 0.0025123707, -0.005782466, 0.016721312, -0.01082005, -0.0031159818, + 0.011404397, 0.008675304, -0.009144066, -0.0038014657, -0.0032379883, + 0.0040679537, -0.004523873, -0.01961094, -0.0005590627, -0.014974693, + -0.0055641388, -0.026635945, -0.006703936, -0.0011759179, + -0.008412027, 0.030540153, -0.0023550466, -0.0067938357, + -0.0011052826, 0.0007472898, 0.0070956415, -0.024915012, -0.029076075, + 0.0044275518, 0.026289191, 0.004568822, 0.001528292, -0.026083706, + -0.00708922, -0.0025717686, 0.018544989, 0.008026743, -0.0024240764, + -0.0009038113, -0.011340183, -0.033365566, -0.00055665465, + 0.025839692, 0.02748357, -0.009933898, 0.008604669, -0.009471557, + 0.014627937, -0.009477979, 0.0064053414, 0.0010571221, -0.003618456, + 0.014846264, 0.022423511, -0.013818841, 0.0059044724, -0.014974693, + 0.019212814, 0.018211076, 0.0023775215, -0.018416561, -0.023271134, + 0.011905266, -0.021190602, 0.0028992598, -0.0032973862, 0.004084007, + -0.019508198, -0.001153443, 0.00041377862, 0.011731888, -0.011609882, + -0.009150487, 0.015115963, 0.008386342, 0.027509255, -0.011757574, + -0.006411763, 0.019919166, -0.005085745, 0.0041000606, 0.021062175, + 0.016284658, -0.009458714, 0.008142329, 0.0012634094, 0.024118759, + 0.002284411, -0.0056925663, -0.021563044, 0.023399564, -0.0043986556, + 0.008200121, 0.004568822, 0.015475562, -0.00055665465, -0.0018204654, + -0.0012682255, 0.020638363, -0.0022860165, -0.015693888, -0.018557832, + -0.013960112, 0.0028238085, 0.009426608, -0.015732417, -0.007493768, + -0.015372819, -0.02591675, -0.0047454108, -0.011564932, 0.002412839, + 0.017106596, -0.0061805923, -0.013382187, 0.0162975, -0.0093816575, + -0.021845585, -0.00038668836, -0.0027852799, -0.024324244, + -0.025236081, -0.036756065, -0.010698044, 0.00003102838, 0.0014014695, + -0.00071839354, -0.0140500115, 0.044769965, -0.018442245, + -0.023643576, 0.0358057, 0.009966005, -0.0071662767, -0.003949158, + 0.0013725731, -0.010415503, -0.0020548464, -0.025724107, -0.036910176, + -0.039838333, -0.011211756, -0.0043729697, 0.003945947, 0.02702123, + 0.0037083556, 0.012977639, 0.002859126, -0.012065801, -0.010190753, + 0.0020933747, -0.0097476775, 0.007757045, 0.021357559, -0.0061324323, + 0.0030373198, 0.010370553, 0.008347813, 0.031644635, -0.025608523, + -0.02108786, 0.011629146, 0.0081102215, -0.008168014, 0.0014424059, + 0.020432878, -0.0059879506, 0.0005377918, -0.015270077, 0.0143068675, + -0.007583667, -0.005589824, -0.01173831, -0.010004533, 0.022115285, + -0.003907419, -0.00093110226, -0.0013525062, -0.013690414, + 0.0017674889, 0.017273553, -0.016310344, -0.00515638, 0.021357559, + 0.02661026, 0.0034258142, 0.023232607, 0.026687317, 0.017761579, + 0.023836218, -0.009921055, 0.00069110264, -0.0065658763, -0.010685201, + 0.006440659, 0.001574847, 0.011975901, 0.00037906296, -0.003711566, + 0.0037372517, 0.013356501, 0.005888419, 0.016836897, -0.0022956487, + -0.006819521, 0.023258291, -0.002827019, -0.017684521, 0.011841052, + -0.018018434, 0.011173227, 0.009580721, 0.021614416, -0.004469291, + 0.01758178, -0.02062552, -0.026867116, -0.016400242, 0.0022812006, + 0.0026167184, -0.0036890914, 0.0059751077, 0.016592884, 0.010511823, + 0.0051114303, 0.018878901, 0.0015395294, -0.0076029315, -0.015128806, + 0.0024369194, 0.019212814, -0.025287453, -0.007988215, -0.0031384567, + -0.005082534, 0.016258972, 0.022616152, -0.0055448744, 0.010325603, + 0.020548463, 0.0024850797, 0.029641159, -0.00092387816, -0.005952633, + 0.022847323, 0.0047935713, -0.002550899, 0.010441188, -0.016220443, + -0.009137644, -0.015026064, -0.0075387177, -0.016284658, + 0.000010064784, 0.00032026708, 0.006559455, -0.022577625, + 0.0017498301, -0.0059590545, -0.023091335, -0.011545667, 0.012329078, + -0.0025541098, -0.013446401, 0.029692529, 0.025569994, 0.023797689, + 0.004138589, -0.008052429, 0.010088012, -0.003300597, -0.008071694, + -0.0041995924, -0.010100854, -0.0056411955, -0.01496185, 0.008534034, + -0.0018670205, -0.0155783035, 0.008347813, 0.0027001966, + -0.0087202545, 0.028716477, -0.002127087, -0.013677571, 0.0031641424, + -0.032851856, -0.0003864877, 0.001675984, -0.003329493, -0.004751832, + -0.015488404, 0.022140969, -0.012123593, -0.006501662, 0.0069543705, + 0.013459244, 0.010017375, -0.008437713, -0.010152225, -0.007936844, + 0.005377918, 0.024902169, 0.0060939034, 0.019187128, 0.0040711644, + 0.016181914, 0.0011076906, 0.005878787, -0.03899071, 0.007879051, + 0.01058888, 0.024889326, 0.0021977224, 0.00082434656, -0.004751832, + -0.012091487, 0.016567199, 0.019970538, 0.011012692, 0.0046908287, + 0.0014536432, -0.010332025, 0.010120118, -0.0358057, -0.008161593, + -0.00049725675, 0.016772684, -0.017055225, 0.013459244, 0.037732117, + -0.014371081, 0.0140500115, -0.009522929, -0.0024882904 + ] + } + ] + }, + { + "name": "untitled", + "title": "Untitled", + "content": "Staying up-to-date and discerning key technological trends amidst constant advancements can be overwhelming. Assistance in navigating these changes can be beneficial.", + "keywords": [ + "technological trends", + "advancements", + "up-to-date", + "navigating changes", + "beneficial" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + 0.026481831, 0.04921357, -0.002366284, -0.021460302, -0.027380828, + -0.0036216667, 0.012605198, -0.006658986, -0.06981341, 0.021896956, + -0.03893934, 0.025441566, -0.013189545, -0.0004924407, 0.0061902246, + 0.013844527, -0.015963588, 0.04998414, 0.0066975146, -0.0013460849, + 0.0354461, -0.0061163786, -0.060566597, 0.044179197, 0.012682254, + 0.00824507, -0.030694267, 0.0064470805, -0.026327718, 0.0017578568, + -0.003660195, -0.001222473, 0.045232303, 0.001574847, 0.009728413, + -0.019392611, 0.0015339106, -0.020548463, 0.029872328, 0.034881018, + 0.011256705, 0.0020708998, 0.018943114, 0.010145804, 0.028896276, + -0.02853668, 0.0003991298, -0.036550578, 0.021190602, 0.023232607, + 0.016002117, -0.014949007, -0.021293346, 0.041456524, 0.011834631, + -0.004276649, 0.011937373, 0.019135756, -0.007423132, -0.050857447, + 0.00016474891, -0.014332553, -0.024452671, 0.009426608, 0.012438241, + 0.0215502, 0.017299237, -0.002600665, 0.04433331, 0.020317294, + 0.029666845, 0.06642291, 0.0055320314, 0.029332932, 0.054299314, + -0.008938582, -0.03149052, 0.01201443, -0.025955278, 0.013651885, + 0.02559568, -0.028100023, -0.020188866, 0.0044949763, -0.0021094282, + 0.0000010096138, 0.011256705, -0.073666245, 0.0030164502, + 0.0036056133, 0.010672358, -0.013728942, -0.011584196, -0.020908061, + -0.044487424, -0.027791796, 0.045566216, -0.010299917, -0.005217383, + 0.024298558, 0.011956637, -0.002674511, -0.0404291, 0.01731208, + 0.0005526413, 0.016438771, -0.0006758518, -0.006209489, -0.016143387, + 0.013112488, 0.025955278, -0.003226751, -0.0071662767, -0.0046651433, + -0.027201029, 0.0028061494, -0.009728413, -0.017800106, 0.044127826, + 0.015115963, -0.0037211983, 0.014743523, -0.03226109, 0.006071429, + -0.021601573, 0.010254968, 0.020432878, -0.017453352, 0.01961094, + -0.032158345, 0.02591675, 0.011353026, -0.0068516284, 0.011333762, + 0.009336708, -0.04225278, -0.0065401904, 0.031798746, 0.028100023, + 0.048263203, -0.06390572, -0.007159855, 0.019790739, -0.00056187203, + -0.0013733758, -0.025467252, -0.016952483, 0.01606633, 0.012733626, + -0.026327718, -0.024195816, -0.013780314, -0.027509255, 0.00446608, + 0.018095491, 0.020047594, 0.0060842717, -0.024414143, 0.05676513, + -0.046901867, -0.018801844, -0.013587671, 0.028228452, -0.036807436, + -0.011166805, -0.0064246054, -0.020201707, -0.00026046782, + 0.0016093621, 0.0034482891, 0.042278465, -0.0054389215, -0.04572033, + -0.07489915, 0.0060329004, -0.025261767, -0.014204125, -0.014846264, + -0.005731095, 0.024375614, 0.008501926, -0.0031464836, 0.028022967, + -0.009779784, -0.020908061, -0.022128126, 0.006890157, 0.023964645, + 0.038554054, -0.021575887, 0.00079223956, 0.051448215, 0.013883055, + 0.0060040043, 0.02905039, 0.0014680914, -0.011725467, -0.033365566, + 0.015937902, 0.026032334, -0.054196574, 0.015385662, -0.02205107, + -0.014782051, 0.0024593943, 0.015937902, -0.009535771, -0.0095743, + 0.020869533, 0.030642897, -0.01035771, -0.017607465, -0.017787263, + 0.042612378, -0.014936164, 0.02817708, -0.017183652, -0.007352497, + -0.017389137, 0.0459515, 0.026404776, 0.037115663, 0.007513032, + -0.026404776, -0.018557832, -0.009304601, -0.06303241, 0.0059751077, + 0.004613772, -0.026918488, 0.032158345, 0.01790285, 0.0072625973, + 0.056919243, -0.0379376, 0.06133716, 0.024555413, 0.0073268116, + -0.01717081, 0.034315933, 0.027406514, 0.02899902, -0.00980547, + -0.02463247, -0.007904737, -0.028074337, 0.023733474, 0.0059141046, + 0.0075708246, 0.028716477, 0.00035177203, 0.0030084234, 0.051936243, + 0.00994674, -0.011930952, 0.009709149, -0.005750359, 0.006633301, + -0.008906474, 0.0073139686, 0.0019874217, 0.010370553, -0.017157966, + 0.017479036, 0.005731095, -0.035112187, 0.0005357851, 0.0032652793, + 0.0019537094, -0.029153133, -0.001703275, -0.004225278, 0.007288283, + 0.017466193, 0.028613735, 0.010903528, 0.042612378, 0.02301428, + -0.03125935, -0.04623404, -0.021113547, 0.009054166, 0.0051242732, + -0.025518622, 0.0045559797, 0.01243182, -0.007943265, 0.027098287, + 0.0022137759, -0.046799127, -0.011423661, 0.0044436054, 0.030899752, + 0.025711264, 0.030129185, -0.0028848117, 0.0030662161, -0.0033487573, + -0.046901867, 0.019508198, 0.018146861, 0.02661026, -0.007866208, + -0.015231549, 0.0122648645, -0.015128806, 0.005050427, -0.00395879, + -0.026160762, -0.05404246, -0.009221123, -0.053374633, 0.0028334404, + 0.025698422, -0.011995166, -0.00033009984, 0.017222181, -0.014139911, + 0.038502682, -0.017440509, -0.020009067, 0.0029875538, 0.018429402, + 0.019906323, -0.012348342, 0.023399564, -0.000100685465, -0.012727205, + 0.01592506, 0.01201443, -0.03457279, 0.004626615, -0.017812949, + -0.05106293, 0.011051221, -0.012759311, 0.018763315, -0.008296442, + 0.0010033429, 0.019970538, -0.004976581, 0.0057664127, 0.009908212, + 0.037192717, 0.026507517, -0.02149883, -0.009818313, -0.0053971824, + -0.013163859, 0.01740198, 0.000666621, 0.003351968, 0.05501851, + -0.0039620004, 0.017363451, -0.006283335, -0.02499207, -0.02283448, + 0.01606633, -0.039581478, -0.024439828, 0.0067809927, -0.009458714, + -0.0022812006, -0.019482512, 0.019495355, 0.00073284167, 0.009400922, + 0.030617211, 0.0062191207, 0.013240916, 0.014011484, -0.0122648645, + 0.011956637, 0.013497772, 0.006960792, -0.017363451, -0.008604669, + -0.012386871, -0.00061725656, 0.032440886, 0.025531465, -0.019944852, + 0.00086769095, -0.0011991955, -0.0030260824, -0.0017899638, + -0.017838635, -0.017337766, 0.011089749, -0.008084536, -0.013382187, + -0.00469725, 0.0030517678, -0.0038464156, 0.017517565, -0.044281937, + -0.0033583895, 0.028896276, 0.0041642743, 0.023232607, -0.03056584, + 0.002181669, 0.031336408, -0.020753948, -0.0052976506, -0.0019970539, + -0.057124726, 0.012662991, -0.014011484, -0.024786584, -0.0030180556, + 0.003393707, -0.012322657, -0.02899902, -0.05008688, -0.016541513, + 0.03714135, 0.015488404, 0.034059078, -0.01449951, -0.037911914, + 0.0063314955, 0.0039555794, -0.015411347, 0.0030276875, 0.012740048, + 0.025698422, -0.0381174, -0.021357559, -0.031464834, -0.009477979, + 0.0027917014, 0.010306339, -0.015000379, 0.050985876, -0.03688449, + 0.0045591905, 0.03978696, 0.013356501, -0.02946136, 0.005962265, + 0.032338142, -0.018095491, -0.013574828, 0.022487724, 0.02684143, + 0.01873763, 0.024337087, 0.056919243, -0.014165597, 0.03434162, + 0.016567199, 0.011622725, 0.007416711, 0.0011245467, -0.042920604, + 0.012496035, -0.007988215, 0.031798746, 0.023399564, -0.023181235, + 0.014345396, -0.0352663, -0.013972955, 0.02513334, 0.0461313, + 0.00971557, 0.000971236, -0.017877163, -0.0072561763, -0.025582837, + -0.039581478, 0.02541588, -0.045925815, -0.0031063498, 0.004482134, + 0.012996903, 0.022962907, -0.015848001, -0.040814385, 0.022718895, + -0.044924077, 0.015950745, -0.0050632698, 0.026147919, -0.025929593, + -0.027663369, 0.0106274085, -0.015167334, 0.052295838, 0.0053522326, + 0.008765204, -0.020227393, 0.010807208, 0.0070378487, -0.03724409, + -0.055223994, 0.02370779, 0.033005968, -0.0033969178, 0.03084838, + 0.037629373, -0.0029153132, 0.003204276, -0.0039427364, -0.02030445, + -0.03521493, 0.013472087, 0.016323186, 0.061491277, 0.0035125029, + -0.028279822, -0.020009067, 0.016913954, -0.015475562, -0.012386871, + -0.010210018, 0.029409988, -0.009895369, -0.009580721, 0.011622725, + -0.0025525044, 0.006543401, -0.019790739, 0.012579513, 0.0033038077, + -0.0009977242, 0.029872328, -0.021241974, -0.033262827, 0.0057696234, + 0.0056090886, 0.028022967, 0.06405983, -0.02513334, -0.02126766, + -0.019238498, -0.0017867531, 0.042715117, -0.03351968, -0.057535697, + 0.012412556, 0.03190149, 0.01850646, 0.006154907, -0.0047807284, + -0.012207071, 0.0028703634, -0.03167032, 0.053888347, -0.014949007, + 0.005888419, 0.0030581893, -0.014653623, -0.007121327, -0.014692151, + -0.02007328, -0.037038606, -0.013330815, 0.02140893, 0.011243862, + 0.021845585, -0.004456448, 0.0062865457, 0.015475562, 0.016002117, + -0.011282391, 0.005410025, 0.0017562514, 0.011218177, 0.012451084, + 0.026327718, 0.0064663445, 0.019906323, 0.011719045, -0.018660573, + -0.015398505, 0.012367606, 0.011423661, -0.008360656, 0.019649468, + 0.031387776, 0.025390195, 0.0005686948, 0.0066782506, -0.00814875, + -0.030077813, -0.0034033393, 0.044769965, 0.036627635, 0.023450933, + -0.0029265506, 0.015115963, 0.0010563195, 0.009060588, -0.027534941, + 0.015681047, -0.028665107, -0.029435674, 0.0068387855, -0.0041450104, + 0.015732417, -0.020458564, -0.024388457, 0.008463399, 0.023643576, + 0.0153086055, 0.018904585, 0.010511823, 0.0024465513, 0.020831006, + 0.0017129071, -0.008026743, -0.032286774, -0.006315442, 0.03403339, + 0.029076075, 0.010800786, -0.022359297, -0.014294025, -0.024414143, + -0.0074552395, -0.009766942, 0.025377352, -0.021421773, 0.012451084, + -0.016323186, -0.0031175872, 0.0018814687, -0.001079597, 0.009914634, + -0.015526933, 0.006459923, -0.01473068, -0.019687997, 0.0061805923, + -0.008418448, -0.02080532, -0.012784997, 0.0026488253, 0.058306266, + 0.016926797, 0.03624235, -0.004639458, 0.00016755828, -0.021100704, + 0.024902169, 0.007191962, 0.004899524, 0.055994563, -0.028048653, + 0.009477979, -0.0011646805, -0.00091825944, 0.01800559, 0.02776611, + -0.030257612, 0.028048653, -0.003241199, -0.016836897, 0.014897636, + 0.0004274241, 0.006877314, 0.012412556, 0.020908061, -0.03737252, + 0.038091715, 0.017928535, -0.0034771853, -0.014332553, 0.017915692, + -0.004116114, 0.011449347, -0.010132961, 0.0067938357, 0.0032797274, + -0.0069351066, -0.010486138, 0.0020548464, 0.010486138, -0.0164773, + -0.02946136, -0.005731095, 0.013818841, -0.021190602, 0.044025082, + 0.03673038, -0.005506346, -0.01758178, 0.019366927, 0.033493996, + -0.0018959168, -0.007827681, 0.031053865, 0.004337652, -0.0011686939, + 0.031233665, 0.01150714, -0.014846264, 0.0020773213, 0.023373878, + 0.01740198, -0.039966762, 0.010736573, 0.009766942, -0.0018333083, + -0.024619628, -0.05224447, -0.008643198, 0.006703936, -0.004626615, + 0.030694267, -0.011076906, 0.053888347, -0.035112187, 0.040583216, + -0.035137873, 0.008495505, 0.039966762, 0.02375916, 0.008707412, + -0.053785603, -0.012117172, -0.03236383, 0.008309285, -0.029230189, + -0.025788322, 0.01473068, 0.0060650073, -0.038399942, 0.009599986, + 0.026250662, -0.009266073, 0.026738688, -0.001855783, -0.013818841, + -0.02499207, 0.022962907, -0.007172698, 0.0067424644, -0.032081287, + 0.04374254, 0.006652565, 0.0006770558, 0.017055225, -0.0025187922, + -0.003660195, -0.059179574, 0.0028510992, -0.038656797, -0.019816425, + 0.038785223, 0.016271815, 0.0037500947, 0.00022956486, 0.012142858, + 0.0070378487, 0.01588653, 0.0358057, 0.023656419, 0.010858579, + 0.00202595, 0.0025492937, 0.0077827303, 0.0022137759, 0.004816046, + -0.010113697, -0.0027130393, 0.0065947725, 0.03323714, -0.0023261502, + -0.005262333, -0.0018188601, 0.022397825, -0.00060922984, 0.010967743, + -0.011988744, 0.019662311, -0.03398202, 0.034649845, -0.006071429, + 0.002319729, -0.0023405985, -0.007577246, -0.019392611, 0.009978848, + -0.002969895, 0.0044725016, -0.013818841, -0.008444134, -0.004597719, + -0.017684521, 0.010537509, -0.0093945, -0.016541513, 0.0115199825, + -0.012194228, 0.044410367, 0.006601194, 0.012309814, 0.019598097, + -0.010017375, 0.010909949, -0.030334668, 0.0030533732, 0.007179119, + 0.013985798, -0.016708469, 0.018211076, 0.040994186, -0.022269398, + 0.020009067, -0.0081102215, -0.036422152, -0.027534941, 0.016567199, + -0.0024096284, 0.00019906324, -0.02003475, 0.034778275, -0.021421773, + 0.0432802, -0.010794365, 0.020908061, 0.02306565, 0.023720633, + -0.026481831, 0.00043344416, 0.018955957, -0.010730151, -0.0024609996, + -0.013703257, -0.009734835, -0.00971557, 0.011564932, 0.0070185843, + 0.0067681503, 0.02928156, -0.009323865, -0.0076157744, -0.015462719, + 0.027971596, -0.0072947047, 0.010210018, 0.04199592, 0.0072112265, + -0.054813027, 0.015038907, -0.04148221, -0.012984061, 0.017890006, + -0.01510312, 0.019726524, -0.025467252, 0.02666163, 0.024683842, + 0.0083542345, 0.0038110977, 0.008450556, 0.007943265, -0.012573091, + 0.03102818, 0.0044500264, 0.004931631, 0.004074375, 0.0023839427, + 0.012001587, -0.018724788, -0.0052655437, 0.015090277, 0.0019248131, + -0.013960112, -0.024645314, -0.007493768, 0.014692151, -0.009291758, + -0.01937977, -0.012252022, 0.013446401, -0.025390195, -0.004116114, + -0.010556773, -0.033262827, 0.024285715, 0.03084838, -0.019970538, + -0.012194228, 0.012149279, -0.021794215, 0.0012915031, 0.004543137, + -0.0024064176, -0.006090693, 0.012457506, 0.025801165, -0.018917428, + -0.020086123, -0.015629675, 0.027689055, 0.020381507, -0.008668883, + 0.0035574527, -0.017235024, -0.0040165824, 0.00022976553, 0.023656419, + 0.0075066104, -0.0037372517, -0.0025974542, -0.00833497, 0.012374028, + 0.0052591222, -0.03369948, -0.00865604, 0.001333242, -0.02840825, + -0.0136262, -0.00015060177, 0.031002494, 0.03231246, 0.020060437, + -0.02817708, 0.0046651433, -0.00045351102, -0.0055866134, 0.014396767, + -0.023502305, -0.01597643, -0.0005907683, 0.015603989, 0.0076992526, + 0.0073717614, 0.006068218, 0.01864773, -0.021370402, -0.038887966, + 0.0018943115, -0.01072373, 0.028973334, -0.008540455, 0.016567199, + -0.023001436, -0.01836519, -0.0067103575, 0.044949763, -0.01868626, + 0.025081968, 0.038476996, -0.0025043439, 0.010909949, 0.013202388, + -0.010800786, 0.011410818, 0.047030296, 0.008277178, -0.025030596, + 0.03218403, -0.01946967, 0.019135756, 0.004957317, 0.0035478207, + -0.030899752, -0.008636776, 0.013651885, -0.008822996, -0.018390875, + -0.06477903, 0.000050116978, -0.0009913028, 0.024157288, + -0.0140500115, -0.015655361, -0.015205863, 0.00051050086, 0.03601118, + -0.007724938, 0.0053522326, 0.024773741, 0.02559568, 0.015642518, + 0.021627259, 0.007564403, 0.042021606, 0.02858805, -0.017568937, + -0.022115285, -0.016053487, -0.004209224, 0.0015001984, 0.022256555, + -0.025608523, -0.0017000643, -0.0069543705, 0.052655436, -0.010614566, + 0.0023405985, 0.0005803336, -0.0041321674, 0.014704994, 0.035291985, + -0.003554242, -0.018609202, -0.00071518286, -0.01634887, 0.02905039, + -0.044179197, 0.0028832064, -0.014627937, -0.0056861453, 0.021370402, + 0.0010282259, 0.032517944, -0.025801165, -0.008360656, -0.025069125, + -0.0075001894, -0.016657097, -0.018519303, -0.00478715, 0.00285431, + 0.018095491, 0.0045367153, -0.012733626, -0.0042284885, 0.010832893, + 0.014627937, 0.010351288, 0.02324545, 0.02393896, 0.021807056, + 0.00014438105, -0.0044949763, 0.014062854, -0.034290247, -0.007930422, + -0.008135907, 0.05589182, 0.0023887588, 0.017697364, 0.012746469, + -0.035985496, 0.021896956, 0.0022009332, 0.010473295, -0.014011484, + -0.008084536, 0.003798255, 0.018172547, 0.01772305, 0.00055344397, + 0.0025075546, -0.00028073534, 0.0032973862, -0.033622425, + -0.008501926, 0.02858805, 0.02545441, 0.0018509671, -0.017568937, + 0.005644406, -0.012528141, 0.006684672, -0.0027290927, 0.02066405, + -0.00060481514, -0.023990331, 0.017607465, 0.0037372517, 0.009580721, + 0.0019681575, -0.009709149, 0.035189245, -0.020766791, 0.019816425, + 0.016104858, -0.01505175, -0.01431971, -0.0036762485, -0.0329546, + -0.0017369873, 0.018211076, -0.018532146, 0.031875804, 0.015770946, + -0.03816877, 0.012560248, 0.006251228, -0.013003324, 0.015501247, + -0.016284658, -0.014576566, 0.0028575207, -0.009182595, 0.05332326, + -0.024837956, 0.0155783035, 0.049316313, -0.013549143, -0.014704994, + -0.016913954, -0.0056989877, 0.0055577173, -0.053374633, 0.0032973862, + 0.004032636, -0.03940168, -0.0002867554, -0.034906704, -0.012964796, + -0.008938582, 0.032517944, -0.021473145, 0.014846264, 0.013330815, + 0.021241974, -0.017979905, -0.0019392612, -0.0050632698, -0.020959433, + -0.011012692, -0.014692151, -0.029384302, 0.0020291607, 0.023104178, + -0.023669261, -0.012849211, -0.0074295537, 0.018339504, -0.0155783035, + -0.0077313595, 0.019777896, -0.016246129, -0.03770643, 0.0075322962, + 0.0030854803, 0.01615623, -0.024593942, 0.028742163, -0.021768529, + 0.015038907, 0.0031962493, -0.013138174, -0.020291608, 0.02158873, + -0.01026781, 0.034007706, -0.0040390575, 0.0032829382, -0.013998641, + -0.028433936, -0.005721463, 0.00814875, -0.0011510351, -0.016888268, + -0.008784468, -0.0067360434, -0.01054393, -0.03411045, 0.022295082, + -0.021935485, -0.001362941, -0.008168014, 0.0105182445, -0.0005365878, + -0.0017626729, 0.021396087, 0.00015551815, 0.0028302297, 0.03246657, + 0.0010458847, 0.0007224069, -0.018172547, -0.029332932, 0.008437713, + -0.026687317, 0.00598474, -0.0076157744, -0.00059116964, 0.011744731, + -0.008945003, -0.01164841, -0.020895218, 0.013151017, -0.013048274, + -0.026353404, -0.008296442, -0.019199971, -0.031978548, -0.0028607314, + 0.010132961, -0.01675984, 0.037834857, 0.023605047, 0.0048642065, + -0.01449951, -0.000033310986, -0.0012296971, 0.012829946, 0.026481831, + -0.017915692, 0.031233665, -0.011372291, -0.009728413, 0.014615094, + -0.017967062, 0.0014416032, -0.011038378, -0.002725882, -0.0219098, + 0.013163859, 0.012271285, 0.014473824, -0.02518471, 0.012842789, + -0.008893631, -0.038143083, 0.014127068, 0.03246657, -0.0035799276, + 0.0042702276, 0.0012642121, -0.0350865, 0.024131602, 0.0062769135, + 0.016554356, -0.019816425, 0.010710887, -0.007269019, 0.005853101, + -0.010100854, 0.013844527, -0.012817104, -0.0036955127, -0.0057888874, + -0.031362094, -0.025403038, 0.029589787, 0.0043665483, 0.02292438, + 0.020792477, -0.006228753, 0.028254136, -0.0032235403, -0.0017385926, + 0.020086123, -0.035780013, -0.032029916, 0.015398505, -0.037860543, + 0.0049348418, 0.0086110905, 0.011385133, -0.019122913, -0.02541588, + -0.011622725, -0.029101761, 0.011808945, -0.020753948, 0.000802273, + 0.019559568, -0.004616983, -0.005695777, 0.029076075, 0.021485986, + 0.008168014, -0.013189545, -0.0007500992, 0.011320919, -0.0008291626, + -0.0012401319, 0.025325982, -0.015270077, -0.015180177, 0.018198233, + 0.019546725, -0.019803582, -0.007737781, 0.03310871, 0.0123997135, + -0.0016583252, 0.008078115, -0.0016647466, -0.0054710284, + -0.002546083, -0.00216401, 0.025980964, -0.008283599, 0.024439828, + 0.00644387, 0.03195286, 0.011494297, 0.012862054, 0.003083875, + -0.02545441, -0.00893216, 0.023797689, -0.04723578, -0.005422868, + -0.001841335, 0.025801165, -0.02587822, -0.0097476775, -0.019996224, + -0.008033165, -0.010601723, -0.019456826, -0.0005947817, -0.009182595, + 0.003157721, 0.020895218, 0.0045463475, -0.022256555, -0.006036111, + 0.004379391, -0.0069543705, 0.018082649, -0.004783939, -0.020497093, + 0.0016181915, -0.050035506, -0.0018172547, -0.013523458, 0.0016872215, + 0.01449951, -0.009831156, 0.02035582, 0.0033327038, -0.00019735756, + 0.02205107, -0.0037725694, 0.0074102897, -0.020728262, -0.0037308303, + -0.012303392, -0.02518471, 0.023258291, -0.02481227, 0.04538642, + 0.0064213946, 0.014255497, 0.02853668, -0.00010665535, 0.016104858, + 0.01173831, 0.004915578, -0.00045190565, -0.019122913, 0.01288774, + -0.027278084, -0.008501926, 0.027534941, -0.023874747, 0.0032540418, + -0.031362094, 0.0115199825, -0.005833837, 0.013112488, 0.015783789, + 0.012720783, 0.0072625973, 0.02789454, 0.023052808, -0.0028880222, + -0.0057664127, -0.015629675, -0.00547745, -0.017658835, -0.010807208, + -0.011044799, 0.019443983, 0.015347133, 0.015038907, 0.021139232, + 0.0032700952, -0.0034258142, -0.0051949085, -0.021884114, + -0.0034129713, -0.0071470123, -0.0029650792, -0.0012634094, + 0.012027272, 0.004658722, 0.013587671, 0.032286774, 0.025017753, + 0.0057439376, 0.01279784, 0.0076029315, 0.0023213343, 0.0012834762, + 0.011083327, 0.0011735099, -0.010768679, 0.0066268793, 0.0047261463, + -0.0077056736, 0.004061532, 0.0025974542, -0.0019023382, + -0.0061805923, 0.001878258, -0.011275969, 0.04949611, -0.0046940395, + 0.0021014016, -0.007423132, -0.013960112, 0.013728942, 0.004581665, + -0.015526933, 0.028279822, -0.035112187, -0.008142329, -0.023810532, + -0.010280653, 0.031798746, 0.007236912, -0.016271815, 0.009028481, + -0.01072373, -0.019803582, 0.009978848, -0.00018702312, -0.036807436, + -0.016772684, -0.02066405, -0.007757045, -0.0012987271, -0.008816575, + 0.011828209, -0.015899373, -0.004485344, 0.009535771, -0.00024240765, + -0.003586349, -0.0023373878, -0.013080381, -0.0056604594, + -0.014294025, -0.010640251, -0.0023839427, 0.026353404, -0.011712624, + 0.0014335763, -0.0077827303, -0.033648107, -0.011719045, + -0.0059911613, -0.027046915, 0.0020548464, -0.012618041, -0.009567878, + 0.023861904, 0.012425399, 0.01666994, 0.02434993, 0.006344338, + -0.017157966, 0.005288019, -0.010967743, 0.011796102, 0.015539776, + 0.0049284203, -0.02016318, -0.00012391285, 0.015372819, -0.008822996, + 0.0064984513, -0.0065530334, 0.0005153169, 0.0048513636, 0.022808794, + 0.013960112, 0.025467252, 0.032158345, 0.014024327, -0.019212814, + -0.014460981, 0.0033969178, 0.016258972, 0.0043858127, 0.0030662161, + -0.003522135, 0.02408023, -0.006658986, 0.0034129713, 0.0012481586, + 0.026918488, -0.02794591, 0.022436354, 0.02237214, -0.012772154, + -0.019944852, -0.0022763845, 0.016734155, 0.015822317, 0.017055225, + 0.028870592, -0.0021190604, 0.01473068, -0.013780314, -0.014512353, + 0.0013284261, 0.013215231, 0.012528141, -0.034726903, -0.0023807322, + -0.011866737, 0.011886002, 0.028485307, -0.007757045, 0.020586992, + -0.03269774, 0.02173, 0.020753948, -0.001878258, 0.018416561, + 0.0023887588, 0.021832742, 0.045823075, 0.029692529, -0.01763315, + 0.014101383, -0.021370402, 0.0040904284, 0.007204805, -0.0015828739, + -0.028921962, -0.0025942435, 0.017979905, -0.02748357, -0.010858579, + -0.0037147768, 0.011430083, -0.011385133, 0.0054581854, -0.0244912, + -0.008668883, -0.008758782, 0.0032620686, 0.016284658, -0.0020692945, + -0.011051221, 0.018275289, 0.0015347133, -0.007879051, -0.021010803, + -0.0072754403, -0.042201407, -0.007885473, 0.004254174, -0.0069094207, + -0.0030967176, 0.019418297, -0.007795573, 0.034906704, 0.000084631974, + 0.009433029, -0.00446608, 0.02481227, 0.010210018, -0.0062865457, + -0.0019039436, -0.011173227, -0.0022812006, -0.011622725, 0.00704427, + -0.014525195, 0.0077827303, 0.007821259, 0.0076221954, 0.003907419, + -0.0242215, -0.007288283, -0.0145637235, -0.0011422056, 0.012040115, + 0.013099645, -0.024735212, -0.0011767206, -0.02269321, -0.004135378, + 0.014371081, 0.013870213, -0.013016167, -0.001703275, 0.02518471, + -0.0021624046, -0.0053811288, 0.0071662767, 0.0016230076, + 0.0006746478, -0.011038378, -0.013433558, 0.02375916, -0.014717837, + -0.009638513, -0.025428724, -0.00584668, 0.01588653, -0.033802222, + 0.0022908326, -0.009587143, -0.0016422718, 0.009683464, -0.004539926, + -0.002513976, -0.0036698272, -0.00855972, -0.019366927, -0.0048642065, + -0.005994372, -0.02223087, 0.016143387, -0.0007814035, -0.013433558, + -0.008521191, 0.006652565, -0.012059379, -0.010813629, -0.012354763, + 0.039324623, -0.022590468, 0.011597039, -0.007140591, -0.014717837, + 0.0066654077, 0.01003664, -0.0013252153, 0.013497772, 0.021049332, + 0.0014761181, -0.0072433334, -0.005088955, 0.019790739, -0.011918109, + 0.007814838, 0.0047807284, 0.041816123, -0.0072754403, -0.013041853, + 0.03560021, 0.025197553, -0.009336708, 0.0014464192, -0.0005991964, + -0.00024702304, -0.0116548315, 0.0018525724, -0.004437184, + 0.0035413993, 0.010145804, -0.007994637, -0.027278084, -0.0058402587, + -0.010563195, 0.00013474896, -0.00088454713, 0.016284658, + 0.0065209265, 0.008964268, -0.0103833955, 0.009240387, 0.020291608, + 0.012008009, -0.016400242, -0.0127914185, -0.024439828, 0.009176173, + -0.0039555794, 0.0022539096, -0.011468611, -0.0060168467, + 0.0060842717, -0.020728262, 0.0143068675, 0.011301654, -0.0064470805, + 0.009214701, -0.009099117, 0.039941076, 0.004722936, 0.0064759767, + 0.0021431404, -0.01449951, -0.0075451387, -0.0040390575, -0.015488404, + 0.0099916905, -0.0032941755, -0.00019805989, -0.009696306, + -0.00038347766, 0.011571353, 0.0046779863, -0.01649014, 0.005419657, + -0.025300296, 0.004010161, -0.013844527, 0.007692831, -0.0009856841, + -0.0058434694, -0.019559568, -0.018121175, 0.010486138, 0.0014600647, + 0.0013220047, 0.00095598516, -0.004957317, 0.006556244, -0.0012706335, + 0.016708469, 0.015398505, 0.023489462, 0.013279445, -0.0001430767, + 0.016130544, 0.0035349778, -0.022436354, -0.011346605, 0.026995543, + 0.005496714, 0.004411498, -0.009811891, 0.008758782, -0.008970688, + -0.026533203, -0.012085065, -0.010858579, -0.0001264212, -0.015732417, + -0.0062640705, -0.013253759, 0.0017787263, -0.00575357, -0.003917051, + 0.006081061, -0.007005742, 0.0060200575, 0.015950745, 0.00382073, + -0.028151395, 0.027149657, -0.008906474, 0.007679988, 0.0033583895, + 0.012226336, 0.004170696, -0.008842261, 0.0023951803, 0.0048353104, + -0.0008765204, -0.032903228, -0.029358618, 0.014576566, 0.0054581854, + 0.0143068675, 0.0026311665, 0.0047582537, 0.0013838105, 0.004437184, + 0.012695097, 0.01464078, 0.029332932, -0.016258972, 0.019187128, + 0.008534034, 0.01464078, 0.030488782, -0.007224069, 0.016926797, + 0.0056604594, -0.004828889, 0.0049637384, 0.008765204, -0.023515148, + 0.01440961, -0.03369948, -0.014062854, -0.018557832, -0.008142329, + -0.030411726, 0.012926267, -0.012457506, 0.0070506916, -0.005262333, + 0.025711264, -0.0008395974, 0.00066341035, -0.014345396, -0.013587671, + 0.012515298, -0.017183652, -0.010409081, 0.03863111, 0.019456826, + 0.03701292, -0.00023919696, -0.012727205, 0.015719574, -0.012733626, + -0.025287453, -0.0035895596, -0.000015940612, 0.006440659, + -0.0045367153, 0.024452671, 0.00043665484, 0.009831156, 0.009555035, + -0.017877163, 0.0111411195, 0.007814838, 0.023181235, 0.006684672, + -0.01141724, 0.010556773, -0.0057439376, -0.00029177213, -0.046259727, + -0.00013384596, 0.014242654, -0.004915578, 0.014139911, -0.0013693625, + 0.010254968, -0.027586313, 0.010935635, -0.000024381234, 0.0048674173, + 0.0008516375, -0.009818313, -0.020908061, 0.013703257, 0.0039106295, + 0.008578983, -0.02454257, 0.010338445, -0.0054838713, -0.0071277483, + 0.005593035, 0.015411347, -0.00013163859, -0.0010306338, -0.003364811, + -0.016310344, 0.030617211, 0.0008267546, 0.013960112, 0.0068516284, + 0.0017947798, -0.006748886, 0.0009961189, -0.008778047, -0.0033198611, + 0.006357181, 0.0009616039, 0.004254174, -0.011160384, -0.027046915, + -0.0011285602, -0.012136436, 0.009587143, 0.011308076, -0.02375916, + 0.017466193, -0.031875804, 0.0040294253, -0.0069672135, -0.010742993, + -0.015758103, -0.0012385265, -0.0070506916, -0.0070250058, + -0.003586349, -0.023219764, -0.00589484, 0.008309285, -0.001671168, + -0.0031705636, 0.022282239, -0.028896276, 0.017812949, -0.026507517, + 0.008450556, 0.017761579, 0.009831156, -0.017299237, -0.008482663, + -0.024439828, -0.009696306, 0.006270492, -0.018583516, -0.0127785755, + -0.00028595273, -0.0065369797, -0.0029843433, 0.0051371157, + 0.0061452747, -0.0055962456, -0.028690793, -0.013998641, 0.008052429, + 0.0162076, -0.0019039436, 0.0016270209, -0.008964268, -0.026815744, + 0.009644935, -0.030000757, -0.0063475487, -0.0029201293, 0.010287074, + -0.00014347804, 0.012444663, 0.0059718974, 0.007879051, 0.013484929, + -0.008277178, -0.0037789908, -0.021845585, -0.015706731, 0.013472087, + -0.010158647, 0.00031605302, -0.01542419, 0.025120497, 0.010595301, + 0.008970688, -0.0067809927, 0.006257649, 0.039735593, -0.0028478885, + -0.0040390575, 0.031130921, -0.032492258, -0.0009118381, 0.025248924, + -0.011539247, 0.022885852, 0.007667145, -0.01040266, 0.0024786585, + 0.004986213, 0.009002795, -0.02053562, -0.021896956, -0.0066268793, + 0.00014387938, 0.009233966, -0.015706731, 0.00072280824, -0.022577625, + -0.007159855, 0.004498187, 0.01772305, 0.03195286, -0.011937373, + 0.021858428, -0.012219914, -0.021922642, 0.004713304, -0.007140591, + 0.0076864096, 0.0014175229, 0.02357936, 0.013125331, 0.023283977, + -0.0003373239, 0.011198913, 0.030180555, 0.008514769, 0.00069351064, + -0.0034001286, 0.011153962, -0.019649468, -0.016554356, -0.020599835, + 0.021575887, -0.021485986, -0.00589484, 0.014062854, 0.00007720724, + -0.019161442, 0.002610297, -0.005326547, -0.0006112365, 0.00076133665, + -0.016708469, 0.0022169866, -0.008367077, -0.024786584, -0.011340183, + 0.010030218, 0.005763202, -0.013960112, -0.0067553073, 0.0026761163, + -0.013960112, -0.0027066178, 0.0055127675, -0.0052527008, 0.004860996, + 0.015154492, 0.0148077365, 0.00003150497, 0.002297254, 0.0067553073, + 0.011385133, 0.0038945759, 0.020407192, -0.02509481, 0.015732417, + 0.009433029, -0.006055375, 0.040994186, 0.0023100968, 0.00244013, + -0.020098966, -0.00855972, 0.01615623, 0.026404776, -0.0014857502, + -0.0016101648, 0.0021158496, 0.010871422, -0.012720783, -0.006819521, + -0.009933898, -0.023052808, -0.018108333, 0.0015917033, 0.0031785904, + 0.013844527, 0.019854954, 0.028254136, 0.023656419, 0.015963588, + -0.01882753, -0.012091487, 0.007891894, 0.011873159, 0.0073589184, + -0.019906323, -0.02016318, 0.004337652, 0.00034193927, 0.006052165, + 0.015668204, 0.002435314, 0.008399185, 0.010306339, 0.0010266205, + -0.015437033, 0.015205863, -0.0013525062, 0.038682483, 0.00063772476, + -0.013613357, 0.026263505, 0.015437033, -0.016130544, 0.007236912, + 0.010980585, 0.012945532, 0.00870099, -0.008187278, 0.0048963134, + 0.014846264, 0.00009371223, 0.013908741, 0.012425399, -0.005069691, + 0.006800257, 0.018673416, -0.0005787282, 0.026327718, -0.020612678, + -0.024889326, -0.0035991918, 0.00019655489, -0.020009067, 0.017247867, + -0.022012541, 0.023682104, 0.0069029992, 0.023027122, -0.012130015, + 0.00515638, -0.0032556471, -0.0038656797, 0.013266602, -0.0029056813, + -0.00764146, -0.012123593, 0.011532825, -0.011218177, -0.008232228, + -0.019880638, 0.023463776, -0.00070153736, 0.009927477, -0.013780314, + -0.0299237, 0.023258291, -0.0045784544, -0.0044628694, 0.005943001, + -0.011590618, -0.004915578, -0.010010955, -0.009677042, 0.0016302316, + -0.016952483, 0.008212964, 0.03190149, -0.001942472, -0.007352497, + -0.021832742, 0.0034611318, -0.01542419, 0.020060437, -0.00354461, + -0.017003853, 0.015000379, 0.0107622575, -0.0011036772, 0.0069543705, + 0.00025886248, -0.0019986592, -0.0008460188, -0.02881922, 0.009047746, + -0.005981529, -0.013960112, 0.0029538416, 0.0071020625, 0.022539096, + -0.0022346454, 0.011083327, 0.019302713, -0.02301428, 0.022539096, + -0.0038271514, -0.0019167863, -0.018814687, -0.023283977, 0.01675984, + 0.0004017385, 0.029204503, -0.003692302, 0.0013059512, 0.005647617, + -0.011622725, -0.01786432, 0.008585405, -0.019867796, 0.0038817332, + 0.01330513, -0.0017113017, 0.016644256, 0.018609202, 0.0017899638, + 0.0047614644, 0.019765053, 0.019803582, -0.020599835, -0.006469555, + 0.020766791, 0.020240236, -0.011333762, 0.0098696835, 0.0020741106, + -0.0051082196, -0.00418996, 0.016002117, 0.000589163, 0.009876105, + 0.0019216024, 0.0006646144, 0.0029265506, -0.037321147, 0.015013221, + -0.04397371, -0.012919846, -0.027201029, 0.006524137, 0.019598097, + 0.012990482, 0.013343658, 0.016965326, -0.015912216, -0.0012192624, + 0.024465514, -0.010184332, -0.003393707, 0.00034033394, -0.01542419, + -0.004138589, 0.020381507, 0.0002448157, -0.0015892952, -0.01473068, + 0.011391554, -0.026790058, 0.0024979226, -0.014435296, 0.015270077, + 0.008456977, 0.0067553073, 0.0148077365, -0.0043023345, -0.021164916, + -0.0035638742, 0.025492938, -0.014666466, 0.0034290247, -0.0013444795, + -0.0136262, 0.0101265395, -0.01196948, 0.005464607, -0.0047582537, + -0.0033230719, -0.023309663, -0.01606633, 0.04068596, 0.005853101, + 0.009612828, -0.018378032, 0.009375236, 0.0071983836, 0.0012738442, + 0.02067689, -0.027611997, -0.011532825, -0.0127914185, 0.021049332, + 0.0039523686, 0.028202767, -0.012046536, -0.003480396, -0.019443983, + 0.014204125, 0.007583667, 0.0020468195, 0.0034418677, -0.023733474, + 0.02187127, -0.011834631, -0.0043472843, -0.0034482891, 0.0020933747, + 0.016014958, -0.01178968, 0.009105538, -0.011430083, -0.0010563195, + -0.017003853, -0.0036955127, -0.0072947047, 0.025261767, 0.009047746, + -0.0000065593545, -0.004902735, 0.001979395, -0.033853594, + -0.004860996, -0.0010466874, 0.034958072, 0.02158873, -0.031233665, + 0.025467252, -0.019765053, -0.011686939, 0.031053865, -0.015437033, + -0.011545667, -0.0046490896, 0.021216288, 0.007468082, -0.008822996, + -0.0016109674, 0.018018434, 0.0021030067, 0.0075194533, -0.017825792, + 0.022179497, 0.009368815, 0.00319946, -0.016811213, 0.044769965, + 0.0038817332, 0.015603989, 0.007435975, 0.008322128, 0.0150645925, + 0.01155851, -0.0010121723, -0.0043440736, 0.01758178, -0.00056187203, + 0.005474239, 0.0032668847, -0.009330287, 0.0092468085, -0.0070378487, + 0.009657778, -0.0021656153, 0.01684974, -0.016323186, 0.01952104, + 0.005474239, -0.0023999964, -0.0065626656, 0.021742843, -0.0046940395, + -0.0005245477, -0.017055225, 0.0011967875, -0.0029570523, + 0.0003947151, 0.013754628, 0.00065377826, -0.0018028066, 0.01408854, + -0.0062897564, 0.013106067, 0.017196495, -0.004912367, 0.021254817, + 0.00010986605, -0.0061934353, -0.016978167, 0.0026985912, + -0.013420715, -0.045412105, -0.00053177174, 0.009407343, -0.016169071, + -0.00051772495, -0.008302864, 0.026083706, 0.0028494939, -0.00621591, + 0.014859107, -0.02269321, 0.02361789, 0.018275289, -0.000034916335, + 0.0053426004, 0.0116548315, -0.03310871, 0.023322506, -0.0008917712, + -0.012457506, 0.010794365, -0.004254174, -0.012457506, 0.008925739, + -0.029307246, -0.0070185843, 0.014435296, 0.003968422, 0.003711566, + 0.007063534, -0.010068747, 0.0070506916, 0.0111411195, -0.0052366476, + 0.013343658, -0.009426608, -0.014949007, 0.0093559725, -0.008675304, + -0.0064502913, 0.016785527, 0.023630733, 0.0028783903, -0.006832364, + -0.016130544, 0.00354461, 0.0021479565, -0.0043344414, 0.0011751152, + 0.006832364, 0.0075451387, -0.015488404, -0.015642518, -0.0053104935, + -0.0076350383, -0.014139911, -0.020497093, 0.011776838, 0.0062897564, + 0.011109013, -0.010531087, 0.009028481, 0.012181386, 0.030052127, + 0.009786205, 0.00759651, 0.008341392, -0.01708091, 0.000027265844, + 0.020098966, -0.005349022, 0.013093224, 0.003250831, -0.02794591, + 0.023425248, 0.0053104935, -0.030694267, 0.008174435, -0.00064575154, + -0.0052462798, 0.009144066, 0.026456147, 0.00040213985, 0.00068026653, + 0.0035638742, -0.012207071, -0.0095743, 0.016888268, -0.001979395, + -0.0030036075, -0.0012730415, 0.02509481, 0.006414973, -0.0009455504, + 0.026212133, -0.006716779, -0.016913954, -0.010646673, -0.015180177, + 0.018146861, 0.03177306, 0.001321202, -0.0024931065, -0.03038604, + 0.014011484, -0.012907004, -0.0016872215, 0.015501247, 0.030000757, + -0.03359674, -0.02002191, -0.006652565, 0.0081102215, 0.022218026, + 0.006620458, -0.017915692, 0.0053233365, 0.005159591, 0.0006810692, + 0.0137931565, -0.020111809, -0.0056283525, 0.022757424, -0.019983381, + -0.02545441, 0.005689356, 0.030976808, 0.0358057, 0.0145508805, + -0.0162076, 0.02219234, 0.016053487, 0.0069543705, 0.005644406, + -0.00009918045, -0.010710887, 0.022076756, -0.0016398637, + -0.000030702293, 0.004382602, 0.0028623368, 0.014846264, 0.014486667, + 0.0011221387, -0.020522777, -0.006992899, -0.003978054, -0.027098287, + -0.007435975, 0.04107124, 0.002536451, 0.012130015, 0.0067360434, + -0.016079172, 0.0061902246, -0.014627937, -0.012091487, -0.0048192567, + -0.0060425326, -0.004655511, 0.03020624, -0.011173227, -0.027637683, + 0.009131223, 0.016464457, -0.014152754, -0.01708091, -0.008810153, + 0.006980056, 0.0068580494, -0.0102613885, 0.0036248774, -0.009799048, + 0.000051346076, -0.017761579, 0.008553298, 0.021884114, 0.028562363, + -0.01864773, -0.00018270875, -0.004658722, 0.029101761, -0.0026568521, + -0.019803582, -0.013446401, 0.032672055, -0.013317972, -0.003024477, + 0.013228074, -0.0023518358, 0.023361035, -0.013690414, -0.0015186599, + -0.0068580494, 0.017658835, 0.0043119667, 0.018390875, 0.016040644, + -0.0050568483, -0.0041032713, -0.004084007, -0.0132922875, + 0.0014624727, 0.0065658763, 0.018442245, -0.0041995924, 0.004816046, + -0.010203596, -0.021293346, -0.007744202, 0.00621591, 0.0030726374, + 0.0022442774, -0.012412556, 0.013202388, 0.0033327038, -0.013131753, + 0.021100704, -0.002896049, -0.017222181, 0.023078494, -0.00488026, + -0.007821259, -0.011391554, -0.020098966, -0.0025252134, 0.0012425398, + -0.012643727, 0.026250662, -0.012393292, 0.005451764, 0.007269019, + 0.010742993, 0.009792627, -0.020227393, -0.013574828, 0.0045046085, + 0.00980547, -0.017196495, 0.0014424059, -0.0070121633, -0.013754628, + 0.0072818617, 0.0036505628, -0.00188789, 0.0050568483, 0.024504043, + 0.0033391253, 0.0043729697, 0.0014175229, 0.0021286923, 0.0066268793, + 0.011391554, 0.0023502305, 0.010768679, -0.01510312, -0.004774307, + 0.00529444, -0.00446608, -0.0021880902, 0.0033070184, -0.022641838, + -0.00024060164, -0.0072433334, 0.009966005, 0.0039844755, + -0.0071855406, 0.00036381214, 0.011950215, 0.026186448, 0.0044403947, + -0.0008893632, 0.021601573, 0.013407872, 0.010935635, 0.002385548, + -0.009272494, -0.0122777065, -0.009728413, -0.00042622007, 0.00704427, + 0.0065498226, -0.008887211, 0.016862582, -0.0067745717, 0.015989274, + 0.013151017, -0.015501247, -0.011076906, -0.012547405, -0.010113697, + 0.005763202, -0.015681047, 0.0036730377, -0.0021431404, 0.0002586618, + -0.0034771853, -0.0140500115, -0.006556244, -0.0041450104, + -0.005612299, -0.028279822, -0.019174285, -0.005538453, 0.008636776, + 0.0136262, -0.022320768, 0.009856841, 0.0020981908, 0.017620306, + 0.0076543028, 0.0067103575, -0.003933104, -0.0038496263, 0.00870099, + 0.035934124, -0.0137931565, 0.00043705618, 0.0036377201, + -0.0065755085, -0.01649014, -0.0010370553, -0.022115285, 0.0018429403, + 0.012136436, 0.00056387874, -0.005349022, 0.0011863527, -0.0030501625, + 0.0015170545, -0.012829946, 0.01408854, 0.0016310343, 0.022205183, + -0.017183652, 0.02016318, 0.003310229, 0.008135907, -0.004369759, + 0.01026781, 0.008270756, -0.008649618, 0.011635567, 0.031747375, + -0.013549143, 0.0191486, 0.021357559, 0.0074038683, 0.03665332, + -0.00976052, -0.016580041, -0.012894161, -0.013240916, -0.018596359, + -0.0014351817, 0.0082193855, -0.0053650755, 0.008546877, + -0.0015981246, 0.009856841, 0.005650827, 0.0070250058, -0.0028783903, + -0.0081937, 0.022474881, -0.006100325, -0.007525875, 0.007468082, + 0.010479717, -0.0060457434, 0.017389137, 0.015693888, 0.0093559725, + -0.001772305, 0.0072754403, 0.015796632, 0.012380449, -0.005666881, + 0.014037169, -0.0077506234, -0.023219764, 0.009041324, 0.013106067, + 0.04084007, 0.006418184, 0.009182595, 0.021563044, 0.009458714, + -0.010768679, 0.0021351138, 0.009362394, -0.0031224033, -0.021665785, + 0.015899373, 0.009009217, 0.0010587275, 0.0037436732, 0.014679309, + 0.0025123707, -0.005782466, 0.016721312, -0.01082005, -0.0031159818, + 0.011404397, 0.008675304, -0.009144066, -0.0038014657, -0.0032379883, + 0.0040679537, -0.004523873, -0.01961094, -0.0005590627, -0.014974693, + -0.0055641388, -0.026635945, -0.006703936, -0.0011759179, + -0.008412027, 0.030540153, -0.0023550466, -0.0067938357, + -0.0011052826, 0.0007472898, 0.0070956415, -0.024915012, -0.029076075, + 0.0044275518, 0.026289191, 0.004568822, 0.001528292, -0.026083706, + -0.00708922, -0.0025717686, 0.018544989, 0.008026743, -0.0024240764, + -0.0009038113, -0.011340183, -0.033365566, -0.00055665465, + 0.025839692, 0.02748357, -0.009933898, 0.008604669, -0.009471557, + 0.014627937, -0.009477979, 0.0064053414, 0.0010571221, -0.003618456, + 0.014846264, 0.022423511, -0.013818841, 0.0059044724, -0.014974693, + 0.019212814, 0.018211076, 0.0023775215, -0.018416561, -0.023271134, + 0.011905266, -0.021190602, 0.0028992598, -0.0032973862, 0.004084007, + -0.019508198, -0.001153443, 0.00041377862, 0.011731888, -0.011609882, + -0.009150487, 0.015115963, 0.008386342, 0.027509255, -0.011757574, + -0.006411763, 0.019919166, -0.005085745, 0.0041000606, 0.021062175, + 0.016284658, -0.009458714, 0.008142329, 0.0012634094, 0.024118759, + 0.002284411, -0.0056925663, -0.021563044, 0.023399564, -0.0043986556, + 0.008200121, 0.004568822, 0.015475562, -0.00055665465, -0.0018204654, + -0.0012682255, 0.020638363, -0.0022860165, -0.015693888, -0.018557832, + -0.013960112, 0.0028238085, 0.009426608, -0.015732417, -0.007493768, + -0.015372819, -0.02591675, -0.0047454108, -0.011564932, 0.002412839, + 0.017106596, -0.0061805923, -0.013382187, 0.0162975, -0.0093816575, + -0.021845585, -0.00038668836, -0.0027852799, -0.024324244, + -0.025236081, -0.036756065, -0.010698044, 0.00003102838, 0.0014014695, + -0.00071839354, -0.0140500115, 0.044769965, -0.018442245, + -0.023643576, 0.0358057, 0.009966005, -0.0071662767, -0.003949158, + 0.0013725731, -0.010415503, -0.0020548464, -0.025724107, -0.036910176, + -0.039838333, -0.011211756, -0.0043729697, 0.003945947, 0.02702123, + 0.0037083556, 0.012977639, 0.002859126, -0.012065801, -0.010190753, + 0.0020933747, -0.0097476775, 0.007757045, 0.021357559, -0.0061324323, + 0.0030373198, 0.010370553, 0.008347813, 0.031644635, -0.025608523, + -0.02108786, 0.011629146, 0.0081102215, -0.008168014, 0.0014424059, + 0.020432878, -0.0059879506, 0.0005377918, -0.015270077, 0.0143068675, + -0.007583667, -0.005589824, -0.01173831, -0.010004533, 0.022115285, + -0.003907419, -0.00093110226, -0.0013525062, -0.013690414, + 0.0017674889, 0.017273553, -0.016310344, -0.00515638, 0.021357559, + 0.02661026, 0.0034258142, 0.023232607, 0.026687317, 0.017761579, + 0.023836218, -0.009921055, 0.00069110264, -0.0065658763, -0.010685201, + 0.006440659, 0.001574847, 0.011975901, 0.00037906296, -0.003711566, + 0.0037372517, 0.013356501, 0.005888419, 0.016836897, -0.0022956487, + -0.006819521, 0.023258291, -0.002827019, -0.017684521, 0.011841052, + -0.018018434, 0.011173227, 0.009580721, 0.021614416, -0.004469291, + 0.01758178, -0.02062552, -0.026867116, -0.016400242, 0.0022812006, + 0.0026167184, -0.0036890914, 0.0059751077, 0.016592884, 0.010511823, + 0.0051114303, 0.018878901, 0.0015395294, -0.0076029315, -0.015128806, + 0.0024369194, 0.019212814, -0.025287453, -0.007988215, -0.0031384567, + -0.005082534, 0.016258972, 0.022616152, -0.0055448744, 0.010325603, + 0.020548463, 0.0024850797, 0.029641159, -0.00092387816, -0.005952633, + 0.022847323, 0.0047935713, -0.002550899, 0.010441188, -0.016220443, + -0.009137644, -0.015026064, -0.0075387177, -0.016284658, + 0.000010064784, 0.00032026708, 0.006559455, -0.022577625, + 0.0017498301, -0.0059590545, -0.023091335, -0.011545667, 0.012329078, + -0.0025541098, -0.013446401, 0.029692529, 0.025569994, 0.023797689, + 0.004138589, -0.008052429, 0.010088012, -0.003300597, -0.008071694, + -0.0041995924, -0.010100854, -0.0056411955, -0.01496185, 0.008534034, + -0.0018670205, -0.0155783035, 0.008347813, 0.0027001966, + -0.0087202545, 0.028716477, -0.002127087, -0.013677571, 0.0031641424, + -0.032851856, -0.0003864877, 0.001675984, -0.003329493, -0.004751832, + -0.015488404, 0.022140969, -0.012123593, -0.006501662, 0.0069543705, + 0.013459244, 0.010017375, -0.008437713, -0.010152225, -0.007936844, + 0.005377918, 0.024902169, 0.0060939034, 0.019187128, 0.0040711644, + 0.016181914, 0.0011076906, 0.005878787, -0.03899071, 0.007879051, + 0.01058888, 0.024889326, 0.0021977224, 0.00082434656, -0.004751832, + -0.012091487, 0.016567199, 0.019970538, 0.011012692, 0.0046908287, + 0.0014536432, -0.010332025, 0.010120118, -0.0358057, -0.008161593, + -0.00049725675, 0.016772684, -0.017055225, 0.013459244, 0.037732117, + -0.014371081, 0.0140500115, -0.009522929, -0.0024882904 + ] + } + ] + } ] diff --git a/src/scrapers/website/examples/www.pavolhejny.com.knowledge.json b/src/scrapers/website/examples/www.pavolhejny.com.knowledge.json index b53f02f687..13e5a976d8 100644 --- a/src/scrapers/website/examples/www.pavolhejny.com.knowledge.json +++ b/src/scrapers/website/examples/www.pavolhejny.com.knowledge.json @@ -1,83 +1,1949 @@ [ - { - "name": "generative-ai-transforming-our-world", - "title": "Generative AI: Transforming Our World", - "content": "---\n\nGenerative AI is emerging as a transformative technology, comparable to the rise of personal computers and the early internet. This wave of innovation is expected to impact virtually every sector of human activity, creating both opportunities and challenges for businesses.", - "keywords": [ - "Generative AI", - "transformative technology", - "personal computers", - "internet", - "innovation", - "impact", - "sectors", - "human activity", - "opportunities", - "challenges", - "businesses" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - -0.03798621, 0.017438458, -0.02202907, -0.025974356, -0.0050086067, 0.0149598215, -0.004062618, 0.009606552, 0.020547755, 0.007388245, 0.039306197, -0.019594433, 0.04622878, 0.0037289555, -0.020709086, 0.011021868, -0.011109867, -0.006632921, -0.0137645025, -0.020958416, -0.012239187, 0.009562552, -0.053943355, -0.0048399423, 0.02157441, 0.006445923, -0.0026399684, -0.0050892727, -0.036343567, 0.025901023, 0.007985905, 0.010515874, 0.023290388, -0.03165029, -0.0062259254, 0.0333516, 0.0040039523, 0.046492778, 0.010963202, 0.007076582, 0.053855356, -0.0057419315, 0.008462566, 0.014827823, 0.027030343, -0.019169105, -0.00011463925, -0.022645064, -0.031679623, 0.008565231, -0.027177008, 0.0050929394, 0.031180961, 0.044175472, -0.020885084, -0.004469613, 0.037751548, -0.027998332, 0.03607957, 0.015605147, -0.0066622538, 0.047842097, -0.06547122, -0.00020395589, -0.025695693, -0.03452492, -0.0019744765, 0.029156985, 0.0129138455, 0.023862382, -0.026649015, -0.032794274, 0.0030304638, 0.005202938, 0.0166758, -0.0012622349, 0.040068854, 0.017717121, 0.0027114677, -0.00250797, 0.015194485, 0.039658193, -0.008249901, 0.008521232, 0.012217187, -0.0010587374, 0.008513899, -0.062068593, 0.004513613, -0.008601897, -0.023715716, 0.024522373, -0.02629702, -0.0023521385, -0.00394162, 0.0065669217, 0.0113738645, 0.019066438, -0.01647047, 0.005653932, 0.019799763, -0.031826288, 0.026986344, -0.011087867, -0.05092206, -0.01927177, -0.0117185265, -0.0018543946, -0.0036904558, 0.062303256, 0.0010413209, -0.02474237, -0.009210557, 0.014952488, -0.03038897, 0.011652527, -0.052447375, -0.0307703, 0.02257173, -0.031796955, -0.037898213, -0.01341984, 0.013214509, -0.031386293, -0.0059325956, 0.04080218, -0.0016068975, -0.03323427, 0.0045062797, 0.03185562, -0.044322137, 0.07685242, 0.017115796, 0.07544443, 0.018289115, -0.016353138, 0.012415185, -0.019169105, 0.008389233, 0.0016637301, -0.025857024, -0.023715716, -0.021838406, -0.035756905, -0.015663812, -0.019579766, -0.019799763, 0.066351205, -0.0015143153, -0.03625557, -0.04505546, 0.018714443, 0.014343829, -0.012143855, -0.0025593028, -0.00018390405, 0.010589207, -0.016338471, 0.0019286437, 0.020797085, 0.018201116, 0.003391626, 0.00614526, 0.048839416, 0.009562552, 0.005397269, 0.030007642, -0.011285865, -0.034260925, 0.04344215, 0.008191235, -0.003189962, -0.012224521, -0.026311686, -0.0005692432, 0.008506565, 0.0079125725, 0.013537171, 0.010779872, 0.003048797, -0.0051589385, -0.005844597, 0.012341852, 0.02040109, 0.029948976, 0.046140783, 0.08301234, -0.02336372, 0.023275722, -0.0015903977, 0.007388245, -0.018171784, 0.03863154, 0.0021449744, 0.042532824, -0.017746454, -0.029670313, -0.0036116235, -0.03461292, -0.0005637433, -0.0008116986, -0.029802311, 0.016543802, 0.028775657, 0.03449559, 0.0072122472, -0.0126645155, -0.011212532, 0.021237079, -0.0094525535, 0.012004524, -0.018905107, 0.00013027969, -0.033498265, 0.009628552, -0.014996488, 0.019755764, 0.006376257, -0.029758312, -0.009078559, -0.018201116, 0.022894394, 0.017497124, -0.03795688, -0.0135445045, -0.08459632, -0.0044549466, -0.02119308, -0.023055725, 0.018611778, -0.018846441, 0.028892988, 0.014079832, -0.006240592, 0.012363852, -0.017071797, 0.047372766, -0.027719669, -0.056289993, 0.020914417, -0.00944522, -0.009063892, -0.056407325, -0.0142998295, 0.045202125, -0.0030359637, 0.015957143, 0.04062618, 0.023481052, 0.02846766, -0.0063432576, 0.013456506, -0.012847846, -0.025783692, -0.038572874, 0.0037766215, 0.031972952, 0.0418875, 0.020958416, 0.0135445045, -0.006046261, 0.013111844, -0.02679568, 0.029538313, -0.061599266, -0.0077952403, 0.023891715, 0.03255961, 0.006086594, 0.026106354, -0.02065042, -0.015898477, 0.0073735784, -0.011212532, -0.025739692, 0.011652527, 0.030506302, 0.015487815, 0.054618016, -0.008022571, 0.0074359113, -0.015018487, 0.0729218, -0.018699776, -0.021515744, -0.0195651, -0.03226628, -0.004421947, -0.064063236, -0.010222545, -0.008719229, 0.04690344, 0.00072140805, 0.03402626, -0.00948922, -0.01285518, 0.016323805, -0.024434375, 0.015003821, -0.029288983, 0.0020936416, -0.021691741, -0.00034580837, 0.017277127, 0.0040002856, 0.02896632, 0.016661134, 0.040098187, 0.016661134, 0.0142998295, -0.013515172, -0.006170926, 0.005162605, 0.021867739, -0.0010706539, 0.05596733, 0.020753086, -0.029142318, -0.011564529, -0.002084475, 0.0012640682, -0.029567646, -0.0530927, 0.032706276, -0.0041066175, 0.0444688, 0.018245116, 0.0126645155, 0.010647872, -0.015693147, -0.0072562466, -0.038954202, 0.0032944607, -0.015487815, 0.010823871, -0.0056025996, 0.0043119485, 0.0077365744, -0.007083915, -0.02248373, 0.018039785, -0.010479208, -0.024405042, -0.009577218, -0.015942477, -0.03173829, -0.027367674, 0.0010248211, -0.0034539588, 0.017893119, 0.0008602814, -0.005551267, -0.015253151, -0.04109551, 0.008557898, 0.00036574563, 0.0363729, -0.0333516, 0.027485006, 0.00011166013, 0.0038462875, -0.027793001, 0.008873227, 0.015693147, -0.023085058, 0.006735586, -0.020665087, -0.012180521, -0.013793835, -0.0032761276, 0.016323805, 0.03992219, -0.0071352483, 0.024111712, -0.00415795, -0.007985905, -0.020709086, -0.009848549, 0.007487244, 0.04743143, -0.011109867, 0.026971677, -0.01504782, 0.0015289817, -0.0057419315, -0.018699776, 0.03135696, -0.0504234, 0.019477101, 0.04886875, -0.015429149, -0.012451851, -0.039452862, -0.03666623, 0.029582312, -0.0004358698, 0.031180961, -0.027602337, 0.036548898, -0.022102403, -0.0025061367, -0.016089141, 0.0151798185, 0.033204935, -0.019741097, 0.022263734, -0.04288482, -0.008528565, 0.0068822512, 0.0068492517, -0.0009533219, -0.017907785, -0.029494314, -0.016015809, 0.005085606, 0.0025189698, 0.00069849164, 0.0035492908, 0.019961094, -0.029831644, 0.007487244, 0.011065868, 0.0166318, 0.005415602, -0.013119177, -0.007897906, 0.045788787, 0.020034427, -0.012202521, -0.01927177, 0.008814561, -0.020122426, 0.035844903, 0.014717824, -0.029098319, 0.0057345983, -0.0144244945, 0.011989857, 0.0051589385, 0.01204119, -0.026825013, 0.021633076, -0.00066045043, -0.0054302686, -0.062537916, 0.0139405, -0.017027797, -0.02010776, -0.019110437, -0.035991568, -0.02144241, 0.049220745, -0.06265525, -0.022513064, 0.008990559, 0.0016545636, 0.030623633, -0.012627849, 0.0017003963, -0.01475449, 0.0149598215, 0.0036134569, 0.010317877, -0.023011725, 0.050804727, -0.010193212, -0.03695956, -0.035874236, 0.011549862, 0.0032852942, 0.016661134, -0.013933167, -0.0045172796, 0.02833566, 0.01036921, -0.015898477, -0.013016511, -0.025901023, 0.0028232995, 0.016763799, 0.039863523, 0.02069442, 0.01423383, -0.019389102, -0.035639573, 0.019477101, 0.022263734, 0.0025336363, 0.010479208, 0.00788324, -0.0364609, 0.0072012474, 0.038455542, -0.0033641264, -0.014329162, -0.040743515, -0.007039916, -0.0027957999, 0.009005226, 0.027441006, -0.039687525, -0.033116937, 0.0028947988, 0.005653932, -0.00091390574, 0.011945858, 0.021339744, 0.0066145877, 0.007028916, 0.0059655956, -0.047284767, 0.019711765, -0.023774382, -0.008704563, 0.016265139, 0.020518422, -0.025050368, 0.0057235984, -0.019975761, 0.0033531266, 0.006669587, -0.00528727, 0.010442542, 0.026883679, -0.021134414, 0.025974356, 0.017629122, 0.01339784, 0.026517017, 0.009723884, -0.037135556, -0.0042239497, -0.024390375, 0.022058403, -0.015663812, 0.015707813, 0.036636896, -0.016895799, -0.03698889, -0.00307263, -0.029098319, 0.01258385, -0.0071719144, 0.024229044, 0.017394459, 0.003573124, 0.02654635, -0.009980547, -0.020034427, 0.012466517, 0.020093093, -0.015825145, 0.03129829, 0.0057529313, -0.0041542836, 0.009437887, 0.013581171, -0.012554517, -0.035287578, -0.049836736, -0.0015858144, -0.03745822, -0.01759979, 0.016499802, 0.0016783966, -0.00061461766, 0.052183375, 0.040362183, -0.02290906, 0.009151891, -0.0015473148, -0.0015903977, -0.044732798, 0.0074175783, 0.054735348, 0.0011632361, 0.0073809116, -0.03481825, -0.008909893, 0.015062487, -0.0062882584, -0.02085575, -0.013757169, 0.03572757, 0.011498529, -0.0011962357, 0.015150486, -0.014893822, -0.005096606, -0.0070545822, -0.005774931, 0.015825145, -0.003397126, -0.01366917, -0.014673824, 0.0075605763, 0.003989286, 0.0068089184, -0.0011054868, -0.03552224, 0.003532791, 0.03235428, 0.047050104, -0.013933167, -0.04376481, 0.0027151343, 0.012048523, -0.0098632155, 0.03657823, -0.014549159, 0.004572279, -0.029670313, 0.016763799, -0.043794144, 0.02031309, 0.0015958976, -0.0001079935, 0.010405876, -0.05752198, -0.013581171, 0.0049536075, 0.0010349043, -0.011212532, -0.010156545, 0.014512493, 0.015942477, 0.0053679356, -0.016954465, 0.007963905, 0.018655777, -0.010515874, -0.004278949, -0.030124974, -0.00695925, 0.011901858, -0.02650235, -0.0037216223, 0.017687788, 0.0035951238, -0.0053092698, -0.027690336, -0.0061085937, 0.007575243, 0.0306823, 0.0019799764, 0.009555219, 0.020841084, -0.0076339087, 0.023789048, 0.008095903, 0.016602468, 0.007758574, -0.014585826, 0.020679753, 0.0072562466, -0.02044509, 0.016895799, -0.005679599, 0.020210424, 0.022219734, 0.0017040629, 0.012539851, -0.010941203, 0.02855566, 0.015839811, 0.0009881549, 0.026517017, -0.034348924, -0.0031881286, 0.009672551, -0.019022439, -0.049602073, -0.010237211, -0.0032541277, 0.015619813, -0.012803847, 0.028702324, 0.0062479256, 0.0065889214, 0.023349054, -0.02295306, -0.036402233, -0.016558468, 0.025754359, -0.014710491, -0.0036171235, -0.009885215, 0.014563826, 0.025857024, 0.015238484, -0.012202521, 0.004198283, 0.018362448, -0.019623766, 0.0118138585, 0.009349888, -0.040596846, -0.02950898, -0.012517851, -0.024053046, -0.006838252, -0.013837835, 0.005353269, 0.009679885, -0.016910465, 0.009973214, 0.02361305, 0.0016637301, -0.026605016, 0.016925132, 0.0030469636, 0.030066308, -0.033498265, 0.024522373, -0.011425197, 0.009995214, 0.012407851, -0.019022439, -0.0195211, -0.007831906, 0.0135738375, 0.023525052, -0.018494446, -0.038895536, 0.015106486, -0.0006806169, 0.000753491, -0.003959953, -0.029860977, -0.0008259068, -0.007941905, -0.018699776, 0.0074139116, 0.0037986212, 0.001477649, -0.009899882, -0.025241032, 0.028848989, 0.005107606, -0.019931762, 0.03572757, -0.020034427, 0.019917095, -0.008557898, -0.011109867, -0.030476969, -0.010383876, -0.007058249, -0.010823871, 0.019227771, 0.03748755, -0.0035914571, 0.026722347, -0.00921789, 0.015121153, -0.0040956177, 0.0029241317, -0.02483037, -0.010919203, 0.011197866, 0.015707813, 0.027455673, -0.010332543, 0.040186185, 0.012158521, -0.019535767, 0.015062487, -0.02478637, -0.005892263, 0.018846441, 0.029816978, -0.0013044011, 0.038191542, -0.0030762965, 0.0055806, -0.015121153, 0.007105915, -0.006027928, 0.007941905, -0.03009564, -0.033028938, 0.0020038094, -0.026517017, 0.019535767, -0.013258508, -0.02474237, -0.003959953, -0.038895536, 0.041770168, 0.021955738, 0.005980262, -0.011923858, -0.01597181, 0.0030286305, 0.00027087177, -0.0033769596, 0.002119308, -0.009027226, 0.007505577, -0.034466255, 0.051479384, 0.0051296055, -0.017438458, 0.0082572345, -0.041124843, -0.017907785, -0.01427783, -0.01005388, 0.033967596, 0.0001905498, 0.011102534, 0.009885215, 0.0021229747, -0.018318448, 0.02207307, -0.01003188, -0.035962235, 0.03933553, -0.035463575, -0.029582312, -0.016074475, -0.033322267, -0.008939226, 0.014285163, -0.03402626, 0.010545207, 0.032295614, -0.018919773, 0.035580907, 0.015663812, 0.0013676503, -0.01843578, -0.019462435, -0.0018635611, -0.020078426, -0.005646599, -0.009628552, -0.0017416459, -0.012393185, 0.007358912, -0.015223818, 0.0041066175, 0.00037582885, -0.006706253, 0.0088878935, -0.007289246, -0.0056392658, -0.010039213, 0.008579898, -0.017379792, 0.0105232075, -0.02135441, -0.004487946, 0.020371757, -0.023422386, 0.01893444, 0.02930365, -0.008440566, -0.00075211603, -0.022219734, 0.001206319, 0.046346113, -0.0019744765, 0.0035126247, 0.0063359244, -0.028408993, -0.014747157, -0.03214895, -0.00028645492, -0.022791728, -0.034348924, 0.039394196, -0.0002940173, 0.03519958, -0.00612326, 0.00978255, 0.01868511, 0.0115938615, -0.019477101, -0.020577088, 0.010310544, 0.017834453, -0.009459887, 0.015223818, 0.013023844, -0.0042239497, -0.02876099, 0.01735046, 0.013529838, 0.005184605, -0.011263865, 0.01088987, 0.016323805, -0.021339744, 0.015942477, 0.036050238, 0.024845036, 0.010200545, 0.033967596, -0.027015677, 0.008858561, -0.01448316, -0.0012649848, 0.025475696, 0.017291794, -0.0012631515, 0.037399553, -0.03173829, 0.004432947, -0.015605147, -0.019594433, 0.042532824, 0.010603873, -0.012048523, 0.017658455, -0.0100832125, 0.01059654, 0.0125691835, 0.03144496, 0.0005972012, 0.03522891, 0.047842097, 0.003743622, -0.05062873, -0.036138237, 0.010779872, -0.025593027, 0.0034576254, -0.0076632416, -0.021075748, -0.020078426, 0.042268828, 0.026781013, 0.016881132, 0.049426075, -0.012371185, -0.014857156, 0.03470092, -0.021310411, 0.010383876, -0.043002155, 0.010757872, 0.0006004095, 0.002412638, 0.017218461, -0.0032761276, 0.002667468, 0.014952488, 0.019506434, 0.034847584, 0.024053046, 0.019843763, -0.000045632267, -0.0000048554107, -0.012869846, -0.010801871, -0.008330567, -0.043911476, -0.008271901, 0.016499802, -0.0043889475, 0.05731665, -0.02207307, -0.014043165, 0.034436923, 0.014314496, 0.0031661289, 0.014974488, -0.019447768, -0.015209151, -0.040186185, -0.0034961249, -0.039863523, 0.024874369, 0.0364609, -0.0036666228, -0.015414483, -0.0022476397, -0.010823871, -0.017951785, 0.028452994, 0.014263163, 0.0065705883, 0.020797085, 0.0035951238, -0.038455542, -0.009591886, -0.017541124, -0.05772731, 0.021134414, -0.026531683, 0.024698371, -0.011652527, -0.021838406, -0.02320239, 0.0008511148, 0.008198569, 0.025109034, 0.0006797002, 0.013647171, -0.0059545957, -0.03833821, 0.0012209854, 0.033527598, -0.015619813, 0.011447197, 0.012554517, 0.03992219, -0.023422386, 0.02587169, 0.020591754, -0.0026876347, 0.017658455, -0.029567646, 0.0022788062, 0.015942477, -0.026737014, -0.0019433101, -0.01281118, -0.031620957, 0.023877049, 0.0053422693, -0.028687658, 0.003490625, 0.0109338695, 0.022747729, 0.015253151, 0.01977043, 0.015942477, -0.029376982, 0.012026523, -0.00037949547, 0.021515744, -0.018787775, 0.024859702, 0.0028012998, 0.013617837, -0.06588188, -0.00642759, 0.0048399423, -0.026267687, -0.0036061236, -0.0052616037, 0.0055366005, -0.04878075, 0.009562552, 0.019433102, 0.00060590944, 0.017159795, 0.010677205, 0.013097177, 0.0010642373, 0.023569051, -0.028658325, -0.022087736, -0.009041892, -0.008997892, 0.014431828, -0.01421183, 0.021897072, -0.017130462, -0.0083525665, 0.014901156, -0.013207176, 0.0055696, 0.030300971, -0.0048766085, 0.003320127, -0.020386424, -0.015165152, 0.006585255, -0.0019506434, -0.010501208, -0.0018021452, -0.0071535814, 0.0019103105, -0.023231722, -0.023730382, 0.0007910739, 0.008719229, 0.0123491855, 0.028188996, -0.0006022428, 0.015707813, -0.01839178, 0.010229878, -0.00446228, 0.012026523, 0.0056209327, 0.002484137, -0.021794407, 0.03675423, 0.021413077, 0.00072140805, -0.0083159, -0.0028581326, 0.004502613, 0.031972952, 0.013786502, -0.023569051, 0.00076632417, -0.021046415, -0.01170386, -0.02846766, 0.014079832, 0.007138915, -0.0222784, 0.010435209, 0.00024566374, 0.0034832917, -0.000894656, -0.0023943046, 0.0086678965, -0.007186581, 0.008748562, 0.043794144, 0.005386269, -0.03695956, 0.01199719, 0.010493875, -0.0035914571, 0.002031309, -0.0070032496, -0.004025952, -0.010970536, -0.012224521, -0.0148351565, 0.0063175913, 0.023129057, -0.0024089713, 0.0016343972, 0.0006586171, 0.0041542836, -0.038132876, 0.04097818, 0.002808633, -0.0042936155, -0.0027921333, 0.0064605894, 0.009591886, -0.00867523, -0.0102738775, -0.027455673, -0.016705133, -0.016895799, -0.0148351565, 0.011087867, -0.008719229, -0.024434375, 0.0015482315, -0.0039269533, -0.0033274603, 0.049895402, -0.0034411256, 0.042826153, 0.00009458741, 0.015194485, 0.00026835097, 0.0034227925, -0.009019893, -0.043031488, 0.02383305, 0.0026308019, 0.009943881, 0.0029882977, -0.015663812, -0.004568612, -0.033204935, 0.0095478855, 0.014549159, 0.0054192687, 0.0087925615, 0.010317877, -0.02161841, 0.006398257, 0.011256532, 0.005125939, 0.010156545, 0.023818383, 0.012261187, 0.015414483, 0.012085189, -0.011513196, -0.01847978, -0.008909893, 0.010757872, -0.010853204, 0.0071535814, 0.010339877, 0.009701884, -0.010589207, -0.0016472304, 0.02742634, 0.025314365, 0.009525886, -0.018714443, -0.0014318163, 0.0014657325, -0.0117185265, 0.009811883, -0.0023209723, 0.01597181, -0.0057052653, -0.0082279015, -0.011175866, -0.010787205, -0.007890573, -0.034348924, -0.006500922, 0.02173574, 0.017497124, 0.03314627, 0.01452716, 0.0032889608, 0.008213235, -0.0074725775, 0.0113738645, -0.017203795, 0.018963773, -0.0067795855, -0.002143141, -0.01864111, 0.0473141, 0.011806525, -0.0064019235, 0.012319853, -0.039951522, 0.011615861, -0.009870549, -0.008931893, -0.008381899, -0.0053826026, -0.01143253, -0.007028916, -0.00724158, 0.013661837, 0.015781146, 0.013911167, 0.0044549466, 0.039276864, -0.0057529313, -0.009694551, -0.0074139116, -0.024801036, 0.03522891, -0.01985843, -0.016499802, -0.022117069, 0.0044439468, 0.022718396, 0.017702455, 0.0051186057, -0.03502358, -0.025314365, -0.011439864, -0.0038719538, 0.01638247, 0.009349888, 0.0037216223, 0.009144558, -0.019125104, -0.008895227, -0.00034924582, -0.00057474314, -0.0031001295, 0.017585123, -0.012767181, 0.0050086067, 0.04097818, -0.009518553, -0.01613314, 0.03531691, 0.008953893, -0.0072562466, -0.012591183, -0.0020881416, 0.008939226, -0.019535767, -0.017526457, -0.00865323, 0.029376982, 0.043207485, -0.0047556097, -0.003255961, 0.013830502, -0.010075879, 0.03305827, -0.030828966, 0.018699776, 0.015077153, -0.038191542, 0.0017874787, 0.0035822906, -0.020137092, -0.009364555, 0.021794407, -0.024845036, -0.0019469768, -0.025197033, -0.012437184, -0.0071095815, 0.00032266282, -0.02562236, -0.0044732797, -0.012173188, 0.008719229, -0.027397007, 0.013221842, -0.036519565, 0.0141091645, -0.022923727, -0.01843578, -0.009679885, 0.010295877, -0.013368507, 0.02950898, 0.037428886, -0.012679182, -0.030858299, -0.030887632, -0.011080534, 0.00036895392, -0.028863655, 0.0062332586, -0.020533089, -0.003538291, 0.0018149783, -0.010970536, 0.015502482, 0.011491196, 0.0005082856, 0.004946274, -0.02453704, -0.03531691, -0.008616564, -0.0017526457, 0.01477649, 0.037047558, 0.030330304, 0.0031716288, 0.026649015, -0.0032522944, -0.00023558052, 0.032002285, -0.010215212, 0.0013933167, 0.0010028214, -0.027925, 0.008645897, -0.0011192366, -0.031620957, 0.006621921, -0.008697229, 0.0017801455, 0.008095903, -0.010222545, 0.019095771, 0.028438326, 0.017438458, 0.01373517, -0.015561148, 0.014901156, -0.016441137, -0.023774382, -0.017526457, -0.0061672595, 0.017203795, -0.019755764, -0.002018476, -0.0097385505, -0.032002285, -0.007215914, -0.019829096, 0.025314365, 0.0014098164, -0.013991833, 0.0006338674, 0.012686515, 0.02595969, -0.0044292803, -0.019169105, -0.037516885, -0.011219866, -0.017687788, -0.012737848, 0.007168248, -0.0057785977, 0.008095903, -0.035756905, -0.029875644, 0.007193914, -0.010875204, 0.0085432315, -0.016837133, 0.006893251, -0.028042331, -0.0016032308, 0.005888596, 0.011667194, 0.014651825, -0.023627717, 0.004608945, -0.01059654, -0.029978309, 0.0062259254, -0.021926405, 0.013346507, 0.023481052, 0.0046309447, -0.00039599527, 0.0075019104, 0.010244545, -0.0056135994, 0.0024749704, 0.011564529, -0.010897203, -0.0014143997, -0.045964785, 0.035375576, 0.015487815, -0.03616757, -0.004198283, -0.012393185, -0.0045576124, -0.0018901441, -0.0091078915, 0.00026835097, -0.022161068, -0.01534115, -0.019301103, -0.012180521, 0.025021035, -0.0011568195, -0.0054522683, 0.0145198265, 0.014248497, -0.026282353, -0.024933035, -0.023906382, -0.005679599, 0.026927678, 0.007292913, -0.025358364, 0.0034997915, -0.0018699777, -0.020195758, -0.0043009487, 0.0042276164, 0.033410266, -0.019697098, -0.008345233, 0.015238484, 0.032119617, -0.022337066, -0.007505577, -0.0069189174, 0.0038169543, 0.013977166, -0.010992535, 0.00013646712, -0.019961094, -0.005532934, -0.023393054, -0.017453125, 0.014131164, 0.018963773, 0.025079701, -0.013757169, -0.012539851, 0.012972511, 0.030506302, 0.010501208, -0.0027518005, 0.0014418995, -0.0053386027, 0.006181926, -0.027954333, -0.01059654, 0.023583718, -0.00867523, -0.0028251328, 0.0034686252, 0.012957845, -0.0014629825, 0.0033531266, -0.02010776, -0.004242283, 0.013075178, -0.037546217, -0.01229052, 0.028819656, 0.011388531, -0.02182374, -0.0034594587, 0.006299258, 0.0014730657, -0.0120338565, -0.008550565, -0.023965048, -0.01475449, 0.012715848, 0.028042331, 0.015370483, 0.0016609802, -0.02311439, -0.0042716158, 0.0078099067, -0.023935715, 0.015737146, -0.004751943, -0.028320994, 0.04376481, 0.012209854, 0.015267817, 0.004289949, -0.0019341436, -0.01482049, -0.020122426, 0.029816978, 0.035844903, -0.0034099591, -0.020034427, -0.0036006237, 0.0040112855, -0.001168736, 0.010413209, -0.024214378, 0.0032357946, 0.011857858, 0.016411804, -0.001930477, -0.0029772979, -0.003202795, -0.015825145, 0.009789883, -0.009657885, -0.0009808217, -0.007853907, -0.0038242876, -0.009855882, -0.018318448, -0.024141045, 0.006009595, 0.020386424, 0.0066622538, -0.01306051, 0.0027169676, 0.02085575, -0.02048909, -0.0025611361, 0.017570456, 0.013016511, 0.011168533, 0.0054009357, 0.0011852358, -0.0011879858, -0.00035382912, -0.031972952, 0.001592231, 0.013434506, 0.009247223, -0.0030469636, 0.007410245, 0.014013832, -0.014079832, 0.019902429, 0.015810478, 0.04100751, -0.021691741, 0.0011769859, 0.006794252, 0.027279673, -0.022498397, 0.005562267, 0.0073809116, 0.015869144, -0.016954465, -0.01303851, 0.019638432, 0.0081325695, 0.0021743074, 0.011007202, 0.054882012, 0.010427875, -0.0077659073, 0.009393888, -0.006368924, -0.0045282794, -0.034084927, -0.041124843, -0.0007681575, -0.013331841, -0.023026392, 0.010493875, 0.0022403065, 0.0037802882, 0.008330567, 0.030300971, -0.009599219, 0.004110284, 0.018083785, 0.013251175, -0.00058024307, -0.01935977, -0.010471875, 0.0011458197, 0.011571862, -0.0022183068, 0.00020166425, -0.019477101, 0.010259211, 0.00090886414, -0.0043706144, -0.007780574, 0.02110508, -0.002425471, 0.0015353983, 0.0026876347, -0.0075972425, 0.00087265624, 0.006746586, 0.008286567, -0.009298556, 0.02173574, 0.022263734, -0.020430423, 0.01889044, -0.0035291244, -0.014417161, 0.0032999606, -0.020841084, 0.022806395, 0.0014217331, 0.007553243, 0.0011109867, -0.009269223, -0.0043486147, -0.00865323, 0.0025794692, -0.008367233, 0.011293198, -0.010479208, 0.015693147, 0.0024273044, -0.013647171, -0.003485125, 0.0068309186, 0.011879858, -0.00017118546, 0.009181224, 0.024214378, 0.038250208, -0.017482458, -0.020181092, 0.03942353, -0.017775787, -0.0072122472, 0.010530541, -0.019697098, 0.007846573, 0.004091951, 0.004091951, -0.009225223, -0.004080951, -0.016030475, -0.00362629, -0.021838406, -0.026326353, 0.0030982962, -0.01003188, 0.008550565, 0.0071315817, -0.009129891, -0.016910465, 0.024727704, 0.0034246258, 0.0024878036, 0.005257937, 0.009811883, -0.015003821, -0.02725034, -0.0017764787, 0.00363179, 0.018039785, -0.012987178, 0.011674527, 0.011249199, -0.008191235, 0.013610504, 0.003967286, -0.0076852413, -0.014497827, -0.0222784, -0.003809621, 0.0068419185, 0.010603873, -0.014197163, -0.0119091915, -0.014864489, 0.011498529, 0.012393185, 0.0139844995, -0.008572564, -0.014710491, 0.0017471458, -0.0022879727, 0.011095201, 0.0031697955, -0.010735871, -0.005756598, -0.022630395, -0.015106486, 0.011263865, -0.009723884, 0.021251746, -0.0062589254, -0.0051992713, -0.0031917952, -0.007971238, -0.024214378, -0.010075879, -0.002638135, 0.017863786, -0.00014322746, -0.003915953, -0.011395864, 0.017394459, 0.009621219, 0.004872942, 0.012994511, 0.011806525, 0.004597945, 0.016807798, -0.0122538535, -0.008909893, 0.0147398235, 0.022087736, -0.010515874, -0.0055439337, -0.0026326352, -0.020973083, -0.0070215827, 0.017585123, -0.0139844995, -0.009555219, 0.005969262, 0.020591754, 0.016353138, 0.024918368, 0.035756905, 0.012935845, -0.005389936, -0.018157117, -0.005815264, -0.021471743, -0.009232556, -0.0034502922, 0.02474237, -0.009093225, -0.009122558, 0.013412506, -0.008183902, 0.005202938, 0.008293901, 0.016162474, -0.020577088, -0.008396566, -0.024141045, 0.0119091915, 0.0017746454, -0.0066145877, 0.0050562727, -0.009005226, 0.0043486147, -0.012415185, -0.004597945, -0.003208295, -0.015663812, -0.026414352, 0.021662408, 0.0014409828, 0.010281211, 0.018289115, -0.0028617992, 0.003208295, 0.009393888, 0.018817108, 0.008946559, 0.01170386, 0.0072269137, 0.0014363995, 0.028775657, -0.03569824, 0.0024089713, 0.011630528, 0.0038169543, 0.0050819395, 0.0008648647, -0.013911167, -0.008763229, 0.0008190319, 0.022630395, 0.009965881, -0.028746324, 0.026825013, 0.009687218, 0.005917929, 0.034759585, -0.0021871405, 0.017922452, 0.034348924, -0.0069849165, -0.012979845, -0.01529715, 0.014226497, -0.0017132296, 0.0060829273, -0.018377114, 0.01847978, -0.023099724, -0.009364555, -0.012407851, 0.00088823936, -0.01534115, -0.005114939, 0.00029676728, 0.0022183068, 0.005510934, -0.0011504029, 0.00786124, -0.002678468, -0.000025293968, 0.0055072675, 0.027734336, -0.008191235, 0.001256735, 0.0034869583, 0.00016064392, -0.0051956046, 0.00726358, 0.01504782, -0.014145831, -0.0054779346, -0.0023704716, -0.007553243, -0.00083828164, 0.008095903, 0.015106486, -0.0054229354, -0.0035859572, 0.011777192, 0.000079405305, -0.007494577, 0.0021138082, 0.0112785315, 0.0047299433, -0.0069005843, -0.00006061386, -0.017907785, -0.0032302947, 0.0028562993, 0.01931577, -0.006453256, -0.0064019235, -0.024199711, -0.0036409565, -0.007905239, -0.014974488, 0.015517148, 0.026062354, 0.010985202, 0.008939226, 0.006170926, 0.0056135994, -0.0052909367, -0.015443816, 0.012627849, 0.020210424, -0.010259211, 0.01170386, 0.0055439337, -0.0046309447, -0.0039342865, -0.017189128, 0.011608528, 0.011359198, 0.011557195, -0.0011412364, 0.026179688, 0.03173829, -0.0011696527, 0.0016783966, -0.019550433, 0.0013988166, 0.0013878167, -0.0076852413, -0.0070252493, 0.0142044965, -0.019741097, 0.009371888, -0.0015738979, -0.0036831226, 0.019594433, 0.014607825, -0.0016123974, -0.0019818097, 0.0028214662, -0.0032394612, -0.007905239, -0.0012439018, -0.02571036, -0.0014684824, 0.027778335, -0.019433102, 0.0012384019, 0.046258114, -0.01448316, 0.016059808, -0.015135819, 0.019374436, -0.007875906, 0.0056319325, -0.002830633, 0.017585123, -0.012613183, 0.011285865, -0.005144272, 0.01314851, 0.00082819845, -0.0029222984, -0.008147236, 0.0058959294, 0.004715277, 0.0008405733, 0.005793264, -0.023935715, 0.003508958, 0.008330567, 0.005437602, -0.0078099067, 0.028071664, -0.0052176043, 0.037546217, -0.0013850668, 0.018963773, -0.0046236115, -0.023730382, -0.019506434, 0.0013639837, -0.016749132, -0.009753217, -0.0029167985, -0.009562552, 0.0030157974, -0.012943178, 0.0062149256, -0.018553112, -0.010794538, -0.002060642, 0.011212532, 0.011689194, -0.004469613, 0.008462566, -0.0078025735, -0.0023338054, 0.012158521, -0.0004995774, -0.0024328043, -0.0012109022, -0.0049829404, 0.012495851, 0.0069555836, -0.02984631, -0.002678468, -0.016573135, -0.0168518, -0.010970536, 0.0038279542, 0.0072379136, 0.007446911, 0.01730646, -0.0022769729, -0.0033696264, 0.005606266, 0.007468911, -0.0016637301, -0.011835858, 0.0071425815, 0.0041946163, 0.0027243008, -0.040274184, -0.0043962807, 0.009159224, -0.0012695681, -0.00043655728, -0.007168248, 0.016807798, 0.014600492, 0.0055182674, 0.004832609, 0.036108904, -0.028482327, -0.012994511, -0.025343698, -0.00057840976, 0.0073919115, 0.009980547, 0.019286437, 0.004579612, -0.0078172395, 0.0049939402, -0.0125691835, 0.02144241, 0.005708932, -0.02153041, 0.022542397, -0.008367233, -0.0040479517, -0.008242568, 0.021383744, 0.0017288127, 0.020914417, -0.022351732, -0.016323805, -0.019037105, 0.021061081, 0.0020496421, -0.0028672991, -0.013221842, 0.031151628, 0.008689896, -0.011755193, 0.024317043, -0.027705003, 0.0049719405, -0.006563255, -0.021897072, 0.0003361835, -0.016587801, 0.0048472756, 0.012143855, -0.010662539, 0.004704277, 0.0131045105, -0.001586731, -0.009775217, 0.027646337, 0.011799192, -0.023730382, -0.011439864, -0.013053177, 0.0057822643, 0.010816538, -0.0050672726, -0.023774382, -0.00923989, -0.004385281, 0.001144903, -0.00788324, 0.003596957, 0.012891846, -0.007897906, 0.0076632416, -0.003255961, 0.007941905, -0.0018048951, 0.02541703, 0.0020294758, -0.00081536523, -0.012195188, -0.012532517, 0.021457076, 0.024478374, 0.015062487, -0.0019139771, 0.002407138, -0.008330567, 0.0016499803, 0.011007202, 0.019873096, 0.011777192, 0.0111392, -0.007971238, -0.0073075793, -0.001613314, 0.010567207, 0.019125104, -0.011051201, 0.00065403385, -0.0016362305, 0.018758442, 0.029039653, 0.0035804573, 0.02483037, -0.0010055713, -0.0031807953, -0.020210424, 0.015282484, 0.00083003176, 0.011403197, -0.013896501, 0.0034447922, 0.0006755753, -0.011938524, 0.019022439, 0.0013621504, -0.019594433, 0.002542803, 0.03223695, -0.029714312, 0.012026523, -0.015869144, 0.0028562993, 0.014087165, 0.033850264, -0.005969262, -0.009364555, -0.024023714, -0.0014162331, 0.007384578, 0.004073618, -0.0051882714, -0.008484566, -0.004638278, -0.017907785, 0.02670768, -0.010640539, -0.012275853, -0.009679885, -0.023305055, -0.00043999474, -0.0067429193, 0.024991702, 0.023187723, -0.014541826, -0.0055256006, 0.016030475, -0.012862513, 0.015165152, 0.010845871, 0.0057125986, -0.0011513196, 0.01814245, 0.0058482634, -0.0166758, 0.011190533, 0.028056998, 0.014446494, -0.008829228, -0.020533089, 0.023862382, 0.011300531, -0.008242568, 0.0011751526, -0.008748562, 0.014043165, 0.019550433, 0.0092032235, -0.011879858, -0.0065522552, 0.008147236, 0.019110437, 0.0029516313, -0.005085606, -0.016103808, -0.012085189, -0.00093498884, 0.0005000357, -0.006346924, 0.016866466, 0.0026821345, -0.0012723182, -0.02837966, -0.0041432837, -0.0039452864, -0.007648575, -0.005661266, 0.011762526, 0.00072782463, 0.0111172, 0.022747729, 0.011747859, -0.023231722, 0.021941071, -0.028188996, 0.008499232, -0.039863523, 0.025945023, 0.017834453, -0.0008259068, 0.00004179377, -0.015531815, -0.0018773109, -0.008059237, 0.0031661289, 0.0014281496, 0.009687218, -0.007318579, -0.0166318, 0.01597181, -0.0016866465, 0.040772848, 0.0046236115, -0.004161617, 0.0015198152, -0.009166557, -0.02629702, 0.0034594587, -0.015634479, -0.015414483, 0.023789048, -0.023481052, 0.014703157, -0.009063892, 0.00017198753, -0.007677908, 0.018333115, 0.00001711568, -0.006533922, 0.0030451303, 0.0054779346, -0.0014794823, 0.011806525, 0.0052139377, -0.025387697, -0.011615861, 0.018215783, -0.0032009617, -0.008645897, -0.013500505, 0.049308743, -0.009085892, -0.016074475, -0.015854478, 0.004220283, 0.007329579, 0.024009047, 0.02336372, -0.027191674, -0.005950929, 0.001339234, -0.007941905, -0.002348472, -0.020503756, -0.004542946, 0.021794407, -0.02207307, 0.007127915, 0.007336912, -0.0060829273, -0.02562236, -0.020679753, -0.0026986345, 0.0032431278, 0.016822465, 0.0011320699, -0.016147807, 0.007751241, 0.004685944, 0.02345172, -0.008953893, -0.012679182, -0.007058249, 0.0037106224, -0.01985843, -0.006057261, -0.000009725144, -0.020357091, -0.0014455661, -0.0010880703, 0.021325078, -0.011498529, 0.021515744, -0.004289949, -0.015135819, -0.0009290306, -0.016573135, -0.00642759, 0.005892263, 0.0023796381, 0.017643789, -0.0007571576, -0.0065669217, 0.0046639442, -0.00500494, -0.0017810621, -0.020166425, 0.0077732406, -0.0094525535, 0.004198283, 0.004286282, 0.02453704, 0.0013841501, -0.019389102, 0.020767752, 0.016954465, 0.023437053, 0.0036702894, -0.00006496797, -0.007919906, -0.014637158, 0.010244545, 0.021457076, 0.004920608, 0.01847978, -0.023877049, -0.007366245, 0.018010452, -0.007985905, 0.008073904, -0.015898477, 0.016441137, 0.008367233, -0.0011183199, 0.026605016, 0.002526303, 0.0125691835, 0.018289115, 0.001474899, -0.012767181, -0.008990559, 0.016514469, -0.0029992976, -0.00528727, -0.0042569493, 0.00047482766, -0.009672551, 0.00027476755, -0.01759979, -0.0051296055, -0.022542397, -0.00028118413, 0.010413209, -0.008293901, -0.0014061498, 0.0016783966, 0.013214509, -0.008433233, 0.009364555, -0.0015427315, -0.0065779216, 0.00394162, 0.024317043, -0.012444518, -0.00014655033, -0.0030836298, 0.014923155, 0.03285294, -0.006592588, -0.006475256, -0.0005669516, -0.0166758, -0.022410398, 0.024551706, 0.018846441, -0.013639837, 0.025006369, 0.039658193, 0.010750539, 0.023803717, -0.0063139247, -0.017643789, 0.017863786, 0.003849954, 0.006651254, -0.0020496421, -0.026766347, 0.014050499, 0.005074606, -0.015795812, 0.01538515, 0.00016614384, -0.0012365686, 0.008836561, -0.0030781298, -0.010413209, -0.0028709657, -0.008733896, 0.0068785846, 0.0111831995, -0.020151759, -0.0069299173, 0.019433102, 0.014329162, 0.023950381, -0.007395578, 0.014409828, 0.015575814, -0.014431828, 0.0029589646, 0.02747034, 0.011227199, -0.0007754907, -0.0057345983, -0.005492601, -0.014541826, 0.0059215957, 0.0059765955, 0.0057052653, -0.023041058, -0.0005687849, 0.015605147, 0.016265139, 0.0028709657, 0.016925132, 0.016793132, -0.013214509, 0.009907215, 0.007138915, 0.027763668, -0.009679885, -0.019242438, -0.0034704586, 0.021119747, 0.012070523, 0.0025299697, -0.0031514624, 0.0040992843, -0.004542946, 0.006783252, -0.005221271, -0.016807798, 0.011263865, 0.025153033, -0.0023264722, 0.011806525, -0.015751813, -0.006097594, 0.0025978023, -0.0083159, -0.003989286, 0.0033989593, -0.0032357946, 0.0022329732, -0.0037472886, 0.007215914, -0.02833566, 0.007146248, 0.013111844, -0.0011549862, -0.003149629, 0.0065705883, -0.01341984, -0.010039213, -0.022263734, -0.005928929, 0.023246389, -0.020635754, -0.011359198, -0.0015408982, -0.010493875, 0.0077732406, -0.0027261341, -0.007897906, -0.005485268, 0.014461161, 0.010207878, -0.009767883, 0.0141091645, -0.009393888, 0.03874887, 0.008755895, -0.012950512, 0.010897203, -0.0048216092, 0.029728979, 0.018611778, 0.00066732534, 0.0072122472, -0.00890256, -0.033410266, -0.006658587, 0.012921179, -0.021809073, 0.0027169676, 0.014123831, -0.0021321413, 0.018553112, 0.0058739297, 0.020958416, -0.002638135, -0.0031111294, 0.01204119, 0.004080951, 0.013947833, -0.013969833, 0.0013025678, -0.0026069689, 0.006640254, 0.008939226, -0.009386554, 0.022791728, -0.0039122864, -0.007721908, 0.013185176, 0.02114908, -0.0005252437, 0.0074249115, 0.019506434, 0.00978255, -0.0073442454, -0.009063892, -0.011087867, 0.0036226234, 0.0072562466, -0.0026216353, -0.0074065784, 0.00223114, 0.011843191, -0.03399693, -0.0035951238, 0.0021853072, -0.021383744, 0.018010452, -0.0196091, -0.003673956, 0.01776112, 0.003584124, 0.0035217912, -0.009973214, -0.0041176174, -0.011549862, 0.017042464, 0.0063175913, 0.013749836, -0.02541703, -0.018919773, 0.009914548, -0.00726358, -0.025285032, 0.006211259, -0.0167198, -0.0015665647, -0.0130091775, 0.008381899, 0.01705713, -0.005551267, 0.008895227, -0.012121855, -0.02968498, -0.010537874, -0.0010211545, 0.015121153, 0.0045832787, 0.010743205, 0.00061645097, -0.011285865, -0.00044366135, -0.010200545, 0.008037237, -0.019081105, 0.0002903507, -0.029200984, -0.018567778, 0.021501077, 0.011535196, -0.009012559, 0.021926405, -0.0018113117, -0.0168518, -0.0016930631, 0.005628266, -0.002872799, 0.0034411256, 0.0031532957, -0.0040112855, 0.024097046, 0.020474423, -0.010075879, 0.009826549, 0.02433171, 0.0039269533, -0.016323805, -0.020547755, -0.007105915, -0.0081325695, 0.004480613, -0.000014007645, 0.007934572, -0.013779169, -0.004957274, -0.0019268104, 0.0168518, 0.0030084641, 0.005492601, 0.010545207, 0.02566636, -0.012341852, -0.0018094784, 0.0103032105, 0.008807228, 0.003959953, 0.007516577, -0.024874369, -0.01032521, -0.00024818454, 0.015253151, 0.023891715, 0.015531815, -0.008345233, 0.017071797, -0.017042464, -0.019330436, -0.005180938, -0.0082279015, -0.011073201, 0.0038132877, -0.025226366, -0.015649145, -0.02336372, -0.0037472886, -0.004260616, 0.011645194, -0.011806525, 0.017643789, -0.02399438, -0.008484566, -0.0056135994, -0.0052652704, -0.01897844, 0.004319282, 0.01642647, -0.0076559084, 0.00007442098, -0.017746454, -0.00041501588, -0.014857156, 0.04798876, -0.0051662717, 0.012107189, -0.007017916, -0.010684539, -0.0058849296, 0.0028379662, 0.019403769, 0.011293198, -0.012767181, 0.0043596146, -0.0037362888, -0.006827252, 0.0036519563, 0.025563695, 0.008044571, 0.0023924713, 0.017775787, 0.0015793978, 0.005356936, 0.0040222853, -0.013815835, -0.00786124, 0.0003730789, 0.008623897, -0.006434923, -0.023173057, -0.015311817, 0.006863918, -0.011733193, -0.002707801, 0.014329162, 0.00030776716, 0.0020258091, 0.012070523, 0.00445128, 0.009085892, -0.0037179557, -0.02135441, 0.0053752693, 0.008073904, 0.0034631253, 0.006728253, -0.0151798185, -0.023349054, -0.0058482634, -0.019902429, -0.0003849954, -0.0042349496, -0.006555922, 0.012371185, -0.017995784, -0.005045273, -0.002390638, -0.012004524, 0.017497124, 0.036431566, 0.005345936, 0.02453704, 0.005173605, -0.005155272, -0.010999869, 0.000919864, -0.0040772846, -0.005998595, -0.010897203, 0.02428771, -0.0101785455, -0.005980262, 0.010860537, 0.010515874, 0.00009630614, -0.004014952, -0.0030506302, 0.002372305, 0.016206473, 0.0070509156, 0.0066182544, 0.021413077, -0.012503184, -0.0037161224, 0.003538291, 0.00394162, -0.0012759848, -0.0007236997, 0.020723753, 0.015223818, 0.012803847, 0.0022751396, 0.015854478, 0.0003714747, 0.00023271597, -0.0023759715, 0.03737022, -0.007868573, 0.010244545, 0.007157248, -0.0041652834, 0.014688491, -0.010897203, 0.016558468, -0.001727896, 0.007446911, -0.000893281, -0.016103808, 0.01418983, 0.00064578396, 0.026663681, 0.016778465, 0.0019781431, -0.0096432185, 0.013258508, -0.00055641, 0.01935977, -0.008308567, 0.007204914, 0.034143593, 0.0012182355, 0.007292913, -0.0010486541, 0.012693848, -0.027529005, -0.004740943, -0.00029241317, -0.0021981404, -0.019902429, 0.013368507, 0.0048876083, 0.009819216, 0.006999583, 0.005617266, -0.0045099463, -0.009569885, -0.009789883, -0.025358364, -0.0037876214, -0.011571862, -0.0069189174, 0.012063189, 0.0010110713, -0.0014767323, -0.009885215, 0.011417864, -0.00474461, 0.023187723, -0.005034273, -0.004260616, -0.012151188, 0.011271198, 0.019433102, -0.0168518, 0.004392614, -0.039804857, -0.00065586716, 0.00043724477, 0.027265007, 0.016015809, -0.0055806, 0.02022509, 0.002478637, 0.008095903, -0.0018195616, -0.0015775645, 0.008609231, -0.0069262506, -0.011696527, 0.018993106, -0.008851227, -0.0018516446, 0.00033916262, -0.0252117, -0.0017627289, -0.0085432315, -0.00976055, 0.030623633, 0.014131164, -0.018553112, 0.013771836, -0.015326483, 0.01839178, 0.0049609407, -0.012158521, -0.01362517, -0.005191938, -0.014497827, 0.018626444, -0.010215212, 0.0003393918, -0.00586293, -0.0039929524, 0.0072782463, 0.009628552, 0.021926405, 0.0009542386, -0.015531815, -0.022923727, 0.008711896, 0.00697025, -0.014307163, -0.0020239758, 0.0101785455, 0.025226366, 0.024918368, -0.008330567, 0.008601897, -0.008836561, -0.005444935, 0.0061305934, 0.0005481601, 0.004036952, 0.013529838, -0.038455542, -0.002907632, -0.009401221, -0.00585193, -0.03176762, -0.006805252, -0.0077659073, -0.02026909, -0.038308874, -0.022381065, 0.0055366005, -0.014695824, -0.017409125, 0.0050599393, -0.010141878, -0.017614456, -0.0005719932, -0.0022458064, -0.00035726657, 0.011652527, -0.0060132616, -0.015062487, 0.014307163, -0.0028397995, -0.007146248, 0.0072855796, -0.019843763, -0.0022751396, -0.0083525665, -0.032207616, 0.045348793, 0.0018204782, 0.009357221, 0.027353007, -0.0046896106, 0.00069665833, 0.00024520542, -0.00086303137, -0.02340772, 0.0011531529, -0.019829096, -0.00037514136, 0.027309008, 0.019550433, -0.014387828, 0.0042569493, 0.013243842, 0.008917226, 0.0022788062, 0.008095903, 0.009635885, -0.009063892, -0.001344734, 0.0070509156, 0.021647742, -0.009628552, 0.011175866, -0.0020936416, 0.008843894, 0.010112545, -0.005928929, 0.008763229, -0.01306051, -0.016397137, 0.006867585, -0.0031349626, 0.009877882, 0.011087867, 0.013192509, 0.0032632942, -0.009694551, 0.011043868, -0.013441839, -0.013999166, -0.0070692487, -0.023026392, 0.019154437, 0.0034062925, -0.004531946, 0.02738234, -0.008059237, 0.028892988, -0.0068822512, 0.015194485, 0.021266412, -0.011879858, 0.0061122603, 0.0026161354, -0.01726246, 0.018714443, 0.00033755848, -0.010185879, -0.008367233, 0.0025831359, -0.0017746454, -0.0014473994, -0.0020716419, 0.00018504987, 0.021867739, 0.0043009487, 0.027646337, -0.00783924, -0.016705133, -0.0060352613, 0.006255259, -0.011806525, -0.026047688, 0.025197033, 0.007105915, -0.0030176307, 0.0024914702, 0.003001131, 0.012327186, -0.026267687, 0.010141878, -0.010699205, 0.0059949285, -0.012165855, 0.0195211, -0.011843191, -0.002489637, 0.00502694, 0.005532934, 0.014182497, 0.022381065, -0.022997059, 0.0039636195, 0.015707813, 0.0077952403, 0.008851227, -0.028540993, 0.0127891805, -0.00250797, 0.0012961512, 0.010427875, -0.008469899, -0.0053716027, -0.004568612, 0.00726358, -0.008807228, -0.006020595, 0.0049352744, -0.0039232867, -0.0073809116, 0.034847584, 0.012825847, -0.003560291, 0.0013438173, 0.01226852, -0.014006499, -0.009540552, -0.013368507, -0.0012099856, 0.0035217912, 0.0012494017, -0.016793132, 0.0020918082, 0.01726246, -0.011542529, 0.022674397, -0.023481052, -0.0027921333, 0.028614325, 0.027074343, 0.02921565, -0.0025886358, 0.0014125664, 0.015502482, -0.013522505, 0.006863918, -0.012437184, 0.0026803012, 0.017115796, 0.024141045, 0.013023844, 0.006500922, -0.0011146533, -0.021970404, -0.028643658, -0.019154437, -0.0029864644, -0.01818645, -0.0036427898, -0.010339877, 0.015011154, -0.0030121307, -0.0012494017, 0.01931577, 0.01418983, -0.0074725775, 0.008220568, 0.013500505, 0.007846573, 0.0017214795, 0.010112545, 0.035170246, 0.018582445, 0.022102403, 0.018333115, 0.0053202696, -0.00039439113, 0.011337198, -0.022205068, 0.004249616, 0.011483863, 0.005991262, 0.011109867, -0.021603743, -0.00472261, 0.0047996095, -0.012510518, -0.014094498, 0.012385852, -0.015209151, 0.014805824, -0.01751179, -0.0013199842, 0.0005981178, 0.0010165712, 0.011806525, -0.014886489, -0.008308567, -0.0051516052, -0.018113118, -0.0035052914, -0.0014932322, -0.000113378854, -0.026033022, -0.022674397, 0.0039269533, -0.023539718, 0.0011430697, -0.009019893, -0.020342425 - ] - } - ] - }, - { - "name": "emerging-technologies-shaping-our-future", - "title": "Emerging Technologies Shaping Our Future", - "content": "Key technologies and concepts in the current tech landscape include ChatGPT, GPT-4, Anthropic Claude, Gemini, AI, ML, LLMs, DALL-E, Midjourney, Stable Diffusion, Sora, VR/AR/XR, TypeScript, blockchain, Web3, and PWAs.", - "keywords": [ - "ChatGPT", - "GPT-4", - "Anthropic Claude", - "Gemini", - "AI", - "ML", - "LLMs", - "DALL-E", - "Midjourney", - "Stable Diffusion", - "Sora", - "VR", - "AR", - "XR", - "TypeScript", - "blockchain", - "Web3", - "PWAs" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - -0.011711601, 0.018536234, -0.031360634, -0.016186133, 0.0018540126, -0.015983807, -0.030551326, 0.012248545, 0.00081952027, -0.0037761158, 0.04189718, 0.0019804668, 0.003637989, -0.006338272, -0.02051281, 0.012186291, -0.02961751, -0.014808755, -0.031329505, -0.02155557, -0.014723156, -0.016668605, -0.048620652, -0.007061979, 0.02255164, -0.0027353007, 0.037352614, -0.0064939074, -0.053601, 0.026862755, 0.02558654, -0.009058009, 0.020746265, -0.043080013, 0.007824594, -0.019859139, 0.013337996, 0.012925561, 0.014621992, -0.02406131, 0.023532147, 0.0037138616, -0.023267565, 0.03442666, -0.012435308, 0.011789419, -0.0044978773, -0.012956688, -0.0016098587, -0.018925324, -0.042488597, -0.019003142, 0.055997793, 0.015337917, 0.02168008, -0.035547238, 0.05606005, -0.006170963, 0.018287217, -0.010855603, 0.027531989, 0.00620209, -0.035734, -0.0023656655, -0.014069485, -0.0016011043, -0.0005422938, 0.028061152, -0.011283603, -0.010723313, -0.040558714, 0.025244141, 0.0014493592, -0.018691871, 0.012232981, -0.0011614328, -0.012863306, 0.02510407, -0.011501492, -0.018536234, 0.038037412, 0.017197765, -0.0021575026, 0.008178666, 0.029991036, -0.0051437663, 0.007540559, -0.08379436, 0.009509353, 0.010412041, -0.0134702865, 0.026489228, -0.027282972, -0.0450099, 0.01270767, 0.03903348, -0.008995755, -0.005770201, 0.0042994414, 0.024481526, 0.048247125, -0.005151548, 0.028730385, -0.013579232, 0.015485771, 0.00832652, -0.0150110815, 0.009369281, -0.001316096, 0.048029236, -0.009556044, -0.008069721, -0.009548262, -0.0046807495, -0.0150110815, 0.022131424, -0.021602262, -0.043422412, -0.0269717, -0.01942336, -0.0017188039, 0.0015865134, 0.013330214, -0.012264108, -0.026255775, 0.032683536, -0.0037741703, -0.0030932634, -0.03174972, 0.010987894, -0.027936643, 0.04454299, -0.008458811, 0.026286902, 0.038535446, 0.025072942, 0.024014618, 0.024963997, 0.019174341, 0.022800658, -0.0121396, -0.011913927, -0.00015600066, -0.035484985, -0.0151745, -0.003560171, -0.005272166, 0.047811344, -0.004540677, 0.01114353, -0.030691398, 0.004505659, 0.013439159, -0.013602577, 0.010661058, -0.031765286, 0.03165634, 0.013244614, -0.019221032, 0.042177327, 0.015579153, -0.04311114, 0.01660635, 0.04270649, 0.056309067, -0.028045587, 0.018925324, 0.0080930665, -0.03872221, 0.03946926, 0.016761985, 0.030442381, -0.013890503, -0.020637318, -0.019298851, -0.008069721, 0.017088821, 0.0011137694, 0.0059258365, 0.016730858, 0.0009654289, -0.004241078, 0.020326048, 0.01760242, 0.010707749, -0.0077506676, 0.06823078, -0.005976418, 0.014108393, -0.014559737, -0.003425935, -0.021882406, 0.010116333, -0.033928625, 0.027438607, -0.016123878, -0.0012110418, -0.019189905, 0.007731213, -0.024403708, 0.008521065, -0.018396161, -0.004132133, 0.039562643, 0.020668447, 0.044325102, 0.00026847192, 0.013369123, 0.04734444, 0.0036360435, -0.037570503, 0.008824555, -0.031547394, -0.004805258, -0.02272284, -0.025337523, 0.0024376472, 0.014528611, -0.011073493, -0.0091047, 0.0042605326, -0.008645574, 0.018147144, -0.019143214, -0.031454016, -0.028279042, -0.009828406, 0.022987422, -0.048216, 0.006731252, -0.004322787, -0.0047974763, 0.004824713, -0.013306869, 0.0269717, -0.033804115, 0.028870458, -0.024248071, -0.014824319, 0.027205154, -0.0023228657, -0.026660427, -0.021415498, -0.03386637, 0.030380126, 0.009198081, -0.013851594, 0.03682345, -0.012279672, 0.00650558, 0.005186566, -0.010357569, -0.0005053303, -0.014816537, -0.03160965, 0.010645495, 0.037757266, -0.0011254421, 0.041430276, 0.0036691162, -0.015664753, -0.05272944, -0.0035115348, 0.0360764, 0.01990583, 0.011945055, 0.022909604, 0.013695958, -0.0006055209, -0.040745478, 0.042737614, 0.010606586, 0.006789616, 0.014240684, -0.019438922, -0.0042722053, 0.021057537, -0.0037352615, 0.043329034, -0.0046846406, 0.022691714, 0.00050873484, 0.036232036, -0.016326206, -0.0242792, 0.007984121, -0.00011532863, 0.01331465, -0.004957003, -0.01283218, -0.026302466, 0.028652567, 0.007081433, -0.014730937, 0.019329978, -0.019407796, 0.013166796, 0.0532586, -0.0044239503, -0.028730385, -0.019049833, -0.010583241, -0.016419586, -0.0076417224, -0.0047546765, 0.034924693, 0.041305766, 0.00053937564, -0.03343059, -0.02493287, 0.030208927, -0.03912686, 0.0019473941, 0.005945291, 0.02575774, 0.035889637, 0.023236439, -0.008209793, -0.015049991, -0.029166166, 0.034239896, -0.021197608, -0.07153025, 0.010575458, 0.036636688, 0.027983334, 0.0002378311, 0.023765601, 0.009065791, -0.003052409, -0.050021376, -0.012653198, -0.030753652, -0.009649426, 0.040963367, -0.013587013, 0.027049517, 0.018131582, -0.01643515, -0.005532856, 0.0058052186, -0.01981245, -0.011929492, -0.038597703, -0.00971168, -0.07383367, 0.0038753338, 0.035765126, 0.00776234, 0.014816537, 0.0009138745, 0.00035650347, 0.0032197174, -0.03364848, 0.01859849, -0.01487101, 0.018925324, -0.049149815, -0.010552114, 0.024948433, 0.011330293, -0.05285395, 0.0123730535, 0.012069563, -0.010559895, -0.046472877, -0.006478344, 0.006941361, -0.030722525, 0.0051282025, 0.0011945055, 0.0066767796, -0.05724288, -0.0018783307, -0.007575577, 0.0034278806, 0.020326048, -0.024388144, -0.005949182, 0.048745163, -0.02272284, 0.016092751, -0.0073732506, 0.04644175, 0.0041866056, 0.0057818736, 0.035422727, -0.036169782, -0.027781006, 0.01912765, -0.01461421, 0.0053227474, -0.033057064, 0.011828328, 0.029944345, -0.0061087087, -0.015882643, 0.027998896, -0.0051787845, -0.051982388, -0.047064293, 0.025679922, 0.022536077, 0.024606034, 0.012956688, -0.0029940454, -0.033555098, -0.04721993, 0.039095737, -0.0041943872, -0.035391603, -0.025742177, -0.0117271645, 0.009610516, 0.009742807, -0.0070269606, 0.0067234705, 0.032060992, 0.0070347423, -0.013555886, 0.014061702, 0.021213172, -0.0057429643, -0.00962608, -0.029275112, 0.013781558, 0.049180944, 0.028434677, -0.0022859022, -0.018193835, -0.015112245, -0.008256484, 0.05416129, -0.038006283, -0.04392045, -0.036200907, 0.028886022, 0.026333593, -0.0003521262, 0.004241078, -0.04613048, 0.02575774, -0.012450871, -0.0044083865, -0.047282185, -0.00015478476, -0.03707247, -0.00802303, -0.009260336, -0.04522779, -0.016139442, 0.027858825, -0.03165634, 0.0018520671, -0.002050503, -0.05098632, 0.018224962, 0.0029512455, 0.038006283, -0.03134507, 0.0151745, 0.017851437, 0.041710418, -0.034582295, -0.012279672, -0.032278884, -0.03841094, -0.03473793, -0.017306712, 0.04961672, 0.005560092, -0.043235652, 0.0367612, 0.028808204, -0.002116648, -0.0239368, 0.011260257, -0.031204997, -0.0029240092, 0.016186133, 0.01981245, 0.049990248, 0.033804115, -0.0034045351, -0.00459515, 0.008124193, 0.004851949, -0.008225357, 0.043515794, 0.017011004, -0.017026566, -0.009524916, 0.009524916, -0.032963682, -0.0015018864, -0.040776603, -0.018878633, -0.019298851, -0.0029843182, 0.027500862, -0.0332127, 0.0017479856, -0.012015091, 0.013213487, -0.032496773, 0.007801249, 0.00065123895, 0.013695958, -0.0151745, 0.019018706, -0.04413834, -0.01569588, -0.005832455, -0.008653356, 0.005906382, 0.008458811, -0.0014843773, -0.0012567599, -0.025353087, -0.010155242, 0.01530679, 0.03039569, 0.03483131, -0.004960894, -0.013376905, 0.038784463, -0.019407796, -0.0028559186, 0.03333721, -0.02129099, -0.047064293, -0.0070580877, -0.042083945, 0.016357332, -0.013018942, 0.0020835756, -0.006890779, -0.003885061, -0.008264266, -0.017415656, -0.017913692, -0.0011244693, -0.0012392508, 0.022629458, 0.0006108709, -0.00849772, 0.019625686, -0.035111457, 0.0007378114, 0.014909918, -0.011898364, 0.000956188, 0.013516977, 0.007999685, -0.01166491, -0.006174854, 0.013081197, -0.0025582649, 0.00007003615, -0.029819837, 0.01244309, 0.011198003, -0.006478344, 0.026987264, -0.008855682, -0.009283681, 0.0379129, 0.025695486, -0.0239368, -0.0022061388, 0.008256484, -0.011524837, -0.034488913, -0.008396557, 0.016684167, 0.008256484, -0.013104542, -0.012731016, -0.0031282813, -0.006190418, 0.0025349197, -0.022598332, 0.022738405, 0.02813897, 0.014653119, 0.023485456, -0.011781637, 0.0019259942, 0.0028131187, -0.021929096, 0.026909446, 0.024201382, -0.0091903, -0.031220559, -0.016793113, 0.017633546, 0.02575774, 0.00997626, 0.0033111535, -0.048589524, 0.01227189, 0.024823925, 0.028107842, -0.011571528, -0.02974202, 0.01574257, 0.005303293, -0.003101045, -0.0026205194, 0.0023792838, -0.022582768, -0.003169136, 0.01964125, 0.0010894513, 0.000083836676, -0.012614289, -0.016014934, 0.026551483, -0.053694382, -0.02007703, 0.011299166, -0.011221347, -0.020668447, -0.013081197, -0.011384766, 0.03707247, 0.0063188174, -0.0038753338, 0.022645023, 0.0038150249, -0.036200907, 0.0053733294, -0.0070269606, -0.009742807, 0.008661137, 0.01114353, 0.01175051, 0.017462347, 0.006863543, 0.008046376, -0.049149815, 0.022411568, -0.004233296, 0.060822506, 0.00034434441, 0.009657207, -0.018754125, 0.013641486, 0.050270393, 0.0013841867, 0.0015203682, 0.017851437, 0.001287887, 0.024123562, -0.0066845617, 0.017493473, -0.0038344793, -0.005563983, -0.007859613, 0.021960225, 0.008318738, -0.023796728, -0.021540007, 0.004914203, 0.017322274, -0.013983885, -0.008583319, -0.011742728, -0.06132054, 0.027734315, 0.028341295, -0.04149253, -0.026224649, 0.019625686, -0.002346211, -0.012419744, 0.0010466514, -0.017291147, 0.0025738284, 0.02701839, -0.025213014, 0.000101953665, -0.012030655, -0.012388617, -0.028310169, 0.0122251995, -0.019719068, -0.006824634, 0.031064924, 0.009081354, -0.003976497, -0.0036068617, 0.0017470128, 0.008155321, 0.01726002, 0.00019296419, 0.021010846, -0.01760242, -0.005276057, -0.036698945, 0.030068854, -0.00089782453, -0.010816694, -0.0042994414, -0.004326678, 0.02020154, -0.0010777785, 0.016886495, -0.0399673, 0.011937273, 0.02497956, 0.03343059, -0.011019021, 0.014061702, 0.00013922117, 0.002025212, 0.011696037, -0.0010369241, -0.025446469, 0.021648953, 0.020964155, 0.014123958, -0.012334145, -0.026333593, -0.032060992, 0.028325733, 0.03221663, -0.011384766, 0.0036088072, -0.0032411173, 0.005560092, -0.026504792, 0.005330529, 0.009143609, -0.01881638, -0.016092751, -0.012310799, 0.03828643, 0.03168747, -0.024683852, 0.03476906, -0.037103597, 0.0054744924, -0.032154374, -0.011649347, -0.04656626, 0.02094859, 0.030131109, -0.015758134, 0.037010215, -0.02306524, -0.017726928, 0.012256326, -0.03265241, -0.0066767796, 0.01053655, -0.014061702, -0.020808518, -0.009898443, -0.03367961, 0.018754125, 0.014544174, 0.00750165, 0.03750825, 0.016014934, 0.0033597897, 0.0040620966, -0.011431457, -0.0084432475, 0.008077503, -0.008202012, -0.0022800658, 0.05761641, -0.019049833, -0.028932713, -0.011594874, 0.008770082, -0.049149815, -0.025228579, -0.03333721, -0.038659956, 0.034022007, -0.032932553, 0.02558654, -0.03243452, -0.00020852778, 0.021929096, -0.039313626, 0.02879264, 0.06910234, 0.024917306, 0.0015057772, 0.03486244, 0.014186212, 0.023874545, -0.010373132, 0.0016166678, 0.036200907, 0.0007256523, -0.019843576, 0.025135197, 0.01825609, 0.015446862, -0.012100691, -0.019936958, -0.037663884, 0.0027547553, -0.041648164, 0.03073809, 0.019236596, 0.02497956, 0.013151233, 0.0040815515, 0.0025329741, -0.009026881, 0.0050854026, -0.026209084, -0.00031808086, -0.0084432475, -0.008723392, 0.009065791, -0.017617984, -0.021664517, -0.0121240355, -0.04410721, 0.0128243975, 0.068355285, -0.012567598, 0.013540323, -0.006217654, -0.034146514, -0.022240369, -0.027858825, 0.00979728, 0.0017265857, 0.020964155, 0.0138204675, 0.0055523105, -0.0032274993, 0.021041973, -0.010443169, -0.00362048, 0.0022119752, -0.0027819916, 0.0021866842, 0.00048393034, -0.0047391127, -0.020357175, 0.01660635, -0.012256326, 0.061694067, -0.0091358265, 0.036947962, 0.034364406, -0.035920765, 0.018800816, 0.010645495, 0.026069012, -0.022971857, -0.028201224, -0.0058869277, -0.015330135, 0.0113769835, 0.02150888, -0.0020796845, -0.042862125, 0.033710733, -0.02354771, -0.002346211, 0.0017304766, -0.005567874, -0.009851752, -0.020621756, 0.0107310945, 0.029664202, 0.017322274, -0.02029492, 0.038908973, 0.00087982917, -0.00268861, 0.031967614, 0.020170411, -0.028387986, 0.014061702, 0.0044122776, -0.014520829, -0.005455038, -0.01075444, 0.0011400329, -0.016933184, -0.01036535, 0.02566436, 0.0069919424, 0.002501847, 0.030115545, 0.025851121, 0.019003142, 0.0057468554, -0.010116333, 0.003499862, 0.004404496, -0.038628828, -0.051079698, -0.016684167, 0.02701839, -0.0071475785, -0.010520986, -0.02406131, 0.013976103, -0.005381111, -0.014520829, 0.021088663, -0.015112245, -0.061507303, 0.009703898, -0.010108551, -0.037881777, 0.015625844, 0.024388144, 0.0040309695, 0.02588225, 0.030426817, 0.017835874, -0.012264108, -0.0074082683, 0.004447296, -0.012606507, -0.02913504, -0.015680317, -0.0041204602, -0.022349315, 0.010855603, 0.025539849, -0.004400605, 0.030411253, -0.01461421, -0.013649268, 0.01036535, 0.005980309, 0.0056962734, -0.006447217, -0.0011731056, -0.009470444, 0.014606428, -0.009384844, 0.001058324, 0.027547553, 0.00620209, 0.04504103, 0.012606507, 0.024855051, 0.034053132, 0.03685458, -0.009252554, -0.021213172, 0.00028476503, -0.023298694, -0.023143057, -0.016621914, 0.01301116, 0.008598883, 0.027002826, 0.005229366, -0.0046340586, 0.042488597, 0.02454378, -0.0022839566, -0.021664517, 0.005318857, 0.008256484, -0.019874703, 0.004478423, -0.005844128, 0.0110268025, 0.02449709, -0.0023637202, -0.031298377, 0.0011098784, -0.010318659, -0.01227189, 0.00066145253, -0.009205863, -0.010777785, 0.0103342235, 0.018271653, -0.048931923, 0.019112088, 0.016979875, -0.0032469537, 0.05151548, -0.021991352, 0.0382553, -0.0066495435, -0.0032566811, -0.02350102, 0.015034427, -0.026675992, 0.016201695, -0.020917464, -0.000483444, 0.011462583, -0.027936643, -0.010030733, 0.01448192, 0.0015213408, 0.017851437, -0.0065367073, 0.046753023, 0.00017107789, 0.025430905, 0.015197845, -0.007061979, 0.020232666, -0.042177327, 0.00046058497, -0.010559895, 0.010801131, -0.00083216565, -0.008217575, 0.011789419, -0.00035528757, 0.024823925, 0.008995755, -0.020403866, 0.0075561227, 0.02471498, 0.0010067697, -0.010513204, -0.00724096, -0.01738453, 0.015921552, -0.010925639, 0.016326206, -0.014334066, -0.028979404, 0.02068401, -0.009968479, -0.08379436, -0.029991036, 0.015376826, -0.034333277, -0.02558654, 0.002632192, 0.030753652, -0.01331465, -0.0034726257, 0.011812764, 0.006742925, 0.002501847, 0.004241078, 0.03364848, 0.018396161, 0.006054236, -0.023360947, -0.018178273, -0.008746737, 0.012481999, 0.010917858, 0.021975787, 0.043235652, -0.004700204, 0.016014934, -0.0017470128, -0.0020193758, 0.0057896553, 0.020808518, -0.009773934, -0.014800973, -0.017135512, -0.0030329544, -0.015983807, -0.013797122, 0.0033014263, -0.0015942951, 0.0025718831, 0.039313626, -0.020715138, -0.012349708, -0.0073421234, 0.0015193954, -0.0011312783, 0.022660585, 0.014209557, 0.008116412, -0.0153612625, -0.006665107, -0.01587486, -0.0008088203, 0.009400408, -0.0029065004, -0.024761671, 0.04015406, 0.0008093066, 0.0066028526, -0.031718593, -0.01183611, -0.002441538, 0.064682275, 0.025617668, -0.0134702865, -0.037881777, 0.0050659482, -0.005155439, -0.014497483, 0.0062565627, 0.030504635, -0.04192831, 0.018878633, 0.020154849, -0.02228706, -0.004809149, -0.026878318, 0.023127494, -0.010816694, 0.013913848, 0.020792956, 0.0077817948, -0.0075639044, 0.022785094, 0.014746501, 0.022738405, -0.008653356, -0.015937116, -0.04622386, -0.011252475, -0.017057694, -0.009213645, 0.0301778, 0.0037975158, 0.017913692, -0.0053110747, 0.002801446, 0.023080803, -0.012513125, 0.03126725, -0.0071397964, -0.0077740126, -0.011688256, 0.001242169, 0.017944818, -0.008256484, 0.016839804, -0.022816222, -0.0002582583, 0.0009766152, -0.026956137, 0.010303096, -0.02913504, -0.009268117, 0.005264384, -0.005525074, 0.011073493, 0.0110812755, 0.003955097, 0.016590785, 0.0030037728, 0.027858825, 0.0063032536, 0.019003142, 0.012909997, -0.017898127, -0.0072915414, 0.010886731, -0.0011273875, -0.020606192, 0.004311114, 0.00620209, -0.024450399, 0.02580443, 0.0051632207, -0.0019347487, -0.02974202, -0.0030232272, -0.015711443, 0.043173395, 0.017306712, -0.022909604, -0.0009853698, 0.029275112, 0.0030601907, 0.033368334, 0.02068401, 0.0016847586, -0.021259863, 0.0029317911, -0.007809031, 0.003793625, 0.028107842, 0.026426975, -0.002046612, -0.0066145253, 0.015415736, 0.028994966, 0.020824082, 0.005828564, 0.017400093, 0.011703819, 0.019189905, 0.007493868, 0.0005539665, -0.01027975, 0.013127888, -0.03070696, 0.031127179, -0.013773777, -0.0006726389, -0.0058363457, 0.014738719, 0.0064666714, 0.012886652, 0.0005807164, 0.014349629, -0.00006353106, 0.042208456, 0.011213566, 0.021026408, 0.017275585, -0.043329034, 0.043266777, -0.001146842, -0.03039569, -0.008287611, 0.05372551, 0.014676465, -0.012022872, 0.008186448, -0.010676622, -0.016715296, 0.011703819, 0.034986947, -0.010318659, 0.0010184423, -0.006711798, -0.018489543, 0.027998896, 0.012980034, -0.0029317911, -0.021399936, -0.03066027, 0.03109605, 0.00863001, 0.0101863695, -0.0036496616, -0.017291147, 0.011890582, 0.008731173, -0.010684404, -0.012676544, -0.021695644, 0.00967277, 0.010077424, 0.020154849, 0.014458574, -0.00044283274, -0.006805179, 0.012287454, 0.0010505422, 0.008598883, -0.008069721, -0.022613894, -0.007797358, -0.015618062, 0.017758055, 0.013306869, 0.032496773, 0.024948433, 0.00048295764, -0.010178587, 0.0074821953, -0.009096918, 0.0047079856, 0.00028063098, -0.0015573316, 0.0016594677, -0.0027528098, 0.03950039, 0.0002623924, 0.00641609, -0.018972015, 0.0011711601, -0.009999606, 0.010349787, -0.017057694, -0.010606586, -0.008287611, -0.015026646, 0.03395975, -0.0061515085, 0.011252475, 0.016295077, -0.016279515, 0.007707868, 0.008762301, -0.019968085, 0.001044706, 0.0024979562, -0.0036885706, 0.0021652842, -0.020761829, 0.009594953, 0.021182045, -0.013431378, -0.013073415, -0.009081354, 0.0052099116, 0.029275112, -0.020668447, 0.018287217, -0.010038515, 0.020434992, -0.042986635, 0.009859534, -0.0032255538, 0.0010923694, 0.0016662767, 0.009805061, 0.0207307, -0.004704095, -0.014186212, -0.014692028, 0.020995282, -0.024263635, 0.005918055, 0.022847349, -0.03716585, -0.0027236282, -0.00040076242, 0.0035309894, 0.022769531, -0.00018056195, 0.0094860075, 0.014458574, -0.0015835952, -0.017509038, -0.012785489, 0.022660585, -0.0068090702, 0.022084733, 0.018629616, 0.017228894, -0.012925561, -0.015166718, -0.0002465856, 0.007855722, 0.0010009333, -0.011166875, -0.00650558, 0.007894631, -0.0027703189, -0.0064199804, -0.026847191, -0.0268005, 0.010824476, -0.0021322118, -0.01638846, 0.01895645, 0.009789498, 0.0075522317, 0.019112088, 0.0046457313, -0.017119948, 0.00088372006, 0.016450714, -0.010287533, 0.015384608, -0.020715138, -0.0018754125, 0.0064705624, 0.0015057772, 0.0027314099, -0.030940415, 0.0027567006, -0.008910155, 0.018660743, 0.015182281, -0.013555886, 0.0016400132, 0.018458417, 0.031469576, -0.004661295, -0.038099665, -0.032621283, 0.012731016, 0.009906225, -0.013143451, -0.017929254, -0.008396557, 0.009260336, -0.019392231, -0.026395848, 0.016668605, 0.0017781401, -0.0008068748, 0.0004318896, 0.010054079, -0.012738798, 0.000004384859, 0.009143609, 0.0003110286, 0.028061152, 0.002801446, -0.009945134, 0.014933264, -0.009415971, 0.006665107, 0.009851752, 0.011252475, 0.00053159386, 0.01726002, -0.009275899, -0.0050698393, -0.00728376, 0.0039181337, -0.0063849622, -0.008124193, -0.00051019393, -0.0075483406, -0.0133224325, 0.03126725, 0.010785568, -0.042924378, -0.01331465, -0.017882563, -0.0017742491, 0.010777785, 0.009198081, 0.017306712, -0.028870458, -0.007540559, -0.00028816957, 0.008038593, -0.0017518766, 0.0073304507, -0.003937588, -0.010178587, 0.0075172135, 0.012497562, -0.010022951, 0.0076339403, -0.016310642, 0.018240526, 0.033897497, -0.036792323, -0.008661137, -0.0062409993, -0.0047585675, -0.005871364, -0.013260178, -0.015096681, 0.00082973385, -0.008256484, -0.017944818, 0.020637318, -0.024372581, -0.007462741, -0.019578995, 0.013337996, -0.0010106605, -0.007202051, -0.019921394, -0.0072137234, 0.005676819, -0.006132054, -0.014240684, 0.017228894, 0.02155557, 0.019501178, -0.008357648, -0.020824082, 0.0034317714, 0.011555965, 0.015376826, 0.026286902, -0.0077117584, 0.009960697, 0.03100267, -0.009618298, -0.0055056196, -0.02762537, 0.0037158069, -0.011633783, 0.0032236085, 0.009065791, -0.010777785, -0.0027528098, -0.007030851, 0.02519745, 0.015509116, -0.010342005, 0.011050148, 0.021306554, 0.003005718, -0.0055834376, -0.00650558, 0.009735025, -0.023360947, 0.02206917, -0.0023695563, -0.004848058, 0.0028267368, 0.0026205194, 0.015135591, 0.004089333, 0.008700047, -0.024372581, -0.009244772, -0.016917622, -0.012637635, 0.010692186, -0.02463716, -0.022613894, 0.023360947, 0.022302624, 0.0052099116, 0.00072419323, 0.008365429, 0.005571765, -0.020855209, 0.032247756, 0.0077584493, -0.013190142, -0.031033797, -0.029166166, 0.010567677, -0.009034663, -0.017104385, -0.019003142, -0.010785568, 0.013750431, -0.008357648, 0.010917858, 0.0033928624, -0.019796886, -0.022162551, -0.0007791522, -0.0054900562, -0.0056223464, 0.005104857, -0.0049025305, -0.00719816, -0.009112481, -0.0036671706, -0.016575223, 0.00819423, 0.011283603, 0.0028111732, 0.0008370293, 0.016621914, -0.01105793, -0.014193993, 0.0062721265, -0.0014036412, 0.008248703, -0.014995518, 0.0006964706, -0.013672614, -0.014474138, -0.03243452, 0.015501335, 0.004431732, 0.017680237, 0.006700125, 0.008466592, 0.0147153735, -0.0069219065, 0.017960383, 0.010489859, 0.04743782, -0.04258198, 0.0036652253, -0.0020757937, 0.033492845, -0.013369123, 0.010030733, 0.00997626, 0.01738453, 0.0205595, -0.019454487, 0.0014872955, -0.014489702, 0.0050153667, 0.035484985, 0.042519726, -0.0026730464, 0.008038593, 0.03070696, -0.012015091, 0.02480836, -0.031064924, -0.009657207, 0.0150655545, 0.004536786, -0.013649268, -0.033897497, -0.0051437663, 0.006128163, 0.026489228, -0.014902136, -0.006665107, 0.013439159, 0.022613894, 0.013664831, -0.02371891, -0.0064705624, 0.015548025, 0.018318344, 0.00056953006, 0.013765994, -0.013057851, -0.01378934, 0.0027528098, 0.007050306, -0.019267723, 0.017337838, 0.029586382, 0.007244851, 0.007921867, 0.015625844, 0.003525153, 0.0014192048, 0.00011709169, -0.0022508842, -0.010933422, 0.017571293, 0.027282972, -0.015789261, -0.002050503, -0.012894434, 0.01097233, -0.00060163, -0.032060992, 0.014170648, -0.001979494, -0.015197845, 0.020248229, -0.003499862, -0.009524916, -0.0055211834, 0.013532541, -0.010684404, -0.0022431023, -0.003213881, 0.00654838, 0.003933697, -0.002289793, 0.013018942, 0.00088907004, -0.013252396, -0.017228894, -0.013501413, 0.01417843, 0.0063421624, 0.005198239, -0.03209212, 0.008949064, -0.0039025699, 0.028590314, -0.0047974763, -0.024139127, -0.014349629, 0.003005718, 0.0029512455, 0.016419586, 0.0042138416, 0.007202051, -0.019236596, -0.013229051, -0.022567205, 0.0023870654, 0.0150889, 0.012194072, -0.0039726063, 0.022862913, -0.022536077, 0.022006916, -0.00023345384, -0.0045640226, -0.022753967, -0.005501729, -0.011384766, -0.004883076, 0.027609807, 0.0043383506, 0.014372975, 0.008575537, -0.011493711, -0.0020135394, 0.0064939074, 0.01105793, -0.013516977, 0.0007212751, 0.003525153, -0.0032625173, -0.009875097, 0.012092909, -0.008272048, 0.0050970754, -0.0036360435, 0.0064238715, 0.015905987, 0.00067701616, -0.004626277, -0.026940573, -0.023392074, 0.007124233, -0.00455235, 0.014909918, -0.009906225, -0.0009124154, -0.0024862834, -0.005676819, -0.02502625, -0.012544253, 0.0150889, 0.0066145253, -0.019874703, 0.0019960303, 0.0056106737, -0.007120342, -0.0031924811, 0.0031749723, -0.009937352, 0.004167151, 0.009805061, -0.0239368, -0.009096918, 0.0016468223, 0.009587171, 0.008840119, -0.007836267, -0.006365508, -0.009244772, 0.0151745, -0.020528374, -0.007661177, -0.004089333, 0.0152134085, 0.0056612557, -0.014925482, -0.015493553, -0.0008686428, 0.0036652253, -0.009805061, -0.0119761815, 0.012894434, 0.0061515085, 0.011221347, 0.0183806, 0.02796777, 0.014147302, 0.0028520278, 0.016279515, -0.014248466, 0.0015816498, -0.028170096, -0.01721333, -0.0034628985, 0.0068596518, -0.0065250346, -0.0023481564, 0.0027742097, -0.0067545976, 0.019298851, -0.0027139008, -0.0032430629, -0.020372739, 0.024263635, 0.019921394, -0.02957082, -0.016108315, -0.0038889518, 0.0021477754, 0.0064199804, 0.034333277, -0.02701839, -0.021820152, 0.00003137036, -0.015236754, 0.012466435, 0.013906067, -0.008785646, 0.0012227144, -0.005194348, -0.005560092, -0.011999527, 0.0024337564, 0.014271812, 0.015182281, -0.003680789, -0.012038436, 0.0060036546, 0.0084899375, -0.04544568, -0.019547869, -0.00385977, 0.006291581, -0.0054706014, 0.009851752, 0.010738877, 0.004241078, -0.0006429708, -0.003560171, -0.0015407953, -0.02840355, 0.0102330595, 0.00001182468, 0.005991982, 0.0060075456, -0.014225121, -0.011415892, 0.0076728496, -0.020154849, -0.015260099, -0.013532541, -0.0010223333, 0.007976339, 0.021633388, 0.00017278016, 0.0051165298, -0.014443011, -0.01934554, -0.015789261, -0.003373408, -0.016466277, -0.00019867894, 0.00047906672, 0.005649583, 0.008808992, 0.009065791, -0.014287375, -0.007879067, -0.00002994167, -0.015968243, 0.002254775, -0.01956343, 0.0003766875, 0.0091358265, 0.011781637, 0.01478541, -0.0011760237, -0.010217496, -0.009065791, 0.01478541, -0.014279593, -0.0028714822, 0.0125753805, 0.00092068355, 0.016061625, -0.010396478, -0.0072487416, -0.00685187, 0.0071864873, -0.0010301151, -0.00039687153, 0.017415656, 0.015789261, -0.011781637, -0.003910352, -0.01825609, -0.008054158, 0.019189905, -0.013073415, -0.013999448, -0.0033383898, -0.0049920212, 0.0069919424, -0.016248386, 0.0007645613, -0.0041943872, -0.005034821, -0.013415813, 0.02675381, 0.019236596, 0.0049453303, -0.0019892212, -0.0050231484, 0.0041476963, -0.0052371477, -0.00048174174, -0.0034512258, 0.01859849, -0.0020349394, -0.017633546, -0.024621598, 0.00455235, 0.019750195, 0.016544096, 0.010715531, 0.014435229, 0.019065397, 0.0036243708, 0.021804588, 0.001097233, 0.010692186, 0.0011273875, 0.012419744, -0.004731331, 0.013501413, -0.0030135, 0.012715452, -0.024606034, 0.003750825, -0.013392469, 0.014123958, -0.005602892, -0.025866685, -0.0035504438, -0.009478225, 0.009805061, -0.01912765, -0.011236912, -0.009587171, 0.006598962, -0.0052099116, 0.011268038, 0.027127337, -0.017151074, 0.012435308, 0.009431535, -0.008155321, -0.008910155, 0.0070386333, -0.029243983, -0.009034663, -0.0052176933, -0.0034667894, -0.006490017, 0.02532196, -0.012155163, 0.02150888, -0.0049064215, 0.021477753, 0.014808755, 0.009182517, -0.006174854, 0.0041515874, 0.013026725, -0.017011004, 0.006805179, 0.004925876, 0.016761985, 0.011906146, 0.00077282946, 0.0022353204, -0.016186133, -0.004766349, -0.007964667, -0.026613738, 0.0096338615, 0.0042605326, 0.003943424, 0.0143651925, -0.0021516662, -0.026302466, 0.0015592771, -0.005902491, -0.020154849, -0.00019916531, 0.0036302072, -0.0058091097, 0.02597563, -0.024621598, -0.014800973, 0.022567205, -0.017633546, -0.006326599, 0.009096918, 0.0004681236, -0.025555413, -0.00863001, 0.027329663, 0.023874545, -0.011587092, 0.014038358, 0.002892882, -0.027563116, 0.012341926, -0.012147381, 0.021446627, 0.018302782, 0.03095598, 0.008474374, -0.0032625173, 0.015921552, 0.008054158, 0.016108315, 0.008357648, -0.0055134012, 0.015112245, -0.010046297, -0.013042288, 0.004361696, -0.0012071509, 0.0027547553, -0.0003200263, -0.011050148, 0.016637476, -0.012987815, 0.018458417, 0.013003379, -0.005451147, -0.020964155, -0.018894197, 0.0034492805, -0.021104228, 0.024123562, -0.0032294448, 0.0063771806, 0.005272166, 0.0019075124, 0.01318236, -0.021399936, 0.007124233, 0.009143609, -0.00011812521, 0.009447099, 0.017493473, -0.008451029, -0.0037041341, -0.0129722515, 0.001608886, -0.0017265857, -0.009540481, -0.0021322118, 0.012559816, 0.030520199, 0.0023637202, 0.005910273, -0.015026646, 0.009851752, -0.012941125, 0.008816773, 0.0012343872, -0.00033753534, -0.016186133, -0.023454329, -0.015462426, -0.0015116136, -0.032060992, -0.009812843, 0.012427526, -0.01144702, 0.0044823135, 0.009501571, 0.011353638, -0.020761829, 0.010194151, -0.010030733, -0.01357145, -0.003217772, -0.011392547, 0.0046029314, 0.009003537, 0.01591377, -0.0064705624, -0.01084004, -0.008793428, -0.0027586462, -0.020792956, 0.0049103126, 0.017026566, 0.004058206, 0.008131975, 0.005186566, -0.019936958, -0.01409283, 0.008163103, 0.00971168, -0.0147153735, 0.004447296, 0.0039570425, 0.029166166, 0.010318659, -0.00975837, -0.00012706211, 0.010124114, -0.013104542, -0.0060931453, 0.0077428855, -0.0061631813, -0.0047546765, 0.0003766875, -0.010801131, 0.0029181729, -0.0059375092, -0.018676307, 0.009096918, -0.00715536, 0.0023053566, 0.014583083, 0.026333593, -0.00039687153, -0.013088979, 0.023189748, -0.0021866842, 0.0013180415, -0.028481368, 0.0002403845, 0.012349708, 0.0037916794, 0.0138671575, -0.008754519, -0.009065791, 0.0066845617, 0.016139442, 0.01252869, -0.0059375092, 0.00849772, 0.0057818736, -0.00424886, 0.004132133, -0.0042722053, -0.018473981, 0.0090191, -0.0010330332, 0.008202012, -0.027998896, -0.004883076, 0.0060347817, 0.0070969965, 0.036449924, 0.012637635, -0.02051281, -0.0072526326, 0.0060347817, 0.020139284, 0.003079645, 0.00034628986, -0.014295157, -0.010271969, 0.005077621, -0.0063460534, 0.012590944, 0.0053538745, 0.008567756, -0.0016886494, -0.0037994613, -0.013727086, 0.008155321, -0.007875176, -0.0026380285, -0.009330371, -0.0019026488, -0.019501178, -0.0026010647, 0.004318896, 0.020481683, 0.005618456, 0.021524444, -0.01973463, 0.004252751, -0.007696195, -0.00949379, 0.009221426, -0.013579232, 0.008910155, 0.00028476503, -0.002350102, -0.0056534735, -0.007836267, -0.012598725, 0.017244456, -0.012100691, -0.006100927, -0.009649426, -0.014824319, -0.01665304, -0.023220874, -0.004848058, -0.00019332896, 0.008855682, 0.0029279003, 0.010248624, -0.008832336, 0.0031166088, -0.006213763, 0.007536668, 0.016404023, -0.0085911015, 0.0125753805, 0.008061939, 0.009587171, -0.032465648, 0.0013277687, -0.016979875, 0.008451029, -0.0091358265, 0.030504635, 0.006205981, -0.006777943, 0.009073572, -0.019750195, 0.00028427868, -0.008061939, -0.003885061, 0.004704095, -0.018411726, 0.006490017, -0.0005923891, 0.0050892937, 0.011112403, 0.016933184, -0.019890267, -0.0012801052, 0.00032051266, -0.0030776998, -0.010995676, 0.0074744136, -0.009563825, 0.0006775025, -0.0007120342, -0.0148632275, 0.0059569636, -0.005229366, 0.0059880908, -0.035360474, -0.0008598883, 0.0010709695, 0.005910273, 0.008318738, -0.02311193, 0.03078478, 0.011159093, 0.009143609, -0.007209833, 0.0013978048, 0.013913848, 0.008544411, -0.0016662767, 0.007120342, 0.01279327, -0.0024395925, -0.015719226, -0.017617984, -0.0044239503, 0.011221347, 0.022427132, 0.023563273, -0.018722998, 0.0071903784, -0.0016643313, -0.0020602301, -0.014551956, 0.005268275, -0.00008845712, 0.01227189, 0.00871561, -0.005980309, 0.010567677, -0.007956885, -0.0077506676, -0.020154849, -0.008030812, -0.012668761, -0.004209951, 0.004972567, -0.0062409993, 0.0014639501, 0.00053791655, -0.005264384, 0.0109801125, 0.006174854, -0.0058674733, -0.01066884, 0.007388814, -0.0015213408, -0.000007572811, 0.0045912587, 0.009369281, 0.0040387516, 0.027531989, 0.0021088663, 0.012847743, 0.0028384095, -0.018878633, -0.0060503455, 0.0072137234, -0.01144702, 0.0027158463, 0.0037838977, 0.0029415183, 0.0066884523, -0.021259863, 0.0110268025, -0.0033053171, -0.0041126786, -0.01995252, -0.018240526, 0.002801446, -0.006365508, 0.003680789, 0.0048714033, 0.0053966744, -0.001277187, -0.0072915414, -0.008100848, 0.005276057, -0.0042177327, 0.023952363, -0.010637714, -0.0056573646, 0.03760163, 0.004505659, -0.0033383898, 0.029212857, -0.0060347817, -0.02029492, -0.008505502, 0.0022041933, 0.016310642, 0.0019182124, -0.0065016896, -0.009875097, -0.0022839566, 0.008754519, -0.005785764, 0.0003329149, -0.013291305, 0.00009538778, 0.0055056196, -0.0002507197, 0.009945134, -0.01682424, -0.01981245, -0.001281078, -0.0091047, -0.011454802, 0.0077817948, 0.0077467766, -0.013252396, -0.016310642, 0.005264384, -0.012046218, -0.0039298064, 0.000961538, -0.006190418, -0.007167033, -0.008567756, 0.009828406, -0.010030733, 0.011711601, -0.0037975158, -0.008225357, -0.028387986, 0.005501729, 0.006097036, -0.0072215055, 0.018302782, -0.014404102, 0.007011397, 0.0010456786, -0.016839804, -0.011945055, -0.0031107725, 0.034955822, -0.008824555, 0.000064199805, 0.03265241, 0.029197292, 0.04080773, -0.0057429643, 0.0014153139, 0.009602735, -0.007167033, 0.0068868883, -0.0062332177, -0.04734444, -0.012754361, 0.013906067, 0.002311193, 0.010427604, 0.01478541, -0.0030679726, 0.008676701, -0.0050892937, -0.006209872, -0.0018715216, -0.01799151, 0.01881638, -0.0017363129, -0.018193835, -0.017322274, 0.030971542, 0.015851516, 0.003278081, -0.021742335, 0.006439435, 0.01660635, 0.018302782, -0.010077424, 0.011260257, 0.014380757, -0.010287533, -0.0005632074, 0.0005053303, -0.010629931, -0.008007467, -0.0022489387, 0.0044122776, -0.00962608, 0.016232824, 0.011268038, 0.02194466, 0.023625528, 0.018738562, -0.007754558, -0.015166718, -0.0023442656, -0.0062293266, 0.030598016, 0.017415656, -0.036169782, -0.0039998423, -0.011517056, 0.026302466, 0.010170805, 0.020108158, -0.009999606, -0.019750195, 0.0067623793, -0.031578522, -0.01990583, 0.021415498, 0.037570503, 0.0087389555, 0.014575302, 0.0029804274, -0.007999685, -0.017711364, 0.011415892, -0.014621992, 0.013493632, 0.026209084, 0.007855722, 0.015633626, -0.013999448, -0.0048675127, 0.021150917, 0.018707434, 0.0047196583, 0.026535919, 0.010559895, -0.014738719, 0.0039920607, -0.0073226686, 0.005042603, 0.0076105953, 0.016715296, -0.0029298456, 0.0032975355, 0.0025602104, 0.012217417, -0.017322274, 0.007287651, -0.001040815, 0.015322354, 0.011703819, -0.005692383, 0.027282972, -0.0074471775, 0.025181888, 0.02774988, -0.0031185541, 0.013688177, 0.0048286035, 0.017540164, 0.019454487, -0.0038870063, -0.003877279, -0.0114781475, -0.006929688, -0.0037741703, -0.007540559, -0.021913534, 0.014933264, 0.011198003, -0.009377062, -0.008536628, 0.0017382584, 0.0022178115, -0.009781715, -0.01431072, 0.042613108, 0.013532541, 0.009244772, -0.008731173, -0.007435505, -0.009244772, 0.011999527, -0.0027819916, -0.009875097, 0.026909446, 0.01270767, 0.022629458, 0.028465804, 0.017851437, -0.0038266976, -0.010770003, 0.014660901, 0.030349, -0.019968085, 0.0015952679, 0.005264384, 0.008303175, 0.014038358, -0.009999606, -0.024108, -0.0047507854, -0.014380757, -0.019080961, -0.0051787845, -0.003126336, -0.006217654, 0.025742177, -0.005649583, 0.02658261, 0.009727243, -0.0068285246, 0.010956767, -0.0062993625, -0.031282816, -0.00027430826, 0.020185975, 0.030053291, -0.0067351433, -0.029944345, -0.0023423203, 0.022598332, -0.0069452515, -0.0060386728, 0.009999606, -0.00053791655, 0.01660635, -0.0033383898, 0.01682424, 0.013026725, 0.0033811897, 0.012847743, 0.0034804076, 0.010886731, -0.0032800264, 0.022691714, -0.010847822, -0.00023977655, 0.004929767, -0.012256326, -0.0027333554, 0.020326048, 0.004315005, 0.023578838, -0.019018706, 0.0016516859, -0.0010651331, -0.0026088466, 0.01738453, -0.0056262375, -0.008972409, 0.015812607, -0.00033631944, -0.020419428, -0.014793191, -0.006019218, 0.008941282, -0.012552035, -0.00042483734, -0.009781715, 0.01764911, -0.0033753533, -0.0036652253, -0.007657286, 0.005124312, 0.007832376, 0.000730516, -0.002597174, 0.0062254355, -0.0041827145, -0.002494065, -0.0018501217, 0.027672062, 0.01760242, 0.01934554, 0.023392074, 0.0073654684, 0.01357145, -0.0035660074, -0.007723431, 0.037383743, -0.0069141244, -0.011898364, -0.005493947, 0.0031068814, 0.007466632, 0.00503093, 0.013742649, -0.018722998, -0.013454723, 0.008272048, 0.02974202, -0.005466711, -0.020108158, 0.0004010056, 0.006626198, -0.018131582, -0.012116254, 0.001732422, -0.018551799, -0.0056534735, -0.012941125, -0.0029979364, -0.014108393, 0.01053655, -0.0066495435, 0.010886731, -0.0024785015, 0.0019016761, -0.013906067, -0.0069608153, 0.0077584493, -0.0021672298, -0.016341768, 0.01456752, -0.0038403156, -0.00018165627, 0.014964391, 0.005634019, -0.007688413, 0.021742335, 0.008505502, -0.004929767, 0.0017022676, -0.0073732506, -0.0023831746, -0.0030037728, 0.0016857312, -0.023469893, 0.011493711, 0.012466435, 0.0075950315, -0.010139679, -0.009283681, 0.007225396, 0.0013657049, 0.0027177918, 0.0030971542, 0.020466119, 0.002741137, -0.010801131, 0.008879027, -0.030022163, -0.018722998, -0.0011925601, -0.0038908974, 0.0147153735, -0.008544411, -0.016155006, 0.000210838, -0.010092988, 0.0028364642, 0.026862755, 0.0013861322, -0.009828406, 0.0034590077, -0.006209872, 0.009937352, -0.034986947, -0.00007994578, 0.013244614, -0.008015248, 0.019221032, 0.00029862637, -0.013765994, -0.011322511, -0.0018977851, -0.020419428, -0.014754282, 0.0153612625, 0.00050581666, 0.0147153735, 0.019874703, 0.0022119752, 0.0022392115, 0.015586935, -0.0011108512, -0.021228736, -0.00032610583, 0.028994966, 0.018084891, -0.0039823335, 0.0013180415, 0.0148632275, -0.018489543, 0.01378934, -0.009571607, 0.015851516, 0.014302938, 0.009174736, -0.0041554784, -0.005455038, 0.008482156, -0.0059414003, -0.017291147, 0.024559343, -0.0054706014, 0.0025096287, 0.008544411, 0.02701839, -0.00572351, 0.004536786, 0.0013005324, 0.0019211306, 0.021446627, 0.019267723, -0.012380836, 0.012131818, 0.033897497, 0.009159172, 0.010645495, -0.016808676, -0.010248624, -0.02541534, 0.019361105, 0.009112481, 0.022053605, 0.0045251134, 0.018909762, 0.008404338, -0.019796886, 0.03881559, 0.0034745713, 0.010170805, 0.011322511, -0.016497405, -0.0050231484, -0.012435308, 0.06057349, 0.008544411, 0.0064588897, -0.020279357, 0.011034585, -0.017835874, 0.0039823335, -0.009968479, 0.0076650674, 0.017026566, 0.0008127112, 0.013361341, -0.0036788434, 0.017929254, 0.006190418, -0.006820743, -0.0012392508, 0.022660585, -0.021197608, -0.008233138, -0.004279987, 0.0036146436, 0.009921788, -0.008567756, 0.0076689585, -0.0073654684, -0.019547869, -0.028201224, -0.008186448, -0.006128163, -0.0025174106, 0.023641093, 0.022100296, -0.0030232272, -0.017462347, 0.008762301, 0.02094859, 0.009859534, 0.0033792444, -0.022536077, -0.0023987382, 0.009641644, 0.020061467, -0.028901584, 0.0015835952, 0.010715531, -0.0036632798, 0.00005465495, 0.023034113, 0.016621914, -0.020310484, 0.0052916203, -0.010793349, 0.008746737, -0.011252475, -0.017446784, -0.002208084, -0.0026788828, 0.00034312851, -0.0119684, -0.0077817948, -0.0058480185, -0.022520514, -0.03299481, -0.0041048964, -0.0038208612, -0.007124233, 0.0077117584, -0.0061125997, 0.011198003, 0.009703898, -0.0006784752, -0.013415813, -0.015898207, 0.0018530398, -0.018442852, -0.0033111535, -0.00074413413, 0.023905672, 0.00039419654, 0.015516899, -0.047842473, -0.018894197, 0.026255775, 0.01998365, 0.019267723, 0.0031574632, -0.002025212, -0.015820388, 0.018769689, 0.019236596, -0.014334066, -0.003447335, -0.005692383, 0.028543623, 0.013431378, 0.010295314, -0.013672614, 0.004478423, -0.020403866, -0.010139679, 0.014832101, 0.0055172923, -0.004190496, -0.020870773, 0.00040830104, -0.0074004866, 0.007921867, -0.0009941242, 0.011812764, 0.011812764, -0.02957082, -0.002706119, -0.02150888, -0.006474453, 0.00044331912, -0.020746265, -0.019174341, 0.009244772, 0.009042446, 0.019298851, 0.008598883, -0.018800816, -0.0054706014, 0.005299402, -0.006657325, -0.01934554, -0.007618377, 0.010606586, 0.018349472, -0.012777707, -0.0011876964, -0.024465961, -0.0117271645, 0.056122303, -0.009587171, 0.015952678, 0.027874388, 0.006326599, -0.006731252, 0.008124193, -0.000043043052, -0.046908658, -0.0071008876, -0.017758055, 0.023812292, -0.00014834046, 0.02012372, -0.028434677, -0.008054158, 0.021757897, -0.018287217, 0.012559816, -0.012466435, 0.019221032, -0.0117271645, -0.004774131, -0.0042994414, 0.020886337, 0.0045718043, -0.018333908, -0.015843734, -0.0028209006, -0.0070386333, 0.009828406, -0.0099840425, -0.02068401, -0.02740748, 0.023827855, -0.0048013674, 0.011898364, 0.007894631, 0.018925324, -0.002480447, -0.0035309894, -0.010458732, -0.017166639, -0.01348585, 0.013143451, 0.002120539, 0.015664753, 0.018287217, -0.0037138616, 0.024963997, -0.0026419193, -0.009198081, 0.010116333, 0.00025217878, 0.0051632207, -0.007929648, -0.0051165298, 0.008668919, 0.0013044233, 0.0029026093, -0.0023753927, 0.002840355, -0.01764911, -0.035267092, -0.004015406, 0.00849772, 0.011711601, -0.009159172, -0.002089412, -0.0037488795, 0.0074004866, -0.021711206, -0.022380441, -0.0023598291, 0.008342084, -0.022567205, -0.0055134012, 0.011843892, -0.004778022, -0.013688177, 0.008342084, 0.016186133, 0.0021263754, -0.012590944, 0.020108158, 0.0027644825, 0.006641762, 0.0009805061, 0.014279593, -0.000982938, -0.00388117, -0.0059686364, -0.010497641, -0.0012509235, 0.02172677, -0.0016001315, 0.0014026684, -0.008365429, 0.008482156, -0.0026419193, -0.031547394, 0.04488539, -0.0019979759, 0.0012820506, 0.0045756954, 0.0066884523, 0.013104542, -0.0037838977, -0.009353717, 0.0017333947, 0.014411883, 0.024917306, 0.012279672, 0.0062682354, 0.025602104, 0.010489859, -0.006509471, 0.0034025896, -0.0028812094, 0.00046982584, -0.0053616567, 0.021010846, 0.0028111732, 0.002462938, 0.011773855, -0.0110268025, -0.0065328167, 0.032247756, 0.0046768584, 0.0016361224, -0.039251372, -0.011734947, 0.018287217, -0.0030251727, -0.0037741703, 0.0027080646, -0.01036535, -0.011454802, 0.0036088072, 0.02740748, 0.013018942, -0.007645613, 0.022878475, 0.012349708, 0.025150761, -0.008178666, -0.005859691, 0.004462859, 0.0026399738, 0.0013355504, -0.015711443, -0.019174341, -0.00015843248, 0.009096918, 0.00004149277, 0.013657049, -0.0024084654, -0.016746422, 0.030815907, 0.013081197, 0.009314808, 0.025835559, 0.010209714, 0.009462662, 0.008349866, 0.02588225, 0.014131739, 0.011641565, 0.010209714, 0.0054005655, 0.0014911864, 0.017026566, -0.0021477754, -0.0058052186, 0.012279672, -0.004774131, 0.008069721, -0.013205705, -0.0099840425, 0.0066456525, 0.022489386, -0.010902294, -0.0049492214, -0.022022478, 0.0042177327, -0.0034317714, 0.0024882287, 0.0032294448, 0.009447099, 0.012839961, -0.008956846, -0.021182045, -0.0004985212, -0.017228894, -0.011812764, 0.021197608, 0.011198003, -0.0126220705, -0.0011760237, -0.00006249754, -0.0019814395, 0.0028714822, -0.039780535, 0.007688413 - ] - } - ] - }, - { - "name": "harnessing-technology-for-business-success", - "title": "Harnessing Technology for Business Success", - "content": "Businesses need to leverage the full potential of technology and generative AI to remain competitive. However, keeping up with the rapid pace of technological advancement and identifying key technologies among the daily noise can be challenging.\n\n---", - "keywords": [ - "Businesses", - "technology", - "generative AI", - "competitive", - "rapid pace", - "technological advancement", - "key technologies" - ], - "index": [ - { - "modelName": "text-embedding-3-large", - "position": [ - -0.005866572, -0.00040896775, -0.02123286, -0.023772236, 0.0036981583, -0.017240226, -0.0035604814, 0.0117637385, 0.036683284, 0.019060621, 0.05237847, 0.0040270537, 0.04521926, -0.00055023015, -0.049747307, 0.016062321, 0.003166572, -0.015848158, 0.0062681297, -0.028223794, -0.010379319, -0.036438525, -0.05403059, -0.01667422, -0.037356373, 0.006233711, -0.027963737, 0.01223796, -0.053449288, 0.044117846, 0.037509345, 0.011618413, 0.02495014, -0.035857223, -0.015343342, 0.030441925, -0.009346741, -0.0082759205, 0.01841813, 0.00088725204, 0.046045322, -0.01595524, -0.011587818, -0.021722378, 0.03472521, 0.020406798, -0.004715439, -0.0102798855, -0.009798016, -0.019305382, -0.03802946, -0.005797733, 0.016811896, 0.02752011, 0.0055759205, -0.03184929, 0.0635762, -0.03496997, 0.024368837, 0.03261416, 0.020743342, 0.025715012, -0.031696316, 0.019596033, -0.006107507, -0.019030027, 0.020926911, 0.03561246, -0.024873653, -0.005116997, -0.053112745, -0.0060042487, 0.018494615, -0.017423796, 0.0073274784, -0.020100849, 0.0011109773, 0.0063560903, -0.006466997, -0.0030116853, 0.022808498, 0.024215862, 0.00077730167, -0.009889801, 0.04833994, -0.0023577195, 0.010930028, -0.06614617, 0.019825494, -0.008444192, -0.02788725, -0.012543908, -0.040630024, -0.019060621, 0.021278752, 0.015603398, -0.025332576, 0.009132577, -0.048370536, -0.020804532, 0.017194333, -0.0018509914, 0.02716827, -0.021447023, 0.0069067986, 0.009079036, -0.029615862, 0.031023227, 0.0065205377, 0.03503116, -0.016842492, -0.022318978, 0.02752011, -0.008283569, -0.037631724, 0.023680452, -0.035153538, -0.064004526, 0.019443057, -0.050634556, -0.029233426, -0.02644929, 0.009316146, -0.03197167, -0.015427478, 0.024720678, 0.013125212, 0.0009885976, 0.015381585, -0.0067423508, -0.060241356, 0.07569178, 0.001415014, 0.05090991, 0.0074842772, 0.02752011, 0.036316145, -0.00527762, 0.023053257, -0.011503682, -0.027076485, -0.005866572, -0.024873653, -0.038427193, 0.003694334, -0.0037842065, -0.002153116, 0.06553427, -0.017255522, 0.005602691, -0.053479884, -0.0065702545, 0.024062889, -0.009996884, 0.03016657, -0.03279773, 0.020666854, -0.03396034, 0.0035317987, 0.022028327, 0.04778923, -0.01085354, -0.0052087815, 0.047728043, 0.04029348, 0.0046160053, 0.04479093, -0.020529177, -0.06767592, 0.057793763, 0.0018490792, -0.0019513809, -0.02326742, -0.043444756, -0.02423116, -0.018968837, 0.028636824, -0.011870821, 0.002713385, 0.00049860124, 0.0069947587, 0.007862889, 0.0010784701, 0.00455864, 0.00614575, 0.01319405, 0.06584022, -0.009163172, 0.03184929, 0.013920679, 0.00770609, -0.011052407, 0.011235977, -0.021079885, 0.025210196, -0.06859376, 0.0061381017, -0.032767136, -0.017898016, 0.007243342, 0.0007682188, -0.012559206, -0.013224645, 0.054428324, 0.01673541, 0.019183, 0.008796033, 0.002480099, 0.017714446, 0.005667705, -0.011717847, -0.00875779, -0.0018500353, -0.010386968, -0.00012285764, 0.015305098, -0.0017114021, 0.018127477, -0.033776768, 0.011962606, -0.011366005, -0.011557223, 0.0026541075, -0.025684418, -0.005036685, -0.047299713, -0.016750706, -0.005281444, -0.0049754954, -0.006459348, -0.021095183, 0.009316146, 0.024980735, -0.00983626, 0.01835694, -0.036438525, 0.03184929, -0.039253253, -0.054367136, -0.0074651553, 0.01067762, -0.0039543905, -0.05378583, -0.014211331, 0.06645212, -0.05500963, 0.0063293194, 0.046565436, 0.005530028, 0.041303113, -0.015664589, -0.024797166, -0.022028327, -0.01511388, 0.0028682717, -0.0045854105, 0.026051557, -0.00037478752, -0.004482153, 0.025332576, 0.002650283, 0.01553456, 0.0023672802, 0.052806795, -0.025332576, -0.011702549, 0.040691216, 0.032155238, 0.027061189, 0.013247591, 0.0032851272, 0.022074219, 0.009515014, -0.011021812, -0.0013538243, -0.007480453, 0.028453255, 0.00905609, 0.042557504, 0.022701414, 0.029600564, -0.016490651, 0.07807818, -0.023634559, -0.02854504, -0.012918696, -0.017653257, -0.01751558, -0.044423793, -0.015037392, -0.038304813, 0.011197733, 0.014609064, 0.025133708, -0.017653257, -0.017913314, 0.03922266, -0.009813313, -0.024445323, -0.015786968, 0.0032736543, -0.02566912, 0.010409914, 0.02051388, 0.007335127, 0.0142342765, 0.0076143052, 0.04096657, 0.01511388, 0.011060056, 0.0045739375, 0.005151416, 0.015136826, -0.0011511331, -0.015343342, 0.050420392, 0.011786684, -0.025760904, -0.0019131373, -0.010241643, -0.0146932, -0.008207082, -0.068349004, 0.012972237, -0.009163172, 0.019397166, -0.0066352687, 0.036071386, -0.01577167, -0.006203116, 0.0069182715, 0.003283215, 0.013086968, 0.019703114, 0.021860056, -0.001910269, -0.0054497165, -0.0015173157, -0.006379036, -0.026586967, 0.0033807363, -0.002713385, -0.011503682, -0.014364305, 0.009193767, -0.00668881, -0.023435693, 0.031176202, 0.0052623227, -0.024307648, 0.00092645176, -0.009132577, -0.005889518, 0.011541925, 0.00079546735, -0.0005358888, 0.019183, -0.012612747, -0.012880452, 0.013943625, -0.0008413597, -0.05702889, -0.02848385, 0.03209405, -0.009629744, 0.011939659, -0.011029461, 0.00213017, -0.012406232, 0.02045269, 0.011595466, 0.040568836, -0.008444192, 0.013354673, -0.027994333, -0.011656657, 0.004180028, 0.02423116, -0.009813313, 0.048003398, -0.023649856, -0.007809348, -0.0010124999, 0.0049449005, 0.006386685, 0.014417846, 0.0047804527, -0.04846232, 0.017530877, 0.05675354, -0.028590932, 0.0014408285, -0.017729744, -0.033899147, -0.0033138099, 0.013499999, 0.03129858, 0.0068876767, 0.03579603, -0.014907365, -0.009155523, 0.0001245308, 0.009010198, 0.0028147306, -0.04102776, -0.0034954671, -0.037448157, -0.017668555, 0.0077099144, 0.017882718, -0.02117167, -0.043628324, -0.040691216, -0.026051557, -0.006466997, -0.00078877475, 0.019764304, 0.010562888, 0.01457082, -0.04329178, 0.0016071883, 0.008122945, 0.0401711, 0.016230594, -0.005912464, 0.009155523, 0.03772351, 0.030717278, 0.0005119865, 0.0057709627, -0.019687817, -0.016873086, 0.028330876, -0.0009905099, -0.007166855, 0.0021244332, 0.019886684, 0.008130594, -0.013744758, 0.017882718, -0.044546172, 0.009706232, -0.00004672893, 0.008375353, -0.039008494, -0.0033233708, -0.002606303, -0.012528611, -0.027596598, -0.068716146, -0.022884984, 0.016490651, -0.030472519, 0.0052202544, 0.004570113, -0.03503116, 0.026847024, 0.009889801, 0.028529743, 0.002386402, 0.03579603, -0.016077619, 0.03383796, -0.019825494, 0.024659488, 0.03524532, -0.024690084, -0.011366005, 0.02441473, 0.04466855, 0.009369688, -0.031512745, 0.003797592, 0.018127477, -0.0046045324, 0.03070198, -0.015611047, -0.006669688, -0.0043406514, -0.011243626, 0.01601643, 0.0017553823, 0.017439092, -0.014723795, -0.026128044, 0.012039092, 0.025592633, 0.00039606052, -0.003904674, 0.011679603, -0.017959205, 0.0008666961, 0.030778468, -0.017316712, -0.01745439, -0.015725778, 0.01205439, 0.0033405805, 0.015756372, 0.006692634, -0.038488384, -0.016169405, -0.040324077, 0.0095226625, -0.01055524, -0.011939659, 0.022166004, 0.029309914, 0.0025087816, 0.020299716, -0.039100282, 0.05632521, -0.0057135974, -0.002373017, 0.042863455, 0.055560336, -0.00014233799, 0.020896316, -0.009690934, -0.031941075, 0.017959205, 0.0028376768, 0.03010538, 0.029355805, -0.0055032573, 0.0015699008, -0.008987252, 0.018372236, 0.03209405, -0.026678752, -0.012574503, -0.0074575064, -0.049318977, 0.014624362, -0.055743907, 0.011534276, 0.01733201, -0.023037959, -0.008061755, -0.0058856937, -0.02891218, 0.006019546, 0.008719546, 0.029524077, 0.014127195, 0.0055567985, 0.048645888, -0.024934843, 0.014892067, 0.013316429, 0.020192632, -0.0046810196, 0.011641359, -0.00023711046, -0.017546175, -0.008719546, 0.03191048, -0.032767136, -0.007323654, -0.0455864, -0.015786968, 0.0033769119, -0.0009341005, -0.00079307717, -0.012781019, 0.0000068382883, 0.022196598, 0.021615297, -0.021217562, 0.010348724, -0.021844758, 0.0029428469, -0.004126487, -0.023405097, 0.031023227, -0.0052508493, -0.005331161, -0.03322606, -0.012452124, 0.006692634, -0.020881018, -0.018005097, 0.020192632, 0.037998863, 0.012406232, -0.007839942, 0.0033635267, -0.02537847, 0.017852124, -0.00542677, -0.0017200069, 0.035306513, -0.03184929, -0.016444758, -0.019030027, 0.02901926, 0.020636259, 0.009048441, -0.0003726363, -0.009943342, -0.0008246281, 0.031757504, 0.049471952, 0.009392634, -0.029065154, 0.026158638, -0.015725778, -0.0042832857, 0.017301416, -0.0092779035, -0.0020001414, -0.021814162, 0.019351274, -0.054948438, 0.017959205, 0.0006635269, -0.015297449, 0.018877052, -0.049166, -0.009843908, -0.010562888, -0.0107694045, 0.005025212, 0.009400283, 0.006008073, 0.053020958, 0.01624589, -0.015343342, -0.0032774785, -0.004222096, -0.0063905097, -0.027902547, -0.03928385, -0.00031288064, 0.0032736543, -0.009063738, 0.009362039, 0.049930874, 0.0008910764, -0.009790367, -0.014769687, 0.00875779, 0.0075033987, 0.07489631, -0.0071936254, -0.008390651, 0.001488633, -0.006111331, 0.06226062, 0.014180736, 0.009262606, 0.019397166, 0.0124980165, 0.02591388, 0.0000712885, -0.011947308, -0.0007299752, 0.023512179, 0.012666288, 0.025470253, 0.026510479, 0.023940507, -0.01463201, 0.005621813, 0.008987252, 0.00461983, 0.03340963, -0.01817337, -0.0063713877, 0.025653822, 0.007947025, -0.038366, 0.007816996, 0.015259205, -0.000422353, -0.01768385, 0.03891671, 0.020682152, -0.013446458, 0.009828611, -0.018678186, -0.023405097, 0.010669971, 0.004298583, -0.06290311, -0.0053732293, -0.040354673, 0.0071706795, 0.020773936, 0.0055262037, -0.003451487, 0.0052164304, 0.021033993, -0.03374617, 0.003862606, 0.007159206, -0.0050290367, -0.052194897, -0.024368837, -0.026892915, 0.005174362, 0.000086287175, -0.014127195, 0.010432861, -0.0024820112, 0.01745439, 0.020299716, -0.0018978398, -0.03135977, -0.011327761, 0.023879318, 0.02254844, -0.0075531155, 0.006176345, 0.011167138, 0.021401132, -0.007870537, 0.020743342, -0.051674783, 0.003049929, 0.018601699, -0.01007337, -0.014884418, -0.032124642, -0.0134923505, 0.01817337, 0.019565437, 0.020697448, -0.01829575, -0.0056065153, -0.00845949, -0.020192632, -0.0042182715, 0.012704532, 0.010103965, -0.014004814, 0.0009780807, 0.026342208, 0.016047025, -0.013316429, 0.014555523, -0.0029485833, -0.0048875352, 0.01864759, -0.009124928, -0.03051841, 0.008528328, -0.0015861543, -0.035000563, 0.015343342, 0.018448723, -0.028682718, 0.02614334, -0.024690084, -0.013125212, -0.013538242, 0.01085354, -0.02147762, 0.0050099147, 0.0008671742, -0.0005942103, 0.00055501057, -0.0008088526, 0.017561471, 0.033011895, -0.011710198, 0.009492068, -0.016536543, 0.016169405, 0.019932577, 0.0067270533, -0.017607365, 0.050971102, -0.018968837, -0.02554674, -0.0056830025, 0.022701414, -0.018846458, -0.0010182365, -0.010096316, -0.050389796, 0.0015708568, -0.030396031, 0.025623228, -0.026510479, -0.03429688, -0.002281232, -0.031160904, 0.046565436, 0.026969403, 0.0074575064, 0.009966289, 0.005717422, 0.024445323, 0.009186118, 0.018494615, 0.024552407, -0.02033031, 0.0038033284, -0.013285834, 0.013652974, -0.0032315862, -0.013056373, -0.0049869684, -0.031160904, -0.020559771, -0.0043483, -0.016291784, 0.03490878, 0.00041518232, 0.015259205, 0.0062107644, -0.0050213877, 0.015450424, 0.0067270533, 0.032644756, -0.04084419, 0.021248156, -0.040691216, -0.004803399, 0.011610764, -0.038855523, 0.0044974503, 0.0051131723, -0.046871386, 0.01487677, 0.019427761, -0.011197733, 0.017316712, 0.011503682, -0.0076678465, -0.0033673511, -0.017377902, 0.025485551, -0.0149685545, -0.018020395, -0.015664589, 0.0022544616, -0.008566571, 0.010065721, -0.002585269, 0.028743908, -0.00025240792, -0.004899008, 0.007205099, -0.0019886685, 0.011947308, -0.012322095, -0.0013021954, 0.009629744, -0.0007108534, -0.030396031, -0.008413597, -0.016154107, -0.019137109, 0.024368837, 0.02141643, -0.021798866, -0.0034591358, -0.0019064447, 0.010593483, 0.0233898, -0.0070138806, -0.006195467, 0.016291784, -0.01427252, -0.0055491496, -0.023359206, -0.0009082861, -0.013958923, -0.024934843, 0.009216713, -0.024628893, 0.05121586, -0.0045777615, -0.016490651, 0.0057135974, 0.010065721, 0.0056524076, -0.0015211401, -0.009140226, 0.009407931, -0.00055214233, 0.016551841, 0.0092702545, -0.015970537, -0.03383796, 0.006409631, -0.0045968834, -0.00428711, -0.01985609, 0.037019826, 0.0019121811, -0.02182946, 0.022441357, 0.008803682, 0.002206657, 0.0054994333, 0.031451557, -0.014616713, 0.000034210116, -0.023894615, -0.019290084, 0.0044171386, 0.010463456, -0.022303682, 0.017989801, -0.02470538, -0.003768909, -0.0024016995, -0.018999431, 0.02926402, 0.022058923, -0.020942207, 0.015274503, -0.0043444755, 0.00050146953, 0.01397422, -0.0027172095, -0.03466402, 0.049992066, 0.027627194, 0.019259488, -0.022762604, -0.051766567, 0.023282718, 0.03094674, 0.021814162, -0.009155523, -0.013025778, -0.0056638806, 0.034388665, 0.03184929, 0.0014742917, 0.0082759205, -0.0012084985, -0.023894615, 0.02147762, -0.03567365, 0.017377902, -0.02447592, 0.019947873, -0.02782606, -0.012704532, 0.017836826, -0.0015718129, -0.005920113, 0.02860623, 0.017668555, 0.031573936, 0.0122532565, 0.032889515, 0.02464419, -0.024720678, -0.0022487252, -0.017928611, -0.011044758, -0.014907365, 0.011932011, 0.0142342765, -0.005269971, 0.039773367, -0.0035470962, -0.0030002124, -0.008581869, 0.0050175632, 0.0020211756, 0.010868838, -0.017484985, -0.005254674, -0.0124980165, 0.0009838172, -0.009369688, 0.036775067, 0.032155238, -0.008237677, 0.0010268412, -0.0050634556, 0.00650524, 0.00044792844, 0.03441926, 0.0002502567, -0.022104815, 0.0053732293, 0.01847932, -0.041150138, -0.021003397, -0.007599008, -0.029539375, 0.01517507, -0.034572236, -0.0035834275, -0.018081585, -0.027657788, -0.011343059, 0.019901982, -0.01541218, 0.004034702, 0.0076907924, 0.007369546, -0.017255522, -0.010868838, 0.0028529742, 0.02147762, -0.014012463, 0.029830026, -0.0035222378, 0.03094674, -0.013683569, 0.008230028, -0.02111048, 0.013951274, 0.016169405, -0.03561246, 0.0038339233, -0.0036044614, -0.005855099, 0.0006095077, -0.022992067, -0.06883852, 0.043047022, -0.0033807363, -0.005660056, -0.018142775, 0.013683569, 0.025898581, 0.0070330026, 0.018265154, 0.0067270533, -0.041853823, -0.0045089233, -0.022441357, 0.0052087815, -0.03184929, 0.020835126, -0.009362039, 0.01061643, -0.072754666, -0.018203964, 0.007847591, -0.0038721669, -0.022456655, 0.024399431, 0.04167025, -0.030839657, 0.005445892, 0.007243342, 0.010478753, 0.02260963, -0.0071400846, 0.007671671, -0.00815354, 0.017943908, -0.0115113305, -0.018020395, -0.027489517, -0.0097674215, 0.015075636, -0.017408498, 0.022120113, -0.0029715295, -0.00059277617, -0.000087840825, -0.017026061, 0.00542677, 0.023864022, -0.0015163596, -0.005686827, -0.025103115, -0.008321812, -0.009583852, -0.020177336, 0.008857223, 0.006818838, -0.0086966, 0.00069938024, -0.032981303, -0.011649008, -0.014310764, 0.007740509, -0.00995864, 0.0032258497, 0.003694334, 0.004593059, -0.016398866, 0.00887252, -0.009216713, 0.0036388808, 0.014012463, -0.017989801, -0.029906513, 0.020116147, 0.0030174218, -0.00088486186, -0.010624078, 0.00025910055, -0.0023577195, 0.03340963, 0.018999431, -0.014639659, 0.004941076, -0.0323694, 0.009943342, -0.024460621, -0.0034744332, 0.006015722, -0.0072356937, 0.016827194, -0.011473087, 0.016092917, -0.010861189, -0.020590367, -0.009813313, -0.009469121, 0.015565155, 0.040018126, 0.022869688, -0.010081019, -0.0065167136, 0.022395466, 0.00011389429, -0.0024571528, -0.018984135, -0.006922096, -0.00031072943, 0.005224079, -0.008184135, 0.01313286, 0.011029461, -0.010188101, 0.010364022, -0.009790367, 0.00056313735, -0.021905947, 0.027428327, -0.01379065, -0.0061992914, -0.008597166, 0.011901416, 0.018800566, -0.007954674, -0.0059392345, -0.04641246, -0.029921811, -0.008145892, -0.00563711, 0.023818128, -0.010593483, 0.006585552, 0.000017732493, -0.0009728222, -0.025959771, 0.039681584, -0.009553257, 0.018968837, 0.019947873, 0.003315722, 0.0029428469, -0.008184135, 0.0020077901, -0.054795463, 0.0057135974, 0.011771387, -0.008666005, -0.0073963166, 0.014341359, 0.01397422, -0.0059277616, 0.006283427, 0.0052737957, 0.008558922, 0.0020747166, -0.0075837104, -0.02831558, -0.011985552, 0.03916147, -0.0007251947, 0.0070635974, 0.039987534, 0.011725495, 0.024338242, 0.015878752, -0.0067920676, -0.018509913, -0.0069947587, 0.0015354814, -0.03579603, -0.008513031, 0.0045662886, -0.0018318696, 0.0012744687, -0.013859489, 0.015756372, 0.020483285, 0.01636827, -0.0027726628, -0.00054927403, -0.003491643, 0.01157252, 0.028223794, -0.021278752, 0.010631727, 0.0023577195, 0.005621813, -0.0014800283, -0.0050519826, -0.0057594897, -0.008222379, 0.0029887392, 0.02710708, 0.019427761, 0.027152972, 0.009254957, 0.013454107, 0.008115297, -0.0026101272, 0.00071467774, -0.02495014, 0.004539518, 0.014517279, -0.014723795, -0.01661303, 0.059966, 0.015297449, -0.014915013, 0.010440509, -0.018127477, 0.026235126, 0.009193767, -0.0071209627, -0.0067423508, 0.0125056645, -0.0132399425, -0.0038377475, -0.0013031515, 0.027152972, 0.02014674, 0.00070559484, 0.00012172228, 0.033562604, -0.0022449007, -0.0027879602, -0.033103682, -0.04479093, 0.011863172, -0.019703114, -0.010708215, -0.024277052, -0.0044209627, 0.0036924218, 0.02195184, 0.014440793, -0.022104815, -0.017071953, -0.008620113, -0.0001198101, 0.0013289659, 0.0203915, 0.0061610476, 0.018433426, -0.025944473, -0.018571103, 0.005920113, 0.024261754, 0.005992776, 0.021737676, -0.009063738, -0.025638524, 0.04466855, -0.00833711, 0.0005009915, 0.02026912, 0.02764249, -0.007101841, 0.008497733, 0.010318129, 0.041945606, -0.010754107, 0.0074536824, 0.011442492, 0.0074575064, 0.052103113, -0.004107365, -0.008413597, -0.015985835, 0.005434419, 0.03974277, -0.024843058, 0.010784701, 0.004562464, -0.035184134, 0.0072318693, 0.018066287, -0.020911613, -0.010226345, 0.013178753, -0.002820467, 0.011251274, -0.029768836, -0.015251557, -0.015442776, -0.0030212463, 0.008383002, -0.020758638, -0.010364022, 0.026173936, -0.026250424, -0.00031957327, -0.0037077193, 0.004589235, -0.03249178, 0.01823456, -0.017026061, 0.0048798863, -0.0053158635, 0.003684773, 0.015335693, -0.0044515575, -0.01973371, -0.01319405, -0.009713881, -0.0060348436, -0.019305382, -0.0031072944, -0.02260963, 0.008612464, -0.0024762745, -0.0053655803, 0.027871953, 0.008566571, -0.002881657, -0.009163172, -0.01624589, -0.008933711, -0.0057059485, -0.013668271, 0.00995864, 0.022823794, 0.007969971, 0.00064679526, 0.03466402, -0.020590367, -0.00015393058, 0.027871953, -0.033501413, -0.01595524, -0.010784701, -0.020712746, -0.012092634, 0.020773936, -0.036101982, 0.009147875, -0.0030766996, -0.025562039, 0.00051581085, -0.00221813, -0.006459348, 0.017561471, 0.003705807, 0.007931728, 0.021737676, 0.021921245, -0.0028950423, -0.025133708, 0.000010621568, -0.006271954, 0.014907365, -0.015878752, -0.0036580025, -0.03334844, -0.03640793, -0.0055376766, -0.0047116145, 0.0010564801, 0.028820395, 0.011649008, -0.0013681656, 0.01811218, 0.01912181, 0.018601699, -0.027856655, -0.023894615, -0.00713626, -0.005300566, -0.016536543, 0.011740793, -0.013935977, 0.02267082, -0.017163739, -0.021814162, 0.017071953, -0.012949291, -0.009492068, -0.014769687, -0.003895113, -0.03766232, -0.010654674, 0.012023795, 0.016521245, 0.02326742, -0.0107617555, 0.0032564446, -0.024430027, -0.013155807, -0.009377336, -0.00010720166, 0.013783002, 0.020865722, -0.01247507, -0.010937676, 0.009996884, -0.0002037668, 0.0046466, 0.015741074, 0.02033031, -0.0031952548, 0.0018701132, -0.029126344, 0.024812464, -0.014356656, -0.021095183, -0.013316429, 0.0017085338, -0.0014283994, 0.01235269, -0.01727082, 0.0034055947, -0.028300282, -0.014960906, -0.04059943, -0.020406798, 0.025531443, 0.015970537, -0.008421246, 0.007702266, 0.0002595786, -0.021905947, -0.03094674, -0.01145779, 0.014662605, 0.023359206, -0.013805948, -0.014762038, 0.011327761, -0.0045433426, -0.010539942, -0.017255522, 0.014310764, 0.040691216, -0.025057223, -0.0043100566, 0.002818555, 0.017699149, -0.0019618978, 0.022334276, -0.024016995, -0.00009835782, -0.0057365433, -0.008864872, -0.010708215, 0.0010211047, -0.032155238, -0.013201699, -0.034358073, 0.014723795, 0.021814162, 0.02141643, -0.017301416, -0.017913314, 0.014517279, 0.041486684, 0.009851557, 0.023190932, -0.0031493623, -0.00070655096, 0.013507647, -0.017760338, -0.0028548865, 0.01751558, 0.005985127, -0.013844192, -0.013469405, 0.02704589, -0.007820821, -0.00060424925, -0.008688951, 0.001878718, 0.0062949006, -0.038702548, -0.002436119, 0.002373017, 0.013102265, -0.045494616, 0.016276486, -0.014616713, 0.0062528322, 0.003965864, 0.0040232292, -0.011977903, 0.002206657, 0.0073504243, 0.0032143765, 0.005331161, 0.007052124, -0.012199716, 0.016551841, 0.015458073, -0.019152407, 0.017117845, -0.009996884, -0.02441473, 0.029722944, 0.019779602, 0.00094366143, 0.0017228753, 0.0018347378, 0.0036159346, -0.004202974, 0.00905609, 0.038304813, -0.014058356, -0.01067762, -0.0010383144, 0.014899716, -0.008956657, 0.008390651, -0.009017847, -0.011541925, 0.012199716, 0.01636827, -0.00419915, -0.010601132, -0.012046741, -0.011817279, 0.0030862605, 0.0016014518, 0.0047881017, 0.0011272308, -0.021079885, -0.023986401, 0.0060424926, 0.014586118, 0.000506728, 0.004631303, 0.016597733, -0.0065205377, -0.0090407925, 0.01457082, -0.003977337, -0.017301416, 0.015863456, 0.0055262037, 0.0090407925, -0.0037287532, 0.025776202, 0.0051667136, -0.025806798, -0.036010195, 0.005690651, 0.0119932005, 0.017484985, -0.012192067, 0.000496211, -0.00289313, -0.006642917, 0.01061643, 0.03135977, 0.043261185, -0.014624362, -0.009698583, -0.002830028, 0.018846458, -0.02735184, 0.0059086396, 0.005560623, 0.013813597, -0.0045089233, 0.00030451486, 0.012857506, -0.0041303113, 0.0062528322, 0.017989801, 0.04433201, 0.015970537, -0.003705807, 0.040140506, 0.006673512, 0.01289575, -0.007369546, -0.038427193, 0.0002605347, -0.009675637, -0.014830877, -0.012566854, -0.0034113312, -0.004921954, -0.0029351981, 0.01751558, 0.009935693, 0.012153824, 0.0038090649, -0.004443909, 0.007801699, -0.00020842775, -0.0073274784, 0.006459348, 0.012215014, 0.019993767, 0.0012438738, -0.012421529, 0.016658923, 0.010180453, 0.012299149, -0.0110141635, 0.031160904, 0.014211331, -0.008329461, 0.00031861718, -0.0041188383, -0.0035853398, 0.0042909347, -0.017010763, -0.012161472, 0.035000563, 0.014884418, -0.014142492, 0.019580735, -0.0049563735, -0.016108215, 0.00084614014, -0.020942207, 0.0137218125, -0.006956515, -0.0014953257, 0.019550141, -0.014081302, -0.0055032573, -0.013935977, -0.0002469104, -0.015220962, 0.020131443, -0.0019771953, -0.0044974503, -0.003747875, -0.013117563, -0.028269686, -0.00069603394, 0.0057556652, 0.0004254603, 0.016199999, 0.010081019, 0.013262888, -0.0097827185, -0.0227932, 0.024078185, -0.01876997, 0.009820962, 0.015802266, -0.006849433, -0.0031627477, 0.0023366853, -0.013683569, -0.0023883143, -0.0074460334, -0.0043980167, 0.00055787887, -0.014524928, -0.004853116, 0.012781019, -0.026801132, 0.0040691216, 0.006585552, -0.015970537, 0.000025814446, 0.011947308, 0.012704532, -0.011641359, -0.0012888102, 0.0051016994, -0.016505947, -0.010409914, 0.018968837, -0.027780168, 0.022303682, 0.008191784, 0.0067844186, -0.008971954, -0.009316146, 0.01733201, 0.013836543, -0.0015039305, -0.018249856, -0.00097234413, -0.020988101, 0.0034706087, 0.011465439, 0.0062949006, -0.017347308, -0.020529177, 0.024399431, 0.0054229456, 0.016047025, -0.00887252, -0.0067270533, 0.0059966003, -0.023053257, -0.0019657223, -0.0062107644, -0.0040041073, -0.00011706134, -0.016811896, -0.008864872, -0.0047881017, -0.0025814446, 0.026801132, 0.007419263, -0.016949574, 0.009820962, 0.002797521, -0.0018137038, -0.030549007, 0.0018060551, 0.0029160762, 0.01817337, -0.00160432, -0.0015039305, -0.010494051, -0.0061878185, -0.0011377478, 0.01097592, 0.007262464, 0.009859206, 0.011947308, -0.011503682, -0.017653257, 0.0026521953, 0.023481585, -0.007652549, 0.0070635974, -0.011740793, -0.008566571, -0.016567137, 0.0011014163, -0.014341359, 0.005407648, -0.0125056645, 0.020529177, 0.031329177, 0.031053822, 0.04179263, 0.016475353, 0.011893767, -0.04102776, -0.00012656249, -0.022594333, -0.0068532573, 0.00791643, 0.025194898, -0.014685552, -0.029248724, 0.018907648, -0.010692917, -0.004791926, 0.0037096315, 0.005036685, -0.020009063, -0.00046489906, -0.008184135, -0.03974277, -0.016766004, 0.009752124, -0.0017372166, 0.00728541, 0.01601643, -0.0068532573, -0.011970254, -0.0006267174, -0.026510479, -0.018800566, 0.014326061, -0.008895467, -0.0037402264, -0.008910764, 0.0032755663, -0.0067538237, 0.0017754602, 0.0047613312, 0.003724929, -0.0018500353, -0.018402832, -0.008061755, 0.01733201, -0.051521808, -0.011893767, -0.0031608355, 0.0010775141, -0.012918696, 0.03172691, -0.005369405, -0.00213017, 0.007664022, 0.01912181, 0.01391303, -0.0009111543, 0.009300849, -0.0034457506, -0.0031703964, 0.020116147, 0.01481558, 0.033654388, 0.03172691, -0.015419829, -0.030426627, -0.01811218, 0.008023513, 0.0018385622, 0.012138526, -0.012650991, 0.011465439, -0.017913314, -0.0137218125, -0.014150141, 0.0046160053, -0.0034553113, -0.012039092, 0.018265154, 0.017469687, -0.013079319, -0.0016138809, -0.0062298863, -0.0007137216, -0.000031222335, 0.012023795, 0.015106231, -0.036928043, 0.0017056656, 0.015366288, -0.01727082, 0.017347308, 0.012589801, 0.0021588525, -0.0075033987, 0.01535099, 0.013148158, -0.011947308, 0.008367705, -0.0020498582, 0.022808498, -0.008681303, 0.006922096, 0.017852124, 0.013018129, -0.011549574, 0.004550991, 0.020804532, 0.015939942, 0.005889518, -0.001847167, -0.018433426, 0.013630027, 0.010647025, -0.0044477335, -0.021615297, 0.0017993625, 0.006788243, -0.016031727, -0.0035165013, 0.0028873936, -0.0011635623, 0.017163739, 0.006945042, 0.014440793, -0.0043444755, 0.004505099, -0.018158073, -0.010593483, -0.016505947, 0.0050519826, -0.012696883, -0.010058073, 0.014723795, -0.00036020714, -0.000020944359, -0.00935439, 0.003355878, 0.031512745, 0.0077634556, 0.0003124026, 0.022640225, 0.026984701, 0.0054497165, 0.01685779, -0.00054927403, -0.011205382, 0.0006554001, 0.0072777616, -0.022701414, 0.016628327, -0.015251557, 0.015419829, -0.0034228042, -0.007847591, 0.012184419, 0.025347874, 0.010692917, -0.00755694, -0.01337762, 0.00009805904, 0.0020689801, -0.015611047, 0.0023577195, -0.0055873934, 0.017286118, -0.0072012744, -0.0038281868, 0.05066515, -0.024858356, 0.018127477, -0.014448441, 0.00083657925, -0.008949008, -0.0075301696, -0.011251274, -0.010188101, 0.0004412358, 0.013507647, -0.0028644474, 0.017668555, 0.0025145183, 0.0045203962, -0.020223228, 0.011503682, 0.0016693341, -0.01912181, 0.005212606, -0.007771104, 0.013630027, -0.007664022, -0.0109988665, -0.0024628893, 0.004007932, -0.0051016994, 0.02320623, -0.000080610385, -0.0008643059, -0.009316146, -0.03051841, -0.018509913, 0.011419546, -0.025363171, -0.0066964584, 0.024093483, -0.013798299, -0.009423229, -0.013522945, 0.016628327, -0.027183568, -0.0031111187, -0.017836826, -0.005331161, -0.00001818066, -0.019075919, 0.013331727, -0.008168838, -0.0010880311, 0.0049563735, 0.0026980876, -0.02170708, -0.01450963, 0.022196598, 0.003684773, -0.0031264161, -0.023588667, 0.0051208213, -0.025240792, -0.017760338, 0.00043956266, -0.0061992914, 0.016414164, 0.009690934, 0.015236259, 0.0124980165, -0.000538279, 0.019397166, 0.007216572, 0.00009710295, -0.015373937, -0.011243626, 0.0066276197, 0.010325778, -0.020070253, -0.011213031, 0.00057986897, 0.0076066568, 0.0040041073, -0.011909064, 0.009859206, 0.0010516996, 0.019198298, 0.016398866, 0.027688384, -0.009262606, -0.009163172, -0.037601132, -0.0076296027, 0.0010105878, 0.011174787, 0.02932521, 0.019565437, -0.010027478, 0.007419263, -0.00650524, 0.0120008495, -0.008383002, -0.015756372, 0.0149915, 0.014188385, 0.007029178, -0.0063063735, 0.01961133, -0.0097827185, 0.00911728, -0.019580735, -0.0050175632, -0.0102722375, 0.02129405, 0.0018232648, 0.008918413, -0.007254815, 0.020850424, 0.004960198, -0.004378895, 0.009400283, -0.021691782, -0.006310198, -0.0023137392, -0.016750706, 0.012230311, -0.012199716, -0.0029696173, 0.020422095, -0.0011855523, 0.007740509, 0.018127477, -0.017852124, -0.030396031, 0.018678186, 0.022502547, -0.020009063, -0.004294759, -0.0056179883, 0.007208923, -0.0063063735, -0.010593483, -0.017010763, -0.00072423863, 0.0034878184, 0.016995465, 0.004726912, 0.016643625, 0.019198298, -0.010096316, -0.0028510622, 0.0036905096, -0.008138243, -0.0026311614, 0.018188667, -0.020406798, -0.009935693, -0.000036002784, 0.0034878184, -0.0030461047, 0.014547874, 0.0012716005, -0.005962181, 0.002344334, -0.018280452, -0.00352415, 0.0037861187, 0.01895354, 0.0026235126, 0.014287817, -0.012574503, 0.0050060903, -0.0059660054, -0.0072892345, 0.023282718, -0.0064746453, -0.00527762, -0.0033138099, 0.013308781, -0.0029849147, 0.0034457506, -0.0024953964, 0.02710708, -0.009759773, -0.013025778, 0.013828894, -0.021814162, -0.003862606, -0.0056103393, 0.008268272, -0.0048569404, -0.011480736, 0.020162039, -0.0026560195, -0.022166004, 0.006593201, 0.037111614, -0.03279773, -0.00578626, 0.005572096, 0.0012782931, 0.024674786, 0.0049640224, 0.012360339, -0.0142189795, -0.024139374, -0.013048724, 0.01541218, 0.0116949, -0.008413597, -0.015626345, -0.0051399427, 0.008566571, -0.0052202544, -0.008122945, -0.016521245, 0.008008215, -0.00770609, -0.009170821, -0.014532577, 0.010417563, 0.011733144, 0.0034476626, 0.015328044, 0.0011052407, -0.023236826, 0.019626627, 0.0073963166, 0.009446175, -0.004899008, 0.023405097, 0.014501982, -0.01667422, -0.0070635974, 0.026281018, 0.005923937, 0.0011310552, -0.020605665, 0.027810764, -0.02123286, -0.007725212, 0.01630708, -0.010922379, 0.015970537, 0.003799504, -0.0072586397, 0.0013423512, -0.0119932005, 0.0070330026, 0.0008322768, 0.004355949, -0.017163739, -0.025500849, 0.0033597024, -0.0053808778, -0.011350707, -0.020009063, 0.014012463, 0.026219828, -0.010172804, -0.008490085, 0.004390368, 0.0042679883, -0.010692917, 0.015481018, -0.007052124, -0.002946671, 0.035887815, 0.018066287, 0.007526345, -0.01704136, 0.005476487, -0.028927477, -0.0020957505, -0.023496883, 0.032430593, 0.02254844, 0.008207082, -0.00012028815, -0.0010899432, -0.019519545, -0.005407648, 0.005740368, -0.007931728, -0.011006515, -0.008314163, -0.014563172, 0.025041925, 0.012750424, 0.024001699, -0.0020077901, 0.0087883845, -0.008191784, 0.0077290363, -0.028958071, 0.01553456, -0.019596033, -0.011488385, 0.008413597, -0.0085359765, 0.009377336, 0.0032411471, -0.0062566567, -0.011289517, 0.009377336, 0.0048951837, -0.00076869683, 0.01624589, -0.00124483, -0.011618413, 0.012314446, -0.0020708921, -0.016536543, 0.0009809489, 0.006719405, 0.0015804177, -0.0069718123, -0.0014675991, 0.019259488, -0.018050991, -0.020284418, -0.0077940505, 0.00126682, 0.010310481, 0.01541218, 0.022961471, -0.013813597, 0.0100198295, -0.006933569, -0.0080388095, 0.0115113305, -0.003965864, 0.0023041782, 0.007281586, -0.01811218, 0.0051552407, 0.00037837285, -0.01163371, -0.022211896, -0.021217562, 0.0035470962, 0.005552974, -0.009446175, -0.010394617, -0.012635693, 0.013232294, 0.00461983, 0.013530594, -0.0046733706, -0.0058206795, 0.00014425017, 0.0051590647, 0.0043444755, 0.009943342, -0.000003417277, -0.006914447, -0.0017056656, 0.0038453962, 0.028560339, -0.0097750705, 0.03258357, 0.00092645176, -0.009339093, -0.0037765577, -0.005893342, 0.0030346315, -0.010562888, -0.0058780448, 0.017561471, -0.00797762, -0.012421529, -0.0033826486, -0.006681161, -0.0005296742, -0.01564929, -0.008834277, 0.0074345605, 0.0068647303, 0.0092779035, -0.000020003208, -0.0023500707, -0.0074842772, -0.015970537, 0.008405948, 0.011985552, -0.009140226, 0.007931728, 0.007901133, -0.019534843, 0.018402832, 0.00043406512, -0.0067270533, 0.01967252, -0.011587818, -0.0015115793, 0.0027783993, 0.0040308777, 0.0061686966, 0.0074345605, 0.017408498, -0.002353895, 0.004531869, 0.0049334276, 0.010562888, 0.008428895, 0.007847591, -0.016597733, -0.011090651, -0.012062039, -0.019137109, 0.0024437676, 0.0034533993, -0.0071745035, 0.009254957, -0.015985835, -0.005751841, -0.001329922, -0.0027879602, -0.02308385, -0.013828894, -0.0043865438, -0.006279603, -0.0046351273, -0.0030021244, -0.0051858355, -0.008138243, 0.0026101272, -0.0022085693, -0.014165438, 0.0109988665, 0.014410198, -0.0077328607, 0.006723229, 0.0018500353, 0.010379319, 0.020697448, -0.0043330025, -0.0007514872, 0.0049066567, -0.021783568, -0.01553456, 0.017255522, 0.032705948, -0.017530877, -0.01133541, 0.01698017, 0.00578626, 0.027474219, -0.009950991, -0.010494051, 0.008513031, -0.0043712463, 0.0143949, -0.023971103, -0.024996033, -0.0122532565, 0.012520962, -0.02404759, 0.020712746, 0.042649288, 0.0050519826, -0.0009049397, -0.012918696, -0.0074766283, -0.0032545323, -0.010815296, 0.018158073, 0.0035968127, -0.015725778, -0.00019301078, 0.022701414, 0.011671954, 0.014448441, 0.005174362, 0.0019714588, 0.02189065, 0.0035337107, 0.015083285, 0.023359206, -0.012574503, -0.010409914, 0.005751841, -0.011174787, -0.01624589, 0.012046741, 0.0052584982, -0.012704532, -0.011434844, 0.011909064, -0.0026196882, 0.014379602, -0.0036694757, 0.009446175, 0.0009427053, 0.01085354, 0.007381019, 0.008933711, 0.021783568, -0.003906586, -0.009384985, -0.0015861543, 0.0021339941, 0.007071246, -0.0020613314, 0.009392634, 0.0018328256, -0.00032339763, -0.0055453256, -0.0036656512, -0.011289517, 0.009729178, 0.021875352, 0.0062566567, 0.0036656512, -0.002183711, -0.0020097024, -0.0023519828, 0.0005296742, 0.002858711, 0.0030881725, 0.011564871, 0.0059392345, 0.0046083564, 0.012803965, -0.02872861, 0.010777053, 0.017653257, -0.020223228, 0.0076907924, -0.009132577, -0.029860621, -0.009155523, -0.019993767, -0.016582435, 0.016154107, -0.016873086, -0.009232011, 0.009216713, -0.010983569, -0.0022716713, 0.0044515575, -0.012207365, -0.006956515, 0.0122456085, 0.015503965, -0.004401841, 0.013576486, -0.011312464, 0.04733031, 0.016475353, -0.0044477335, 0.010471104, 0.001804143, 0.030028893, -0.004600708, 0.008306515, 0.0021167845, -0.0068417843, -0.00614575, -0.01727082, -0.008926062, -0.0070062317, -0.0037631725, 0.011564871, 0.0063560903, 0.013798299, 0.02033031, 0.015045041, -0.006088385, -0.016414164, 0.022196598, 0.015741074, 0.011312464, -0.0011434844, 0.0020785409, -0.0092702545, 0.020804532, 0.0019810197, -0.027627194, 0.009323795, -0.017699149, 0.020590367, 0.008252974, 0.008184135, 0.024965437, -0.0030365437, 0.009966289, 0.017821528, 0.0149762025, -0.02014674, -0.012888101, -0.010417563, 0.005116997, -0.007610481, -0.0019265225, 0.010081019, 0.009996884, -0.043199997, -0.0046810196, 0.011090651, -0.025026627, -0.005912464, -0.009484419, -0.009254957, 0.0069373935, 0.007870537, -0.010310481, 0.008880169, 0.0054497165, -0.0142342765, 0.02932521, 0.011488385, 0.008375353, -0.045311045, -0.01571048, 0.010126911, 0.0017056656, -0.00611898, 0.01361473, -0.009637393, 0.008994901, 0.0038702546, 0.0047230874, 0.013163455, -0.0023787534, -0.0025183426, 0.0033042491, 0.0024016995, 0.0061686966, 0.005212606, 0.01691898, 0.009576203, 0.009928045, 0.0024648015, -0.005078753, -0.0056065153, 0.0072471667, -0.026954105, -0.021691782, 0.0005287181, -0.018464021, 0.00063054176, 0.0053349854, -0.0087883845, 0.003652266, 0.00728541, -0.0034533993, 0.016888384, -0.010570537, 0.01559575, -0.004482153, -0.026709346, 0.0036140224, -0.012406232, 0.0028893058, -0.004784277, -0.006008073, 0.0033539657, 0.022961471, 0.008949008, -0.014601415, -0.009798016, 0.0015240083, -0.004046175, 0.01223796, -0.00013385268, 0.009384985, -0.0045662886, -0.0000012959509, 0.003365439, 0.0010756019, -0.003166572, -0.0038243623, 0.0203915, 0.026005663, -0.002946671, -0.003556657, 0.0083983, -0.013645325, 0.014058356, 0.0021263454, 0.003399858, -0.01553456, 0.00014735747, 0.011404249, 0.012199716, 0.018265154, -0.00023866411, 0.014165438, 0.009224362, -0.024384134, -0.0008064624, -0.012046741, -0.036560904, 0.007778753, -0.010371671, 0.0015546033, -0.01733201, -0.002047946, -0.019412464, 0.019091217, 0.014700849, 0.023940507, -0.017133143, -0.010371671, -0.021186966, -0.014433144, 0.0046810196, -0.0013433073, 0.01133541, -0.019366572, 0.005560623, -0.015083285, -0.018096883, -0.005048158, 0.016720112, -0.0019599856, -0.002353895, 0.0046466, -0.018035693, -0.012283851, -0.0012553469, 0.0087883845, -0.006799716, -0.01301048, -0.0024246457, -0.008704249, -0.0076907924, -0.0031302406, 0.005797733, 0.0048875352, 0.007155382, 0.014287817, -0.0100351265, -0.010960623, 0.011595466, -0.012926345, 0.015741074, -0.018142775, -0.00095800275, 0.0042259204, -0.016521245, -0.014280169, 0.012635693, -0.0185864, -0.016062321, 0.005369405, 0.0010411827, 0.015565155, 0.016092917, 0.006015722, 0.010746459, -0.00068312674, -0.019427761, 0.0008055063, -0.0028434135, -0.017913314, 0.016643625, 0.0010058073, -0.0032067278, -0.003820538, -0.03903909, -0.0044286116, 0.01541218, -0.0020154389, 0.009690934, -0.014662605, 0.013828894, 0.0056524076, 0.00029614905, 0.010861189, 0.017561471, 0.008650708, 0.021967137, 0.022502547, -0.0015890226, 0.0009666076, -0.0011683427, -0.03064079, 0.0120084975, 0.009996884, 0.01463201, 0.0046618977, 0.009101982, 0.013415863, -0.003977337, 0.0006209808, -0.0046924925, 0.024200564, 0.000119451564, -0.0032583568, 0.012184419, 0.003874079, 0.016658923, -0.005763314, 0.0065167136, -0.00995864, 0.00911728, -0.004325354, 0.0035222378, -0.0048263455, 0.020758638, 0.016995465, 0.01577167, 0.009851557, -0.006447875, -0.005625637, -0.0029179885, 0.03209405, 0.0016798512, 0.0016110126, 0.009866855, -0.015297449, 0.0038989375, -0.0067079314, 0.008161189, -0.0070330026, 0.0129875345, -0.012727478, -0.019886684, 0.021385834, -0.0016492562, 0.01745439, 0.016873086, 0.008077053, -0.013637676, 0.010027478, 0.019106515, 0.031390365, -0.0071439086, 0.010570537, 0.016766004, 0.01589405, 0.0063828607, -0.0018223086, 0.017362606, -0.016322378, -0.009002549, 0.019228894, -0.0105322935, 0.00032746102, 0.017790934, 0.017286118, 0.0044171386, 0.0092855515, 0.023634559, 0.0038721669, -0.0167966, -0.02189065, -0.026173936, -0.0018777619, -0.018372236, 0.01475439, -0.0075110476, -0.0020957505, 0.0032889517, -0.014020112, 0.03249178, -0.011626061, 0.011450141, -0.0070865434, -0.013691217, 0.0013280098, 0.0076869684, 0.01145779, -0.010777053, 0.023940507, -0.020223228, 0.001203718, 0.012735127, 0.015450424, 0.011098299, -0.006470821, -0.0022104813, -0.00068025844, 0.0057480168, -0.0018337817, 0.0011071529, -0.022181302, -0.010746459, -0.008597166, -0.015136826, 0.004455382, -0.001879674, -0.017898016, -0.013706515, -0.00064631726, 0.006975637, 0.006428753, 0.0092702545, -0.0043865438, -0.0039008495, 0.00881133, 0.0015957152, 0.013836543, 0.006711756, -0.01823456, -0.011916713, -0.009377336, -0.014379602, 0.018877052, -0.006099858, -0.004168555, 0.0037325777, -0.0012457861, 0.019335976, 0.016888384, 0.012222662, 0.0077749286, -0.011182436, -0.026541075, -0.003556657, -0.010960623, -0.025776202, -0.025898581, 0.008688951, 0.033134274, 0.027397731, -0.014066005, 0.0046810196, 0.00045605522, -0.012781019, 0.0016932364, 0.0056524076, -0.00162631, 0.009193767, -0.014241925, 0.01007337, -0.0031933426, 0.003399858, -0.008734844, -0.006799716, -0.001847167, -0.030778468, -0.031237392, -0.025684418, -0.014066005, -0.0056103393, -0.024185268, 0.008627761, -0.020896316, -0.0045165718, -0.0014589942, 0.00046418197, -0.011167138, 0.0067155804, -0.0054803113, -0.013102265, 0.009820962, -0.0007008144, 0.01025694, 0.004115014, -0.008122945, -0.002742068, -0.024032293, -0.019901982, 0.055223793, 0.015863456, 0.021492917, 0.015664589, -0.0047689797, -0.008268272, -0.0014006727, 0.009683286, -0.03010538, 0.01217677, -0.015320395, -0.0037211045, 0.0085359765, 0.009071387, -0.008658356, -0.014150141, 0.0022640226, -0.0033424927, 0.008145892, 0.019397166, -0.0025298158, -0.01704136, 0.0008910764, 0.009362039, 0.012620396, -0.015266854, -0.007342776, -0.019045325, 0.010233994, -0.014142492, 0.012903399, 0.00045677228, -0.00785524, -0.020605665, 0.010149858, 0.00025910055, -0.010379319, 0.01745439, -0.005109348, 0.007931728, -0.008933711, 0.0061992914, -0.009622096, -0.0009847733, -0.0046504247, -0.03184929, 0.009339093, 0.0102722375, -0.0044898014, 0.022257788, -0.005319688, 0.0074651553, -0.00093457854, 0.009124928, 0.013836543, -0.0033195466, 0.01457082, -0.008643059, 0.0071515576, 0.007185977, -0.004941076, -0.023313314, -0.004359773, -0.00905609, -0.014387251, -0.021018695, -0.005212606, -0.009124928, 0.025026627, 0.0047766287, 0.012039092, 0.0016435197, -0.027091783, -0.013148158, 0.00352415, -0.017928611, -0.0047804527, 0.012941642, 0.021186966, -0.013347024, 0.008184135, -0.01163371, 0.022701414, -0.02681643, -0.0082912175, -0.004550991, 0.0021665012, -0.0071515576, 0.011190084, -0.007040651, 0.00027678823, 0.02117167, 0.018540509, 0.025133708, 0.019596033, -0.009254957, -0.0016693341, 0.0032239375, 0.0032124643, 0.0009020715, -0.008138243, 0.01768385, 0.00077013095, 0.00183665, -0.010509348, -0.0041417843, 0.0027631018, 0.010861189, -0.0015641642, -0.008956657, 0.005541501, 0.0050519826, 0.011832577, -0.005560623, 0.038733143, -0.01223796, -0.0071745035, 0.0013672095, -0.0040194048, -0.018096883, -0.029065154, -0.008413597, 0.01847932, 0.009652691, 0.005981303, 0.009560905, 0.001066041, 0.027718978, -0.003472521, 0.0009173689, -0.0015488667, 0.0007237606, 0.024797166, 0.010249291, 0.016154107, 0.0015450424, -0.014264871, 0.0035929885, -0.01961133, -0.0070444755, 0.002246813, -0.02716827, 0.010930028, 0.0215847, 0.0035872518, -0.017179037, 0.021844758, -0.00095131015, -0.024996033, -0.044179033, 0.0015393058, -0.023527477, 0.0036293198, 0.025424361, 0.009706232, 0.008658356, -0.008428895, 0.0043483, 0.0167966, -0.0044477335, 0.006428753, 0.028652122, 0.01427252, 0.014532577, 0.003851133, 0.022380168, 0.01889235, -0.00035327548, 0.021615297, 0.0063943337, -0.0058818692, 0.01145779, -0.011687252, 0.007862889, 0.01319405, -0.0020422095, 0.0035509204, -0.012169121, 0.005182011, -0.0052737957, -0.010348724, -0.016475353, -0.0185864, -0.015588101, 0.0120008495, -0.010142209, -0.016291784, 0.00983626, -0.006788243, 0.005143767, -0.0019179177, -0.0036159346, 0.019947873, -0.0048263455, -0.017959205, -0.001720963, 0.013354673, -0.013385268, -0.011427195, 0.00863541, -0.0102645885, 0.010524645, -0.021033993, -0.01073881 - ] - } - ] - } + { + "name": "generative-ai-transforming-our-world", + "title": "Generative AI: Transforming Our World", + "content": "---\n\nGenerative AI is emerging as a transformative technology, comparable to the rise of personal computers and the early internet. This wave of innovation is expected to impact virtually every sector of human activity, creating both opportunities and challenges for businesses.", + "keywords": [ + "Generative AI", + "transformative technology", + "personal computers", + "internet", + "innovation", + "impact", + "sectors", + "human activity", + "opportunities", + "challenges", + "businesses" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + -0.03798621, 0.017438458, -0.02202907, -0.025974356, -0.0050086067, + 0.0149598215, -0.004062618, 0.009606552, 0.020547755, 0.007388245, + 0.039306197, -0.019594433, 0.04622878, 0.0037289555, -0.020709086, + 0.011021868, -0.011109867, -0.006632921, -0.0137645025, -0.020958416, + -0.012239187, 0.009562552, -0.053943355, -0.0048399423, 0.02157441, + 0.006445923, -0.0026399684, -0.0050892727, -0.036343567, 0.025901023, + 0.007985905, 0.010515874, 0.023290388, -0.03165029, -0.0062259254, + 0.0333516, 0.0040039523, 0.046492778, 0.010963202, 0.007076582, + 0.053855356, -0.0057419315, 0.008462566, 0.014827823, 0.027030343, + -0.019169105, -0.00011463925, -0.022645064, -0.031679623, 0.008565231, + -0.027177008, 0.0050929394, 0.031180961, 0.044175472, -0.020885084, + -0.004469613, 0.037751548, -0.027998332, 0.03607957, 0.015605147, + -0.0066622538, 0.047842097, -0.06547122, -0.00020395589, -0.025695693, + -0.03452492, -0.0019744765, 0.029156985, 0.0129138455, 0.023862382, + -0.026649015, -0.032794274, 0.0030304638, 0.005202938, 0.0166758, + -0.0012622349, 0.040068854, 0.017717121, 0.0027114677, -0.00250797, + 0.015194485, 0.039658193, -0.008249901, 0.008521232, 0.012217187, + -0.0010587374, 0.008513899, -0.062068593, 0.004513613, -0.008601897, + -0.023715716, 0.024522373, -0.02629702, -0.0023521385, -0.00394162, + 0.0065669217, 0.0113738645, 0.019066438, -0.01647047, 0.005653932, + 0.019799763, -0.031826288, 0.026986344, -0.011087867, -0.05092206, + -0.01927177, -0.0117185265, -0.0018543946, -0.0036904558, 0.062303256, + 0.0010413209, -0.02474237, -0.009210557, 0.014952488, -0.03038897, + 0.011652527, -0.052447375, -0.0307703, 0.02257173, -0.031796955, + -0.037898213, -0.01341984, 0.013214509, -0.031386293, -0.0059325956, + 0.04080218, -0.0016068975, -0.03323427, 0.0045062797, 0.03185562, + -0.044322137, 0.07685242, 0.017115796, 0.07544443, 0.018289115, + -0.016353138, 0.012415185, -0.019169105, 0.008389233, 0.0016637301, + -0.025857024, -0.023715716, -0.021838406, -0.035756905, -0.015663812, + -0.019579766, -0.019799763, 0.066351205, -0.0015143153, -0.03625557, + -0.04505546, 0.018714443, 0.014343829, -0.012143855, -0.0025593028, + -0.00018390405, 0.010589207, -0.016338471, 0.0019286437, 0.020797085, + 0.018201116, 0.003391626, 0.00614526, 0.048839416, 0.009562552, + 0.005397269, 0.030007642, -0.011285865, -0.034260925, 0.04344215, + 0.008191235, -0.003189962, -0.012224521, -0.026311686, -0.0005692432, + 0.008506565, 0.0079125725, 0.013537171, 0.010779872, 0.003048797, + -0.0051589385, -0.005844597, 0.012341852, 0.02040109, 0.029948976, + 0.046140783, 0.08301234, -0.02336372, 0.023275722, -0.0015903977, + 0.007388245, -0.018171784, 0.03863154, 0.0021449744, 0.042532824, + -0.017746454, -0.029670313, -0.0036116235, -0.03461292, -0.0005637433, + -0.0008116986, -0.029802311, 0.016543802, 0.028775657, 0.03449559, + 0.0072122472, -0.0126645155, -0.011212532, 0.021237079, -0.0094525535, + 0.012004524, -0.018905107, 0.00013027969, -0.033498265, 0.009628552, + -0.014996488, 0.019755764, 0.006376257, -0.029758312, -0.009078559, + -0.018201116, 0.022894394, 0.017497124, -0.03795688, -0.0135445045, + -0.08459632, -0.0044549466, -0.02119308, -0.023055725, 0.018611778, + -0.018846441, 0.028892988, 0.014079832, -0.006240592, 0.012363852, + -0.017071797, 0.047372766, -0.027719669, -0.056289993, 0.020914417, + -0.00944522, -0.009063892, -0.056407325, -0.0142998295, 0.045202125, + -0.0030359637, 0.015957143, 0.04062618, 0.023481052, 0.02846766, + -0.0063432576, 0.013456506, -0.012847846, -0.025783692, -0.038572874, + 0.0037766215, 0.031972952, 0.0418875, 0.020958416, 0.0135445045, + -0.006046261, 0.013111844, -0.02679568, 0.029538313, -0.061599266, + -0.0077952403, 0.023891715, 0.03255961, 0.006086594, 0.026106354, + -0.02065042, -0.015898477, 0.0073735784, -0.011212532, -0.025739692, + 0.011652527, 0.030506302, 0.015487815, 0.054618016, -0.008022571, + 0.0074359113, -0.015018487, 0.0729218, -0.018699776, -0.021515744, + -0.0195651, -0.03226628, -0.004421947, -0.064063236, -0.010222545, + -0.008719229, 0.04690344, 0.00072140805, 0.03402626, -0.00948922, + -0.01285518, 0.016323805, -0.024434375, 0.015003821, -0.029288983, + 0.0020936416, -0.021691741, -0.00034580837, 0.017277127, 0.0040002856, + 0.02896632, 0.016661134, 0.040098187, 0.016661134, 0.0142998295, + -0.013515172, -0.006170926, 0.005162605, 0.021867739, -0.0010706539, + 0.05596733, 0.020753086, -0.029142318, -0.011564529, -0.002084475, + 0.0012640682, -0.029567646, -0.0530927, 0.032706276, -0.0041066175, + 0.0444688, 0.018245116, 0.0126645155, 0.010647872, -0.015693147, + -0.0072562466, -0.038954202, 0.0032944607, -0.015487815, 0.010823871, + -0.0056025996, 0.0043119485, 0.0077365744, -0.007083915, -0.02248373, + 0.018039785, -0.010479208, -0.024405042, -0.009577218, -0.015942477, + -0.03173829, -0.027367674, 0.0010248211, -0.0034539588, 0.017893119, + 0.0008602814, -0.005551267, -0.015253151, -0.04109551, 0.008557898, + 0.00036574563, 0.0363729, -0.0333516, 0.027485006, 0.00011166013, + 0.0038462875, -0.027793001, 0.008873227, 0.015693147, -0.023085058, + 0.006735586, -0.020665087, -0.012180521, -0.013793835, -0.0032761276, + 0.016323805, 0.03992219, -0.0071352483, 0.024111712, -0.00415795, + -0.007985905, -0.020709086, -0.009848549, 0.007487244, 0.04743143, + -0.011109867, 0.026971677, -0.01504782, 0.0015289817, -0.0057419315, + -0.018699776, 0.03135696, -0.0504234, 0.019477101, 0.04886875, + -0.015429149, -0.012451851, -0.039452862, -0.03666623, 0.029582312, + -0.0004358698, 0.031180961, -0.027602337, 0.036548898, -0.022102403, + -0.0025061367, -0.016089141, 0.0151798185, 0.033204935, -0.019741097, + 0.022263734, -0.04288482, -0.008528565, 0.0068822512, 0.0068492517, + -0.0009533219, -0.017907785, -0.029494314, -0.016015809, 0.005085606, + 0.0025189698, 0.00069849164, 0.0035492908, 0.019961094, -0.029831644, + 0.007487244, 0.011065868, 0.0166318, 0.005415602, -0.013119177, + -0.007897906, 0.045788787, 0.020034427, -0.012202521, -0.01927177, + 0.008814561, -0.020122426, 0.035844903, 0.014717824, -0.029098319, + 0.0057345983, -0.0144244945, 0.011989857, 0.0051589385, 0.01204119, + -0.026825013, 0.021633076, -0.00066045043, -0.0054302686, + -0.062537916, 0.0139405, -0.017027797, -0.02010776, -0.019110437, + -0.035991568, -0.02144241, 0.049220745, -0.06265525, -0.022513064, + 0.008990559, 0.0016545636, 0.030623633, -0.012627849, 0.0017003963, + -0.01475449, 0.0149598215, 0.0036134569, 0.010317877, -0.023011725, + 0.050804727, -0.010193212, -0.03695956, -0.035874236, 0.011549862, + 0.0032852942, 0.016661134, -0.013933167, -0.0045172796, 0.02833566, + 0.01036921, -0.015898477, -0.013016511, -0.025901023, 0.0028232995, + 0.016763799, 0.039863523, 0.02069442, 0.01423383, -0.019389102, + -0.035639573, 0.019477101, 0.022263734, 0.0025336363, 0.010479208, + 0.00788324, -0.0364609, 0.0072012474, 0.038455542, -0.0033641264, + -0.014329162, -0.040743515, -0.007039916, -0.0027957999, 0.009005226, + 0.027441006, -0.039687525, -0.033116937, 0.0028947988, 0.005653932, + -0.00091390574, 0.011945858, 0.021339744, 0.0066145877, 0.007028916, + 0.0059655956, -0.047284767, 0.019711765, -0.023774382, -0.008704563, + 0.016265139, 0.020518422, -0.025050368, 0.0057235984, -0.019975761, + 0.0033531266, 0.006669587, -0.00528727, 0.010442542, 0.026883679, + -0.021134414, 0.025974356, 0.017629122, 0.01339784, 0.026517017, + 0.009723884, -0.037135556, -0.0042239497, -0.024390375, 0.022058403, + -0.015663812, 0.015707813, 0.036636896, -0.016895799, -0.03698889, + -0.00307263, -0.029098319, 0.01258385, -0.0071719144, 0.024229044, + 0.017394459, 0.003573124, 0.02654635, -0.009980547, -0.020034427, + 0.012466517, 0.020093093, -0.015825145, 0.03129829, 0.0057529313, + -0.0041542836, 0.009437887, 0.013581171, -0.012554517, -0.035287578, + -0.049836736, -0.0015858144, -0.03745822, -0.01759979, 0.016499802, + 0.0016783966, -0.00061461766, 0.052183375, 0.040362183, -0.02290906, + 0.009151891, -0.0015473148, -0.0015903977, -0.044732798, 0.0074175783, + 0.054735348, 0.0011632361, 0.0073809116, -0.03481825, -0.008909893, + 0.015062487, -0.0062882584, -0.02085575, -0.013757169, 0.03572757, + 0.011498529, -0.0011962357, 0.015150486, -0.014893822, -0.005096606, + -0.0070545822, -0.005774931, 0.015825145, -0.003397126, -0.01366917, + -0.014673824, 0.0075605763, 0.003989286, 0.0068089184, -0.0011054868, + -0.03552224, 0.003532791, 0.03235428, 0.047050104, -0.013933167, + -0.04376481, 0.0027151343, 0.012048523, -0.0098632155, 0.03657823, + -0.014549159, 0.004572279, -0.029670313, 0.016763799, -0.043794144, + 0.02031309, 0.0015958976, -0.0001079935, 0.010405876, -0.05752198, + -0.013581171, 0.0049536075, 0.0010349043, -0.011212532, -0.010156545, + 0.014512493, 0.015942477, 0.0053679356, -0.016954465, 0.007963905, + 0.018655777, -0.010515874, -0.004278949, -0.030124974, -0.00695925, + 0.011901858, -0.02650235, -0.0037216223, 0.017687788, 0.0035951238, + -0.0053092698, -0.027690336, -0.0061085937, 0.007575243, 0.0306823, + 0.0019799764, 0.009555219, 0.020841084, -0.0076339087, 0.023789048, + 0.008095903, 0.016602468, 0.007758574, -0.014585826, 0.020679753, + 0.0072562466, -0.02044509, 0.016895799, -0.005679599, 0.020210424, + 0.022219734, 0.0017040629, 0.012539851, -0.010941203, 0.02855566, + 0.015839811, 0.0009881549, 0.026517017, -0.034348924, -0.0031881286, + 0.009672551, -0.019022439, -0.049602073, -0.010237211, -0.0032541277, + 0.015619813, -0.012803847, 0.028702324, 0.0062479256, 0.0065889214, + 0.023349054, -0.02295306, -0.036402233, -0.016558468, 0.025754359, + -0.014710491, -0.0036171235, -0.009885215, 0.014563826, 0.025857024, + 0.015238484, -0.012202521, 0.004198283, 0.018362448, -0.019623766, + 0.0118138585, 0.009349888, -0.040596846, -0.02950898, -0.012517851, + -0.024053046, -0.006838252, -0.013837835, 0.005353269, 0.009679885, + -0.016910465, 0.009973214, 0.02361305, 0.0016637301, -0.026605016, + 0.016925132, 0.0030469636, 0.030066308, -0.033498265, 0.024522373, + -0.011425197, 0.009995214, 0.012407851, -0.019022439, -0.0195211, + -0.007831906, 0.0135738375, 0.023525052, -0.018494446, -0.038895536, + 0.015106486, -0.0006806169, 0.000753491, -0.003959953, -0.029860977, + -0.0008259068, -0.007941905, -0.018699776, 0.0074139116, 0.0037986212, + 0.001477649, -0.009899882, -0.025241032, 0.028848989, 0.005107606, + -0.019931762, 0.03572757, -0.020034427, 0.019917095, -0.008557898, + -0.011109867, -0.030476969, -0.010383876, -0.007058249, -0.010823871, + 0.019227771, 0.03748755, -0.0035914571, 0.026722347, -0.00921789, + 0.015121153, -0.0040956177, 0.0029241317, -0.02483037, -0.010919203, + 0.011197866, 0.015707813, 0.027455673, -0.010332543, 0.040186185, + 0.012158521, -0.019535767, 0.015062487, -0.02478637, -0.005892263, + 0.018846441, 0.029816978, -0.0013044011, 0.038191542, -0.0030762965, + 0.0055806, -0.015121153, 0.007105915, -0.006027928, 0.007941905, + -0.03009564, -0.033028938, 0.0020038094, -0.026517017, 0.019535767, + -0.013258508, -0.02474237, -0.003959953, -0.038895536, 0.041770168, + 0.021955738, 0.005980262, -0.011923858, -0.01597181, 0.0030286305, + 0.00027087177, -0.0033769596, 0.002119308, -0.009027226, 0.007505577, + -0.034466255, 0.051479384, 0.0051296055, -0.017438458, 0.0082572345, + -0.041124843, -0.017907785, -0.01427783, -0.01005388, 0.033967596, + 0.0001905498, 0.011102534, 0.009885215, 0.0021229747, -0.018318448, + 0.02207307, -0.01003188, -0.035962235, 0.03933553, -0.035463575, + -0.029582312, -0.016074475, -0.033322267, -0.008939226, 0.014285163, + -0.03402626, 0.010545207, 0.032295614, -0.018919773, 0.035580907, + 0.015663812, 0.0013676503, -0.01843578, -0.019462435, -0.0018635611, + -0.020078426, -0.005646599, -0.009628552, -0.0017416459, -0.012393185, + 0.007358912, -0.015223818, 0.0041066175, 0.00037582885, -0.006706253, + 0.0088878935, -0.007289246, -0.0056392658, -0.010039213, 0.008579898, + -0.017379792, 0.0105232075, -0.02135441, -0.004487946, 0.020371757, + -0.023422386, 0.01893444, 0.02930365, -0.008440566, -0.00075211603, + -0.022219734, 0.001206319, 0.046346113, -0.0019744765, 0.0035126247, + 0.0063359244, -0.028408993, -0.014747157, -0.03214895, -0.00028645492, + -0.022791728, -0.034348924, 0.039394196, -0.0002940173, 0.03519958, + -0.00612326, 0.00978255, 0.01868511, 0.0115938615, -0.019477101, + -0.020577088, 0.010310544, 0.017834453, -0.009459887, 0.015223818, + 0.013023844, -0.0042239497, -0.02876099, 0.01735046, 0.013529838, + 0.005184605, -0.011263865, 0.01088987, 0.016323805, -0.021339744, + 0.015942477, 0.036050238, 0.024845036, 0.010200545, 0.033967596, + -0.027015677, 0.008858561, -0.01448316, -0.0012649848, 0.025475696, + 0.017291794, -0.0012631515, 0.037399553, -0.03173829, 0.004432947, + -0.015605147, -0.019594433, 0.042532824, 0.010603873, -0.012048523, + 0.017658455, -0.0100832125, 0.01059654, 0.0125691835, 0.03144496, + 0.0005972012, 0.03522891, 0.047842097, 0.003743622, -0.05062873, + -0.036138237, 0.010779872, -0.025593027, 0.0034576254, -0.0076632416, + -0.021075748, -0.020078426, 0.042268828, 0.026781013, 0.016881132, + 0.049426075, -0.012371185, -0.014857156, 0.03470092, -0.021310411, + 0.010383876, -0.043002155, 0.010757872, 0.0006004095, 0.002412638, + 0.017218461, -0.0032761276, 0.002667468, 0.014952488, 0.019506434, + 0.034847584, 0.024053046, 0.019843763, -0.000045632267, + -0.0000048554107, -0.012869846, -0.010801871, -0.008330567, + -0.043911476, -0.008271901, 0.016499802, -0.0043889475, 0.05731665, + -0.02207307, -0.014043165, 0.034436923, 0.014314496, 0.0031661289, + 0.014974488, -0.019447768, -0.015209151, -0.040186185, -0.0034961249, + -0.039863523, 0.024874369, 0.0364609, -0.0036666228, -0.015414483, + -0.0022476397, -0.010823871, -0.017951785, 0.028452994, 0.014263163, + 0.0065705883, 0.020797085, 0.0035951238, -0.038455542, -0.009591886, + -0.017541124, -0.05772731, 0.021134414, -0.026531683, 0.024698371, + -0.011652527, -0.021838406, -0.02320239, 0.0008511148, 0.008198569, + 0.025109034, 0.0006797002, 0.013647171, -0.0059545957, -0.03833821, + 0.0012209854, 0.033527598, -0.015619813, 0.011447197, 0.012554517, + 0.03992219, -0.023422386, 0.02587169, 0.020591754, -0.0026876347, + 0.017658455, -0.029567646, 0.0022788062, 0.015942477, -0.026737014, + -0.0019433101, -0.01281118, -0.031620957, 0.023877049, 0.0053422693, + -0.028687658, 0.003490625, 0.0109338695, 0.022747729, 0.015253151, + 0.01977043, 0.015942477, -0.029376982, 0.012026523, -0.00037949547, + 0.021515744, -0.018787775, 0.024859702, 0.0028012998, 0.013617837, + -0.06588188, -0.00642759, 0.0048399423, -0.026267687, -0.0036061236, + -0.0052616037, 0.0055366005, -0.04878075, 0.009562552, 0.019433102, + 0.00060590944, 0.017159795, 0.010677205, 0.013097177, 0.0010642373, + 0.023569051, -0.028658325, -0.022087736, -0.009041892, -0.008997892, + 0.014431828, -0.01421183, 0.021897072, -0.017130462, -0.0083525665, + 0.014901156, -0.013207176, 0.0055696, 0.030300971, -0.0048766085, + 0.003320127, -0.020386424, -0.015165152, 0.006585255, -0.0019506434, + -0.010501208, -0.0018021452, -0.0071535814, 0.0019103105, + -0.023231722, -0.023730382, 0.0007910739, 0.008719229, 0.0123491855, + 0.028188996, -0.0006022428, 0.015707813, -0.01839178, 0.010229878, + -0.00446228, 0.012026523, 0.0056209327, 0.002484137, -0.021794407, + 0.03675423, 0.021413077, 0.00072140805, -0.0083159, -0.0028581326, + 0.004502613, 0.031972952, 0.013786502, -0.023569051, 0.00076632417, + -0.021046415, -0.01170386, -0.02846766, 0.014079832, 0.007138915, + -0.0222784, 0.010435209, 0.00024566374, 0.0034832917, -0.000894656, + -0.0023943046, 0.0086678965, -0.007186581, 0.008748562, 0.043794144, + 0.005386269, -0.03695956, 0.01199719, 0.010493875, -0.0035914571, + 0.002031309, -0.0070032496, -0.004025952, -0.010970536, -0.012224521, + -0.0148351565, 0.0063175913, 0.023129057, -0.0024089713, 0.0016343972, + 0.0006586171, 0.0041542836, -0.038132876, 0.04097818, 0.002808633, + -0.0042936155, -0.0027921333, 0.0064605894, 0.009591886, -0.00867523, + -0.0102738775, -0.027455673, -0.016705133, -0.016895799, + -0.0148351565, 0.011087867, -0.008719229, -0.024434375, 0.0015482315, + -0.0039269533, -0.0033274603, 0.049895402, -0.0034411256, 0.042826153, + 0.00009458741, 0.015194485, 0.00026835097, 0.0034227925, -0.009019893, + -0.043031488, 0.02383305, 0.0026308019, 0.009943881, 0.0029882977, + -0.015663812, -0.004568612, -0.033204935, 0.0095478855, 0.014549159, + 0.0054192687, 0.0087925615, 0.010317877, -0.02161841, 0.006398257, + 0.011256532, 0.005125939, 0.010156545, 0.023818383, 0.012261187, + 0.015414483, 0.012085189, -0.011513196, -0.01847978, -0.008909893, + 0.010757872, -0.010853204, 0.0071535814, 0.010339877, 0.009701884, + -0.010589207, -0.0016472304, 0.02742634, 0.025314365, 0.009525886, + -0.018714443, -0.0014318163, 0.0014657325, -0.0117185265, 0.009811883, + -0.0023209723, 0.01597181, -0.0057052653, -0.0082279015, -0.011175866, + -0.010787205, -0.007890573, -0.034348924, -0.006500922, 0.02173574, + 0.017497124, 0.03314627, 0.01452716, 0.0032889608, 0.008213235, + -0.0074725775, 0.0113738645, -0.017203795, 0.018963773, -0.0067795855, + -0.002143141, -0.01864111, 0.0473141, 0.011806525, -0.0064019235, + 0.012319853, -0.039951522, 0.011615861, -0.009870549, -0.008931893, + -0.008381899, -0.0053826026, -0.01143253, -0.007028916, -0.00724158, + 0.013661837, 0.015781146, 0.013911167, 0.0044549466, 0.039276864, + -0.0057529313, -0.009694551, -0.0074139116, -0.024801036, 0.03522891, + -0.01985843, -0.016499802, -0.022117069, 0.0044439468, 0.022718396, + 0.017702455, 0.0051186057, -0.03502358, -0.025314365, -0.011439864, + -0.0038719538, 0.01638247, 0.009349888, 0.0037216223, 0.009144558, + -0.019125104, -0.008895227, -0.00034924582, -0.00057474314, + -0.0031001295, 0.017585123, -0.012767181, 0.0050086067, 0.04097818, + -0.009518553, -0.01613314, 0.03531691, 0.008953893, -0.0072562466, + -0.012591183, -0.0020881416, 0.008939226, -0.019535767, -0.017526457, + -0.00865323, 0.029376982, 0.043207485, -0.0047556097, -0.003255961, + 0.013830502, -0.010075879, 0.03305827, -0.030828966, 0.018699776, + 0.015077153, -0.038191542, 0.0017874787, 0.0035822906, -0.020137092, + -0.009364555, 0.021794407, -0.024845036, -0.0019469768, -0.025197033, + -0.012437184, -0.0071095815, 0.00032266282, -0.02562236, + -0.0044732797, -0.012173188, 0.008719229, -0.027397007, 0.013221842, + -0.036519565, 0.0141091645, -0.022923727, -0.01843578, -0.009679885, + 0.010295877, -0.013368507, 0.02950898, 0.037428886, -0.012679182, + -0.030858299, -0.030887632, -0.011080534, 0.00036895392, -0.028863655, + 0.0062332586, -0.020533089, -0.003538291, 0.0018149783, -0.010970536, + 0.015502482, 0.011491196, 0.0005082856, 0.004946274, -0.02453704, + -0.03531691, -0.008616564, -0.0017526457, 0.01477649, 0.037047558, + 0.030330304, 0.0031716288, 0.026649015, -0.0032522944, -0.00023558052, + 0.032002285, -0.010215212, 0.0013933167, 0.0010028214, -0.027925, + 0.008645897, -0.0011192366, -0.031620957, 0.006621921, -0.008697229, + 0.0017801455, 0.008095903, -0.010222545, 0.019095771, 0.028438326, + 0.017438458, 0.01373517, -0.015561148, 0.014901156, -0.016441137, + -0.023774382, -0.017526457, -0.0061672595, 0.017203795, -0.019755764, + -0.002018476, -0.0097385505, -0.032002285, -0.007215914, -0.019829096, + 0.025314365, 0.0014098164, -0.013991833, 0.0006338674, 0.012686515, + 0.02595969, -0.0044292803, -0.019169105, -0.037516885, -0.011219866, + -0.017687788, -0.012737848, 0.007168248, -0.0057785977, 0.008095903, + -0.035756905, -0.029875644, 0.007193914, -0.010875204, 0.0085432315, + -0.016837133, 0.006893251, -0.028042331, -0.0016032308, 0.005888596, + 0.011667194, 0.014651825, -0.023627717, 0.004608945, -0.01059654, + -0.029978309, 0.0062259254, -0.021926405, 0.013346507, 0.023481052, + 0.0046309447, -0.00039599527, 0.0075019104, 0.010244545, + -0.0056135994, 0.0024749704, 0.011564529, -0.010897203, -0.0014143997, + -0.045964785, 0.035375576, 0.015487815, -0.03616757, -0.004198283, + -0.012393185, -0.0045576124, -0.0018901441, -0.0091078915, + 0.00026835097, -0.022161068, -0.01534115, -0.019301103, -0.012180521, + 0.025021035, -0.0011568195, -0.0054522683, 0.0145198265, 0.014248497, + -0.026282353, -0.024933035, -0.023906382, -0.005679599, 0.026927678, + 0.007292913, -0.025358364, 0.0034997915, -0.0018699777, -0.020195758, + -0.0043009487, 0.0042276164, 0.033410266, -0.019697098, -0.008345233, + 0.015238484, 0.032119617, -0.022337066, -0.007505577, -0.0069189174, + 0.0038169543, 0.013977166, -0.010992535, 0.00013646712, -0.019961094, + -0.005532934, -0.023393054, -0.017453125, 0.014131164, 0.018963773, + 0.025079701, -0.013757169, -0.012539851, 0.012972511, 0.030506302, + 0.010501208, -0.0027518005, 0.0014418995, -0.0053386027, 0.006181926, + -0.027954333, -0.01059654, 0.023583718, -0.00867523, -0.0028251328, + 0.0034686252, 0.012957845, -0.0014629825, 0.0033531266, -0.02010776, + -0.004242283, 0.013075178, -0.037546217, -0.01229052, 0.028819656, + 0.011388531, -0.02182374, -0.0034594587, 0.006299258, 0.0014730657, + -0.0120338565, -0.008550565, -0.023965048, -0.01475449, 0.012715848, + 0.028042331, 0.015370483, 0.0016609802, -0.02311439, -0.0042716158, + 0.0078099067, -0.023935715, 0.015737146, -0.004751943, -0.028320994, + 0.04376481, 0.012209854, 0.015267817, 0.004289949, -0.0019341436, + -0.01482049, -0.020122426, 0.029816978, 0.035844903, -0.0034099591, + -0.020034427, -0.0036006237, 0.0040112855, -0.001168736, 0.010413209, + -0.024214378, 0.0032357946, 0.011857858, 0.016411804, -0.001930477, + -0.0029772979, -0.003202795, -0.015825145, 0.009789883, -0.009657885, + -0.0009808217, -0.007853907, -0.0038242876, -0.009855882, + -0.018318448, -0.024141045, 0.006009595, 0.020386424, 0.0066622538, + -0.01306051, 0.0027169676, 0.02085575, -0.02048909, -0.0025611361, + 0.017570456, 0.013016511, 0.011168533, 0.0054009357, 0.0011852358, + -0.0011879858, -0.00035382912, -0.031972952, 0.001592231, 0.013434506, + 0.009247223, -0.0030469636, 0.007410245, 0.014013832, -0.014079832, + 0.019902429, 0.015810478, 0.04100751, -0.021691741, 0.0011769859, + 0.006794252, 0.027279673, -0.022498397, 0.005562267, 0.0073809116, + 0.015869144, -0.016954465, -0.01303851, 0.019638432, 0.0081325695, + 0.0021743074, 0.011007202, 0.054882012, 0.010427875, -0.0077659073, + 0.009393888, -0.006368924, -0.0045282794, -0.034084927, -0.041124843, + -0.0007681575, -0.013331841, -0.023026392, 0.010493875, 0.0022403065, + 0.0037802882, 0.008330567, 0.030300971, -0.009599219, 0.004110284, + 0.018083785, 0.013251175, -0.00058024307, -0.01935977, -0.010471875, + 0.0011458197, 0.011571862, -0.0022183068, 0.00020166425, -0.019477101, + 0.010259211, 0.00090886414, -0.0043706144, -0.007780574, 0.02110508, + -0.002425471, 0.0015353983, 0.0026876347, -0.0075972425, + 0.00087265624, 0.006746586, 0.008286567, -0.009298556, 0.02173574, + 0.022263734, -0.020430423, 0.01889044, -0.0035291244, -0.014417161, + 0.0032999606, -0.020841084, 0.022806395, 0.0014217331, 0.007553243, + 0.0011109867, -0.009269223, -0.0043486147, -0.00865323, 0.0025794692, + -0.008367233, 0.011293198, -0.010479208, 0.015693147, 0.0024273044, + -0.013647171, -0.003485125, 0.0068309186, 0.011879858, -0.00017118546, + 0.009181224, 0.024214378, 0.038250208, -0.017482458, -0.020181092, + 0.03942353, -0.017775787, -0.0072122472, 0.010530541, -0.019697098, + 0.007846573, 0.004091951, 0.004091951, -0.009225223, -0.004080951, + -0.016030475, -0.00362629, -0.021838406, -0.026326353, 0.0030982962, + -0.01003188, 0.008550565, 0.0071315817, -0.009129891, -0.016910465, + 0.024727704, 0.0034246258, 0.0024878036, 0.005257937, 0.009811883, + -0.015003821, -0.02725034, -0.0017764787, 0.00363179, 0.018039785, + -0.012987178, 0.011674527, 0.011249199, -0.008191235, 0.013610504, + 0.003967286, -0.0076852413, -0.014497827, -0.0222784, -0.003809621, + 0.0068419185, 0.010603873, -0.014197163, -0.0119091915, -0.014864489, + 0.011498529, 0.012393185, 0.0139844995, -0.008572564, -0.014710491, + 0.0017471458, -0.0022879727, 0.011095201, 0.0031697955, -0.010735871, + -0.005756598, -0.022630395, -0.015106486, 0.011263865, -0.009723884, + 0.021251746, -0.0062589254, -0.0051992713, -0.0031917952, + -0.007971238, -0.024214378, -0.010075879, -0.002638135, 0.017863786, + -0.00014322746, -0.003915953, -0.011395864, 0.017394459, 0.009621219, + 0.004872942, 0.012994511, 0.011806525, 0.004597945, 0.016807798, + -0.0122538535, -0.008909893, 0.0147398235, 0.022087736, -0.010515874, + -0.0055439337, -0.0026326352, -0.020973083, -0.0070215827, + 0.017585123, -0.0139844995, -0.009555219, 0.005969262, 0.020591754, + 0.016353138, 0.024918368, 0.035756905, 0.012935845, -0.005389936, + -0.018157117, -0.005815264, -0.021471743, -0.009232556, -0.0034502922, + 0.02474237, -0.009093225, -0.009122558, 0.013412506, -0.008183902, + 0.005202938, 0.008293901, 0.016162474, -0.020577088, -0.008396566, + -0.024141045, 0.0119091915, 0.0017746454, -0.0066145877, 0.0050562727, + -0.009005226, 0.0043486147, -0.012415185, -0.004597945, -0.003208295, + -0.015663812, -0.026414352, 0.021662408, 0.0014409828, 0.010281211, + 0.018289115, -0.0028617992, 0.003208295, 0.009393888, 0.018817108, + 0.008946559, 0.01170386, 0.0072269137, 0.0014363995, 0.028775657, + -0.03569824, 0.0024089713, 0.011630528, 0.0038169543, 0.0050819395, + 0.0008648647, -0.013911167, -0.008763229, 0.0008190319, 0.022630395, + 0.009965881, -0.028746324, 0.026825013, 0.009687218, 0.005917929, + 0.034759585, -0.0021871405, 0.017922452, 0.034348924, -0.0069849165, + -0.012979845, -0.01529715, 0.014226497, -0.0017132296, 0.0060829273, + -0.018377114, 0.01847978, -0.023099724, -0.009364555, -0.012407851, + 0.00088823936, -0.01534115, -0.005114939, 0.00029676728, 0.0022183068, + 0.005510934, -0.0011504029, 0.00786124, -0.002678468, -0.000025293968, + 0.0055072675, 0.027734336, -0.008191235, 0.001256735, 0.0034869583, + 0.00016064392, -0.0051956046, 0.00726358, 0.01504782, -0.014145831, + -0.0054779346, -0.0023704716, -0.007553243, -0.00083828164, + 0.008095903, 0.015106486, -0.0054229354, -0.0035859572, 0.011777192, + 0.000079405305, -0.007494577, 0.0021138082, 0.0112785315, + 0.0047299433, -0.0069005843, -0.00006061386, -0.017907785, + -0.0032302947, 0.0028562993, 0.01931577, -0.006453256, -0.0064019235, + -0.024199711, -0.0036409565, -0.007905239, -0.014974488, 0.015517148, + 0.026062354, 0.010985202, 0.008939226, 0.006170926, 0.0056135994, + -0.0052909367, -0.015443816, 0.012627849, 0.020210424, -0.010259211, + 0.01170386, 0.0055439337, -0.0046309447, -0.0039342865, -0.017189128, + 0.011608528, 0.011359198, 0.011557195, -0.0011412364, 0.026179688, + 0.03173829, -0.0011696527, 0.0016783966, -0.019550433, 0.0013988166, + 0.0013878167, -0.0076852413, -0.0070252493, 0.0142044965, + -0.019741097, 0.009371888, -0.0015738979, -0.0036831226, 0.019594433, + 0.014607825, -0.0016123974, -0.0019818097, 0.0028214662, + -0.0032394612, -0.007905239, -0.0012439018, -0.02571036, + -0.0014684824, 0.027778335, -0.019433102, 0.0012384019, 0.046258114, + -0.01448316, 0.016059808, -0.015135819, 0.019374436, -0.007875906, + 0.0056319325, -0.002830633, 0.017585123, -0.012613183, 0.011285865, + -0.005144272, 0.01314851, 0.00082819845, -0.0029222984, -0.008147236, + 0.0058959294, 0.004715277, 0.0008405733, 0.005793264, -0.023935715, + 0.003508958, 0.008330567, 0.005437602, -0.0078099067, 0.028071664, + -0.0052176043, 0.037546217, -0.0013850668, 0.018963773, -0.0046236115, + -0.023730382, -0.019506434, 0.0013639837, -0.016749132, -0.009753217, + -0.0029167985, -0.009562552, 0.0030157974, -0.012943178, 0.0062149256, + -0.018553112, -0.010794538, -0.002060642, 0.011212532, 0.011689194, + -0.004469613, 0.008462566, -0.0078025735, -0.0023338054, 0.012158521, + -0.0004995774, -0.0024328043, -0.0012109022, -0.0049829404, + 0.012495851, 0.0069555836, -0.02984631, -0.002678468, -0.016573135, + -0.0168518, -0.010970536, 0.0038279542, 0.0072379136, 0.007446911, + 0.01730646, -0.0022769729, -0.0033696264, 0.005606266, 0.007468911, + -0.0016637301, -0.011835858, 0.0071425815, 0.0041946163, 0.0027243008, + -0.040274184, -0.0043962807, 0.009159224, -0.0012695681, + -0.00043655728, -0.007168248, 0.016807798, 0.014600492, 0.0055182674, + 0.004832609, 0.036108904, -0.028482327, -0.012994511, -0.025343698, + -0.00057840976, 0.0073919115, 0.009980547, 0.019286437, 0.004579612, + -0.0078172395, 0.0049939402, -0.0125691835, 0.02144241, 0.005708932, + -0.02153041, 0.022542397, -0.008367233, -0.0040479517, -0.008242568, + 0.021383744, 0.0017288127, 0.020914417, -0.022351732, -0.016323805, + -0.019037105, 0.021061081, 0.0020496421, -0.0028672991, -0.013221842, + 0.031151628, 0.008689896, -0.011755193, 0.024317043, -0.027705003, + 0.0049719405, -0.006563255, -0.021897072, 0.0003361835, -0.016587801, + 0.0048472756, 0.012143855, -0.010662539, 0.004704277, 0.0131045105, + -0.001586731, -0.009775217, 0.027646337, 0.011799192, -0.023730382, + -0.011439864, -0.013053177, 0.0057822643, 0.010816538, -0.0050672726, + -0.023774382, -0.00923989, -0.004385281, 0.001144903, -0.00788324, + 0.003596957, 0.012891846, -0.007897906, 0.0076632416, -0.003255961, + 0.007941905, -0.0018048951, 0.02541703, 0.0020294758, -0.00081536523, + -0.012195188, -0.012532517, 0.021457076, 0.024478374, 0.015062487, + -0.0019139771, 0.002407138, -0.008330567, 0.0016499803, 0.011007202, + 0.019873096, 0.011777192, 0.0111392, -0.007971238, -0.0073075793, + -0.001613314, 0.010567207, 0.019125104, -0.011051201, 0.00065403385, + -0.0016362305, 0.018758442, 0.029039653, 0.0035804573, 0.02483037, + -0.0010055713, -0.0031807953, -0.020210424, 0.015282484, + 0.00083003176, 0.011403197, -0.013896501, 0.0034447922, 0.0006755753, + -0.011938524, 0.019022439, 0.0013621504, -0.019594433, 0.002542803, + 0.03223695, -0.029714312, 0.012026523, -0.015869144, 0.0028562993, + 0.014087165, 0.033850264, -0.005969262, -0.009364555, -0.024023714, + -0.0014162331, 0.007384578, 0.004073618, -0.0051882714, -0.008484566, + -0.004638278, -0.017907785, 0.02670768, -0.010640539, -0.012275853, + -0.009679885, -0.023305055, -0.00043999474, -0.0067429193, + 0.024991702, 0.023187723, -0.014541826, -0.0055256006, 0.016030475, + -0.012862513, 0.015165152, 0.010845871, 0.0057125986, -0.0011513196, + 0.01814245, 0.0058482634, -0.0166758, 0.011190533, 0.028056998, + 0.014446494, -0.008829228, -0.020533089, 0.023862382, 0.011300531, + -0.008242568, 0.0011751526, -0.008748562, 0.014043165, 0.019550433, + 0.0092032235, -0.011879858, -0.0065522552, 0.008147236, 0.019110437, + 0.0029516313, -0.005085606, -0.016103808, -0.012085189, + -0.00093498884, 0.0005000357, -0.006346924, 0.016866466, 0.0026821345, + -0.0012723182, -0.02837966, -0.0041432837, -0.0039452864, + -0.007648575, -0.005661266, 0.011762526, 0.00072782463, 0.0111172, + 0.022747729, 0.011747859, -0.023231722, 0.021941071, -0.028188996, + 0.008499232, -0.039863523, 0.025945023, 0.017834453, -0.0008259068, + 0.00004179377, -0.015531815, -0.0018773109, -0.008059237, + 0.0031661289, 0.0014281496, 0.009687218, -0.007318579, -0.0166318, + 0.01597181, -0.0016866465, 0.040772848, 0.0046236115, -0.004161617, + 0.0015198152, -0.009166557, -0.02629702, 0.0034594587, -0.015634479, + -0.015414483, 0.023789048, -0.023481052, 0.014703157, -0.009063892, + 0.00017198753, -0.007677908, 0.018333115, 0.00001711568, -0.006533922, + 0.0030451303, 0.0054779346, -0.0014794823, 0.011806525, 0.0052139377, + -0.025387697, -0.011615861, 0.018215783, -0.0032009617, -0.008645897, + -0.013500505, 0.049308743, -0.009085892, -0.016074475, -0.015854478, + 0.004220283, 0.007329579, 0.024009047, 0.02336372, -0.027191674, + -0.005950929, 0.001339234, -0.007941905, -0.002348472, -0.020503756, + -0.004542946, 0.021794407, -0.02207307, 0.007127915, 0.007336912, + -0.0060829273, -0.02562236, -0.020679753, -0.0026986345, 0.0032431278, + 0.016822465, 0.0011320699, -0.016147807, 0.007751241, 0.004685944, + 0.02345172, -0.008953893, -0.012679182, -0.007058249, 0.0037106224, + -0.01985843, -0.006057261, -0.000009725144, -0.020357091, + -0.0014455661, -0.0010880703, 0.021325078, -0.011498529, 0.021515744, + -0.004289949, -0.015135819, -0.0009290306, -0.016573135, -0.00642759, + 0.005892263, 0.0023796381, 0.017643789, -0.0007571576, -0.0065669217, + 0.0046639442, -0.00500494, -0.0017810621, -0.020166425, 0.0077732406, + -0.0094525535, 0.004198283, 0.004286282, 0.02453704, 0.0013841501, + -0.019389102, 0.020767752, 0.016954465, 0.023437053, 0.0036702894, + -0.00006496797, -0.007919906, -0.014637158, 0.010244545, 0.021457076, + 0.004920608, 0.01847978, -0.023877049, -0.007366245, 0.018010452, + -0.007985905, 0.008073904, -0.015898477, 0.016441137, 0.008367233, + -0.0011183199, 0.026605016, 0.002526303, 0.0125691835, 0.018289115, + 0.001474899, -0.012767181, -0.008990559, 0.016514469, -0.0029992976, + -0.00528727, -0.0042569493, 0.00047482766, -0.009672551, + 0.00027476755, -0.01759979, -0.0051296055, -0.022542397, + -0.00028118413, 0.010413209, -0.008293901, -0.0014061498, + 0.0016783966, 0.013214509, -0.008433233, 0.009364555, -0.0015427315, + -0.0065779216, 0.00394162, 0.024317043, -0.012444518, -0.00014655033, + -0.0030836298, 0.014923155, 0.03285294, -0.006592588, -0.006475256, + -0.0005669516, -0.0166758, -0.022410398, 0.024551706, 0.018846441, + -0.013639837, 0.025006369, 0.039658193, 0.010750539, 0.023803717, + -0.0063139247, -0.017643789, 0.017863786, 0.003849954, 0.006651254, + -0.0020496421, -0.026766347, 0.014050499, 0.005074606, -0.015795812, + 0.01538515, 0.00016614384, -0.0012365686, 0.008836561, -0.0030781298, + -0.010413209, -0.0028709657, -0.008733896, 0.0068785846, 0.0111831995, + -0.020151759, -0.0069299173, 0.019433102, 0.014329162, 0.023950381, + -0.007395578, 0.014409828, 0.015575814, -0.014431828, 0.0029589646, + 0.02747034, 0.011227199, -0.0007754907, -0.0057345983, -0.005492601, + -0.014541826, 0.0059215957, 0.0059765955, 0.0057052653, -0.023041058, + -0.0005687849, 0.015605147, 0.016265139, 0.0028709657, 0.016925132, + 0.016793132, -0.013214509, 0.009907215, 0.007138915, 0.027763668, + -0.009679885, -0.019242438, -0.0034704586, 0.021119747, 0.012070523, + 0.0025299697, -0.0031514624, 0.0040992843, -0.004542946, 0.006783252, + -0.005221271, -0.016807798, 0.011263865, 0.025153033, -0.0023264722, + 0.011806525, -0.015751813, -0.006097594, 0.0025978023, -0.0083159, + -0.003989286, 0.0033989593, -0.0032357946, 0.0022329732, + -0.0037472886, 0.007215914, -0.02833566, 0.007146248, 0.013111844, + -0.0011549862, -0.003149629, 0.0065705883, -0.01341984, -0.010039213, + -0.022263734, -0.005928929, 0.023246389, -0.020635754, -0.011359198, + -0.0015408982, -0.010493875, 0.0077732406, -0.0027261341, + -0.007897906, -0.005485268, 0.014461161, 0.010207878, -0.009767883, + 0.0141091645, -0.009393888, 0.03874887, 0.008755895, -0.012950512, + 0.010897203, -0.0048216092, 0.029728979, 0.018611778, 0.00066732534, + 0.0072122472, -0.00890256, -0.033410266, -0.006658587, 0.012921179, + -0.021809073, 0.0027169676, 0.014123831, -0.0021321413, 0.018553112, + 0.0058739297, 0.020958416, -0.002638135, -0.0031111294, 0.01204119, + 0.004080951, 0.013947833, -0.013969833, 0.0013025678, -0.0026069689, + 0.006640254, 0.008939226, -0.009386554, 0.022791728, -0.0039122864, + -0.007721908, 0.013185176, 0.02114908, -0.0005252437, 0.0074249115, + 0.019506434, 0.00978255, -0.0073442454, -0.009063892, -0.011087867, + 0.0036226234, 0.0072562466, -0.0026216353, -0.0074065784, 0.00223114, + 0.011843191, -0.03399693, -0.0035951238, 0.0021853072, -0.021383744, + 0.018010452, -0.0196091, -0.003673956, 0.01776112, 0.003584124, + 0.0035217912, -0.009973214, -0.0041176174, -0.011549862, 0.017042464, + 0.0063175913, 0.013749836, -0.02541703, -0.018919773, 0.009914548, + -0.00726358, -0.025285032, 0.006211259, -0.0167198, -0.0015665647, + -0.0130091775, 0.008381899, 0.01705713, -0.005551267, 0.008895227, + -0.012121855, -0.02968498, -0.010537874, -0.0010211545, 0.015121153, + 0.0045832787, 0.010743205, 0.00061645097, -0.011285865, + -0.00044366135, -0.010200545, 0.008037237, -0.019081105, 0.0002903507, + -0.029200984, -0.018567778, 0.021501077, 0.011535196, -0.009012559, + 0.021926405, -0.0018113117, -0.0168518, -0.0016930631, 0.005628266, + -0.002872799, 0.0034411256, 0.0031532957, -0.0040112855, 0.024097046, + 0.020474423, -0.010075879, 0.009826549, 0.02433171, 0.0039269533, + -0.016323805, -0.020547755, -0.007105915, -0.0081325695, 0.004480613, + -0.000014007645, 0.007934572, -0.013779169, -0.004957274, + -0.0019268104, 0.0168518, 0.0030084641, 0.005492601, 0.010545207, + 0.02566636, -0.012341852, -0.0018094784, 0.0103032105, 0.008807228, + 0.003959953, 0.007516577, -0.024874369, -0.01032521, -0.00024818454, + 0.015253151, 0.023891715, 0.015531815, -0.008345233, 0.017071797, + -0.017042464, -0.019330436, -0.005180938, -0.0082279015, -0.011073201, + 0.0038132877, -0.025226366, -0.015649145, -0.02336372, -0.0037472886, + -0.004260616, 0.011645194, -0.011806525, 0.017643789, -0.02399438, + -0.008484566, -0.0056135994, -0.0052652704, -0.01897844, 0.004319282, + 0.01642647, -0.0076559084, 0.00007442098, -0.017746454, + -0.00041501588, -0.014857156, 0.04798876, -0.0051662717, 0.012107189, + -0.007017916, -0.010684539, -0.0058849296, 0.0028379662, 0.019403769, + 0.011293198, -0.012767181, 0.0043596146, -0.0037362888, -0.006827252, + 0.0036519563, 0.025563695, 0.008044571, 0.0023924713, 0.017775787, + 0.0015793978, 0.005356936, 0.0040222853, -0.013815835, -0.00786124, + 0.0003730789, 0.008623897, -0.006434923, -0.023173057, -0.015311817, + 0.006863918, -0.011733193, -0.002707801, 0.014329162, 0.00030776716, + 0.0020258091, 0.012070523, 0.00445128, 0.009085892, -0.0037179557, + -0.02135441, 0.0053752693, 0.008073904, 0.0034631253, 0.006728253, + -0.0151798185, -0.023349054, -0.0058482634, -0.019902429, + -0.0003849954, -0.0042349496, -0.006555922, 0.012371185, -0.017995784, + -0.005045273, -0.002390638, -0.012004524, 0.017497124, 0.036431566, + 0.005345936, 0.02453704, 0.005173605, -0.005155272, -0.010999869, + 0.000919864, -0.0040772846, -0.005998595, -0.010897203, 0.02428771, + -0.0101785455, -0.005980262, 0.010860537, 0.010515874, 0.00009630614, + -0.004014952, -0.0030506302, 0.002372305, 0.016206473, 0.0070509156, + 0.0066182544, 0.021413077, -0.012503184, -0.0037161224, 0.003538291, + 0.00394162, -0.0012759848, -0.0007236997, 0.020723753, 0.015223818, + 0.012803847, 0.0022751396, 0.015854478, 0.0003714747, 0.00023271597, + -0.0023759715, 0.03737022, -0.007868573, 0.010244545, 0.007157248, + -0.0041652834, 0.014688491, -0.010897203, 0.016558468, -0.001727896, + 0.007446911, -0.000893281, -0.016103808, 0.01418983, 0.00064578396, + 0.026663681, 0.016778465, 0.0019781431, -0.0096432185, 0.013258508, + -0.00055641, 0.01935977, -0.008308567, 0.007204914, 0.034143593, + 0.0012182355, 0.007292913, -0.0010486541, 0.012693848, -0.027529005, + -0.004740943, -0.00029241317, -0.0021981404, -0.019902429, + 0.013368507, 0.0048876083, 0.009819216, 0.006999583, 0.005617266, + -0.0045099463, -0.009569885, -0.009789883, -0.025358364, + -0.0037876214, -0.011571862, -0.0069189174, 0.012063189, 0.0010110713, + -0.0014767323, -0.009885215, 0.011417864, -0.00474461, 0.023187723, + -0.005034273, -0.004260616, -0.012151188, 0.011271198, 0.019433102, + -0.0168518, 0.004392614, -0.039804857, -0.00065586716, 0.00043724477, + 0.027265007, 0.016015809, -0.0055806, 0.02022509, 0.002478637, + 0.008095903, -0.0018195616, -0.0015775645, 0.008609231, -0.0069262506, + -0.011696527, 0.018993106, -0.008851227, -0.0018516446, 0.00033916262, + -0.0252117, -0.0017627289, -0.0085432315, -0.00976055, 0.030623633, + 0.014131164, -0.018553112, 0.013771836, -0.015326483, 0.01839178, + 0.0049609407, -0.012158521, -0.01362517, -0.005191938, -0.014497827, + 0.018626444, -0.010215212, 0.0003393918, -0.00586293, -0.0039929524, + 0.0072782463, 0.009628552, 0.021926405, 0.0009542386, -0.015531815, + -0.022923727, 0.008711896, 0.00697025, -0.014307163, -0.0020239758, + 0.0101785455, 0.025226366, 0.024918368, -0.008330567, 0.008601897, + -0.008836561, -0.005444935, 0.0061305934, 0.0005481601, 0.004036952, + 0.013529838, -0.038455542, -0.002907632, -0.009401221, -0.00585193, + -0.03176762, -0.006805252, -0.0077659073, -0.02026909, -0.038308874, + -0.022381065, 0.0055366005, -0.014695824, -0.017409125, 0.0050599393, + -0.010141878, -0.017614456, -0.0005719932, -0.0022458064, + -0.00035726657, 0.011652527, -0.0060132616, -0.015062487, 0.014307163, + -0.0028397995, -0.007146248, 0.0072855796, -0.019843763, + -0.0022751396, -0.0083525665, -0.032207616, 0.045348793, 0.0018204782, + 0.009357221, 0.027353007, -0.0046896106, 0.00069665833, 0.00024520542, + -0.00086303137, -0.02340772, 0.0011531529, -0.019829096, + -0.00037514136, 0.027309008, 0.019550433, -0.014387828, 0.0042569493, + 0.013243842, 0.008917226, 0.0022788062, 0.008095903, 0.009635885, + -0.009063892, -0.001344734, 0.0070509156, 0.021647742, -0.009628552, + 0.011175866, -0.0020936416, 0.008843894, 0.010112545, -0.005928929, + 0.008763229, -0.01306051, -0.016397137, 0.006867585, -0.0031349626, + 0.009877882, 0.011087867, 0.013192509, 0.0032632942, -0.009694551, + 0.011043868, -0.013441839, -0.013999166, -0.0070692487, -0.023026392, + 0.019154437, 0.0034062925, -0.004531946, 0.02738234, -0.008059237, + 0.028892988, -0.0068822512, 0.015194485, 0.021266412, -0.011879858, + 0.0061122603, 0.0026161354, -0.01726246, 0.018714443, 0.00033755848, + -0.010185879, -0.008367233, 0.0025831359, -0.0017746454, + -0.0014473994, -0.0020716419, 0.00018504987, 0.021867739, + 0.0043009487, 0.027646337, -0.00783924, -0.016705133, -0.0060352613, + 0.006255259, -0.011806525, -0.026047688, 0.025197033, 0.007105915, + -0.0030176307, 0.0024914702, 0.003001131, 0.012327186, -0.026267687, + 0.010141878, -0.010699205, 0.0059949285, -0.012165855, 0.0195211, + -0.011843191, -0.002489637, 0.00502694, 0.005532934, 0.014182497, + 0.022381065, -0.022997059, 0.0039636195, 0.015707813, 0.0077952403, + 0.008851227, -0.028540993, 0.0127891805, -0.00250797, 0.0012961512, + 0.010427875, -0.008469899, -0.0053716027, -0.004568612, 0.00726358, + -0.008807228, -0.006020595, 0.0049352744, -0.0039232867, + -0.0073809116, 0.034847584, 0.012825847, -0.003560291, 0.0013438173, + 0.01226852, -0.014006499, -0.009540552, -0.013368507, -0.0012099856, + 0.0035217912, 0.0012494017, -0.016793132, 0.0020918082, 0.01726246, + -0.011542529, 0.022674397, -0.023481052, -0.0027921333, 0.028614325, + 0.027074343, 0.02921565, -0.0025886358, 0.0014125664, 0.015502482, + -0.013522505, 0.006863918, -0.012437184, 0.0026803012, 0.017115796, + 0.024141045, 0.013023844, 0.006500922, -0.0011146533, -0.021970404, + -0.028643658, -0.019154437, -0.0029864644, -0.01818645, -0.0036427898, + -0.010339877, 0.015011154, -0.0030121307, -0.0012494017, 0.01931577, + 0.01418983, -0.0074725775, 0.008220568, 0.013500505, 0.007846573, + 0.0017214795, 0.010112545, 0.035170246, 0.018582445, 0.022102403, + 0.018333115, 0.0053202696, -0.00039439113, 0.011337198, -0.022205068, + 0.004249616, 0.011483863, 0.005991262, 0.011109867, -0.021603743, + -0.00472261, 0.0047996095, -0.012510518, -0.014094498, 0.012385852, + -0.015209151, 0.014805824, -0.01751179, -0.0013199842, 0.0005981178, + 0.0010165712, 0.011806525, -0.014886489, -0.008308567, -0.0051516052, + -0.018113118, -0.0035052914, -0.0014932322, -0.000113378854, + -0.026033022, -0.022674397, 0.0039269533, -0.023539718, 0.0011430697, + -0.009019893, -0.020342425 + ] + } + ] + }, + { + "name": "emerging-technologies-shaping-our-future", + "title": "Emerging Technologies Shaping Our Future", + "content": "Key technologies and concepts in the current tech landscape include ChatGPT, GPT-4, Anthropic Claude, Gemini, AI, ML, LLMs, DALL-E, Midjourney, Stable Diffusion, Sora, VR/AR/XR, TypeScript, blockchain, Web3, and PWAs.", + "keywords": [ + "ChatGPT", + "GPT-4", + "Anthropic Claude", + "Gemini", + "AI", + "ML", + "LLMs", + "DALL-E", + "Midjourney", + "Stable Diffusion", + "Sora", + "VR", + "AR", + "XR", + "TypeScript", + "blockchain", + "Web3", + "PWAs" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + -0.011711601, 0.018536234, -0.031360634, -0.016186133, 0.0018540126, + -0.015983807, -0.030551326, 0.012248545, 0.00081952027, -0.0037761158, + 0.04189718, 0.0019804668, 0.003637989, -0.006338272, -0.02051281, + 0.012186291, -0.02961751, -0.014808755, -0.031329505, -0.02155557, + -0.014723156, -0.016668605, -0.048620652, -0.007061979, 0.02255164, + -0.0027353007, 0.037352614, -0.0064939074, -0.053601, 0.026862755, + 0.02558654, -0.009058009, 0.020746265, -0.043080013, 0.007824594, + -0.019859139, 0.013337996, 0.012925561, 0.014621992, -0.02406131, + 0.023532147, 0.0037138616, -0.023267565, 0.03442666, -0.012435308, + 0.011789419, -0.0044978773, -0.012956688, -0.0016098587, -0.018925324, + -0.042488597, -0.019003142, 0.055997793, 0.015337917, 0.02168008, + -0.035547238, 0.05606005, -0.006170963, 0.018287217, -0.010855603, + 0.027531989, 0.00620209, -0.035734, -0.0023656655, -0.014069485, + -0.0016011043, -0.0005422938, 0.028061152, -0.011283603, -0.010723313, + -0.040558714, 0.025244141, 0.0014493592, -0.018691871, 0.012232981, + -0.0011614328, -0.012863306, 0.02510407, -0.011501492, -0.018536234, + 0.038037412, 0.017197765, -0.0021575026, 0.008178666, 0.029991036, + -0.0051437663, 0.007540559, -0.08379436, 0.009509353, 0.010412041, + -0.0134702865, 0.026489228, -0.027282972, -0.0450099, 0.01270767, + 0.03903348, -0.008995755, -0.005770201, 0.0042994414, 0.024481526, + 0.048247125, -0.005151548, 0.028730385, -0.013579232, 0.015485771, + 0.00832652, -0.0150110815, 0.009369281, -0.001316096, 0.048029236, + -0.009556044, -0.008069721, -0.009548262, -0.0046807495, + -0.0150110815, 0.022131424, -0.021602262, -0.043422412, -0.0269717, + -0.01942336, -0.0017188039, 0.0015865134, 0.013330214, -0.012264108, + -0.026255775, 0.032683536, -0.0037741703, -0.0030932634, -0.03174972, + 0.010987894, -0.027936643, 0.04454299, -0.008458811, 0.026286902, + 0.038535446, 0.025072942, 0.024014618, 0.024963997, 0.019174341, + 0.022800658, -0.0121396, -0.011913927, -0.00015600066, -0.035484985, + -0.0151745, -0.003560171, -0.005272166, 0.047811344, -0.004540677, + 0.01114353, -0.030691398, 0.004505659, 0.013439159, -0.013602577, + 0.010661058, -0.031765286, 0.03165634, 0.013244614, -0.019221032, + 0.042177327, 0.015579153, -0.04311114, 0.01660635, 0.04270649, + 0.056309067, -0.028045587, 0.018925324, 0.0080930665, -0.03872221, + 0.03946926, 0.016761985, 0.030442381, -0.013890503, -0.020637318, + -0.019298851, -0.008069721, 0.017088821, 0.0011137694, 0.0059258365, + 0.016730858, 0.0009654289, -0.004241078, 0.020326048, 0.01760242, + 0.010707749, -0.0077506676, 0.06823078, -0.005976418, 0.014108393, + -0.014559737, -0.003425935, -0.021882406, 0.010116333, -0.033928625, + 0.027438607, -0.016123878, -0.0012110418, -0.019189905, 0.007731213, + -0.024403708, 0.008521065, -0.018396161, -0.004132133, 0.039562643, + 0.020668447, 0.044325102, 0.00026847192, 0.013369123, 0.04734444, + 0.0036360435, -0.037570503, 0.008824555, -0.031547394, -0.004805258, + -0.02272284, -0.025337523, 0.0024376472, 0.014528611, -0.011073493, + -0.0091047, 0.0042605326, -0.008645574, 0.018147144, -0.019143214, + -0.031454016, -0.028279042, -0.009828406, 0.022987422, -0.048216, + 0.006731252, -0.004322787, -0.0047974763, 0.004824713, -0.013306869, + 0.0269717, -0.033804115, 0.028870458, -0.024248071, -0.014824319, + 0.027205154, -0.0023228657, -0.026660427, -0.021415498, -0.03386637, + 0.030380126, 0.009198081, -0.013851594, 0.03682345, -0.012279672, + 0.00650558, 0.005186566, -0.010357569, -0.0005053303, -0.014816537, + -0.03160965, 0.010645495, 0.037757266, -0.0011254421, 0.041430276, + 0.0036691162, -0.015664753, -0.05272944, -0.0035115348, 0.0360764, + 0.01990583, 0.011945055, 0.022909604, 0.013695958, -0.0006055209, + -0.040745478, 0.042737614, 0.010606586, 0.006789616, 0.014240684, + -0.019438922, -0.0042722053, 0.021057537, -0.0037352615, 0.043329034, + -0.0046846406, 0.022691714, 0.00050873484, 0.036232036, -0.016326206, + -0.0242792, 0.007984121, -0.00011532863, 0.01331465, -0.004957003, + -0.01283218, -0.026302466, 0.028652567, 0.007081433, -0.014730937, + 0.019329978, -0.019407796, 0.013166796, 0.0532586, -0.0044239503, + -0.028730385, -0.019049833, -0.010583241, -0.016419586, -0.0076417224, + -0.0047546765, 0.034924693, 0.041305766, 0.00053937564, -0.03343059, + -0.02493287, 0.030208927, -0.03912686, 0.0019473941, 0.005945291, + 0.02575774, 0.035889637, 0.023236439, -0.008209793, -0.015049991, + -0.029166166, 0.034239896, -0.021197608, -0.07153025, 0.010575458, + 0.036636688, 0.027983334, 0.0002378311, 0.023765601, 0.009065791, + -0.003052409, -0.050021376, -0.012653198, -0.030753652, -0.009649426, + 0.040963367, -0.013587013, 0.027049517, 0.018131582, -0.01643515, + -0.005532856, 0.0058052186, -0.01981245, -0.011929492, -0.038597703, + -0.00971168, -0.07383367, 0.0038753338, 0.035765126, 0.00776234, + 0.014816537, 0.0009138745, 0.00035650347, 0.0032197174, -0.03364848, + 0.01859849, -0.01487101, 0.018925324, -0.049149815, -0.010552114, + 0.024948433, 0.011330293, -0.05285395, 0.0123730535, 0.012069563, + -0.010559895, -0.046472877, -0.006478344, 0.006941361, -0.030722525, + 0.0051282025, 0.0011945055, 0.0066767796, -0.05724288, -0.0018783307, + -0.007575577, 0.0034278806, 0.020326048, -0.024388144, -0.005949182, + 0.048745163, -0.02272284, 0.016092751, -0.0073732506, 0.04644175, + 0.0041866056, 0.0057818736, 0.035422727, -0.036169782, -0.027781006, + 0.01912765, -0.01461421, 0.0053227474, -0.033057064, 0.011828328, + 0.029944345, -0.0061087087, -0.015882643, 0.027998896, -0.0051787845, + -0.051982388, -0.047064293, 0.025679922, 0.022536077, 0.024606034, + 0.012956688, -0.0029940454, -0.033555098, -0.04721993, 0.039095737, + -0.0041943872, -0.035391603, -0.025742177, -0.0117271645, 0.009610516, + 0.009742807, -0.0070269606, 0.0067234705, 0.032060992, 0.0070347423, + -0.013555886, 0.014061702, 0.021213172, -0.0057429643, -0.00962608, + -0.029275112, 0.013781558, 0.049180944, 0.028434677, -0.0022859022, + -0.018193835, -0.015112245, -0.008256484, 0.05416129, -0.038006283, + -0.04392045, -0.036200907, 0.028886022, 0.026333593, -0.0003521262, + 0.004241078, -0.04613048, 0.02575774, -0.012450871, -0.0044083865, + -0.047282185, -0.00015478476, -0.03707247, -0.00802303, -0.009260336, + -0.04522779, -0.016139442, 0.027858825, -0.03165634, 0.0018520671, + -0.002050503, -0.05098632, 0.018224962, 0.0029512455, 0.038006283, + -0.03134507, 0.0151745, 0.017851437, 0.041710418, -0.034582295, + -0.012279672, -0.032278884, -0.03841094, -0.03473793, -0.017306712, + 0.04961672, 0.005560092, -0.043235652, 0.0367612, 0.028808204, + -0.002116648, -0.0239368, 0.011260257, -0.031204997, -0.0029240092, + 0.016186133, 0.01981245, 0.049990248, 0.033804115, -0.0034045351, + -0.00459515, 0.008124193, 0.004851949, -0.008225357, 0.043515794, + 0.017011004, -0.017026566, -0.009524916, 0.009524916, -0.032963682, + -0.0015018864, -0.040776603, -0.018878633, -0.019298851, + -0.0029843182, 0.027500862, -0.0332127, 0.0017479856, -0.012015091, + 0.013213487, -0.032496773, 0.007801249, 0.00065123895, 0.013695958, + -0.0151745, 0.019018706, -0.04413834, -0.01569588, -0.005832455, + -0.008653356, 0.005906382, 0.008458811, -0.0014843773, -0.0012567599, + -0.025353087, -0.010155242, 0.01530679, 0.03039569, 0.03483131, + -0.004960894, -0.013376905, 0.038784463, -0.019407796, -0.0028559186, + 0.03333721, -0.02129099, -0.047064293, -0.0070580877, -0.042083945, + 0.016357332, -0.013018942, 0.0020835756, -0.006890779, -0.003885061, + -0.008264266, -0.017415656, -0.017913692, -0.0011244693, + -0.0012392508, 0.022629458, 0.0006108709, -0.00849772, 0.019625686, + -0.035111457, 0.0007378114, 0.014909918, -0.011898364, 0.000956188, + 0.013516977, 0.007999685, -0.01166491, -0.006174854, 0.013081197, + -0.0025582649, 0.00007003615, -0.029819837, 0.01244309, 0.011198003, + -0.006478344, 0.026987264, -0.008855682, -0.009283681, 0.0379129, + 0.025695486, -0.0239368, -0.0022061388, 0.008256484, -0.011524837, + -0.034488913, -0.008396557, 0.016684167, 0.008256484, -0.013104542, + -0.012731016, -0.0031282813, -0.006190418, 0.0025349197, -0.022598332, + 0.022738405, 0.02813897, 0.014653119, 0.023485456, -0.011781637, + 0.0019259942, 0.0028131187, -0.021929096, 0.026909446, 0.024201382, + -0.0091903, -0.031220559, -0.016793113, 0.017633546, 0.02575774, + 0.00997626, 0.0033111535, -0.048589524, 0.01227189, 0.024823925, + 0.028107842, -0.011571528, -0.02974202, 0.01574257, 0.005303293, + -0.003101045, -0.0026205194, 0.0023792838, -0.022582768, -0.003169136, + 0.01964125, 0.0010894513, 0.000083836676, -0.012614289, -0.016014934, + 0.026551483, -0.053694382, -0.02007703, 0.011299166, -0.011221347, + -0.020668447, -0.013081197, -0.011384766, 0.03707247, 0.0063188174, + -0.0038753338, 0.022645023, 0.0038150249, -0.036200907, 0.0053733294, + -0.0070269606, -0.009742807, 0.008661137, 0.01114353, 0.01175051, + 0.017462347, 0.006863543, 0.008046376, -0.049149815, 0.022411568, + -0.004233296, 0.060822506, 0.00034434441, 0.009657207, -0.018754125, + 0.013641486, 0.050270393, 0.0013841867, 0.0015203682, 0.017851437, + 0.001287887, 0.024123562, -0.0066845617, 0.017493473, -0.0038344793, + -0.005563983, -0.007859613, 0.021960225, 0.008318738, -0.023796728, + -0.021540007, 0.004914203, 0.017322274, -0.013983885, -0.008583319, + -0.011742728, -0.06132054, 0.027734315, 0.028341295, -0.04149253, + -0.026224649, 0.019625686, -0.002346211, -0.012419744, 0.0010466514, + -0.017291147, 0.0025738284, 0.02701839, -0.025213014, 0.000101953665, + -0.012030655, -0.012388617, -0.028310169, 0.0122251995, -0.019719068, + -0.006824634, 0.031064924, 0.009081354, -0.003976497, -0.0036068617, + 0.0017470128, 0.008155321, 0.01726002, 0.00019296419, 0.021010846, + -0.01760242, -0.005276057, -0.036698945, 0.030068854, -0.00089782453, + -0.010816694, -0.0042994414, -0.004326678, 0.02020154, -0.0010777785, + 0.016886495, -0.0399673, 0.011937273, 0.02497956, 0.03343059, + -0.011019021, 0.014061702, 0.00013922117, 0.002025212, 0.011696037, + -0.0010369241, -0.025446469, 0.021648953, 0.020964155, 0.014123958, + -0.012334145, -0.026333593, -0.032060992, 0.028325733, 0.03221663, + -0.011384766, 0.0036088072, -0.0032411173, 0.005560092, -0.026504792, + 0.005330529, 0.009143609, -0.01881638, -0.016092751, -0.012310799, + 0.03828643, 0.03168747, -0.024683852, 0.03476906, -0.037103597, + 0.0054744924, -0.032154374, -0.011649347, -0.04656626, 0.02094859, + 0.030131109, -0.015758134, 0.037010215, -0.02306524, -0.017726928, + 0.012256326, -0.03265241, -0.0066767796, 0.01053655, -0.014061702, + -0.020808518, -0.009898443, -0.03367961, 0.018754125, 0.014544174, + 0.00750165, 0.03750825, 0.016014934, 0.0033597897, 0.0040620966, + -0.011431457, -0.0084432475, 0.008077503, -0.008202012, -0.0022800658, + 0.05761641, -0.019049833, -0.028932713, -0.011594874, 0.008770082, + -0.049149815, -0.025228579, -0.03333721, -0.038659956, 0.034022007, + -0.032932553, 0.02558654, -0.03243452, -0.00020852778, 0.021929096, + -0.039313626, 0.02879264, 0.06910234, 0.024917306, 0.0015057772, + 0.03486244, 0.014186212, 0.023874545, -0.010373132, 0.0016166678, + 0.036200907, 0.0007256523, -0.019843576, 0.025135197, 0.01825609, + 0.015446862, -0.012100691, -0.019936958, -0.037663884, 0.0027547553, + -0.041648164, 0.03073809, 0.019236596, 0.02497956, 0.013151233, + 0.0040815515, 0.0025329741, -0.009026881, 0.0050854026, -0.026209084, + -0.00031808086, -0.0084432475, -0.008723392, 0.009065791, + -0.017617984, -0.021664517, -0.0121240355, -0.04410721, 0.0128243975, + 0.068355285, -0.012567598, 0.013540323, -0.006217654, -0.034146514, + -0.022240369, -0.027858825, 0.00979728, 0.0017265857, 0.020964155, + 0.0138204675, 0.0055523105, -0.0032274993, 0.021041973, -0.010443169, + -0.00362048, 0.0022119752, -0.0027819916, 0.0021866842, 0.00048393034, + -0.0047391127, -0.020357175, 0.01660635, -0.012256326, 0.061694067, + -0.0091358265, 0.036947962, 0.034364406, -0.035920765, 0.018800816, + 0.010645495, 0.026069012, -0.022971857, -0.028201224, -0.0058869277, + -0.015330135, 0.0113769835, 0.02150888, -0.0020796845, -0.042862125, + 0.033710733, -0.02354771, -0.002346211, 0.0017304766, -0.005567874, + -0.009851752, -0.020621756, 0.0107310945, 0.029664202, 0.017322274, + -0.02029492, 0.038908973, 0.00087982917, -0.00268861, 0.031967614, + 0.020170411, -0.028387986, 0.014061702, 0.0044122776, -0.014520829, + -0.005455038, -0.01075444, 0.0011400329, -0.016933184, -0.01036535, + 0.02566436, 0.0069919424, 0.002501847, 0.030115545, 0.025851121, + 0.019003142, 0.0057468554, -0.010116333, 0.003499862, 0.004404496, + -0.038628828, -0.051079698, -0.016684167, 0.02701839, -0.0071475785, + -0.010520986, -0.02406131, 0.013976103, -0.005381111, -0.014520829, + 0.021088663, -0.015112245, -0.061507303, 0.009703898, -0.010108551, + -0.037881777, 0.015625844, 0.024388144, 0.0040309695, 0.02588225, + 0.030426817, 0.017835874, -0.012264108, -0.0074082683, 0.004447296, + -0.012606507, -0.02913504, -0.015680317, -0.0041204602, -0.022349315, + 0.010855603, 0.025539849, -0.004400605, 0.030411253, -0.01461421, + -0.013649268, 0.01036535, 0.005980309, 0.0056962734, -0.006447217, + -0.0011731056, -0.009470444, 0.014606428, -0.009384844, 0.001058324, + 0.027547553, 0.00620209, 0.04504103, 0.012606507, 0.024855051, + 0.034053132, 0.03685458, -0.009252554, -0.021213172, 0.00028476503, + -0.023298694, -0.023143057, -0.016621914, 0.01301116, 0.008598883, + 0.027002826, 0.005229366, -0.0046340586, 0.042488597, 0.02454378, + -0.0022839566, -0.021664517, 0.005318857, 0.008256484, -0.019874703, + 0.004478423, -0.005844128, 0.0110268025, 0.02449709, -0.0023637202, + -0.031298377, 0.0011098784, -0.010318659, -0.01227189, 0.00066145253, + -0.009205863, -0.010777785, 0.0103342235, 0.018271653, -0.048931923, + 0.019112088, 0.016979875, -0.0032469537, 0.05151548, -0.021991352, + 0.0382553, -0.0066495435, -0.0032566811, -0.02350102, 0.015034427, + -0.026675992, 0.016201695, -0.020917464, -0.000483444, 0.011462583, + -0.027936643, -0.010030733, 0.01448192, 0.0015213408, 0.017851437, + -0.0065367073, 0.046753023, 0.00017107789, 0.025430905, 0.015197845, + -0.007061979, 0.020232666, -0.042177327, 0.00046058497, -0.010559895, + 0.010801131, -0.00083216565, -0.008217575, 0.011789419, + -0.00035528757, 0.024823925, 0.008995755, -0.020403866, 0.0075561227, + 0.02471498, 0.0010067697, -0.010513204, -0.00724096, -0.01738453, + 0.015921552, -0.010925639, 0.016326206, -0.014334066, -0.028979404, + 0.02068401, -0.009968479, -0.08379436, -0.029991036, 0.015376826, + -0.034333277, -0.02558654, 0.002632192, 0.030753652, -0.01331465, + -0.0034726257, 0.011812764, 0.006742925, 0.002501847, 0.004241078, + 0.03364848, 0.018396161, 0.006054236, -0.023360947, -0.018178273, + -0.008746737, 0.012481999, 0.010917858, 0.021975787, 0.043235652, + -0.004700204, 0.016014934, -0.0017470128, -0.0020193758, 0.0057896553, + 0.020808518, -0.009773934, -0.014800973, -0.017135512, -0.0030329544, + -0.015983807, -0.013797122, 0.0033014263, -0.0015942951, 0.0025718831, + 0.039313626, -0.020715138, -0.012349708, -0.0073421234, 0.0015193954, + -0.0011312783, 0.022660585, 0.014209557, 0.008116412, -0.0153612625, + -0.006665107, -0.01587486, -0.0008088203, 0.009400408, -0.0029065004, + -0.024761671, 0.04015406, 0.0008093066, 0.0066028526, -0.031718593, + -0.01183611, -0.002441538, 0.064682275, 0.025617668, -0.0134702865, + -0.037881777, 0.0050659482, -0.005155439, -0.014497483, 0.0062565627, + 0.030504635, -0.04192831, 0.018878633, 0.020154849, -0.02228706, + -0.004809149, -0.026878318, 0.023127494, -0.010816694, 0.013913848, + 0.020792956, 0.0077817948, -0.0075639044, 0.022785094, 0.014746501, + 0.022738405, -0.008653356, -0.015937116, -0.04622386, -0.011252475, + -0.017057694, -0.009213645, 0.0301778, 0.0037975158, 0.017913692, + -0.0053110747, 0.002801446, 0.023080803, -0.012513125, 0.03126725, + -0.0071397964, -0.0077740126, -0.011688256, 0.001242169, 0.017944818, + -0.008256484, 0.016839804, -0.022816222, -0.0002582583, 0.0009766152, + -0.026956137, 0.010303096, -0.02913504, -0.009268117, 0.005264384, + -0.005525074, 0.011073493, 0.0110812755, 0.003955097, 0.016590785, + 0.0030037728, 0.027858825, 0.0063032536, 0.019003142, 0.012909997, + -0.017898127, -0.0072915414, 0.010886731, -0.0011273875, -0.020606192, + 0.004311114, 0.00620209, -0.024450399, 0.02580443, 0.0051632207, + -0.0019347487, -0.02974202, -0.0030232272, -0.015711443, 0.043173395, + 0.017306712, -0.022909604, -0.0009853698, 0.029275112, 0.0030601907, + 0.033368334, 0.02068401, 0.0016847586, -0.021259863, 0.0029317911, + -0.007809031, 0.003793625, 0.028107842, 0.026426975, -0.002046612, + -0.0066145253, 0.015415736, 0.028994966, 0.020824082, 0.005828564, + 0.017400093, 0.011703819, 0.019189905, 0.007493868, 0.0005539665, + -0.01027975, 0.013127888, -0.03070696, 0.031127179, -0.013773777, + -0.0006726389, -0.0058363457, 0.014738719, 0.0064666714, 0.012886652, + 0.0005807164, 0.014349629, -0.00006353106, 0.042208456, 0.011213566, + 0.021026408, 0.017275585, -0.043329034, 0.043266777, -0.001146842, + -0.03039569, -0.008287611, 0.05372551, 0.014676465, -0.012022872, + 0.008186448, -0.010676622, -0.016715296, 0.011703819, 0.034986947, + -0.010318659, 0.0010184423, -0.006711798, -0.018489543, 0.027998896, + 0.012980034, -0.0029317911, -0.021399936, -0.03066027, 0.03109605, + 0.00863001, 0.0101863695, -0.0036496616, -0.017291147, 0.011890582, + 0.008731173, -0.010684404, -0.012676544, -0.021695644, 0.00967277, + 0.010077424, 0.020154849, 0.014458574, -0.00044283274, -0.006805179, + 0.012287454, 0.0010505422, 0.008598883, -0.008069721, -0.022613894, + -0.007797358, -0.015618062, 0.017758055, 0.013306869, 0.032496773, + 0.024948433, 0.00048295764, -0.010178587, 0.0074821953, -0.009096918, + 0.0047079856, 0.00028063098, -0.0015573316, 0.0016594677, + -0.0027528098, 0.03950039, 0.0002623924, 0.00641609, -0.018972015, + 0.0011711601, -0.009999606, 0.010349787, -0.017057694, -0.010606586, + -0.008287611, -0.015026646, 0.03395975, -0.0061515085, 0.011252475, + 0.016295077, -0.016279515, 0.007707868, 0.008762301, -0.019968085, + 0.001044706, 0.0024979562, -0.0036885706, 0.0021652842, -0.020761829, + 0.009594953, 0.021182045, -0.013431378, -0.013073415, -0.009081354, + 0.0052099116, 0.029275112, -0.020668447, 0.018287217, -0.010038515, + 0.020434992, -0.042986635, 0.009859534, -0.0032255538, 0.0010923694, + 0.0016662767, 0.009805061, 0.0207307, -0.004704095, -0.014186212, + -0.014692028, 0.020995282, -0.024263635, 0.005918055, 0.022847349, + -0.03716585, -0.0027236282, -0.00040076242, 0.0035309894, 0.022769531, + -0.00018056195, 0.0094860075, 0.014458574, -0.0015835952, + -0.017509038, -0.012785489, 0.022660585, -0.0068090702, 0.022084733, + 0.018629616, 0.017228894, -0.012925561, -0.015166718, -0.0002465856, + 0.007855722, 0.0010009333, -0.011166875, -0.00650558, 0.007894631, + -0.0027703189, -0.0064199804, -0.026847191, -0.0268005, 0.010824476, + -0.0021322118, -0.01638846, 0.01895645, 0.009789498, 0.0075522317, + 0.019112088, 0.0046457313, -0.017119948, 0.00088372006, 0.016450714, + -0.010287533, 0.015384608, -0.020715138, -0.0018754125, 0.0064705624, + 0.0015057772, 0.0027314099, -0.030940415, 0.0027567006, -0.008910155, + 0.018660743, 0.015182281, -0.013555886, 0.0016400132, 0.018458417, + 0.031469576, -0.004661295, -0.038099665, -0.032621283, 0.012731016, + 0.009906225, -0.013143451, -0.017929254, -0.008396557, 0.009260336, + -0.019392231, -0.026395848, 0.016668605, 0.0017781401, -0.0008068748, + 0.0004318896, 0.010054079, -0.012738798, 0.000004384859, 0.009143609, + 0.0003110286, 0.028061152, 0.002801446, -0.009945134, 0.014933264, + -0.009415971, 0.006665107, 0.009851752, 0.011252475, 0.00053159386, + 0.01726002, -0.009275899, -0.0050698393, -0.00728376, 0.0039181337, + -0.0063849622, -0.008124193, -0.00051019393, -0.0075483406, + -0.0133224325, 0.03126725, 0.010785568, -0.042924378, -0.01331465, + -0.017882563, -0.0017742491, 0.010777785, 0.009198081, 0.017306712, + -0.028870458, -0.007540559, -0.00028816957, 0.008038593, + -0.0017518766, 0.0073304507, -0.003937588, -0.010178587, 0.0075172135, + 0.012497562, -0.010022951, 0.0076339403, -0.016310642, 0.018240526, + 0.033897497, -0.036792323, -0.008661137, -0.0062409993, -0.0047585675, + -0.005871364, -0.013260178, -0.015096681, 0.00082973385, -0.008256484, + -0.017944818, 0.020637318, -0.024372581, -0.007462741, -0.019578995, + 0.013337996, -0.0010106605, -0.007202051, -0.019921394, -0.0072137234, + 0.005676819, -0.006132054, -0.014240684, 0.017228894, 0.02155557, + 0.019501178, -0.008357648, -0.020824082, 0.0034317714, 0.011555965, + 0.015376826, 0.026286902, -0.0077117584, 0.009960697, 0.03100267, + -0.009618298, -0.0055056196, -0.02762537, 0.0037158069, -0.011633783, + 0.0032236085, 0.009065791, -0.010777785, -0.0027528098, -0.007030851, + 0.02519745, 0.015509116, -0.010342005, 0.011050148, 0.021306554, + 0.003005718, -0.0055834376, -0.00650558, 0.009735025, -0.023360947, + 0.02206917, -0.0023695563, -0.004848058, 0.0028267368, 0.0026205194, + 0.015135591, 0.004089333, 0.008700047, -0.024372581, -0.009244772, + -0.016917622, -0.012637635, 0.010692186, -0.02463716, -0.022613894, + 0.023360947, 0.022302624, 0.0052099116, 0.00072419323, 0.008365429, + 0.005571765, -0.020855209, 0.032247756, 0.0077584493, -0.013190142, + -0.031033797, -0.029166166, 0.010567677, -0.009034663, -0.017104385, + -0.019003142, -0.010785568, 0.013750431, -0.008357648, 0.010917858, + 0.0033928624, -0.019796886, -0.022162551, -0.0007791522, + -0.0054900562, -0.0056223464, 0.005104857, -0.0049025305, -0.00719816, + -0.009112481, -0.0036671706, -0.016575223, 0.00819423, 0.011283603, + 0.0028111732, 0.0008370293, 0.016621914, -0.01105793, -0.014193993, + 0.0062721265, -0.0014036412, 0.008248703, -0.014995518, 0.0006964706, + -0.013672614, -0.014474138, -0.03243452, 0.015501335, 0.004431732, + 0.017680237, 0.006700125, 0.008466592, 0.0147153735, -0.0069219065, + 0.017960383, 0.010489859, 0.04743782, -0.04258198, 0.0036652253, + -0.0020757937, 0.033492845, -0.013369123, 0.010030733, 0.00997626, + 0.01738453, 0.0205595, -0.019454487, 0.0014872955, -0.014489702, + 0.0050153667, 0.035484985, 0.042519726, -0.0026730464, 0.008038593, + 0.03070696, -0.012015091, 0.02480836, -0.031064924, -0.009657207, + 0.0150655545, 0.004536786, -0.013649268, -0.033897497, -0.0051437663, + 0.006128163, 0.026489228, -0.014902136, -0.006665107, 0.013439159, + 0.022613894, 0.013664831, -0.02371891, -0.0064705624, 0.015548025, + 0.018318344, 0.00056953006, 0.013765994, -0.013057851, -0.01378934, + 0.0027528098, 0.007050306, -0.019267723, 0.017337838, 0.029586382, + 0.007244851, 0.007921867, 0.015625844, 0.003525153, 0.0014192048, + 0.00011709169, -0.0022508842, -0.010933422, 0.017571293, 0.027282972, + -0.015789261, -0.002050503, -0.012894434, 0.01097233, -0.00060163, + -0.032060992, 0.014170648, -0.001979494, -0.015197845, 0.020248229, + -0.003499862, -0.009524916, -0.0055211834, 0.013532541, -0.010684404, + -0.0022431023, -0.003213881, 0.00654838, 0.003933697, -0.002289793, + 0.013018942, 0.00088907004, -0.013252396, -0.017228894, -0.013501413, + 0.01417843, 0.0063421624, 0.005198239, -0.03209212, 0.008949064, + -0.0039025699, 0.028590314, -0.0047974763, -0.024139127, -0.014349629, + 0.003005718, 0.0029512455, 0.016419586, 0.0042138416, 0.007202051, + -0.019236596, -0.013229051, -0.022567205, 0.0023870654, 0.0150889, + 0.012194072, -0.0039726063, 0.022862913, -0.022536077, 0.022006916, + -0.00023345384, -0.0045640226, -0.022753967, -0.005501729, + -0.011384766, -0.004883076, 0.027609807, 0.0043383506, 0.014372975, + 0.008575537, -0.011493711, -0.0020135394, 0.0064939074, 0.01105793, + -0.013516977, 0.0007212751, 0.003525153, -0.0032625173, -0.009875097, + 0.012092909, -0.008272048, 0.0050970754, -0.0036360435, 0.0064238715, + 0.015905987, 0.00067701616, -0.004626277, -0.026940573, -0.023392074, + 0.007124233, -0.00455235, 0.014909918, -0.009906225, -0.0009124154, + -0.0024862834, -0.005676819, -0.02502625, -0.012544253, 0.0150889, + 0.0066145253, -0.019874703, 0.0019960303, 0.0056106737, -0.007120342, + -0.0031924811, 0.0031749723, -0.009937352, 0.004167151, 0.009805061, + -0.0239368, -0.009096918, 0.0016468223, 0.009587171, 0.008840119, + -0.007836267, -0.006365508, -0.009244772, 0.0151745, -0.020528374, + -0.007661177, -0.004089333, 0.0152134085, 0.0056612557, -0.014925482, + -0.015493553, -0.0008686428, 0.0036652253, -0.009805061, + -0.0119761815, 0.012894434, 0.0061515085, 0.011221347, 0.0183806, + 0.02796777, 0.014147302, 0.0028520278, 0.016279515, -0.014248466, + 0.0015816498, -0.028170096, -0.01721333, -0.0034628985, 0.0068596518, + -0.0065250346, -0.0023481564, 0.0027742097, -0.0067545976, + 0.019298851, -0.0027139008, -0.0032430629, -0.020372739, 0.024263635, + 0.019921394, -0.02957082, -0.016108315, -0.0038889518, 0.0021477754, + 0.0064199804, 0.034333277, -0.02701839, -0.021820152, 0.00003137036, + -0.015236754, 0.012466435, 0.013906067, -0.008785646, 0.0012227144, + -0.005194348, -0.005560092, -0.011999527, 0.0024337564, 0.014271812, + 0.015182281, -0.003680789, -0.012038436, 0.0060036546, 0.0084899375, + -0.04544568, -0.019547869, -0.00385977, 0.006291581, -0.0054706014, + 0.009851752, 0.010738877, 0.004241078, -0.0006429708, -0.003560171, + -0.0015407953, -0.02840355, 0.0102330595, 0.00001182468, 0.005991982, + 0.0060075456, -0.014225121, -0.011415892, 0.0076728496, -0.020154849, + -0.015260099, -0.013532541, -0.0010223333, 0.007976339, 0.021633388, + 0.00017278016, 0.0051165298, -0.014443011, -0.01934554, -0.015789261, + -0.003373408, -0.016466277, -0.00019867894, 0.00047906672, + 0.005649583, 0.008808992, 0.009065791, -0.014287375, -0.007879067, + -0.00002994167, -0.015968243, 0.002254775, -0.01956343, 0.0003766875, + 0.0091358265, 0.011781637, 0.01478541, -0.0011760237, -0.010217496, + -0.009065791, 0.01478541, -0.014279593, -0.0028714822, 0.0125753805, + 0.00092068355, 0.016061625, -0.010396478, -0.0072487416, -0.00685187, + 0.0071864873, -0.0010301151, -0.00039687153, 0.017415656, 0.015789261, + -0.011781637, -0.003910352, -0.01825609, -0.008054158, 0.019189905, + -0.013073415, -0.013999448, -0.0033383898, -0.0049920212, + 0.0069919424, -0.016248386, 0.0007645613, -0.0041943872, -0.005034821, + -0.013415813, 0.02675381, 0.019236596, 0.0049453303, -0.0019892212, + -0.0050231484, 0.0041476963, -0.0052371477, -0.00048174174, + -0.0034512258, 0.01859849, -0.0020349394, -0.017633546, -0.024621598, + 0.00455235, 0.019750195, 0.016544096, 0.010715531, 0.014435229, + 0.019065397, 0.0036243708, 0.021804588, 0.001097233, 0.010692186, + 0.0011273875, 0.012419744, -0.004731331, 0.013501413, -0.0030135, + 0.012715452, -0.024606034, 0.003750825, -0.013392469, 0.014123958, + -0.005602892, -0.025866685, -0.0035504438, -0.009478225, 0.009805061, + -0.01912765, -0.011236912, -0.009587171, 0.006598962, -0.0052099116, + 0.011268038, 0.027127337, -0.017151074, 0.012435308, 0.009431535, + -0.008155321, -0.008910155, 0.0070386333, -0.029243983, -0.009034663, + -0.0052176933, -0.0034667894, -0.006490017, 0.02532196, -0.012155163, + 0.02150888, -0.0049064215, 0.021477753, 0.014808755, 0.009182517, + -0.006174854, 0.0041515874, 0.013026725, -0.017011004, 0.006805179, + 0.004925876, 0.016761985, 0.011906146, 0.00077282946, 0.0022353204, + -0.016186133, -0.004766349, -0.007964667, -0.026613738, 0.0096338615, + 0.0042605326, 0.003943424, 0.0143651925, -0.0021516662, -0.026302466, + 0.0015592771, -0.005902491, -0.020154849, -0.00019916531, + 0.0036302072, -0.0058091097, 0.02597563, -0.024621598, -0.014800973, + 0.022567205, -0.017633546, -0.006326599, 0.009096918, 0.0004681236, + -0.025555413, -0.00863001, 0.027329663, 0.023874545, -0.011587092, + 0.014038358, 0.002892882, -0.027563116, 0.012341926, -0.012147381, + 0.021446627, 0.018302782, 0.03095598, 0.008474374, -0.0032625173, + 0.015921552, 0.008054158, 0.016108315, 0.008357648, -0.0055134012, + 0.015112245, -0.010046297, -0.013042288, 0.004361696, -0.0012071509, + 0.0027547553, -0.0003200263, -0.011050148, 0.016637476, -0.012987815, + 0.018458417, 0.013003379, -0.005451147, -0.020964155, -0.018894197, + 0.0034492805, -0.021104228, 0.024123562, -0.0032294448, 0.0063771806, + 0.005272166, 0.0019075124, 0.01318236, -0.021399936, 0.007124233, + 0.009143609, -0.00011812521, 0.009447099, 0.017493473, -0.008451029, + -0.0037041341, -0.0129722515, 0.001608886, -0.0017265857, + -0.009540481, -0.0021322118, 0.012559816, 0.030520199, 0.0023637202, + 0.005910273, -0.015026646, 0.009851752, -0.012941125, 0.008816773, + 0.0012343872, -0.00033753534, -0.016186133, -0.023454329, + -0.015462426, -0.0015116136, -0.032060992, -0.009812843, 0.012427526, + -0.01144702, 0.0044823135, 0.009501571, 0.011353638, -0.020761829, + 0.010194151, -0.010030733, -0.01357145, -0.003217772, -0.011392547, + 0.0046029314, 0.009003537, 0.01591377, -0.0064705624, -0.01084004, + -0.008793428, -0.0027586462, -0.020792956, 0.0049103126, 0.017026566, + 0.004058206, 0.008131975, 0.005186566, -0.019936958, -0.01409283, + 0.008163103, 0.00971168, -0.0147153735, 0.004447296, 0.0039570425, + 0.029166166, 0.010318659, -0.00975837, -0.00012706211, 0.010124114, + -0.013104542, -0.0060931453, 0.0077428855, -0.0061631813, + -0.0047546765, 0.0003766875, -0.010801131, 0.0029181729, + -0.0059375092, -0.018676307, 0.009096918, -0.00715536, 0.0023053566, + 0.014583083, 0.026333593, -0.00039687153, -0.013088979, 0.023189748, + -0.0021866842, 0.0013180415, -0.028481368, 0.0002403845, 0.012349708, + 0.0037916794, 0.0138671575, -0.008754519, -0.009065791, 0.0066845617, + 0.016139442, 0.01252869, -0.0059375092, 0.00849772, 0.0057818736, + -0.00424886, 0.004132133, -0.0042722053, -0.018473981, 0.0090191, + -0.0010330332, 0.008202012, -0.027998896, -0.004883076, 0.0060347817, + 0.0070969965, 0.036449924, 0.012637635, -0.02051281, -0.0072526326, + 0.0060347817, 0.020139284, 0.003079645, 0.00034628986, -0.014295157, + -0.010271969, 0.005077621, -0.0063460534, 0.012590944, 0.0053538745, + 0.008567756, -0.0016886494, -0.0037994613, -0.013727086, 0.008155321, + -0.007875176, -0.0026380285, -0.009330371, -0.0019026488, + -0.019501178, -0.0026010647, 0.004318896, 0.020481683, 0.005618456, + 0.021524444, -0.01973463, 0.004252751, -0.007696195, -0.00949379, + 0.009221426, -0.013579232, 0.008910155, 0.00028476503, -0.002350102, + -0.0056534735, -0.007836267, -0.012598725, 0.017244456, -0.012100691, + -0.006100927, -0.009649426, -0.014824319, -0.01665304, -0.023220874, + -0.004848058, -0.00019332896, 0.008855682, 0.0029279003, 0.010248624, + -0.008832336, 0.0031166088, -0.006213763, 0.007536668, 0.016404023, + -0.0085911015, 0.0125753805, 0.008061939, 0.009587171, -0.032465648, + 0.0013277687, -0.016979875, 0.008451029, -0.0091358265, 0.030504635, + 0.006205981, -0.006777943, 0.009073572, -0.019750195, 0.00028427868, + -0.008061939, -0.003885061, 0.004704095, -0.018411726, 0.006490017, + -0.0005923891, 0.0050892937, 0.011112403, 0.016933184, -0.019890267, + -0.0012801052, 0.00032051266, -0.0030776998, -0.010995676, + 0.0074744136, -0.009563825, 0.0006775025, -0.0007120342, + -0.0148632275, 0.0059569636, -0.005229366, 0.0059880908, -0.035360474, + -0.0008598883, 0.0010709695, 0.005910273, 0.008318738, -0.02311193, + 0.03078478, 0.011159093, 0.009143609, -0.007209833, 0.0013978048, + 0.013913848, 0.008544411, -0.0016662767, 0.007120342, 0.01279327, + -0.0024395925, -0.015719226, -0.017617984, -0.0044239503, 0.011221347, + 0.022427132, 0.023563273, -0.018722998, 0.0071903784, -0.0016643313, + -0.0020602301, -0.014551956, 0.005268275, -0.00008845712, 0.01227189, + 0.00871561, -0.005980309, 0.010567677, -0.007956885, -0.0077506676, + -0.020154849, -0.008030812, -0.012668761, -0.004209951, 0.004972567, + -0.0062409993, 0.0014639501, 0.00053791655, -0.005264384, + 0.0109801125, 0.006174854, -0.0058674733, -0.01066884, 0.007388814, + -0.0015213408, -0.000007572811, 0.0045912587, 0.009369281, + 0.0040387516, 0.027531989, 0.0021088663, 0.012847743, 0.0028384095, + -0.018878633, -0.0060503455, 0.0072137234, -0.01144702, 0.0027158463, + 0.0037838977, 0.0029415183, 0.0066884523, -0.021259863, 0.0110268025, + -0.0033053171, -0.0041126786, -0.01995252, -0.018240526, 0.002801446, + -0.006365508, 0.003680789, 0.0048714033, 0.0053966744, -0.001277187, + -0.0072915414, -0.008100848, 0.005276057, -0.0042177327, 0.023952363, + -0.010637714, -0.0056573646, 0.03760163, 0.004505659, -0.0033383898, + 0.029212857, -0.0060347817, -0.02029492, -0.008505502, 0.0022041933, + 0.016310642, 0.0019182124, -0.0065016896, -0.009875097, -0.0022839566, + 0.008754519, -0.005785764, 0.0003329149, -0.013291305, 0.00009538778, + 0.0055056196, -0.0002507197, 0.009945134, -0.01682424, -0.01981245, + -0.001281078, -0.0091047, -0.011454802, 0.0077817948, 0.0077467766, + -0.013252396, -0.016310642, 0.005264384, -0.012046218, -0.0039298064, + 0.000961538, -0.006190418, -0.007167033, -0.008567756, 0.009828406, + -0.010030733, 0.011711601, -0.0037975158, -0.008225357, -0.028387986, + 0.005501729, 0.006097036, -0.0072215055, 0.018302782, -0.014404102, + 0.007011397, 0.0010456786, -0.016839804, -0.011945055, -0.0031107725, + 0.034955822, -0.008824555, 0.000064199805, 0.03265241, 0.029197292, + 0.04080773, -0.0057429643, 0.0014153139, 0.009602735, -0.007167033, + 0.0068868883, -0.0062332177, -0.04734444, -0.012754361, 0.013906067, + 0.002311193, 0.010427604, 0.01478541, -0.0030679726, 0.008676701, + -0.0050892937, -0.006209872, -0.0018715216, -0.01799151, 0.01881638, + -0.0017363129, -0.018193835, -0.017322274, 0.030971542, 0.015851516, + 0.003278081, -0.021742335, 0.006439435, 0.01660635, 0.018302782, + -0.010077424, 0.011260257, 0.014380757, -0.010287533, -0.0005632074, + 0.0005053303, -0.010629931, -0.008007467, -0.0022489387, 0.0044122776, + -0.00962608, 0.016232824, 0.011268038, 0.02194466, 0.023625528, + 0.018738562, -0.007754558, -0.015166718, -0.0023442656, -0.0062293266, + 0.030598016, 0.017415656, -0.036169782, -0.0039998423, -0.011517056, + 0.026302466, 0.010170805, 0.020108158, -0.009999606, -0.019750195, + 0.0067623793, -0.031578522, -0.01990583, 0.021415498, 0.037570503, + 0.0087389555, 0.014575302, 0.0029804274, -0.007999685, -0.017711364, + 0.011415892, -0.014621992, 0.013493632, 0.026209084, 0.007855722, + 0.015633626, -0.013999448, -0.0048675127, 0.021150917, 0.018707434, + 0.0047196583, 0.026535919, 0.010559895, -0.014738719, 0.0039920607, + -0.0073226686, 0.005042603, 0.0076105953, 0.016715296, -0.0029298456, + 0.0032975355, 0.0025602104, 0.012217417, -0.017322274, 0.007287651, + -0.001040815, 0.015322354, 0.011703819, -0.005692383, 0.027282972, + -0.0074471775, 0.025181888, 0.02774988, -0.0031185541, 0.013688177, + 0.0048286035, 0.017540164, 0.019454487, -0.0038870063, -0.003877279, + -0.0114781475, -0.006929688, -0.0037741703, -0.007540559, + -0.021913534, 0.014933264, 0.011198003, -0.009377062, -0.008536628, + 0.0017382584, 0.0022178115, -0.009781715, -0.01431072, 0.042613108, + 0.013532541, 0.009244772, -0.008731173, -0.007435505, -0.009244772, + 0.011999527, -0.0027819916, -0.009875097, 0.026909446, 0.01270767, + 0.022629458, 0.028465804, 0.017851437, -0.0038266976, -0.010770003, + 0.014660901, 0.030349, -0.019968085, 0.0015952679, 0.005264384, + 0.008303175, 0.014038358, -0.009999606, -0.024108, -0.0047507854, + -0.014380757, -0.019080961, -0.0051787845, -0.003126336, -0.006217654, + 0.025742177, -0.005649583, 0.02658261, 0.009727243, -0.0068285246, + 0.010956767, -0.0062993625, -0.031282816, -0.00027430826, 0.020185975, + 0.030053291, -0.0067351433, -0.029944345, -0.0023423203, 0.022598332, + -0.0069452515, -0.0060386728, 0.009999606, -0.00053791655, 0.01660635, + -0.0033383898, 0.01682424, 0.013026725, 0.0033811897, 0.012847743, + 0.0034804076, 0.010886731, -0.0032800264, 0.022691714, -0.010847822, + -0.00023977655, 0.004929767, -0.012256326, -0.0027333554, 0.020326048, + 0.004315005, 0.023578838, -0.019018706, 0.0016516859, -0.0010651331, + -0.0026088466, 0.01738453, -0.0056262375, -0.008972409, 0.015812607, + -0.00033631944, -0.020419428, -0.014793191, -0.006019218, 0.008941282, + -0.012552035, -0.00042483734, -0.009781715, 0.01764911, -0.0033753533, + -0.0036652253, -0.007657286, 0.005124312, 0.007832376, 0.000730516, + -0.002597174, 0.0062254355, -0.0041827145, -0.002494065, + -0.0018501217, 0.027672062, 0.01760242, 0.01934554, 0.023392074, + 0.0073654684, 0.01357145, -0.0035660074, -0.007723431, 0.037383743, + -0.0069141244, -0.011898364, -0.005493947, 0.0031068814, 0.007466632, + 0.00503093, 0.013742649, -0.018722998, -0.013454723, 0.008272048, + 0.02974202, -0.005466711, -0.020108158, 0.0004010056, 0.006626198, + -0.018131582, -0.012116254, 0.001732422, -0.018551799, -0.0056534735, + -0.012941125, -0.0029979364, -0.014108393, 0.01053655, -0.0066495435, + 0.010886731, -0.0024785015, 0.0019016761, -0.013906067, -0.0069608153, + 0.0077584493, -0.0021672298, -0.016341768, 0.01456752, -0.0038403156, + -0.00018165627, 0.014964391, 0.005634019, -0.007688413, 0.021742335, + 0.008505502, -0.004929767, 0.0017022676, -0.0073732506, -0.0023831746, + -0.0030037728, 0.0016857312, -0.023469893, 0.011493711, 0.012466435, + 0.0075950315, -0.010139679, -0.009283681, 0.007225396, 0.0013657049, + 0.0027177918, 0.0030971542, 0.020466119, 0.002741137, -0.010801131, + 0.008879027, -0.030022163, -0.018722998, -0.0011925601, -0.0038908974, + 0.0147153735, -0.008544411, -0.016155006, 0.000210838, -0.010092988, + 0.0028364642, 0.026862755, 0.0013861322, -0.009828406, 0.0034590077, + -0.006209872, 0.009937352, -0.034986947, -0.00007994578, 0.013244614, + -0.008015248, 0.019221032, 0.00029862637, -0.013765994, -0.011322511, + -0.0018977851, -0.020419428, -0.014754282, 0.0153612625, + 0.00050581666, 0.0147153735, 0.019874703, 0.0022119752, 0.0022392115, + 0.015586935, -0.0011108512, -0.021228736, -0.00032610583, 0.028994966, + 0.018084891, -0.0039823335, 0.0013180415, 0.0148632275, -0.018489543, + 0.01378934, -0.009571607, 0.015851516, 0.014302938, 0.009174736, + -0.0041554784, -0.005455038, 0.008482156, -0.0059414003, -0.017291147, + 0.024559343, -0.0054706014, 0.0025096287, 0.008544411, 0.02701839, + -0.00572351, 0.004536786, 0.0013005324, 0.0019211306, 0.021446627, + 0.019267723, -0.012380836, 0.012131818, 0.033897497, 0.009159172, + 0.010645495, -0.016808676, -0.010248624, -0.02541534, 0.019361105, + 0.009112481, 0.022053605, 0.0045251134, 0.018909762, 0.008404338, + -0.019796886, 0.03881559, 0.0034745713, 0.010170805, 0.011322511, + -0.016497405, -0.0050231484, -0.012435308, 0.06057349, 0.008544411, + 0.0064588897, -0.020279357, 0.011034585, -0.017835874, 0.0039823335, + -0.009968479, 0.0076650674, 0.017026566, 0.0008127112, 0.013361341, + -0.0036788434, 0.017929254, 0.006190418, -0.006820743, -0.0012392508, + 0.022660585, -0.021197608, -0.008233138, -0.004279987, 0.0036146436, + 0.009921788, -0.008567756, 0.0076689585, -0.0073654684, -0.019547869, + -0.028201224, -0.008186448, -0.006128163, -0.0025174106, 0.023641093, + 0.022100296, -0.0030232272, -0.017462347, 0.008762301, 0.02094859, + 0.009859534, 0.0033792444, -0.022536077, -0.0023987382, 0.009641644, + 0.020061467, -0.028901584, 0.0015835952, 0.010715531, -0.0036632798, + 0.00005465495, 0.023034113, 0.016621914, -0.020310484, 0.0052916203, + -0.010793349, 0.008746737, -0.011252475, -0.017446784, -0.002208084, + -0.0026788828, 0.00034312851, -0.0119684, -0.0077817948, + -0.0058480185, -0.022520514, -0.03299481, -0.0041048964, + -0.0038208612, -0.007124233, 0.0077117584, -0.0061125997, 0.011198003, + 0.009703898, -0.0006784752, -0.013415813, -0.015898207, 0.0018530398, + -0.018442852, -0.0033111535, -0.00074413413, 0.023905672, + 0.00039419654, 0.015516899, -0.047842473, -0.018894197, 0.026255775, + 0.01998365, 0.019267723, 0.0031574632, -0.002025212, -0.015820388, + 0.018769689, 0.019236596, -0.014334066, -0.003447335, -0.005692383, + 0.028543623, 0.013431378, 0.010295314, -0.013672614, 0.004478423, + -0.020403866, -0.010139679, 0.014832101, 0.0055172923, -0.004190496, + -0.020870773, 0.00040830104, -0.0074004866, 0.007921867, + -0.0009941242, 0.011812764, 0.011812764, -0.02957082, -0.002706119, + -0.02150888, -0.006474453, 0.00044331912, -0.020746265, -0.019174341, + 0.009244772, 0.009042446, 0.019298851, 0.008598883, -0.018800816, + -0.0054706014, 0.005299402, -0.006657325, -0.01934554, -0.007618377, + 0.010606586, 0.018349472, -0.012777707, -0.0011876964, -0.024465961, + -0.0117271645, 0.056122303, -0.009587171, 0.015952678, 0.027874388, + 0.006326599, -0.006731252, 0.008124193, -0.000043043052, -0.046908658, + -0.0071008876, -0.017758055, 0.023812292, -0.00014834046, 0.02012372, + -0.028434677, -0.008054158, 0.021757897, -0.018287217, 0.012559816, + -0.012466435, 0.019221032, -0.0117271645, -0.004774131, -0.0042994414, + 0.020886337, 0.0045718043, -0.018333908, -0.015843734, -0.0028209006, + -0.0070386333, 0.009828406, -0.0099840425, -0.02068401, -0.02740748, + 0.023827855, -0.0048013674, 0.011898364, 0.007894631, 0.018925324, + -0.002480447, -0.0035309894, -0.010458732, -0.017166639, -0.01348585, + 0.013143451, 0.002120539, 0.015664753, 0.018287217, -0.0037138616, + 0.024963997, -0.0026419193, -0.009198081, 0.010116333, 0.00025217878, + 0.0051632207, -0.007929648, -0.0051165298, 0.008668919, 0.0013044233, + 0.0029026093, -0.0023753927, 0.002840355, -0.01764911, -0.035267092, + -0.004015406, 0.00849772, 0.011711601, -0.009159172, -0.002089412, + -0.0037488795, 0.0074004866, -0.021711206, -0.022380441, + -0.0023598291, 0.008342084, -0.022567205, -0.0055134012, 0.011843892, + -0.004778022, -0.013688177, 0.008342084, 0.016186133, 0.0021263754, + -0.012590944, 0.020108158, 0.0027644825, 0.006641762, 0.0009805061, + 0.014279593, -0.000982938, -0.00388117, -0.0059686364, -0.010497641, + -0.0012509235, 0.02172677, -0.0016001315, 0.0014026684, -0.008365429, + 0.008482156, -0.0026419193, -0.031547394, 0.04488539, -0.0019979759, + 0.0012820506, 0.0045756954, 0.0066884523, 0.013104542, -0.0037838977, + -0.009353717, 0.0017333947, 0.014411883, 0.024917306, 0.012279672, + 0.0062682354, 0.025602104, 0.010489859, -0.006509471, 0.0034025896, + -0.0028812094, 0.00046982584, -0.0053616567, 0.021010846, + 0.0028111732, 0.002462938, 0.011773855, -0.0110268025, -0.0065328167, + 0.032247756, 0.0046768584, 0.0016361224, -0.039251372, -0.011734947, + 0.018287217, -0.0030251727, -0.0037741703, 0.0027080646, -0.01036535, + -0.011454802, 0.0036088072, 0.02740748, 0.013018942, -0.007645613, + 0.022878475, 0.012349708, 0.025150761, -0.008178666, -0.005859691, + 0.004462859, 0.0026399738, 0.0013355504, -0.015711443, -0.019174341, + -0.00015843248, 0.009096918, 0.00004149277, 0.013657049, + -0.0024084654, -0.016746422, 0.030815907, 0.013081197, 0.009314808, + 0.025835559, 0.010209714, 0.009462662, 0.008349866, 0.02588225, + 0.014131739, 0.011641565, 0.010209714, 0.0054005655, 0.0014911864, + 0.017026566, -0.0021477754, -0.0058052186, 0.012279672, -0.004774131, + 0.008069721, -0.013205705, -0.0099840425, 0.0066456525, 0.022489386, + -0.010902294, -0.0049492214, -0.022022478, 0.0042177327, + -0.0034317714, 0.0024882287, 0.0032294448, 0.009447099, 0.012839961, + -0.008956846, -0.021182045, -0.0004985212, -0.017228894, -0.011812764, + 0.021197608, 0.011198003, -0.0126220705, -0.0011760237, + -0.00006249754, -0.0019814395, 0.0028714822, -0.039780535, 0.007688413 + ] + } + ] + }, + { + "name": "harnessing-technology-for-business-success", + "title": "Harnessing Technology for Business Success", + "content": "Businesses need to leverage the full potential of technology and generative AI to remain competitive. However, keeping up with the rapid pace of technological advancement and identifying key technologies among the daily noise can be challenging.\n\n---", + "keywords": [ + "Businesses", + "technology", + "generative AI", + "competitive", + "rapid pace", + "technological advancement", + "key technologies" + ], + "index": [ + { + "modelName": "text-embedding-3-large", + "position": [ + -0.005866572, -0.00040896775, -0.02123286, -0.023772236, 0.0036981583, + -0.017240226, -0.0035604814, 0.0117637385, 0.036683284, 0.019060621, + 0.05237847, 0.0040270537, 0.04521926, -0.00055023015, -0.049747307, + 0.016062321, 0.003166572, -0.015848158, 0.0062681297, -0.028223794, + -0.010379319, -0.036438525, -0.05403059, -0.01667422, -0.037356373, + 0.006233711, -0.027963737, 0.01223796, -0.053449288, 0.044117846, + 0.037509345, 0.011618413, 0.02495014, -0.035857223, -0.015343342, + 0.030441925, -0.009346741, -0.0082759205, 0.01841813, 0.00088725204, + 0.046045322, -0.01595524, -0.011587818, -0.021722378, 0.03472521, + 0.020406798, -0.004715439, -0.0102798855, -0.009798016, -0.019305382, + -0.03802946, -0.005797733, 0.016811896, 0.02752011, 0.0055759205, + -0.03184929, 0.0635762, -0.03496997, 0.024368837, 0.03261416, + 0.020743342, 0.025715012, -0.031696316, 0.019596033, -0.006107507, + -0.019030027, 0.020926911, 0.03561246, -0.024873653, -0.005116997, + -0.053112745, -0.0060042487, 0.018494615, -0.017423796, 0.0073274784, + -0.020100849, 0.0011109773, 0.0063560903, -0.006466997, -0.0030116853, + 0.022808498, 0.024215862, 0.00077730167, -0.009889801, 0.04833994, + -0.0023577195, 0.010930028, -0.06614617, 0.019825494, -0.008444192, + -0.02788725, -0.012543908, -0.040630024, -0.019060621, 0.021278752, + 0.015603398, -0.025332576, 0.009132577, -0.048370536, -0.020804532, + 0.017194333, -0.0018509914, 0.02716827, -0.021447023, 0.0069067986, + 0.009079036, -0.029615862, 0.031023227, 0.0065205377, 0.03503116, + -0.016842492, -0.022318978, 0.02752011, -0.008283569, -0.037631724, + 0.023680452, -0.035153538, -0.064004526, 0.019443057, -0.050634556, + -0.029233426, -0.02644929, 0.009316146, -0.03197167, -0.015427478, + 0.024720678, 0.013125212, 0.0009885976, 0.015381585, -0.0067423508, + -0.060241356, 0.07569178, 0.001415014, 0.05090991, 0.0074842772, + 0.02752011, 0.036316145, -0.00527762, 0.023053257, -0.011503682, + -0.027076485, -0.005866572, -0.024873653, -0.038427193, 0.003694334, + -0.0037842065, -0.002153116, 0.06553427, -0.017255522, 0.005602691, + -0.053479884, -0.0065702545, 0.024062889, -0.009996884, 0.03016657, + -0.03279773, 0.020666854, -0.03396034, 0.0035317987, 0.022028327, + 0.04778923, -0.01085354, -0.0052087815, 0.047728043, 0.04029348, + 0.0046160053, 0.04479093, -0.020529177, -0.06767592, 0.057793763, + 0.0018490792, -0.0019513809, -0.02326742, -0.043444756, -0.02423116, + -0.018968837, 0.028636824, -0.011870821, 0.002713385, 0.00049860124, + 0.0069947587, 0.007862889, 0.0010784701, 0.00455864, 0.00614575, + 0.01319405, 0.06584022, -0.009163172, 0.03184929, 0.013920679, + 0.00770609, -0.011052407, 0.011235977, -0.021079885, 0.025210196, + -0.06859376, 0.0061381017, -0.032767136, -0.017898016, 0.007243342, + 0.0007682188, -0.012559206, -0.013224645, 0.054428324, 0.01673541, + 0.019183, 0.008796033, 0.002480099, 0.017714446, 0.005667705, + -0.011717847, -0.00875779, -0.0018500353, -0.010386968, + -0.00012285764, 0.015305098, -0.0017114021, 0.018127477, -0.033776768, + 0.011962606, -0.011366005, -0.011557223, 0.0026541075, -0.025684418, + -0.005036685, -0.047299713, -0.016750706, -0.005281444, -0.0049754954, + -0.006459348, -0.021095183, 0.009316146, 0.024980735, -0.00983626, + 0.01835694, -0.036438525, 0.03184929, -0.039253253, -0.054367136, + -0.0074651553, 0.01067762, -0.0039543905, -0.05378583, -0.014211331, + 0.06645212, -0.05500963, 0.0063293194, 0.046565436, 0.005530028, + 0.041303113, -0.015664589, -0.024797166, -0.022028327, -0.01511388, + 0.0028682717, -0.0045854105, 0.026051557, -0.00037478752, + -0.004482153, 0.025332576, 0.002650283, 0.01553456, 0.0023672802, + 0.052806795, -0.025332576, -0.011702549, 0.040691216, 0.032155238, + 0.027061189, 0.013247591, 0.0032851272, 0.022074219, 0.009515014, + -0.011021812, -0.0013538243, -0.007480453, 0.028453255, 0.00905609, + 0.042557504, 0.022701414, 0.029600564, -0.016490651, 0.07807818, + -0.023634559, -0.02854504, -0.012918696, -0.017653257, -0.01751558, + -0.044423793, -0.015037392, -0.038304813, 0.011197733, 0.014609064, + 0.025133708, -0.017653257, -0.017913314, 0.03922266, -0.009813313, + -0.024445323, -0.015786968, 0.0032736543, -0.02566912, 0.010409914, + 0.02051388, 0.007335127, 0.0142342765, 0.0076143052, 0.04096657, + 0.01511388, 0.011060056, 0.0045739375, 0.005151416, 0.015136826, + -0.0011511331, -0.015343342, 0.050420392, 0.011786684, -0.025760904, + -0.0019131373, -0.010241643, -0.0146932, -0.008207082, -0.068349004, + 0.012972237, -0.009163172, 0.019397166, -0.0066352687, 0.036071386, + -0.01577167, -0.006203116, 0.0069182715, 0.003283215, 0.013086968, + 0.019703114, 0.021860056, -0.001910269, -0.0054497165, -0.0015173157, + -0.006379036, -0.026586967, 0.0033807363, -0.002713385, -0.011503682, + -0.014364305, 0.009193767, -0.00668881, -0.023435693, 0.031176202, + 0.0052623227, -0.024307648, 0.00092645176, -0.009132577, -0.005889518, + 0.011541925, 0.00079546735, -0.0005358888, 0.019183, -0.012612747, + -0.012880452, 0.013943625, -0.0008413597, -0.05702889, -0.02848385, + 0.03209405, -0.009629744, 0.011939659, -0.011029461, 0.00213017, + -0.012406232, 0.02045269, 0.011595466, 0.040568836, -0.008444192, + 0.013354673, -0.027994333, -0.011656657, 0.004180028, 0.02423116, + -0.009813313, 0.048003398, -0.023649856, -0.007809348, -0.0010124999, + 0.0049449005, 0.006386685, 0.014417846, 0.0047804527, -0.04846232, + 0.017530877, 0.05675354, -0.028590932, 0.0014408285, -0.017729744, + -0.033899147, -0.0033138099, 0.013499999, 0.03129858, 0.0068876767, + 0.03579603, -0.014907365, -0.009155523, 0.0001245308, 0.009010198, + 0.0028147306, -0.04102776, -0.0034954671, -0.037448157, -0.017668555, + 0.0077099144, 0.017882718, -0.02117167, -0.043628324, -0.040691216, + -0.026051557, -0.006466997, -0.00078877475, 0.019764304, 0.010562888, + 0.01457082, -0.04329178, 0.0016071883, 0.008122945, 0.0401711, + 0.016230594, -0.005912464, 0.009155523, 0.03772351, 0.030717278, + 0.0005119865, 0.0057709627, -0.019687817, -0.016873086, 0.028330876, + -0.0009905099, -0.007166855, 0.0021244332, 0.019886684, 0.008130594, + -0.013744758, 0.017882718, -0.044546172, 0.009706232, -0.00004672893, + 0.008375353, -0.039008494, -0.0033233708, -0.002606303, -0.012528611, + -0.027596598, -0.068716146, -0.022884984, 0.016490651, -0.030472519, + 0.0052202544, 0.004570113, -0.03503116, 0.026847024, 0.009889801, + 0.028529743, 0.002386402, 0.03579603, -0.016077619, 0.03383796, + -0.019825494, 0.024659488, 0.03524532, -0.024690084, -0.011366005, + 0.02441473, 0.04466855, 0.009369688, -0.031512745, 0.003797592, + 0.018127477, -0.0046045324, 0.03070198, -0.015611047, -0.006669688, + -0.0043406514, -0.011243626, 0.01601643, 0.0017553823, 0.017439092, + -0.014723795, -0.026128044, 0.012039092, 0.025592633, 0.00039606052, + -0.003904674, 0.011679603, -0.017959205, 0.0008666961, 0.030778468, + -0.017316712, -0.01745439, -0.015725778, 0.01205439, 0.0033405805, + 0.015756372, 0.006692634, -0.038488384, -0.016169405, -0.040324077, + 0.0095226625, -0.01055524, -0.011939659, 0.022166004, 0.029309914, + 0.0025087816, 0.020299716, -0.039100282, 0.05632521, -0.0057135974, + -0.002373017, 0.042863455, 0.055560336, -0.00014233799, 0.020896316, + -0.009690934, -0.031941075, 0.017959205, 0.0028376768, 0.03010538, + 0.029355805, -0.0055032573, 0.0015699008, -0.008987252, 0.018372236, + 0.03209405, -0.026678752, -0.012574503, -0.0074575064, -0.049318977, + 0.014624362, -0.055743907, 0.011534276, 0.01733201, -0.023037959, + -0.008061755, -0.0058856937, -0.02891218, 0.006019546, 0.008719546, + 0.029524077, 0.014127195, 0.0055567985, 0.048645888, -0.024934843, + 0.014892067, 0.013316429, 0.020192632, -0.0046810196, 0.011641359, + -0.00023711046, -0.017546175, -0.008719546, 0.03191048, -0.032767136, + -0.007323654, -0.0455864, -0.015786968, 0.0033769119, -0.0009341005, + -0.00079307717, -0.012781019, 0.0000068382883, 0.022196598, + 0.021615297, -0.021217562, 0.010348724, -0.021844758, 0.0029428469, + -0.004126487, -0.023405097, 0.031023227, -0.0052508493, -0.005331161, + -0.03322606, -0.012452124, 0.006692634, -0.020881018, -0.018005097, + 0.020192632, 0.037998863, 0.012406232, -0.007839942, 0.0033635267, + -0.02537847, 0.017852124, -0.00542677, -0.0017200069, 0.035306513, + -0.03184929, -0.016444758, -0.019030027, 0.02901926, 0.020636259, + 0.009048441, -0.0003726363, -0.009943342, -0.0008246281, 0.031757504, + 0.049471952, 0.009392634, -0.029065154, 0.026158638, -0.015725778, + -0.0042832857, 0.017301416, -0.0092779035, -0.0020001414, + -0.021814162, 0.019351274, -0.054948438, 0.017959205, 0.0006635269, + -0.015297449, 0.018877052, -0.049166, -0.009843908, -0.010562888, + -0.0107694045, 0.005025212, 0.009400283, 0.006008073, 0.053020958, + 0.01624589, -0.015343342, -0.0032774785, -0.004222096, -0.0063905097, + -0.027902547, -0.03928385, -0.00031288064, 0.0032736543, -0.009063738, + 0.009362039, 0.049930874, 0.0008910764, -0.009790367, -0.014769687, + 0.00875779, 0.0075033987, 0.07489631, -0.0071936254, -0.008390651, + 0.001488633, -0.006111331, 0.06226062, 0.014180736, 0.009262606, + 0.019397166, 0.0124980165, 0.02591388, 0.0000712885, -0.011947308, + -0.0007299752, 0.023512179, 0.012666288, 0.025470253, 0.026510479, + 0.023940507, -0.01463201, 0.005621813, 0.008987252, 0.00461983, + 0.03340963, -0.01817337, -0.0063713877, 0.025653822, 0.007947025, + -0.038366, 0.007816996, 0.015259205, -0.000422353, -0.01768385, + 0.03891671, 0.020682152, -0.013446458, 0.009828611, -0.018678186, + -0.023405097, 0.010669971, 0.004298583, -0.06290311, -0.0053732293, + -0.040354673, 0.0071706795, 0.020773936, 0.0055262037, -0.003451487, + 0.0052164304, 0.021033993, -0.03374617, 0.003862606, 0.007159206, + -0.0050290367, -0.052194897, -0.024368837, -0.026892915, 0.005174362, + 0.000086287175, -0.014127195, 0.010432861, -0.0024820112, 0.01745439, + 0.020299716, -0.0018978398, -0.03135977, -0.011327761, 0.023879318, + 0.02254844, -0.0075531155, 0.006176345, 0.011167138, 0.021401132, + -0.007870537, 0.020743342, -0.051674783, 0.003049929, 0.018601699, + -0.01007337, -0.014884418, -0.032124642, -0.0134923505, 0.01817337, + 0.019565437, 0.020697448, -0.01829575, -0.0056065153, -0.00845949, + -0.020192632, -0.0042182715, 0.012704532, 0.010103965, -0.014004814, + 0.0009780807, 0.026342208, 0.016047025, -0.013316429, 0.014555523, + -0.0029485833, -0.0048875352, 0.01864759, -0.009124928, -0.03051841, + 0.008528328, -0.0015861543, -0.035000563, 0.015343342, 0.018448723, + -0.028682718, 0.02614334, -0.024690084, -0.013125212, -0.013538242, + 0.01085354, -0.02147762, 0.0050099147, 0.0008671742, -0.0005942103, + 0.00055501057, -0.0008088526, 0.017561471, 0.033011895, -0.011710198, + 0.009492068, -0.016536543, 0.016169405, 0.019932577, 0.0067270533, + -0.017607365, 0.050971102, -0.018968837, -0.02554674, -0.0056830025, + 0.022701414, -0.018846458, -0.0010182365, -0.010096316, -0.050389796, + 0.0015708568, -0.030396031, 0.025623228, -0.026510479, -0.03429688, + -0.002281232, -0.031160904, 0.046565436, 0.026969403, 0.0074575064, + 0.009966289, 0.005717422, 0.024445323, 0.009186118, 0.018494615, + 0.024552407, -0.02033031, 0.0038033284, -0.013285834, 0.013652974, + -0.0032315862, -0.013056373, -0.0049869684, -0.031160904, + -0.020559771, -0.0043483, -0.016291784, 0.03490878, 0.00041518232, + 0.015259205, 0.0062107644, -0.0050213877, 0.015450424, 0.0067270533, + 0.032644756, -0.04084419, 0.021248156, -0.040691216, -0.004803399, + 0.011610764, -0.038855523, 0.0044974503, 0.0051131723, -0.046871386, + 0.01487677, 0.019427761, -0.011197733, 0.017316712, 0.011503682, + -0.0076678465, -0.0033673511, -0.017377902, 0.025485551, + -0.0149685545, -0.018020395, -0.015664589, 0.0022544616, -0.008566571, + 0.010065721, -0.002585269, 0.028743908, -0.00025240792, -0.004899008, + 0.007205099, -0.0019886685, 0.011947308, -0.012322095, -0.0013021954, + 0.009629744, -0.0007108534, -0.030396031, -0.008413597, -0.016154107, + -0.019137109, 0.024368837, 0.02141643, -0.021798866, -0.0034591358, + -0.0019064447, 0.010593483, 0.0233898, -0.0070138806, -0.006195467, + 0.016291784, -0.01427252, -0.0055491496, -0.023359206, -0.0009082861, + -0.013958923, -0.024934843, 0.009216713, -0.024628893, 0.05121586, + -0.0045777615, -0.016490651, 0.0057135974, 0.010065721, 0.0056524076, + -0.0015211401, -0.009140226, 0.009407931, -0.00055214233, 0.016551841, + 0.0092702545, -0.015970537, -0.03383796, 0.006409631, -0.0045968834, + -0.00428711, -0.01985609, 0.037019826, 0.0019121811, -0.02182946, + 0.022441357, 0.008803682, 0.002206657, 0.0054994333, 0.031451557, + -0.014616713, 0.000034210116, -0.023894615, -0.019290084, + 0.0044171386, 0.010463456, -0.022303682, 0.017989801, -0.02470538, + -0.003768909, -0.0024016995, -0.018999431, 0.02926402, 0.022058923, + -0.020942207, 0.015274503, -0.0043444755, 0.00050146953, 0.01397422, + -0.0027172095, -0.03466402, 0.049992066, 0.027627194, 0.019259488, + -0.022762604, -0.051766567, 0.023282718, 0.03094674, 0.021814162, + -0.009155523, -0.013025778, -0.0056638806, 0.034388665, 0.03184929, + 0.0014742917, 0.0082759205, -0.0012084985, -0.023894615, 0.02147762, + -0.03567365, 0.017377902, -0.02447592, 0.019947873, -0.02782606, + -0.012704532, 0.017836826, -0.0015718129, -0.005920113, 0.02860623, + 0.017668555, 0.031573936, 0.0122532565, 0.032889515, 0.02464419, + -0.024720678, -0.0022487252, -0.017928611, -0.011044758, -0.014907365, + 0.011932011, 0.0142342765, -0.005269971, 0.039773367, -0.0035470962, + -0.0030002124, -0.008581869, 0.0050175632, 0.0020211756, 0.010868838, + -0.017484985, -0.005254674, -0.0124980165, 0.0009838172, -0.009369688, + 0.036775067, 0.032155238, -0.008237677, 0.0010268412, -0.0050634556, + 0.00650524, 0.00044792844, 0.03441926, 0.0002502567, -0.022104815, + 0.0053732293, 0.01847932, -0.041150138, -0.021003397, -0.007599008, + -0.029539375, 0.01517507, -0.034572236, -0.0035834275, -0.018081585, + -0.027657788, -0.011343059, 0.019901982, -0.01541218, 0.004034702, + 0.0076907924, 0.007369546, -0.017255522, -0.010868838, 0.0028529742, + 0.02147762, -0.014012463, 0.029830026, -0.0035222378, 0.03094674, + -0.013683569, 0.008230028, -0.02111048, 0.013951274, 0.016169405, + -0.03561246, 0.0038339233, -0.0036044614, -0.005855099, 0.0006095077, + -0.022992067, -0.06883852, 0.043047022, -0.0033807363, -0.005660056, + -0.018142775, 0.013683569, 0.025898581, 0.0070330026, 0.018265154, + 0.0067270533, -0.041853823, -0.0045089233, -0.022441357, 0.0052087815, + -0.03184929, 0.020835126, -0.009362039, 0.01061643, -0.072754666, + -0.018203964, 0.007847591, -0.0038721669, -0.022456655, 0.024399431, + 0.04167025, -0.030839657, 0.005445892, 0.007243342, 0.010478753, + 0.02260963, -0.0071400846, 0.007671671, -0.00815354, 0.017943908, + -0.0115113305, -0.018020395, -0.027489517, -0.0097674215, 0.015075636, + -0.017408498, 0.022120113, -0.0029715295, -0.00059277617, + -0.000087840825, -0.017026061, 0.00542677, 0.023864022, -0.0015163596, + -0.005686827, -0.025103115, -0.008321812, -0.009583852, -0.020177336, + 0.008857223, 0.006818838, -0.0086966, 0.00069938024, -0.032981303, + -0.011649008, -0.014310764, 0.007740509, -0.00995864, 0.0032258497, + 0.003694334, 0.004593059, -0.016398866, 0.00887252, -0.009216713, + 0.0036388808, 0.014012463, -0.017989801, -0.029906513, 0.020116147, + 0.0030174218, -0.00088486186, -0.010624078, 0.00025910055, + -0.0023577195, 0.03340963, 0.018999431, -0.014639659, 0.004941076, + -0.0323694, 0.009943342, -0.024460621, -0.0034744332, 0.006015722, + -0.0072356937, 0.016827194, -0.011473087, 0.016092917, -0.010861189, + -0.020590367, -0.009813313, -0.009469121, 0.015565155, 0.040018126, + 0.022869688, -0.010081019, -0.0065167136, 0.022395466, 0.00011389429, + -0.0024571528, -0.018984135, -0.006922096, -0.00031072943, + 0.005224079, -0.008184135, 0.01313286, 0.011029461, -0.010188101, + 0.010364022, -0.009790367, 0.00056313735, -0.021905947, 0.027428327, + -0.01379065, -0.0061992914, -0.008597166, 0.011901416, 0.018800566, + -0.007954674, -0.0059392345, -0.04641246, -0.029921811, -0.008145892, + -0.00563711, 0.023818128, -0.010593483, 0.006585552, 0.000017732493, + -0.0009728222, -0.025959771, 0.039681584, -0.009553257, 0.018968837, + 0.019947873, 0.003315722, 0.0029428469, -0.008184135, 0.0020077901, + -0.054795463, 0.0057135974, 0.011771387, -0.008666005, -0.0073963166, + 0.014341359, 0.01397422, -0.0059277616, 0.006283427, 0.0052737957, + 0.008558922, 0.0020747166, -0.0075837104, -0.02831558, -0.011985552, + 0.03916147, -0.0007251947, 0.0070635974, 0.039987534, 0.011725495, + 0.024338242, 0.015878752, -0.0067920676, -0.018509913, -0.0069947587, + 0.0015354814, -0.03579603, -0.008513031, 0.0045662886, -0.0018318696, + 0.0012744687, -0.013859489, 0.015756372, 0.020483285, 0.01636827, + -0.0027726628, -0.00054927403, -0.003491643, 0.01157252, 0.028223794, + -0.021278752, 0.010631727, 0.0023577195, 0.005621813, -0.0014800283, + -0.0050519826, -0.0057594897, -0.008222379, 0.0029887392, 0.02710708, + 0.019427761, 0.027152972, 0.009254957, 0.013454107, 0.008115297, + -0.0026101272, 0.00071467774, -0.02495014, 0.004539518, 0.014517279, + -0.014723795, -0.01661303, 0.059966, 0.015297449, -0.014915013, + 0.010440509, -0.018127477, 0.026235126, 0.009193767, -0.0071209627, + -0.0067423508, 0.0125056645, -0.0132399425, -0.0038377475, + -0.0013031515, 0.027152972, 0.02014674, 0.00070559484, 0.00012172228, + 0.033562604, -0.0022449007, -0.0027879602, -0.033103682, -0.04479093, + 0.011863172, -0.019703114, -0.010708215, -0.024277052, -0.0044209627, + 0.0036924218, 0.02195184, 0.014440793, -0.022104815, -0.017071953, + -0.008620113, -0.0001198101, 0.0013289659, 0.0203915, 0.0061610476, + 0.018433426, -0.025944473, -0.018571103, 0.005920113, 0.024261754, + 0.005992776, 0.021737676, -0.009063738, -0.025638524, 0.04466855, + -0.00833711, 0.0005009915, 0.02026912, 0.02764249, -0.007101841, + 0.008497733, 0.010318129, 0.041945606, -0.010754107, 0.0074536824, + 0.011442492, 0.0074575064, 0.052103113, -0.004107365, -0.008413597, + -0.015985835, 0.005434419, 0.03974277, -0.024843058, 0.010784701, + 0.004562464, -0.035184134, 0.0072318693, 0.018066287, -0.020911613, + -0.010226345, 0.013178753, -0.002820467, 0.011251274, -0.029768836, + -0.015251557, -0.015442776, -0.0030212463, 0.008383002, -0.020758638, + -0.010364022, 0.026173936, -0.026250424, -0.00031957327, + -0.0037077193, 0.004589235, -0.03249178, 0.01823456, -0.017026061, + 0.0048798863, -0.0053158635, 0.003684773, 0.015335693, -0.0044515575, + -0.01973371, -0.01319405, -0.009713881, -0.0060348436, -0.019305382, + -0.0031072944, -0.02260963, 0.008612464, -0.0024762745, -0.0053655803, + 0.027871953, 0.008566571, -0.002881657, -0.009163172, -0.01624589, + -0.008933711, -0.0057059485, -0.013668271, 0.00995864, 0.022823794, + 0.007969971, 0.00064679526, 0.03466402, -0.020590367, -0.00015393058, + 0.027871953, -0.033501413, -0.01595524, -0.010784701, -0.020712746, + -0.012092634, 0.020773936, -0.036101982, 0.009147875, -0.0030766996, + -0.025562039, 0.00051581085, -0.00221813, -0.006459348, 0.017561471, + 0.003705807, 0.007931728, 0.021737676, 0.021921245, -0.0028950423, + -0.025133708, 0.000010621568, -0.006271954, 0.014907365, -0.015878752, + -0.0036580025, -0.03334844, -0.03640793, -0.0055376766, -0.0047116145, + 0.0010564801, 0.028820395, 0.011649008, -0.0013681656, 0.01811218, + 0.01912181, 0.018601699, -0.027856655, -0.023894615, -0.00713626, + -0.005300566, -0.016536543, 0.011740793, -0.013935977, 0.02267082, + -0.017163739, -0.021814162, 0.017071953, -0.012949291, -0.009492068, + -0.014769687, -0.003895113, -0.03766232, -0.010654674, 0.012023795, + 0.016521245, 0.02326742, -0.0107617555, 0.0032564446, -0.024430027, + -0.013155807, -0.009377336, -0.00010720166, 0.013783002, 0.020865722, + -0.01247507, -0.010937676, 0.009996884, -0.0002037668, 0.0046466, + 0.015741074, 0.02033031, -0.0031952548, 0.0018701132, -0.029126344, + 0.024812464, -0.014356656, -0.021095183, -0.013316429, 0.0017085338, + -0.0014283994, 0.01235269, -0.01727082, 0.0034055947, -0.028300282, + -0.014960906, -0.04059943, -0.020406798, 0.025531443, 0.015970537, + -0.008421246, 0.007702266, 0.0002595786, -0.021905947, -0.03094674, + -0.01145779, 0.014662605, 0.023359206, -0.013805948, -0.014762038, + 0.011327761, -0.0045433426, -0.010539942, -0.017255522, 0.014310764, + 0.040691216, -0.025057223, -0.0043100566, 0.002818555, 0.017699149, + -0.0019618978, 0.022334276, -0.024016995, -0.00009835782, + -0.0057365433, -0.008864872, -0.010708215, 0.0010211047, -0.032155238, + -0.013201699, -0.034358073, 0.014723795, 0.021814162, 0.02141643, + -0.017301416, -0.017913314, 0.014517279, 0.041486684, 0.009851557, + 0.023190932, -0.0031493623, -0.00070655096, 0.013507647, -0.017760338, + -0.0028548865, 0.01751558, 0.005985127, -0.013844192, -0.013469405, + 0.02704589, -0.007820821, -0.00060424925, -0.008688951, 0.001878718, + 0.0062949006, -0.038702548, -0.002436119, 0.002373017, 0.013102265, + -0.045494616, 0.016276486, -0.014616713, 0.0062528322, 0.003965864, + 0.0040232292, -0.011977903, 0.002206657, 0.0073504243, 0.0032143765, + 0.005331161, 0.007052124, -0.012199716, 0.016551841, 0.015458073, + -0.019152407, 0.017117845, -0.009996884, -0.02441473, 0.029722944, + 0.019779602, 0.00094366143, 0.0017228753, 0.0018347378, 0.0036159346, + -0.004202974, 0.00905609, 0.038304813, -0.014058356, -0.01067762, + -0.0010383144, 0.014899716, -0.008956657, 0.008390651, -0.009017847, + -0.011541925, 0.012199716, 0.01636827, -0.00419915, -0.010601132, + -0.012046741, -0.011817279, 0.0030862605, 0.0016014518, 0.0047881017, + 0.0011272308, -0.021079885, -0.023986401, 0.0060424926, 0.014586118, + 0.000506728, 0.004631303, 0.016597733, -0.0065205377, -0.0090407925, + 0.01457082, -0.003977337, -0.017301416, 0.015863456, 0.0055262037, + 0.0090407925, -0.0037287532, 0.025776202, 0.0051667136, -0.025806798, + -0.036010195, 0.005690651, 0.0119932005, 0.017484985, -0.012192067, + 0.000496211, -0.00289313, -0.006642917, 0.01061643, 0.03135977, + 0.043261185, -0.014624362, -0.009698583, -0.002830028, 0.018846458, + -0.02735184, 0.0059086396, 0.005560623, 0.013813597, -0.0045089233, + 0.00030451486, 0.012857506, -0.0041303113, 0.0062528322, 0.017989801, + 0.04433201, 0.015970537, -0.003705807, 0.040140506, 0.006673512, + 0.01289575, -0.007369546, -0.038427193, 0.0002605347, -0.009675637, + -0.014830877, -0.012566854, -0.0034113312, -0.004921954, + -0.0029351981, 0.01751558, 0.009935693, 0.012153824, 0.0038090649, + -0.004443909, 0.007801699, -0.00020842775, -0.0073274784, 0.006459348, + 0.012215014, 0.019993767, 0.0012438738, -0.012421529, 0.016658923, + 0.010180453, 0.012299149, -0.0110141635, 0.031160904, 0.014211331, + -0.008329461, 0.00031861718, -0.0041188383, -0.0035853398, + 0.0042909347, -0.017010763, -0.012161472, 0.035000563, 0.014884418, + -0.014142492, 0.019580735, -0.0049563735, -0.016108215, 0.00084614014, + -0.020942207, 0.0137218125, -0.006956515, -0.0014953257, 0.019550141, + -0.014081302, -0.0055032573, -0.013935977, -0.0002469104, + -0.015220962, 0.020131443, -0.0019771953, -0.0044974503, -0.003747875, + -0.013117563, -0.028269686, -0.00069603394, 0.0057556652, + 0.0004254603, 0.016199999, 0.010081019, 0.013262888, -0.0097827185, + -0.0227932, 0.024078185, -0.01876997, 0.009820962, 0.015802266, + -0.006849433, -0.0031627477, 0.0023366853, -0.013683569, + -0.0023883143, -0.0074460334, -0.0043980167, 0.00055787887, + -0.014524928, -0.004853116, 0.012781019, -0.026801132, 0.0040691216, + 0.006585552, -0.015970537, 0.000025814446, 0.011947308, 0.012704532, + -0.011641359, -0.0012888102, 0.0051016994, -0.016505947, -0.010409914, + 0.018968837, -0.027780168, 0.022303682, 0.008191784, 0.0067844186, + -0.008971954, -0.009316146, 0.01733201, 0.013836543, -0.0015039305, + -0.018249856, -0.00097234413, -0.020988101, 0.0034706087, 0.011465439, + 0.0062949006, -0.017347308, -0.020529177, 0.024399431, 0.0054229456, + 0.016047025, -0.00887252, -0.0067270533, 0.0059966003, -0.023053257, + -0.0019657223, -0.0062107644, -0.0040041073, -0.00011706134, + -0.016811896, -0.008864872, -0.0047881017, -0.0025814446, 0.026801132, + 0.007419263, -0.016949574, 0.009820962, 0.002797521, -0.0018137038, + -0.030549007, 0.0018060551, 0.0029160762, 0.01817337, -0.00160432, + -0.0015039305, -0.010494051, -0.0061878185, -0.0011377478, 0.01097592, + 0.007262464, 0.009859206, 0.011947308, -0.011503682, -0.017653257, + 0.0026521953, 0.023481585, -0.007652549, 0.0070635974, -0.011740793, + -0.008566571, -0.016567137, 0.0011014163, -0.014341359, 0.005407648, + -0.0125056645, 0.020529177, 0.031329177, 0.031053822, 0.04179263, + 0.016475353, 0.011893767, -0.04102776, -0.00012656249, -0.022594333, + -0.0068532573, 0.00791643, 0.025194898, -0.014685552, -0.029248724, + 0.018907648, -0.010692917, -0.004791926, 0.0037096315, 0.005036685, + -0.020009063, -0.00046489906, -0.008184135, -0.03974277, -0.016766004, + 0.009752124, -0.0017372166, 0.00728541, 0.01601643, -0.0068532573, + -0.011970254, -0.0006267174, -0.026510479, -0.018800566, 0.014326061, + -0.008895467, -0.0037402264, -0.008910764, 0.0032755663, + -0.0067538237, 0.0017754602, 0.0047613312, 0.003724929, -0.0018500353, + -0.018402832, -0.008061755, 0.01733201, -0.051521808, -0.011893767, + -0.0031608355, 0.0010775141, -0.012918696, 0.03172691, -0.005369405, + -0.00213017, 0.007664022, 0.01912181, 0.01391303, -0.0009111543, + 0.009300849, -0.0034457506, -0.0031703964, 0.020116147, 0.01481558, + 0.033654388, 0.03172691, -0.015419829, -0.030426627, -0.01811218, + 0.008023513, 0.0018385622, 0.012138526, -0.012650991, 0.011465439, + -0.017913314, -0.0137218125, -0.014150141, 0.0046160053, + -0.0034553113, -0.012039092, 0.018265154, 0.017469687, -0.013079319, + -0.0016138809, -0.0062298863, -0.0007137216, -0.000031222335, + 0.012023795, 0.015106231, -0.036928043, 0.0017056656, 0.015366288, + -0.01727082, 0.017347308, 0.012589801, 0.0021588525, -0.0075033987, + 0.01535099, 0.013148158, -0.011947308, 0.008367705, -0.0020498582, + 0.022808498, -0.008681303, 0.006922096, 0.017852124, 0.013018129, + -0.011549574, 0.004550991, 0.020804532, 0.015939942, 0.005889518, + -0.001847167, -0.018433426, 0.013630027, 0.010647025, -0.0044477335, + -0.021615297, 0.0017993625, 0.006788243, -0.016031727, -0.0035165013, + 0.0028873936, -0.0011635623, 0.017163739, 0.006945042, 0.014440793, + -0.0043444755, 0.004505099, -0.018158073, -0.010593483, -0.016505947, + 0.0050519826, -0.012696883, -0.010058073, 0.014723795, -0.00036020714, + -0.000020944359, -0.00935439, 0.003355878, 0.031512745, 0.0077634556, + 0.0003124026, 0.022640225, 0.026984701, 0.0054497165, 0.01685779, + -0.00054927403, -0.011205382, 0.0006554001, 0.0072777616, + -0.022701414, 0.016628327, -0.015251557, 0.015419829, -0.0034228042, + -0.007847591, 0.012184419, 0.025347874, 0.010692917, -0.00755694, + -0.01337762, 0.00009805904, 0.0020689801, -0.015611047, 0.0023577195, + -0.0055873934, 0.017286118, -0.0072012744, -0.0038281868, 0.05066515, + -0.024858356, 0.018127477, -0.014448441, 0.00083657925, -0.008949008, + -0.0075301696, -0.011251274, -0.010188101, 0.0004412358, 0.013507647, + -0.0028644474, 0.017668555, 0.0025145183, 0.0045203962, -0.020223228, + 0.011503682, 0.0016693341, -0.01912181, 0.005212606, -0.007771104, + 0.013630027, -0.007664022, -0.0109988665, -0.0024628893, 0.004007932, + -0.0051016994, 0.02320623, -0.000080610385, -0.0008643059, + -0.009316146, -0.03051841, -0.018509913, 0.011419546, -0.025363171, + -0.0066964584, 0.024093483, -0.013798299, -0.009423229, -0.013522945, + 0.016628327, -0.027183568, -0.0031111187, -0.017836826, -0.005331161, + -0.00001818066, -0.019075919, 0.013331727, -0.008168838, + -0.0010880311, 0.0049563735, 0.0026980876, -0.02170708, -0.01450963, + 0.022196598, 0.003684773, -0.0031264161, -0.023588667, 0.0051208213, + -0.025240792, -0.017760338, 0.00043956266, -0.0061992914, 0.016414164, + 0.009690934, 0.015236259, 0.0124980165, -0.000538279, 0.019397166, + 0.007216572, 0.00009710295, -0.015373937, -0.011243626, 0.0066276197, + 0.010325778, -0.020070253, -0.011213031, 0.00057986897, 0.0076066568, + 0.0040041073, -0.011909064, 0.009859206, 0.0010516996, 0.019198298, + 0.016398866, 0.027688384, -0.009262606, -0.009163172, -0.037601132, + -0.0076296027, 0.0010105878, 0.011174787, 0.02932521, 0.019565437, + -0.010027478, 0.007419263, -0.00650524, 0.0120008495, -0.008383002, + -0.015756372, 0.0149915, 0.014188385, 0.007029178, -0.0063063735, + 0.01961133, -0.0097827185, 0.00911728, -0.019580735, -0.0050175632, + -0.0102722375, 0.02129405, 0.0018232648, 0.008918413, -0.007254815, + 0.020850424, 0.004960198, -0.004378895, 0.009400283, -0.021691782, + -0.006310198, -0.0023137392, -0.016750706, 0.012230311, -0.012199716, + -0.0029696173, 0.020422095, -0.0011855523, 0.007740509, 0.018127477, + -0.017852124, -0.030396031, 0.018678186, 0.022502547, -0.020009063, + -0.004294759, -0.0056179883, 0.007208923, -0.0063063735, -0.010593483, + -0.017010763, -0.00072423863, 0.0034878184, 0.016995465, 0.004726912, + 0.016643625, 0.019198298, -0.010096316, -0.0028510622, 0.0036905096, + -0.008138243, -0.0026311614, 0.018188667, -0.020406798, -0.009935693, + -0.000036002784, 0.0034878184, -0.0030461047, 0.014547874, + 0.0012716005, -0.005962181, 0.002344334, -0.018280452, -0.00352415, + 0.0037861187, 0.01895354, 0.0026235126, 0.014287817, -0.012574503, + 0.0050060903, -0.0059660054, -0.0072892345, 0.023282718, + -0.0064746453, -0.00527762, -0.0033138099, 0.013308781, -0.0029849147, + 0.0034457506, -0.0024953964, 0.02710708, -0.009759773, -0.013025778, + 0.013828894, -0.021814162, -0.003862606, -0.0056103393, 0.008268272, + -0.0048569404, -0.011480736, 0.020162039, -0.0026560195, -0.022166004, + 0.006593201, 0.037111614, -0.03279773, -0.00578626, 0.005572096, + 0.0012782931, 0.024674786, 0.0049640224, 0.012360339, -0.0142189795, + -0.024139374, -0.013048724, 0.01541218, 0.0116949, -0.008413597, + -0.015626345, -0.0051399427, 0.008566571, -0.0052202544, -0.008122945, + -0.016521245, 0.008008215, -0.00770609, -0.009170821, -0.014532577, + 0.010417563, 0.011733144, 0.0034476626, 0.015328044, 0.0011052407, + -0.023236826, 0.019626627, 0.0073963166, 0.009446175, -0.004899008, + 0.023405097, 0.014501982, -0.01667422, -0.0070635974, 0.026281018, + 0.005923937, 0.0011310552, -0.020605665, 0.027810764, -0.02123286, + -0.007725212, 0.01630708, -0.010922379, 0.015970537, 0.003799504, + -0.0072586397, 0.0013423512, -0.0119932005, 0.0070330026, + 0.0008322768, 0.004355949, -0.017163739, -0.025500849, 0.0033597024, + -0.0053808778, -0.011350707, -0.020009063, 0.014012463, 0.026219828, + -0.010172804, -0.008490085, 0.004390368, 0.0042679883, -0.010692917, + 0.015481018, -0.007052124, -0.002946671, 0.035887815, 0.018066287, + 0.007526345, -0.01704136, 0.005476487, -0.028927477, -0.0020957505, + -0.023496883, 0.032430593, 0.02254844, 0.008207082, -0.00012028815, + -0.0010899432, -0.019519545, -0.005407648, 0.005740368, -0.007931728, + -0.011006515, -0.008314163, -0.014563172, 0.025041925, 0.012750424, + 0.024001699, -0.0020077901, 0.0087883845, -0.008191784, 0.0077290363, + -0.028958071, 0.01553456, -0.019596033, -0.011488385, 0.008413597, + -0.0085359765, 0.009377336, 0.0032411471, -0.0062566567, -0.011289517, + 0.009377336, 0.0048951837, -0.00076869683, 0.01624589, -0.00124483, + -0.011618413, 0.012314446, -0.0020708921, -0.016536543, 0.0009809489, + 0.006719405, 0.0015804177, -0.0069718123, -0.0014675991, 0.019259488, + -0.018050991, -0.020284418, -0.0077940505, 0.00126682, 0.010310481, + 0.01541218, 0.022961471, -0.013813597, 0.0100198295, -0.006933569, + -0.0080388095, 0.0115113305, -0.003965864, 0.0023041782, 0.007281586, + -0.01811218, 0.0051552407, 0.00037837285, -0.01163371, -0.022211896, + -0.021217562, 0.0035470962, 0.005552974, -0.009446175, -0.010394617, + -0.012635693, 0.013232294, 0.00461983, 0.013530594, -0.0046733706, + -0.0058206795, 0.00014425017, 0.0051590647, 0.0043444755, 0.009943342, + -0.000003417277, -0.006914447, -0.0017056656, 0.0038453962, + 0.028560339, -0.0097750705, 0.03258357, 0.00092645176, -0.009339093, + -0.0037765577, -0.005893342, 0.0030346315, -0.010562888, + -0.0058780448, 0.017561471, -0.00797762, -0.012421529, -0.0033826486, + -0.006681161, -0.0005296742, -0.01564929, -0.008834277, 0.0074345605, + 0.0068647303, 0.0092779035, -0.000020003208, -0.0023500707, + -0.0074842772, -0.015970537, 0.008405948, 0.011985552, -0.009140226, + 0.007931728, 0.007901133, -0.019534843, 0.018402832, 0.00043406512, + -0.0067270533, 0.01967252, -0.011587818, -0.0015115793, 0.0027783993, + 0.0040308777, 0.0061686966, 0.0074345605, 0.017408498, -0.002353895, + 0.004531869, 0.0049334276, 0.010562888, 0.008428895, 0.007847591, + -0.016597733, -0.011090651, -0.012062039, -0.019137109, 0.0024437676, + 0.0034533993, -0.0071745035, 0.009254957, -0.015985835, -0.005751841, + -0.001329922, -0.0027879602, -0.02308385, -0.013828894, -0.0043865438, + -0.006279603, -0.0046351273, -0.0030021244, -0.0051858355, + -0.008138243, 0.0026101272, -0.0022085693, -0.014165438, 0.0109988665, + 0.014410198, -0.0077328607, 0.006723229, 0.0018500353, 0.010379319, + 0.020697448, -0.0043330025, -0.0007514872, 0.0049066567, -0.021783568, + -0.01553456, 0.017255522, 0.032705948, -0.017530877, -0.01133541, + 0.01698017, 0.00578626, 0.027474219, -0.009950991, -0.010494051, + 0.008513031, -0.0043712463, 0.0143949, -0.023971103, -0.024996033, + -0.0122532565, 0.012520962, -0.02404759, 0.020712746, 0.042649288, + 0.0050519826, -0.0009049397, -0.012918696, -0.0074766283, + -0.0032545323, -0.010815296, 0.018158073, 0.0035968127, -0.015725778, + -0.00019301078, 0.022701414, 0.011671954, 0.014448441, 0.005174362, + 0.0019714588, 0.02189065, 0.0035337107, 0.015083285, 0.023359206, + -0.012574503, -0.010409914, 0.005751841, -0.011174787, -0.01624589, + 0.012046741, 0.0052584982, -0.012704532, -0.011434844, 0.011909064, + -0.0026196882, 0.014379602, -0.0036694757, 0.009446175, 0.0009427053, + 0.01085354, 0.007381019, 0.008933711, 0.021783568, -0.003906586, + -0.009384985, -0.0015861543, 0.0021339941, 0.007071246, -0.0020613314, + 0.009392634, 0.0018328256, -0.00032339763, -0.0055453256, + -0.0036656512, -0.011289517, 0.009729178, 0.021875352, 0.0062566567, + 0.0036656512, -0.002183711, -0.0020097024, -0.0023519828, + 0.0005296742, 0.002858711, 0.0030881725, 0.011564871, 0.0059392345, + 0.0046083564, 0.012803965, -0.02872861, 0.010777053, 0.017653257, + -0.020223228, 0.0076907924, -0.009132577, -0.029860621, -0.009155523, + -0.019993767, -0.016582435, 0.016154107, -0.016873086, -0.009232011, + 0.009216713, -0.010983569, -0.0022716713, 0.0044515575, -0.012207365, + -0.006956515, 0.0122456085, 0.015503965, -0.004401841, 0.013576486, + -0.011312464, 0.04733031, 0.016475353, -0.0044477335, 0.010471104, + 0.001804143, 0.030028893, -0.004600708, 0.008306515, 0.0021167845, + -0.0068417843, -0.00614575, -0.01727082, -0.008926062, -0.0070062317, + -0.0037631725, 0.011564871, 0.0063560903, 0.013798299, 0.02033031, + 0.015045041, -0.006088385, -0.016414164, 0.022196598, 0.015741074, + 0.011312464, -0.0011434844, 0.0020785409, -0.0092702545, 0.020804532, + 0.0019810197, -0.027627194, 0.009323795, -0.017699149, 0.020590367, + 0.008252974, 0.008184135, 0.024965437, -0.0030365437, 0.009966289, + 0.017821528, 0.0149762025, -0.02014674, -0.012888101, -0.010417563, + 0.005116997, -0.007610481, -0.0019265225, 0.010081019, 0.009996884, + -0.043199997, -0.0046810196, 0.011090651, -0.025026627, -0.005912464, + -0.009484419, -0.009254957, 0.0069373935, 0.007870537, -0.010310481, + 0.008880169, 0.0054497165, -0.0142342765, 0.02932521, 0.011488385, + 0.008375353, -0.045311045, -0.01571048, 0.010126911, 0.0017056656, + -0.00611898, 0.01361473, -0.009637393, 0.008994901, 0.0038702546, + 0.0047230874, 0.013163455, -0.0023787534, -0.0025183426, 0.0033042491, + 0.0024016995, 0.0061686966, 0.005212606, 0.01691898, 0.009576203, + 0.009928045, 0.0024648015, -0.005078753, -0.0056065153, 0.0072471667, + -0.026954105, -0.021691782, 0.0005287181, -0.018464021, 0.00063054176, + 0.0053349854, -0.0087883845, 0.003652266, 0.00728541, -0.0034533993, + 0.016888384, -0.010570537, 0.01559575, -0.004482153, -0.026709346, + 0.0036140224, -0.012406232, 0.0028893058, -0.004784277, -0.006008073, + 0.0033539657, 0.022961471, 0.008949008, -0.014601415, -0.009798016, + 0.0015240083, -0.004046175, 0.01223796, -0.00013385268, 0.009384985, + -0.0045662886, -0.0000012959509, 0.003365439, 0.0010756019, + -0.003166572, -0.0038243623, 0.0203915, 0.026005663, -0.002946671, + -0.003556657, 0.0083983, -0.013645325, 0.014058356, 0.0021263454, + 0.003399858, -0.01553456, 0.00014735747, 0.011404249, 0.012199716, + 0.018265154, -0.00023866411, 0.014165438, 0.009224362, -0.024384134, + -0.0008064624, -0.012046741, -0.036560904, 0.007778753, -0.010371671, + 0.0015546033, -0.01733201, -0.002047946, -0.019412464, 0.019091217, + 0.014700849, 0.023940507, -0.017133143, -0.010371671, -0.021186966, + -0.014433144, 0.0046810196, -0.0013433073, 0.01133541, -0.019366572, + 0.005560623, -0.015083285, -0.018096883, -0.005048158, 0.016720112, + -0.0019599856, -0.002353895, 0.0046466, -0.018035693, -0.012283851, + -0.0012553469, 0.0087883845, -0.006799716, -0.01301048, -0.0024246457, + -0.008704249, -0.0076907924, -0.0031302406, 0.005797733, 0.0048875352, + 0.007155382, 0.014287817, -0.0100351265, -0.010960623, 0.011595466, + -0.012926345, 0.015741074, -0.018142775, -0.00095800275, 0.0042259204, + -0.016521245, -0.014280169, 0.012635693, -0.0185864, -0.016062321, + 0.005369405, 0.0010411827, 0.015565155, 0.016092917, 0.006015722, + 0.010746459, -0.00068312674, -0.019427761, 0.0008055063, + -0.0028434135, -0.017913314, 0.016643625, 0.0010058073, -0.0032067278, + -0.003820538, -0.03903909, -0.0044286116, 0.01541218, -0.0020154389, + 0.009690934, -0.014662605, 0.013828894, 0.0056524076, 0.00029614905, + 0.010861189, 0.017561471, 0.008650708, 0.021967137, 0.022502547, + -0.0015890226, 0.0009666076, -0.0011683427, -0.03064079, 0.0120084975, + 0.009996884, 0.01463201, 0.0046618977, 0.009101982, 0.013415863, + -0.003977337, 0.0006209808, -0.0046924925, 0.024200564, + 0.000119451564, -0.0032583568, 0.012184419, 0.003874079, 0.016658923, + -0.005763314, 0.0065167136, -0.00995864, 0.00911728, -0.004325354, + 0.0035222378, -0.0048263455, 0.020758638, 0.016995465, 0.01577167, + 0.009851557, -0.006447875, -0.005625637, -0.0029179885, 0.03209405, + 0.0016798512, 0.0016110126, 0.009866855, -0.015297449, 0.0038989375, + -0.0067079314, 0.008161189, -0.0070330026, 0.0129875345, -0.012727478, + -0.019886684, 0.021385834, -0.0016492562, 0.01745439, 0.016873086, + 0.008077053, -0.013637676, 0.010027478, 0.019106515, 0.031390365, + -0.0071439086, 0.010570537, 0.016766004, 0.01589405, 0.0063828607, + -0.0018223086, 0.017362606, -0.016322378, -0.009002549, 0.019228894, + -0.0105322935, 0.00032746102, 0.017790934, 0.017286118, 0.0044171386, + 0.0092855515, 0.023634559, 0.0038721669, -0.0167966, -0.02189065, + -0.026173936, -0.0018777619, -0.018372236, 0.01475439, -0.0075110476, + -0.0020957505, 0.0032889517, -0.014020112, 0.03249178, -0.011626061, + 0.011450141, -0.0070865434, -0.013691217, 0.0013280098, 0.0076869684, + 0.01145779, -0.010777053, 0.023940507, -0.020223228, 0.001203718, + 0.012735127, 0.015450424, 0.011098299, -0.006470821, -0.0022104813, + -0.00068025844, 0.0057480168, -0.0018337817, 0.0011071529, + -0.022181302, -0.010746459, -0.008597166, -0.015136826, 0.004455382, + -0.001879674, -0.017898016, -0.013706515, -0.00064631726, 0.006975637, + 0.006428753, 0.0092702545, -0.0043865438, -0.0039008495, 0.00881133, + 0.0015957152, 0.013836543, 0.006711756, -0.01823456, -0.011916713, + -0.009377336, -0.014379602, 0.018877052, -0.006099858, -0.004168555, + 0.0037325777, -0.0012457861, 0.019335976, 0.016888384, 0.012222662, + 0.0077749286, -0.011182436, -0.026541075, -0.003556657, -0.010960623, + -0.025776202, -0.025898581, 0.008688951, 0.033134274, 0.027397731, + -0.014066005, 0.0046810196, 0.00045605522, -0.012781019, 0.0016932364, + 0.0056524076, -0.00162631, 0.009193767, -0.014241925, 0.01007337, + -0.0031933426, 0.003399858, -0.008734844, -0.006799716, -0.001847167, + -0.030778468, -0.031237392, -0.025684418, -0.014066005, -0.0056103393, + -0.024185268, 0.008627761, -0.020896316, -0.0045165718, -0.0014589942, + 0.00046418197, -0.011167138, 0.0067155804, -0.0054803113, + -0.013102265, 0.009820962, -0.0007008144, 0.01025694, 0.004115014, + -0.008122945, -0.002742068, -0.024032293, -0.019901982, 0.055223793, + 0.015863456, 0.021492917, 0.015664589, -0.0047689797, -0.008268272, + -0.0014006727, 0.009683286, -0.03010538, 0.01217677, -0.015320395, + -0.0037211045, 0.0085359765, 0.009071387, -0.008658356, -0.014150141, + 0.0022640226, -0.0033424927, 0.008145892, 0.019397166, -0.0025298158, + -0.01704136, 0.0008910764, 0.009362039, 0.012620396, -0.015266854, + -0.007342776, -0.019045325, 0.010233994, -0.014142492, 0.012903399, + 0.00045677228, -0.00785524, -0.020605665, 0.010149858, 0.00025910055, + -0.010379319, 0.01745439, -0.005109348, 0.007931728, -0.008933711, + 0.0061992914, -0.009622096, -0.0009847733, -0.0046504247, -0.03184929, + 0.009339093, 0.0102722375, -0.0044898014, 0.022257788, -0.005319688, + 0.0074651553, -0.00093457854, 0.009124928, 0.013836543, -0.0033195466, + 0.01457082, -0.008643059, 0.0071515576, 0.007185977, -0.004941076, + -0.023313314, -0.004359773, -0.00905609, -0.014387251, -0.021018695, + -0.005212606, -0.009124928, 0.025026627, 0.0047766287, 0.012039092, + 0.0016435197, -0.027091783, -0.013148158, 0.00352415, -0.017928611, + -0.0047804527, 0.012941642, 0.021186966, -0.013347024, 0.008184135, + -0.01163371, 0.022701414, -0.02681643, -0.0082912175, -0.004550991, + 0.0021665012, -0.0071515576, 0.011190084, -0.007040651, 0.00027678823, + 0.02117167, 0.018540509, 0.025133708, 0.019596033, -0.009254957, + -0.0016693341, 0.0032239375, 0.0032124643, 0.0009020715, -0.008138243, + 0.01768385, 0.00077013095, 0.00183665, -0.010509348, -0.0041417843, + 0.0027631018, 0.010861189, -0.0015641642, -0.008956657, 0.005541501, + 0.0050519826, 0.011832577, -0.005560623, 0.038733143, -0.01223796, + -0.0071745035, 0.0013672095, -0.0040194048, -0.018096883, + -0.029065154, -0.008413597, 0.01847932, 0.009652691, 0.005981303, + 0.009560905, 0.001066041, 0.027718978, -0.003472521, 0.0009173689, + -0.0015488667, 0.0007237606, 0.024797166, 0.010249291, 0.016154107, + 0.0015450424, -0.014264871, 0.0035929885, -0.01961133, -0.0070444755, + 0.002246813, -0.02716827, 0.010930028, 0.0215847, 0.0035872518, + -0.017179037, 0.021844758, -0.00095131015, -0.024996033, -0.044179033, + 0.0015393058, -0.023527477, 0.0036293198, 0.025424361, 0.009706232, + 0.008658356, -0.008428895, 0.0043483, 0.0167966, -0.0044477335, + 0.006428753, 0.028652122, 0.01427252, 0.014532577, 0.003851133, + 0.022380168, 0.01889235, -0.00035327548, 0.021615297, 0.0063943337, + -0.0058818692, 0.01145779, -0.011687252, 0.007862889, 0.01319405, + -0.0020422095, 0.0035509204, -0.012169121, 0.005182011, -0.0052737957, + -0.010348724, -0.016475353, -0.0185864, -0.015588101, 0.0120008495, + -0.010142209, -0.016291784, 0.00983626, -0.006788243, 0.005143767, + -0.0019179177, -0.0036159346, 0.019947873, -0.0048263455, + -0.017959205, -0.001720963, 0.013354673, -0.013385268, -0.011427195, + 0.00863541, -0.0102645885, 0.010524645, -0.021033993, -0.01073881 + ] + } + ] + } ] diff --git a/src/scrapers/website/playground/tsconfig.json b/src/scrapers/website/playground/tsconfig.json index e25e758d17..581f158fdc 100644 --- a/src/scrapers/website/playground/tsconfig.json +++ b/src/scrapers/website/playground/tsconfig.json @@ -1,19 +1,19 @@ { - "compilerOptions": { - "target": "esnext", - "module": "commonjs", - "downlevelIteration": true, - "allowJs": true, - "moduleResolution": "node", - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": true, - "noImplicitThis": true, - "noImplicitAny": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noUnusedLocals": false, - "resolveJsonModule": true, - "esModuleInterop": true - }, - "exclude": ["node_modules"] + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true, + "allowJs": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitAny": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "noUnusedLocals": false, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "exclude": ["node_modules"] } diff --git a/src/scrapers/website/playground/website-scraper-playground.ts b/src/scrapers/website/playground/website-scraper-playground.ts index 6983226ca5..01c59edd7f 100644 --- a/src/scrapers/website/playground/website-scraper-playground.ts +++ b/src/scrapers/website/playground/website-scraper-playground.ts @@ -1,79 +1,85 @@ #!/usr/bin/env ts-node -import * as dotenv from 'dotenv'; +import * as dotenv from "dotenv"; -dotenv.config({ path: '.env' }); +dotenv.config({ path: ".env" }); -import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser -import { writeFile } from 'fs/promises'; -import { join } from 'path'; -import { usageToHuman } from '../../../execution/utils/usageToHuman'; -import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground'; -import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson'; -import { titleToName } from '../../../utils/normalization/titleToName'; -import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode'; -import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler'; -import { WebsiteScraper } from '../WebsiteScraper'; +import { join } from "path"; +import colors from "colors"; // <- TODO: [🔶] Make system to put color and style to both node and browser +import { writeFile } from "fs/promises"; +import { usageToHuman } from "../../../execution/utils/usageToHuman"; +import { $provideLlmToolsForTestingAndScriptsAndPlayground } from "../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground"; +import { stringifyPipelineJson } from "../../../utils/editable/utils/stringifyPipelineJson"; +import { titleToName } from "../../../utils/normalization/titleToName"; +import { $provideFilesystemForNode } from "../../_common/register/$provideFilesystemForNode"; +import { makeKnowledgeSourceHandler } from "../../_common/utils/makeKnowledgeSourceHandler"; +import { WebsiteScraper } from "../WebsiteScraper"; playground() - .catch((error) => { - console.error(colors.bgRed(error.name || 'NamelessError')); - console.error(error); - process.exit(1); - }) - .then(() => { - process.exit(0); - }); + .catch((error) => { + console.error(colors.bgRed(error.name || "NamelessError")); + console.error(error); + process.exit(1); + }) + .then(() => { + process.exit(0); + }); async function playground() { - console.info(`🧸 Scrape knowledge from websites (playground)`); - - // Do here stuff you want to test - //========================================> - - //const example = 'https://www.pavolhejny.com/'; // <- TODO: Not scraping really important information, just one-two paragraph - const example = 'https://koralkykatlas.cz/cs/blog/prispevek/-rijna-zhorseni-kvality-kovove-bizuterie.html'; - // <- TODO: [👩🏿‍🤝‍👩🏼] Read here website-scraper-playground.ts and itterate - - const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true }); - const rootDirname = join(__dirname, 'examples'); - - const websiteScraper = new WebsiteScraper( - { llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), fs: $provideFilesystemForNode() }, - { - rootDirname, - }, - ); - - const source = await makeKnowledgeSourceHandler( - { knowledgeSourceContent: example }, - { fs: $provideFilesystemForNode() }, - { rootDirname }, - ); - - const converted = await websiteScraper.$convert(source); - - console.info(colors.bgGreen(' Converted: '), converted); - - const knowledge = await websiteScraper.scrape(source); - - console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); - console.info(colors.bgGreen(' Knowledge: ')); - console.info(knowledge); - - await writeFile( - join( - __dirname, - `../examples/${titleToName( - example, - )}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, - ), - stringifyPipelineJson(knowledge), - 'utf-8', - ); - /**/ - - //========================================/ + console.info(`🧸 Scrape knowledge from websites (playground)`); + + // Do here stuff you want to test + //========================================> + + //const example = 'https://www.pavolhejny.com/'; // <- TODO: Not scraping really important information, just one-two paragraph + const example = + "https://koralkykatlas.cz/cs/blog/prispevek/-rijna-zhorseni-kvality-kovove-bizuterie.html"; + // <- TODO: [👩🏿‍🤝‍👩🏼] Read here website-scraper-playground.ts and itterate + + const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ + isCacheReloaded: true, + }); + const rootDirname = join(__dirname, "examples"); + + const websiteScraper = new WebsiteScraper( + { + llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), + fs: $provideFilesystemForNode(), + }, + { + rootDirname, + }, + ); + + const source = await makeKnowledgeSourceHandler( + { knowledgeSourceContent: example }, + { fs: $provideFilesystemForNode() }, + { rootDirname }, + ); + + const converted = await websiteScraper.$convert(source); + + console.info(colors.bgGreen(" Converted: "), converted); + + const knowledge = await websiteScraper.scrape(source); + + console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage()))); + console.info(colors.bgGreen(" Knowledge: ")); + console.info(knowledge); + + await writeFile( + join( + __dirname, + `../examples/${titleToName( + example, + )}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */, + ), + stringifyPipelineJson(knowledge), + "utf-8", + ); + /**/ + + //========================================/ } /** diff --git a/src/scrapers/website/register-constructor.ts b/src/scrapers/website/register-constructor.ts index 605c918d74..ab1246965b 100644 --- a/src/scrapers/website/register-constructor.ts +++ b/src/scrapers/website/register-constructor.ts @@ -1,6 +1,6 @@ -import type { Registration } from '../../utils/$Register'; -import { $scrapersRegister } from '../_common/register/$scrapersRegister'; -import { createWebsiteScraper } from './createWebsiteScraper'; +import type { Registration } from "../../utils/$Register"; +import { $scrapersRegister } from "../_common/register/$scrapersRegister"; +import { createWebsiteScraper } from "./createWebsiteScraper"; /** * Registration of known scraper @@ -11,7 +11,8 @@ import { createWebsiteScraper } from './createWebsiteScraper'; * @public exported from `@promptbook/wizzard` * @public exported from `@promptbook/cli` */ -export const _WebsiteScraperRegistration: Registration = $scrapersRegister.register(createWebsiteScraper); +export const _WebsiteScraperRegistration: Registration = + $scrapersRegister.register(createWebsiteScraper); /** * TODO: [🎶] Naming "constructor" vs "creator" vs "factory" diff --git a/src/scrapers/website/register-metadata.ts b/src/scrapers/website/register-metadata.ts index c43952a728..a9b4323ac5 100644 --- a/src/scrapers/website/register-metadata.ts +++ b/src/scrapers/website/register-metadata.ts @@ -1,8 +1,8 @@ -import type { Registration } from '../../utils/$Register'; -import { keepTypeImported } from '../../utils/organization/keepTypeImported'; -import { $deepFreeze } from '../../utils/serialization/$deepFreeze'; -import { $scrapersMetadataRegister } from '../_common/register/$scrapersMetadataRegister'; -import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata'; +import type { Registration } from "../../utils/$Register"; +import { keepTypeImported } from "../../utils/organization/keepTypeImported"; +import { $deepFreeze } from "../../utils/serialization/$deepFreeze"; +import { $scrapersMetadataRegister } from "../_common/register/$scrapersMetadataRegister"; +import type { ScraperAndConverterMetadata } from "../_common/register/ScraperAndConverterMetadata"; keepTypeImported(); @@ -12,14 +12,14 @@ keepTypeImported(); * @private within the scraper directory */ export const websiteScraperMetadata = $deepFreeze({ - title: 'Website scraper', - packageName: '@promptbook/website-crawler', - className: 'WebsiteScraper', - mimeTypes: ['text/html'], - documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@', - isAvilableInBrowser: false, - // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server - requiredExecutables: [], + title: "Website scraper", + packageName: "@promptbook/website-crawler", + className: "WebsiteScraper", + mimeTypes: ["text/html"], + documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@", + isAvilableInBrowser: false, + // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server + requiredExecutables: [], }) satisfies ScraperAndConverterMetadata; /* <- Note: [🤛] */ /** @@ -32,7 +32,7 @@ export const websiteScraperMetadata = $deepFreeze({ * @public exported from `@promptbook/cli` */ export const _WebsiteScraperMetadataRegistration: Registration = - $scrapersMetadataRegister.register(websiteScraperMetadata); + $scrapersMetadataRegister.register(websiteScraperMetadata); /** * Note: [💞] Ignore a discrepancy between file name and entity name diff --git a/src/scrapers/website/utils/createShowdownConverter.test.ts b/src/scrapers/website/utils/createShowdownConverter.test.ts index 41dc7675ea..2a9ed796a2 100644 --- a/src/scrapers/website/utils/createShowdownConverter.test.ts +++ b/src/scrapers/website/utils/createShowdownConverter.test.ts @@ -1,68 +1,72 @@ -import { describe, expect, it } from '@jest/globals'; -import { JSDOM } from 'jsdom'; -import { spaceTrim } from 'spacetrim'; -import { createShowdownConverter } from './createShowdownConverter'; +import { describe, expect, it } from "@jest/globals"; +import { JSDOM } from "jsdom"; +import { spaceTrim } from "spacetrim"; +import { createShowdownConverter } from "./createShowdownConverter"; describe(`markdownConverter`, () => { - const showdownConverter = createShowdownConverter(); - const jsdom = new JSDOM(); - - it(`preserves supersimple markdown to html`, () => { - expect(showdownConverter.makeHtml('hello')).toEqual('

hello

'); - }); - - it(`preserves supersimple html to markdown`, () => { - expect(spaceTrim(showdownConverter.makeMarkdown('

hello

', jsdom.window.document))).toEqual('hello'); - }); - - it(`converts simple markdown to html`, () => { - expect( - spaceTrim( - showdownConverter.makeHtml( - spaceTrim(` + const showdownConverter = createShowdownConverter(); + const jsdom = new JSDOM(); + + it(`preserves supersimple markdown to html`, () => { + expect(showdownConverter.makeHtml("hello")).toEqual("

hello

"); + }); + + it(`preserves supersimple html to markdown`, () => { + expect( + spaceTrim( + showdownConverter.makeMarkdown("

hello

", jsdom.window.document), + ), + ).toEqual("hello"); + }); + + it(`converts simple markdown to html`, () => { + expect( + spaceTrim( + showdownConverter.makeHtml( + spaceTrim(` # Hello World! This is a **markdown** text. `), - ), - ), - ).toEqual( - spaceTrim(` + ), + ), + ).toEqual( + spaceTrim(`

Hello World!

This is a markdown text.

`), - ); - }); + ); + }); - it(`converts simple html to markdown`, () => { - expect( - spaceTrim( - showdownConverter.makeMarkdown( - spaceTrim(` + it(`converts simple html to markdown`, () => { + expect( + spaceTrim( + showdownConverter.makeMarkdown( + spaceTrim(`

Hello World!

This is a markdown text.

`), - jsdom.window.document, - ), - ), - ).toEqual( - spaceTrim(` + jsdom.window.document, + ), + ), + ).toEqual( + spaceTrim(` # Hello World! This is a **markdown** text. `), - ); - }); + ); + }); - /*/ + /*/ Note: [🧠][🎐] Code can not be converted back from html to markdown it(`preserves advanced markdown when converted to html and back `, () => { diff --git a/src/scrapers/website/utils/createShowdownConverter.ts b/src/scrapers/website/utils/createShowdownConverter.ts index 14bc6ebbe3..08b6175780 100644 --- a/src/scrapers/website/utils/createShowdownConverter.ts +++ b/src/scrapers/website/utils/createShowdownConverter.ts @@ -1,4 +1,4 @@ -import { Converter as ShowdownConverter } from 'showdown'; +import { Converter as ShowdownConverter } from "showdown"; /** * Create a new showdown converter instance @@ -6,9 +6,9 @@ import { Converter as ShowdownConverter } from 'showdown'; * @private utility of `WebsiteScraper` */ export function createShowdownConverter() { - return new ShowdownConverter({ - flavor: 'github', - /* + return new ShowdownConverter({ + flavor: "github", + /* > import showdownHighlight from 'showdown-highlight'; > extensions: [ > showdownHighlight({ @@ -19,5 +19,5 @@ export function createShowdownConverter() { > }), > ], */ - }); + }); } diff --git a/src/scripting/_test/custom-function-async.test.ts.test.ts b/src/scripting/_test/custom-function-async.test.ts.test.ts index 21ddd86928..efa8bb7ee1 100644 --- a/src/scripting/_test/custom-function-async.test.ts.test.ts +++ b/src/scripting/_test/custom-function-async.test.ts.test.ts @@ -1,60 +1,68 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { forTime } from 'waitasecond'; -import { compilePipeline } from '../../conversion/compilePipeline'; -import { CallbackInterfaceTools } from '../../dialogs/callback/CallbackInterfaceTools'; -import { createPipelineExecutor } from '../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import { MockedEchoLlmExecutionTools } from '../../llm-providers/mocked/MockedEchoLlmExecutionTools'; -import type { PipelineString } from '../../pipeline/PipelineString'; -import { JavascriptExecutionTools } from '../javascript/JavascriptExecutionTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { forTime } from "waitasecond"; +import { compilePipeline } from "../../conversion/compilePipeline"; +import { CallbackInterfaceTools } from "../../dialogs/callback/CallbackInterfaceTools"; +import { createPipelineExecutor } from "../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { MockedEchoLlmExecutionTools } from "../../llm-providers/mocked/MockedEchoLlmExecutionTools"; +import type { PipelineString } from "../../pipeline/PipelineString"; +import { JavascriptExecutionTools } from "../javascript/JavascriptExecutionTools"; -describe('createPipelineExecutor + custom async function ', () => { - it('should use custom postprocessing function', () => { - expect( - getPipelineExecutor().then((pipelineExecutor) => pipelineExecutor({ yourName: 'Matthew' }).asPromise()), - ).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'Hello Matthew the Evangelist', - }, - }); +describe("createPipelineExecutor + custom async function ", () => { + it("should use custom postprocessing function", () => { + expect( + getPipelineExecutor().then((pipelineExecutor) => + pipelineExecutor({ yourName: "Matthew" }).asPromise(), + ), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "Hello Matthew the Evangelist", + }, + }); - expect( - getPipelineExecutor().then((pipelineExecutor) => pipelineExecutor({ yourName: 'Mark' }).asPromise()), - ).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'Hello Mark the Evangelist', - }, - }); + expect( + getPipelineExecutor().then((pipelineExecutor) => + pipelineExecutor({ yourName: "Mark" }).asPromise(), + ), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "Hello Mark the Evangelist", + }, + }); - expect( - getPipelineExecutor().then((pipelineExecutor) => pipelineExecutor({ yourName: 'Luke' }).asPromise()), - ).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'Hello Luke the Evangelist', - }, - }); + expect( + getPipelineExecutor().then((pipelineExecutor) => + pipelineExecutor({ yourName: "Luke" }).asPromise(), + ), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "Hello Luke the Evangelist", + }, + }); - expect( - getPipelineExecutor().then((pipelineExecutor) => pipelineExecutor({ yourName: 'John' }).asPromise()), - ).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'Hello John the Evangelist', - }, - }); - }); + expect( + getPipelineExecutor().then((pipelineExecutor) => + pipelineExecutor({ yourName: "John" }).asPromise(), + ), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "Hello John the Evangelist", + }, + }); + }); }); async function getPipelineExecutor() { - const pipeline = await compilePipeline( - spaceTrim(` + const pipeline = await compilePipeline( + spaceTrim(` # Custom functions Show how to use custom postprocessing async function @@ -74,34 +82,34 @@ async function getPipelineExecutor() { -> {greeting} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - ); + // <- TODO: [📼] Use`book\`` string literal notation + ); - const pipelineExecutor = createPipelineExecutor({ - pipeline, - tools: { - llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), - script: [ - new JavascriptExecutionTools({ - isVerbose: true, + const pipelineExecutor = createPipelineExecutor({ + pipeline, + tools: { + llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), + script: [ + new JavascriptExecutionTools({ + isVerbose: true, - // Note: [🕎] - functions: { - async addHello(value) { - await forTime(1000); - return `Hello ${value}`; - }, - }, - }), - ], - userInterface: new CallbackInterfaceTools({ - isVerbose: true, - async callback() { - return 'Hello'; - }, - }), - }, - }); + // Note: [🕎] + functions: { + async addHello(value) { + await forTime(1000); + return `Hello ${value}`; + }, + }, + }), + ], + userInterface: new CallbackInterfaceTools({ + isVerbose: true, + async callback() { + return "Hello"; + }, + }), + }, + }); - return pipelineExecutor; + return pipelineExecutor; } diff --git a/src/scripting/_test/custom-function-missing.test.ts b/src/scripting/_test/custom-function-missing.test.ts index f92c6bf358..490d286fd0 100644 --- a/src/scripting/_test/custom-function-missing.test.ts +++ b/src/scripting/_test/custom-function-missing.test.ts @@ -1,16 +1,16 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { compilePipeline } from '../../conversion/compilePipeline'; -import { CallbackInterfaceTools } from '../../dialogs/callback/CallbackInterfaceTools'; -import { createPipelineExecutor } from '../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import { MockedEchoLlmExecutionTools } from '../../llm-providers/mocked/MockedEchoLlmExecutionTools'; -import type { PipelineString } from '../../pipeline/PipelineString'; -import { JavascriptExecutionTools } from '../javascript/JavascriptExecutionTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { compilePipeline } from "../../conversion/compilePipeline"; +import { CallbackInterfaceTools } from "../../dialogs/callback/CallbackInterfaceTools"; +import { createPipelineExecutor } from "../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { MockedEchoLlmExecutionTools } from "../../llm-providers/mocked/MockedEchoLlmExecutionTools"; +import type { PipelineString } from "../../pipeline/PipelineString"; +import { JavascriptExecutionTools } from "../javascript/JavascriptExecutionTools"; -describe('createPipelineExecutor + missing custom function', () => { - async function getPipelineExecutor() { - const pipeline = await compilePipeline( - spaceTrim(` +describe("createPipelineExecutor + missing custom function", () => { + async function getPipelineExecutor() { + const pipeline = await compilePipeline( + spaceTrim(` # Custom functions Show how to use custom postprocessing functions @@ -30,41 +30,41 @@ describe('createPipelineExecutor + missing custom function', () => { -> {greeting} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - ); + // <- TODO: [📼] Use`book\`` string literal notation + ); - const pipelineExecutor = createPipelineExecutor({ - pipeline, - tools: { - llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), - script: [ - new JavascriptExecutionTools({ - isVerbose: true, + const pipelineExecutor = createPipelineExecutor({ + pipeline, + tools: { + llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), + script: [ + new JavascriptExecutionTools({ + isVerbose: true, - // Note: [🕎] - functions: { - addHelloWithTypo(value) { - return `Hello ${value}`; - }, - }, - }), - ], - userInterface: new CallbackInterfaceTools({ - isVerbose: true, - async callback() { - return 'Hello'; - }, - }), - }, - }); + // Note: [🕎] + functions: { + addHelloWithTypo(value) { + return `Hello ${value}`; + }, + }, + }), + ], + userInterface: new CallbackInterfaceTools({ + isVerbose: true, + async callback() { + return "Hello"; + }, + }), + }, + }); - return pipelineExecutor; - } + return pipelineExecutor; + } - it('should throw error when custom postprocessing function does not exist', async () => { - const pipelineExecutor = await getPipelineExecutor(); - expect(() => pipelineExecutor({ yourName: 'Matthew' }).asPromise()).rejects.toThrowError( - /Function addHello\(\) is not defined/, - ); - }); + it("should throw error when custom postprocessing function does not exist", async () => { + const pipelineExecutor = await getPipelineExecutor(); + expect(() => + pipelineExecutor({ yourName: "Matthew" }).asPromise(), + ).rejects.toThrowError(/Function addHello\(\) is not defined/); + }); }); diff --git a/src/scripting/_test/custom-function-with-dependencies.test.ts b/src/scripting/_test/custom-function-with-dependencies.test.ts index f50e28fae1..66f2470f3e 100644 --- a/src/scripting/_test/custom-function-with-dependencies.test.ts +++ b/src/scripting/_test/custom-function-with-dependencies.test.ts @@ -1,55 +1,63 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { compilePipeline } from '../../conversion/compilePipeline'; -import { CallbackInterfaceTools } from '../../dialogs/callback/CallbackInterfaceTools'; -import { createPipelineExecutor } from '../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import { MockedEchoLlmExecutionTools } from '../../llm-providers/mocked/MockedEchoLlmExecutionTools'; -import type { PipelineString } from '../../pipeline/PipelineString'; -import { countCharacters } from '../../utils/expectation-counters/countCharacters'; -import { countWords } from '../../utils/expectation-counters/countWords'; -import { JavascriptExecutionTools } from '../javascript/JavascriptExecutionTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { compilePipeline } from "../../conversion/compilePipeline"; +import { CallbackInterfaceTools } from "../../dialogs/callback/CallbackInterfaceTools"; +import { createPipelineExecutor } from "../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { MockedEchoLlmExecutionTools } from "../../llm-providers/mocked/MockedEchoLlmExecutionTools"; +import type { PipelineString } from "../../pipeline/PipelineString"; +import { countCharacters } from "../../utils/expectation-counters/countCharacters"; +import { countWords } from "../../utils/expectation-counters/countWords"; +import { JavascriptExecutionTools } from "../javascript/JavascriptExecutionTools"; -describe('createPipelineExecutor + custom function with dependencies', () => { - it('should use custom postprocessing function', async () => { - const pipelineExecutor = await getPipelineExecutor(); +describe("createPipelineExecutor + custom function with dependencies", () => { + it("should use custom postprocessing function", async () => { + const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({ yourName: 'Matthew' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'Hello Matthew the Evangelist (28 characters, 4 words)', - }, - }); + expect( + pipelineExecutor({ yourName: "Matthew" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "Hello Matthew the Evangelist (28 characters, 4 words)", + }, + }); - expect(pipelineExecutor({ yourName: 'Mark' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'Hello Mark the Evangelist (25 characters, 4 words)', - }, - }); + expect( + pipelineExecutor({ yourName: "Mark" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "Hello Mark the Evangelist (25 characters, 4 words)", + }, + }); - expect(pipelineExecutor({ yourName: 'Luke' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'Hello Luke the Evangelist (25 characters, 4 words)', - }, - }); + expect( + pipelineExecutor({ yourName: "Luke" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "Hello Luke the Evangelist (25 characters, 4 words)", + }, + }); - expect(pipelineExecutor({ yourName: 'John' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'Hello John the Evangelist (25 characters, 4 words)', - }, - }); - }); + expect( + pipelineExecutor({ yourName: "John" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "Hello John the Evangelist (25 characters, 4 words)", + }, + }); + }); }); async function getPipelineExecutor() { - const pipeline = await compilePipeline( - spaceTrim(` + const pipeline = await compilePipeline( + spaceTrim(` # Custom functions Show how to use custom postprocessing functions with dependencies @@ -70,35 +78,38 @@ async function getPipelineExecutor() { -> {greeting} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - ); + // <- TODO: [📼] Use`book\`` string literal notation + ); - return createPipelineExecutor({ - pipeline, - tools: { - llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), - script: [ - new JavascriptExecutionTools({ - isVerbose: true, + return createPipelineExecutor({ + pipeline, + tools: { + llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), + script: [ + new JavascriptExecutionTools({ + isVerbose: true, - // Note: [🕎] - functions: { - addHello(value) { - return `Hello ${value}`; - }, - withStatistics(value) { - // Note: Testing custom function with dependencies - return value + ` (${countCharacters(value)} characters, ${countWords(value)} words)`; - }, - }, - }), - ], - userInterface: new CallbackInterfaceTools({ - isVerbose: true, - async callback() { - return 'Hello'; - }, - }), - }, - }); + // Note: [🕎] + functions: { + addHello(value) { + return `Hello ${value}`; + }, + withStatistics(value) { + // Note: Testing custom function with dependencies + return ( + value + + ` (${countCharacters(value)} characters, ${countWords(value)} words)` + ); + }, + }, + }), + ], + userInterface: new CallbackInterfaceTools({ + isVerbose: true, + async callback() { + return "Hello"; + }, + }), + }, + }); } diff --git a/src/scripting/_test/custom-function.test.ts b/src/scripting/_test/custom-function.test.ts index 4021e971f1..370fe228dd 100644 --- a/src/scripting/_test/custom-function.test.ts +++ b/src/scripting/_test/custom-function.test.ts @@ -1,53 +1,61 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { compilePipeline } from '../../conversion/compilePipeline'; -import { CallbackInterfaceTools } from '../../dialogs/callback/CallbackInterfaceTools'; -import { createPipelineExecutor } from '../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import { MockedEchoLlmExecutionTools } from '../../llm-providers/mocked/MockedEchoLlmExecutionTools'; -import type { PipelineString } from '../../pipeline/PipelineString'; -import { JavascriptExecutionTools } from '../javascript/JavascriptExecutionTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { compilePipeline } from "../../conversion/compilePipeline"; +import { CallbackInterfaceTools } from "../../dialogs/callback/CallbackInterfaceTools"; +import { createPipelineExecutor } from "../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { MockedEchoLlmExecutionTools } from "../../llm-providers/mocked/MockedEchoLlmExecutionTools"; +import type { PipelineString } from "../../pipeline/PipelineString"; +import { JavascriptExecutionTools } from "../javascript/JavascriptExecutionTools"; -describe('createPipelineExecutor + custom function without dependencies', () => { - it('should use custom postprocessing function', async () => { - const pipelineExecutor = await getPipelineExecutor(); +describe("createPipelineExecutor + custom function without dependencies", () => { + it("should use custom postprocessing function", async () => { + const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({ yourName: 'Matthew' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'Hello Matthew the Evangelist', - }, - }); + expect( + pipelineExecutor({ yourName: "Matthew" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "Hello Matthew the Evangelist", + }, + }); - expect(pipelineExecutor({ yourName: 'Mark' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'Hello Mark the Evangelist', - }, - }); + expect( + pipelineExecutor({ yourName: "Mark" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "Hello Mark the Evangelist", + }, + }); - expect(pipelineExecutor({ yourName: 'Luke' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'Hello Luke the Evangelist', - }, - }); + expect( + pipelineExecutor({ yourName: "Luke" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "Hello Luke the Evangelist", + }, + }); - expect(pipelineExecutor({ yourName: 'John' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'Hello John the Evangelist', - }, - }); - }); + expect( + pipelineExecutor({ yourName: "John" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "Hello John the Evangelist", + }, + }); + }); }); async function getPipelineExecutor() { - const pipeline = await compilePipeline( - spaceTrim(` + const pipeline = await compilePipeline( + spaceTrim(` # Custom functions Show how to use custom postprocessing functions @@ -67,31 +75,31 @@ async function getPipelineExecutor() { -> {greeting} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - ); + // <- TODO: [📼] Use`book\`` string literal notation + ); - return createPipelineExecutor({ - pipeline, - tools: { - llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), - script: [ - new JavascriptExecutionTools({ - isVerbose: true, + return createPipelineExecutor({ + pipeline, + tools: { + llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), + script: [ + new JavascriptExecutionTools({ + isVerbose: true, - // Note: [🕎] - functions: { - addHello(value) { - return `Hello ${value}`; - }, - }, - }), - ], - userInterface: new CallbackInterfaceTools({ - isVerbose: true, - async callback() { - return 'Hello'; - }, - }), - }, - }); + // Note: [🕎] + functions: { + addHello(value) { + return `Hello ${value}`; + }, + }, + }), + ], + userInterface: new CallbackInterfaceTools({ + isVerbose: true, + async callback() { + return "Hello"; + }, + }), + }, + }); } diff --git a/src/scripting/_test/postprocessing.test.ts b/src/scripting/_test/postprocessing.test.ts index 2190c1f75a..83e014d55a 100644 --- a/src/scripting/_test/postprocessing.test.ts +++ b/src/scripting/_test/postprocessing.test.ts @@ -1,53 +1,61 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { compilePipeline } from '../../conversion/compilePipeline'; -import { CallbackInterfaceTools } from '../../dialogs/callback/CallbackInterfaceTools'; -import { createPipelineExecutor } from '../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import { MockedEchoLlmExecutionTools } from '../../llm-providers/mocked/MockedEchoLlmExecutionTools'; -import type { PipelineString } from '../../pipeline/PipelineString'; -import { JavascriptExecutionTools } from '../javascript/JavascriptExecutionTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { compilePipeline } from "../../conversion/compilePipeline"; +import { CallbackInterfaceTools } from "../../dialogs/callback/CallbackInterfaceTools"; +import { createPipelineExecutor } from "../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { MockedEchoLlmExecutionTools } from "../../llm-providers/mocked/MockedEchoLlmExecutionTools"; +import type { PipelineString } from "../../pipeline/PipelineString"; +import { JavascriptExecutionTools } from "../javascript/JavascriptExecutionTools"; -describe('createPipelineExecutor + postprocessing', () => { - it('should work when every INPUT PARAMETER defined', async () => { - const pipelineExecutor = await getPipelineExecutor(); +describe("createPipelineExecutor + postprocessing", () => { + it("should work when every INPUT PARAMETER defined", async () => { + const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({ yourName: 'Paůl' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'LUA_P_OLLE_H_DIAS_UO_Y', - }, - }); + expect( + pipelineExecutor({ yourName: "Paůl" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "LUA_P_OLLE_H_DIAS_UO_Y", + }, + }); - expect(pipelineExecutor({ yourName: 'Adam' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'MAD_A_OLLE_H_DIAS_UO_Y', - }, - }); + expect( + pipelineExecutor({ yourName: "Adam" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "MAD_A_OLLE_H_DIAS_UO_Y", + }, + }); - expect(pipelineExecutor({ yourName: 'John' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'NHO_J_OLLE_H_DIAS_UO_Y', - }, - }); + expect( + pipelineExecutor({ yourName: "John" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "NHO_J_OLLE_H_DIAS_UO_Y", + }, + }); - expect(pipelineExecutor({ yourName: 'DAVID' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - greeting: 'DIVAD_OLLE_H_DIAS_UO_Y', - }, - }); - }); + expect( + pipelineExecutor({ yourName: "DAVID" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + greeting: "DIVAD_OLLE_H_DIAS_UO_Y", + }, + }); + }); }); async function getPipelineExecutor() { - const pipeline = await compilePipeline( - spaceTrim(` + const pipeline = await compilePipeline( + spaceTrim(` # Example prompt Show how to use postprocessing @@ -70,27 +78,27 @@ async function getPipelineExecutor() { -> {greeting} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - ); + // <- TODO: [📼] Use`book\`` string literal notation + ); - const pipelineExecutor = createPipelineExecutor({ - pipeline, - tools: { - llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), - script: [ - new JavascriptExecutionTools({ - isVerbose: true, - // Note: [🕎] Custom functions are tested elsewhere - }), - ], - userInterface: new CallbackInterfaceTools({ - isVerbose: true, - async callback() { - return 'Hello'; - }, - }), - }, - }); + const pipelineExecutor = createPipelineExecutor({ + pipeline, + tools: { + llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), + script: [ + new JavascriptExecutionTools({ + isVerbose: true, + // Note: [🕎] Custom functions are tested elsewhere + }), + ], + userInterface: new CallbackInterfaceTools({ + isVerbose: true, + async callback() { + return "Hello"; + }, + }), + }, + }); - return pipelineExecutor; + return pipelineExecutor; } diff --git a/src/scripting/_test/script-execution-errors.test.ts b/src/scripting/_test/script-execution-errors.test.ts index 57eb4f6f07..e2ac2cb0a3 100644 --- a/src/scripting/_test/script-execution-errors.test.ts +++ b/src/scripting/_test/script-execution-errors.test.ts @@ -1,59 +1,75 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { compilePipeline } from '../../conversion/compilePipeline'; -import { CallbackInterfaceTools } from '../../dialogs/callback/CallbackInterfaceTools'; -import { createPipelineExecutor } from '../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import { MockedEchoLlmExecutionTools } from '../../llm-providers/mocked/MockedEchoLlmExecutionTools'; -import type { PipelineString } from '../../pipeline/PipelineString'; -import { JavascriptExecutionTools } from '../javascript/JavascriptExecutionTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { compilePipeline } from "../../conversion/compilePipeline"; +import { CallbackInterfaceTools } from "../../dialogs/callback/CallbackInterfaceTools"; +import { createPipelineExecutor } from "../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { MockedEchoLlmExecutionTools } from "../../llm-providers/mocked/MockedEchoLlmExecutionTools"; +import type { PipelineString } from "../../pipeline/PipelineString"; +import { JavascriptExecutionTools } from "../javascript/JavascriptExecutionTools"; -describe('createPipelineExecutor + executing scripts in promptbook', () => { - it('should work when every INPUT PARAMETER is allowed', async () => { - const pipelineExecutor = await getPipelineExecutor(); +describe("createPipelineExecutor + executing scripts in promptbook", () => { + it("should work when every INPUT PARAMETER is allowed", async () => { + const pipelineExecutor = await getPipelineExecutor(); - expect(pipelineExecutor({ thing: 'a cup of coffee' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - bhing: 'b cup of coffee', - }, - }); - expect(pipelineExecutor({ thing: 'arrow' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - bhing: 'brrow', - }, - }); - expect(pipelineExecutor({ thing: 'aaa' }).asPromise()).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - bhing: 'bbb', - }, - }); - }); + expect( + pipelineExecutor({ thing: "a cup of coffee" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + bhing: "b cup of coffee", + }, + }); + expect( + pipelineExecutor({ thing: "arrow" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + bhing: "brrow", + }, + }); + expect( + pipelineExecutor({ thing: "aaa" }).asPromise(), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + bhing: "bbb", + }, + }); + }); - it('should fail when INPUT PARAMETER is NOT allowed', async () => { - const pipelineExecutor = await getPipelineExecutor(); + it("should fail when INPUT PARAMETER is NOT allowed", async () => { + const pipelineExecutor = await getPipelineExecutor(); - for (const thing of ['apple', 'apples', 'an apple', 'Apple', 'The Apple', '🍏 Apple', 'Apple 🍎']) { - expect(pipelineExecutor({ thing }).asPromise()).resolves.toMatchObject({ - isSuccessful: false, - errors: [/Error: I do not like Apples!/i], - warnings: [ - /PipelineExecutionError: Parameter `{bhing}` should be an output parameter, but it was not generated/i, - ], - }); + for (const thing of [ + "apple", + "apples", + "an apple", + "Apple", + "The Apple", + "🍏 Apple", + "Apple 🍎", + ]) { + expect(pipelineExecutor({ thing }).asPromise()).resolves.toMatchObject({ + isSuccessful: false, + errors: [/Error: I do not like Apples!/i], + warnings: [ + /PipelineExecutionError: Parameter `{bhing}` should be an output parameter, but it was not generated/i, + ], + }); - expect(() => pipelineExecutor({ thing }).asPromise()).rejects.toThrowError(/I do not like Apples!/i); - } - }); + expect(() => + pipelineExecutor({ thing }).asPromise(), + ).rejects.toThrowError(/I do not like Apples!/i); + } + }); }); async function getPipelineExecutor() { - const pipeline = await compilePipeline( - spaceTrim(` + const pipeline = await compilePipeline( + spaceTrim(` # Example prompt Show how to execute a script @@ -75,26 +91,26 @@ async function getPipelineExecutor() { -> {bhing} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - ); - const pipelineExecutor = createPipelineExecutor({ - pipeline, - tools: { - llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), - script: [ - new JavascriptExecutionTools({ - isVerbose: true, - // Note: [🕎] Custom functions are tested elsewhere - }), - ], - userInterface: new CallbackInterfaceTools({ - isVerbose: true, - async callback() { - return 'Hello'; - }, - }), - }, - }); + // <- TODO: [📼] Use`book\`` string literal notation + ); + const pipelineExecutor = createPipelineExecutor({ + pipeline, + tools: { + llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), + script: [ + new JavascriptExecutionTools({ + isVerbose: true, + // Note: [🕎] Custom functions are tested elsewhere + }), + ], + userInterface: new CallbackInterfaceTools({ + isVerbose: true, + async callback() { + return "Hello"; + }, + }), + }, + }); - return pipelineExecutor; + return pipelineExecutor; } diff --git a/src/scripting/_test/script-execution-tools.test.ts b/src/scripting/_test/script-execution-tools.test.ts index a8efd7242b..fef5bfc800 100644 --- a/src/scripting/_test/script-execution-tools.test.ts +++ b/src/scripting/_test/script-execution-tools.test.ts @@ -1,42 +1,46 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { compilePipeline } from '../../conversion/compilePipeline'; -import { CallbackInterfaceTools } from '../../dialogs/callback/CallbackInterfaceTools'; -import { createPipelineExecutor } from '../../execution/createPipelineExecutor/00-createPipelineExecutor'; -import { MockedEchoLlmExecutionTools } from '../../llm-providers/mocked/MockedEchoLlmExecutionTools'; -import type { PipelineString } from '../../pipeline/PipelineString'; -import { JavascriptExecutionTools } from '../javascript/JavascriptExecutionTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { compilePipeline } from "../../conversion/compilePipeline"; +import { CallbackInterfaceTools } from "../../dialogs/callback/CallbackInterfaceTools"; +import { createPipelineExecutor } from "../../execution/createPipelineExecutor/00-createPipelineExecutor"; +import { MockedEchoLlmExecutionTools } from "../../llm-providers/mocked/MockedEchoLlmExecutionTools"; +import type { PipelineString } from "../../pipeline/PipelineString"; +import { JavascriptExecutionTools } from "../javascript/JavascriptExecutionTools"; -describe('createPipelineExecutor + executing scripts in promptbook', () => { - it('should work when every INPUT PARAMETER defined', () => { - expect( - getPipelineExecutor().then((pipelineExecutor) => pipelineExecutor({ thing: 'apple' }).asPromise()), - ).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - bhing: 'bpple', - }, - }); - expect( - getPipelineExecutor().then((pipelineExecutor) => - pipelineExecutor({ thing: 'a cup of coffee' }).asPromise(), - ), - ).resolves.toMatchObject({ - isSuccessful: true, - errors: [], - outputParameters: { - bhing: 'b cup of coffee', - }, - }); - }); +describe("createPipelineExecutor + executing scripts in promptbook", () => { + it("should work when every INPUT PARAMETER defined", () => { + expect( + getPipelineExecutor().then((pipelineExecutor) => + pipelineExecutor({ thing: "apple" }).asPromise(), + ), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + bhing: "bpple", + }, + }); + expect( + getPipelineExecutor().then((pipelineExecutor) => + pipelineExecutor({ thing: "a cup of coffee" }).asPromise(), + ), + ).resolves.toMatchObject({ + isSuccessful: true, + errors: [], + outputParameters: { + bhing: "b cup of coffee", + }, + }); + }); - it('should fail when some INPUT PARAMETER is missing', () => { - expect( - getPipelineExecutor().then((pipelineExecutor) => pipelineExecutor({}).asPromise()), - ).resolves.toMatchObject({ - isSuccessful: false, - /* + it("should fail when some INPUT PARAMETER is missing", () => { + expect( + getPipelineExecutor().then((pipelineExecutor) => + pipelineExecutor({}).asPromise(), + ), + ).resolves.toMatchObject({ + isSuccessful: false, + /* TODO: errors: [ new PipelineExecutionError( @@ -56,17 +60,21 @@ describe('createPipelineExecutor + executing scripts in promptbook', () => { ), ], */ - }); + }); - expect(() => - getPipelineExecutor().then((pipelineExecutor) => pipelineExecutor({}).asPromise()), - ).rejects.toThrowError(/Parameter `\{thing\}` is required as an input parameter/); - }); + expect(() => + getPipelineExecutor().then((pipelineExecutor) => + pipelineExecutor({}).asPromise(), + ), + ).rejects.toThrowError( + /Parameter `\{thing\}` is required as an input parameter/, + ); + }); }); async function getPipelineExecutor() { - const pipeline = await compilePipeline( - spaceTrim(` + const pipeline = await compilePipeline( + spaceTrim(` # Example prompt Show how to execute a script @@ -85,26 +93,26 @@ async function getPipelineExecutor() { -> {bhing} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - ); - const pipelineExecutor = createPipelineExecutor({ - pipeline, - tools: { - llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), - script: [ - new JavascriptExecutionTools({ - isVerbose: true, - // Note: [🕎] Custom functions are tested elsewhere - }), - ], - userInterface: new CallbackInterfaceTools({ - isVerbose: true, - async callback() { - return 'Hello'; - }, - }), - }, - }); + // <- TODO: [📼] Use`book\`` string literal notation + ); + const pipelineExecutor = createPipelineExecutor({ + pipeline, + tools: { + llm: new MockedEchoLlmExecutionTools({ isVerbose: true }), + script: [ + new JavascriptExecutionTools({ + isVerbose: true, + // Note: [🕎] Custom functions are tested elsewhere + }), + ], + userInterface: new CallbackInterfaceTools({ + isVerbose: true, + async callback() { + return "Hello"; + }, + }), + }, + }); - return pipelineExecutor; + return pipelineExecutor; } diff --git a/src/scripting/javascript/JavascriptEvalExecutionTools.test.ts b/src/scripting/javascript/JavascriptEvalExecutionTools.test.ts index 035de70d1d..88ce1e5049 100644 --- a/src/scripting/javascript/JavascriptEvalExecutionTools.test.ts +++ b/src/scripting/javascript/JavascriptEvalExecutionTools.test.ts @@ -1,80 +1,80 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { JavascriptEvalExecutionTools } from './JavascriptEvalExecutionTools'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { JavascriptEvalExecutionTools } from "./JavascriptEvalExecutionTools"; -describe('JavascriptEvalExecutionTools', () => { - const javascriptEvalExecutionTools = new JavascriptEvalExecutionTools( - // <- TODO: [🧱] Implement in a functional (not new Class) way - { - isVerbose: true, - // Note: [🕎] Custom functions are tested elsewhere - }, - ); +describe("JavascriptEvalExecutionTools", () => { + const javascriptEvalExecutionTools = new JavascriptEvalExecutionTools( + // <- TODO: [🧱] Implement in a functional (not new Class) way + { + isVerbose: true, + // Note: [🕎] Custom functions are tested elsewhere + }, + ); - it('should evaluate supersimple statement', () => { - expect( - javascriptEvalExecutionTools.execute({ - scriptLanguage: 'javascript', - parameters: { - animal: 'cat', - }, - script: 'animal', - }), - ).resolves.toEqual('cat'); - expect( - javascriptEvalExecutionTools.execute({ - scriptLanguage: 'javascript', - parameters: { - animal: 'cat', - }, - script: 'return animal', - }), - ).resolves.toEqual('cat'); - }); + it("should evaluate supersimple statement", () => { + expect( + javascriptEvalExecutionTools.execute({ + scriptLanguage: "javascript", + parameters: { + animal: "cat", + }, + script: "animal", + }), + ).resolves.toEqual("cat"); + expect( + javascriptEvalExecutionTools.execute({ + scriptLanguage: "javascript", + parameters: { + animal: "cat", + }, + script: "return animal", + }), + ).resolves.toEqual("cat"); + }); - it('should evaluate single statement', () => { - expect( - javascriptEvalExecutionTools.execute({ - scriptLanguage: 'javascript', - parameters: { - animal: 'cat', - }, - script: "return animal.split('').reverse().join('-')", - }), - ).resolves.toEqual('t-a-c'); - }); + it("should evaluate single statement", () => { + expect( + javascriptEvalExecutionTools.execute({ + scriptLanguage: "javascript", + parameters: { + animal: "cat", + }, + script: "return animal.split('').reverse().join('-')", + }), + ).resolves.toEqual("t-a-c"); + }); - it('should evaluate build-in function', () => { - expect( - javascriptEvalExecutionTools.execute({ - scriptLanguage: 'javascript', - parameters: { - animal: '"cat"', - }, - script: 'return removeQuotes(animal)', - }), - ).resolves.toEqual('cat'); + it("should evaluate build-in function", () => { + expect( + javascriptEvalExecutionTools.execute({ + scriptLanguage: "javascript", + parameters: { + animal: '"cat"', + }, + script: "return removeQuotes(animal)", + }), + ).resolves.toEqual("cat"); - expect( - javascriptEvalExecutionTools.execute({ - scriptLanguage: 'javascript', - parameters: { - animal: 'The animal is: "dog"', - }, - script: 'return unwrapResult(animal)', - }), - ).resolves.toEqual('dog'); - }); + expect( + javascriptEvalExecutionTools.execute({ + scriptLanguage: "javascript", + parameters: { + animal: 'The animal is: "dog"', + }, + script: "return unwrapResult(animal)", + }), + ).resolves.toEqual("dog"); + }); - it('should evaluate multiple statements', () => { - expect( - javascriptEvalExecutionTools.execute({ - scriptLanguage: 'javascript', - parameters: { - animal: 'cat', - sound: 'meow', - }, - script: spaceTrim(` + it("should evaluate multiple statements", () => { + expect( + javascriptEvalExecutionTools.execute({ + scriptLanguage: "javascript", + parameters: { + animal: "cat", + sound: "meow", + }, + script: spaceTrim(` const sentence1 = animal + ' makes ' + sound + '.'; const sentence2 = \`Two \${animal}s makes \${sound} \${sound}.\`; const sentence3 = \`Three \${animal}s makes \${sound} \${sound} \${sound}.\`; @@ -84,58 +84,58 @@ describe('JavascriptEvalExecutionTools', () => { \${sentence3} \`); `), - }), - ).resolves.toEqual( - spaceTrim(` + }), + ).resolves.toEqual( + spaceTrim(` cat makes meow. Two cats makes meow meow. Three cats makes meow meow meow. `), - ); - }); + ); + }); - it('should throw error from script', () => { - () => - expect( - javascriptEvalExecutionTools.execute({ - scriptLanguage: 'javascript', - parameters: {}, - script: "throw new Error('Some error')", - }), - ).rejects.toThrowError('Some error'); - }); + it("should throw error from script", () => { + () => + expect( + javascriptEvalExecutionTools.execute({ + scriptLanguage: "javascript", + parameters: {}, + script: "throw new Error('Some error')", + }), + ).rejects.toThrowError("Some error"); + }); - it('should evaluate custom function', () => { - expect( - javascriptEvalExecutionTools.execute({ - scriptLanguage: 'javascript', - parameters: { - animal: 'cat', - sound: 'meow', - }, - script: spaceTrim(` + it("should evaluate custom function", () => { + expect( + javascriptEvalExecutionTools.execute({ + scriptLanguage: "javascript", + parameters: { + animal: "cat", + sound: "meow", + }, + script: spaceTrim(` function makeSentence(animal, sound) { return animal + ' makes ' + sound + '.'; } return makeSentence(animal, sound); `), - }), - ).resolves.toEqual('cat makes meow.'); - }); + }), + ).resolves.toEqual("cat makes meow."); + }); - it('should fail on python script', () => { - expect( - javascriptEvalExecutionTools.execute({ - scriptLanguage: 'python', - parameters: { - animal: 'cat', - }, - script: spaceTrim(` + it("should fail on python script", () => { + expect( + javascriptEvalExecutionTools.execute({ + scriptLanguage: "python", + parameters: { + animal: "cat", + }, + script: spaceTrim(` print(animal); `), - }), - ).rejects.toThrowError(/not supported/i); - }); + }), + ).rejects.toThrowError(/not supported/i); + }); }); /** diff --git a/src/scripting/javascript/JavascriptEvalExecutionTools.ts b/src/scripting/javascript/JavascriptEvalExecutionTools.ts index 894c5c35fc..70dcf30168 100644 --- a/src/scripting/javascript/JavascriptEvalExecutionTools.ts +++ b/src/scripting/javascript/JavascriptEvalExecutionTools.ts @@ -1,30 +1,33 @@ // Note: [💎] -import _spaceTrim from 'spacetrim'; -import { valueToString } from '../../_packages/utils.index'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import type { ScriptExecutionTools, ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptExecutionTools'; -import { extractBlock } from '../../postprocessing/utils/extractBlock'; -import { prettifyMarkdown as _prettifyMarkdown } from '../../utils/markdown/prettifyMarkdown'; -import { capitalize as _capitalize } from '../../utils/normalization/capitalize'; -import { decapitalize as _decapitalize } from '../../utils/normalization/decapitalize'; -import { nameToUriPart as _nameToUriPart } from '../../utils/normalization/nameToUriPart'; -import { nameToUriParts as _nameToUriParts } from '../../utils/normalization/nameToUriParts'; -import { normalizeToKebabCase as _normalizeToKebabCase } from '../../utils/normalization/normalize-to-kebab-case'; -import { normalizeTo_PascalCase as _normalizeTo_PascalCase } from '../../utils/normalization/normalizeTo_PascalCase'; -import { normalizeTo_SCREAMING_CASE as _normalizeTo_SCREAMING_CASE } from '../../utils/normalization/normalizeTo_SCREAMING_CASE'; -import { normalizeTo_camelCase as _normalizeTo_camelCase } from '../../utils/normalization/normalizeTo_camelCase'; -import { normalizeTo_snake_case as _normalizeTo_snake_case } from '../../utils/normalization/normalizeTo_snake_case'; -import { normalizeWhitespaces as _normalizeWhitespaces } from '../../utils/normalization/normalizeWhitespaces'; -import { parseKeywordsFromString } from '../../utils/normalization/parseKeywordsFromString'; -import { removeDiacritics as _removeDiacritics } from '../../utils/normalization/removeDiacritics'; -import { TODO_any } from '../../utils/organization/TODO_any'; -import { removeEmojis as _removeEmojis } from '../../utils/removeEmojis'; -import { removeQuotes as _removeQuotes } from '../../utils/removeQuotes'; -import { trimCodeBlock as _trimCodeBlock } from '../../utils/trimCodeBlock'; -import { trimEndOfCodeBlock as _trimEndOfCodeBlock } from '../../utils/trimEndOfCodeBlock'; -import { unwrapResult as _unwrapResult } from '../../utils/unwrapResult'; -import type { JavascriptExecutionToolsOptions } from './JavascriptExecutionToolsOptions'; -import { preserve } from './utils/preserve'; +import _spaceTrim from "spacetrim"; +import { valueToString } from "../../_packages/utils.index"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import type { + ScriptExecutionTools, + ScriptExecutionToolsExecuteOptions, +} from "../../execution/ScriptExecutionTools"; +import { extractBlock } from "../../postprocessing/utils/extractBlock"; +import { prettifyMarkdown as _prettifyMarkdown } from "../../utils/markdown/prettifyMarkdown"; +import { capitalize as _capitalize } from "../../utils/normalization/capitalize"; +import { decapitalize as _decapitalize } from "../../utils/normalization/decapitalize"; +import { nameToUriPart as _nameToUriPart } from "../../utils/normalization/nameToUriPart"; +import { nameToUriParts as _nameToUriParts } from "../../utils/normalization/nameToUriParts"; +import { normalizeToKebabCase as _normalizeToKebabCase } from "../../utils/normalization/normalize-to-kebab-case"; +import { normalizeTo_PascalCase as _normalizeTo_PascalCase } from "../../utils/normalization/normalizeTo_PascalCase"; +import { normalizeTo_SCREAMING_CASE as _normalizeTo_SCREAMING_CASE } from "../../utils/normalization/normalizeTo_SCREAMING_CASE"; +import { normalizeTo_camelCase as _normalizeTo_camelCase } from "../../utils/normalization/normalizeTo_camelCase"; +import { normalizeTo_snake_case as _normalizeTo_snake_case } from "../../utils/normalization/normalizeTo_snake_case"; +import { normalizeWhitespaces as _normalizeWhitespaces } from "../../utils/normalization/normalizeWhitespaces"; +import { parseKeywordsFromString } from "../../utils/normalization/parseKeywordsFromString"; +import { removeDiacritics as _removeDiacritics } from "../../utils/normalization/removeDiacritics"; +import type { TODO_any } from "../../utils/organization/TODO_any"; +import { removeEmojis as _removeEmojis } from "../../utils/removeEmojis"; +import { removeQuotes as _removeQuotes } from "../../utils/removeQuotes"; +import { trimCodeBlock as _trimCodeBlock } from "../../utils/trimCodeBlock"; +import { trimEndOfCodeBlock as _trimEndOfCodeBlock } from "../../utils/trimEndOfCodeBlock"; +import { unwrapResult as _unwrapResult } from "../../utils/unwrapResult"; +import type { JavascriptExecutionToolsOptions } from "./JavascriptExecutionToolsOptions"; +import { preserve } from "./utils/preserve"; /** * ScriptExecutionTools for JavaScript implemented via eval @@ -35,200 +38,207 @@ import { preserve } from './utils/preserve'; * @public exported from `@promptbook/execute-javascript` */ export class JavascriptEvalExecutionTools implements ScriptExecutionTools { - protected readonly options: JavascriptExecutionToolsOptions; - - public constructor(options?: JavascriptExecutionToolsOptions) { - this.options = options || {}; - } - - /** - * Executes a JavaScript - */ - public async execute(options: ScriptExecutionToolsExecuteOptions): Promise { - const { scriptLanguage, parameters } = options; - let { script } = options; - - if (scriptLanguage !== 'javascript') { - throw new PipelineExecutionError( - `Script language ${scriptLanguage} not supported to be executed by JavascriptEvalExecutionTools`, - ); - } - - // Note: [💎] - // Note: Using direct eval, following variables are in same scope as eval call so they are accessible from inside the evaluated script: - - const spaceTrim = (_: TODO_any) => _spaceTrim(_); - preserve(spaceTrim); - - const removeQuotes = _removeQuotes; - preserve(removeQuotes); - - const unwrapResult = _unwrapResult; - preserve(unwrapResult); - - const trimEndOfCodeBlock = _trimEndOfCodeBlock; - preserve(trimEndOfCodeBlock); - - const trimCodeBlock = _trimCodeBlock; - preserve(trimCodeBlock); - - // TODO: DRY [🍯] - const trim = (str: string) => str.trim(); - preserve(trim); - - // TODO: DRY [🍯] - const reverse = (str: string) => str.split('').reverse().join(''); - preserve(reverse); - - const removeEmojis = _removeEmojis; - preserve(removeEmojis); - - const prettifyMarkdown = _prettifyMarkdown; - preserve(prettifyMarkdown); - - //-------[n12:]--- - const capitalize = _capitalize; - const decapitalize = _decapitalize; - const nameToUriPart = _nameToUriPart; - const nameToUriParts = _nameToUriParts; - const removeDiacritics = _removeDiacritics; - const normalizeWhitespaces = _normalizeWhitespaces; - const normalizeToKebabCase = _normalizeToKebabCase; - const normalizeTo_camelCase = _normalizeTo_camelCase; - const normalizeTo_snake_case = _normalizeTo_snake_case; - const normalizeTo_PascalCase = _normalizeTo_PascalCase; - const parseKeywords = (input: string) => - // TODO: DRY [🍯] - Array.from(parseKeywordsFromString(input)).join( - ', ', - ); /* <- TODO: [🧠] What is the best format comma list, bullet list,...? */ - const normalizeTo_SCREAMING_CASE = _normalizeTo_SCREAMING_CASE; - preserve(capitalize); - preserve(decapitalize); - preserve(nameToUriPart); - preserve(nameToUriParts); - preserve(removeDiacritics); - preserve(normalizeWhitespaces); - preserve(normalizeToKebabCase); - preserve(normalizeTo_camelCase); - preserve(normalizeTo_snake_case); - preserve(normalizeTo_PascalCase); - preserve(parseKeywords); - preserve(normalizeTo_SCREAMING_CASE); - //-------[/n12]--- - - if (!script.includes('return')) { - script = `return ${script}`; - } - - // TODO: DRY [🍯] - const buildinFunctions = { - // TODO: [🍯] DRY all these functions across the file - spaceTrim, - removeQuotes, - unwrapResult, - trimEndOfCodeBlock, - trimCodeBlock, - trim, - reverse, - removeEmojis, - prettifyMarkdown, - capitalize, - decapitalize, - nameToUriPart, - nameToUriParts, - removeDiacritics, - normalizeWhitespaces, - normalizeToKebabCase, - normalizeTo_camelCase, - normalizeTo_snake_case, - normalizeTo_PascalCase, - parseKeywords, - normalizeTo_SCREAMING_CASE, - extractBlock, // <- [🍓] Remove balast in all other functions, use this one as example - }; - const buildinFunctionsStatement = Object.keys(buildinFunctions) - .map( - (functionName) => - // Note: Custom functions are exposed to the current scope as variables - `const ${functionName} = buildinFunctions.${functionName};`, - ) - .join('\n'); - - // TODO: DRY [🍯] - const customFunctions = this.options.functions || {}; - const customFunctionsStatement = Object.keys(customFunctions) - .map( - (functionName) => - // Note: Custom functions are exposed to the current scope as variables - `const ${functionName} = customFunctions.${functionName};`, - ) - .join('\n'); - - const statementToEvaluate = _spaceTrim( - (block) => ` + protected readonly options: JavascriptExecutionToolsOptions; + + public constructor(options?: JavascriptExecutionToolsOptions) { + this.options = options || {}; + } + + /** + * Executes a JavaScript + */ + public async execute( + options: ScriptExecutionToolsExecuteOptions, + ): Promise { + const { scriptLanguage, parameters } = options; + let { script } = options; + + if (scriptLanguage !== "javascript") { + throw new PipelineExecutionError( + `Script language ${scriptLanguage} not supported to be executed by JavascriptEvalExecutionTools`, + ); + } + + // Note: [💎] + // Note: Using direct eval, following variables are in same scope as eval call so they are accessible from inside the evaluated script: + + const spaceTrim = (_: TODO_any) => _spaceTrim(_); + preserve(spaceTrim); + + const removeQuotes = _removeQuotes; + preserve(removeQuotes); + + const unwrapResult = _unwrapResult; + preserve(unwrapResult); + + const trimEndOfCodeBlock = _trimEndOfCodeBlock; + preserve(trimEndOfCodeBlock); + + const trimCodeBlock = _trimCodeBlock; + preserve(trimCodeBlock); + + // TODO: DRY [🍯] + const trim = (str: string) => str.trim(); + preserve(trim); + + // TODO: DRY [🍯] + const reverse = (str: string) => str.split("").reverse().join(""); + preserve(reverse); + + const removeEmojis = _removeEmojis; + preserve(removeEmojis); + + const prettifyMarkdown = _prettifyMarkdown; + preserve(prettifyMarkdown); + + //-------[n12:]--- + const capitalize = _capitalize; + const decapitalize = _decapitalize; + const nameToUriPart = _nameToUriPart; + const nameToUriParts = _nameToUriParts; + const removeDiacritics = _removeDiacritics; + const normalizeWhitespaces = _normalizeWhitespaces; + const normalizeToKebabCase = _normalizeToKebabCase; + const normalizeTo_camelCase = _normalizeTo_camelCase; + const normalizeTo_snake_case = _normalizeTo_snake_case; + const normalizeTo_PascalCase = _normalizeTo_PascalCase; + const parseKeywords = (input: string) => + // TODO: DRY [🍯] + Array.from(parseKeywordsFromString(input)).join( + ", ", + ); /* <- TODO: [🧠] What is the best format comma list, bullet list,...? */ + const normalizeTo_SCREAMING_CASE = _normalizeTo_SCREAMING_CASE; + preserve(capitalize); + preserve(decapitalize); + preserve(nameToUriPart); + preserve(nameToUriParts); + preserve(removeDiacritics); + preserve(normalizeWhitespaces); + preserve(normalizeToKebabCase); + preserve(normalizeTo_camelCase); + preserve(normalizeTo_snake_case); + preserve(normalizeTo_PascalCase); + preserve(parseKeywords); + preserve(normalizeTo_SCREAMING_CASE); + //-------[/n12]--- + + if (!script.includes("return")) { + script = `return ${script}`; + } + + // TODO: DRY [🍯] + const buildinFunctions = { + // TODO: [🍯] DRY all these functions across the file + spaceTrim, + removeQuotes, + unwrapResult, + trimEndOfCodeBlock, + trimCodeBlock, + trim, + reverse, + removeEmojis, + prettifyMarkdown, + capitalize, + decapitalize, + nameToUriPart, + nameToUriParts, + removeDiacritics, + normalizeWhitespaces, + normalizeToKebabCase, + normalizeTo_camelCase, + normalizeTo_snake_case, + normalizeTo_PascalCase, + parseKeywords, + normalizeTo_SCREAMING_CASE, + extractBlock, // <- [🍓] Remove balast in all other functions, use this one as example + }; + const buildinFunctionsStatement = Object.keys(buildinFunctions) + .map( + (functionName) => + // Note: Custom functions are exposed to the current scope as variables + `const ${functionName} = buildinFunctions.${functionName};`, + ) + .join("\n"); + + // TODO: DRY [🍯] + const customFunctions = this.options.functions || {}; + const customFunctionsStatement = Object.keys(customFunctions) + .map( + (functionName) => + // Note: Custom functions are exposed to the current scope as variables + `const ${functionName} = customFunctions.${functionName};`, + ) + .join("\n"); + + const statementToEvaluate = _spaceTrim( + (block) => ` // Build-in functions: ${block(buildinFunctionsStatement)} // Custom functions: - ${block(customFunctionsStatement || '// -- No custom functions --')} + ${block(customFunctionsStatement || "// -- No custom functions --")} // The script: ${block( - Object.entries(parameters) - .map(([key, value]) => `const ${key} = ${JSON.stringify(value)};`) - .join('\n'), - )} + Object.entries(parameters) + .map( + ([key, value]) => + `const ${key} = ${JSON.stringify(value)};`, + ) + .join("\n"), + )} (()=>{ ${script} })() `, - ); + ); - if (this.options.isVerbose) { - console.info( - _spaceTrim( - (block) => ` + if (this.options.isVerbose) { + console.info( + _spaceTrim( + (block) => ` 🚀 Evaluating ${scriptLanguage} script: ${block(statementToEvaluate)}`, - ), - ); - } - - let result: TODO_any; - try { - result = await eval(statementToEvaluate); - - if (typeof result !== 'string') { - throw new PipelineExecutionError(`Script must return a string, but returned ${valueToString(result)}`); - } - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - if (error instanceof ReferenceError) { - const undefinedName = error.message.split(' ')[0]; - /* + ), + ); + } + + let result: TODO_any; + try { + result = await eval(statementToEvaluate); + + if (typeof result !== "string") { + throw new PipelineExecutionError( + `Script must return a string, but returned ${valueToString(result)}`, + ); + } + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + if (error instanceof ReferenceError) { + const undefinedName = error.message.split(" ")[0]; + /* Note: Remapping error From: [PipelineUrlError: thing is not defined], To: [PipelineExecutionError: Parameter `{thing}` is not defined], */ - if (!statementToEvaluate.includes(undefinedName + '(')) { - throw new PipelineExecutionError( - _spaceTrim( - (block) => ` + if (!statementToEvaluate.includes(undefinedName + "(")) { + throw new PipelineExecutionError( + _spaceTrim( + (block) => ` Parameter \`{${undefinedName}}\` is not defined This happen during evaluation of the javascript, which has access to the following parameters as javascript variables: ${block( - Object.keys(parameters) - .map((key) => ` - ${key}\n`) - .join(''), - )} + Object.keys(parameters) + .map((key) => ` - ${key}\n`) + .join(""), + )} The script is: \`\`\`javascript @@ -240,12 +250,12 @@ export class JavascriptEvalExecutionTools implements ScriptExecutionTools { `, - ), - ); - } else { - throw new PipelineExecutionError( - _spaceTrim( - (block) => ` + ), + ); + } else { + throw new PipelineExecutionError( + _spaceTrim( + (block) => ` Function ${undefinedName}() is not defined - Make sure that the function is one of built-in functions @@ -255,20 +265,22 @@ export class JavascriptEvalExecutionTools implements ScriptExecutionTools { ${block((error as Error).message)} `, - ), - ); - } - } - - throw error; - } - - if (typeof result !== 'string') { - throw new PipelineExecutionError(`Script must return a string, but ${valueToString(result)}`); - } - - return result; - } + ), + ); + } + } + + throw error; + } + + if (typeof result !== "string") { + throw new PipelineExecutionError( + `Script must return a string, but ${valueToString(result)}`, + ); + } + + return result; + } } /** diff --git a/src/scripting/javascript/JavascriptExecutionTools.ts b/src/scripting/javascript/JavascriptExecutionTools.ts index 5e6655106c..5edb1fc992 100644 --- a/src/scripting/javascript/JavascriptExecutionTools.ts +++ b/src/scripting/javascript/JavascriptExecutionTools.ts @@ -1,4 +1,4 @@ -import { JavascriptEvalExecutionTools } from './JavascriptEvalExecutionTools'; +import { JavascriptEvalExecutionTools } from "./JavascriptEvalExecutionTools"; /** * Placeholder for better implementation of JavascriptExecutionTools - some propper sandboxing diff --git a/src/scripting/javascript/JavascriptExecutionToolsOptions.ts b/src/scripting/javascript/JavascriptExecutionToolsOptions.ts index 1b29ff0933..334f6d2329 100644 --- a/src/scripting/javascript/JavascriptExecutionToolsOptions.ts +++ b/src/scripting/javascript/JavascriptExecutionToolsOptions.ts @@ -1,32 +1,35 @@ -import type { Promisable } from 'type-fest'; -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; -import type { string_postprocessing_function_name } from '../../types/typeAliases'; +import type { Promisable } from "type-fest"; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; +import type { string_postprocessing_function_name } from "../../types/typeAliases"; /** * Options for `JavascriptExecutionTools` */ export type JavascriptExecutionToolsOptions = CommonToolsOptions & { - /** - * Functions to be executed in the JavaScript evaluation. - * - * This can be used in two ways: - * 1. To provide custom postprocessing functions. For this case function must receive one string and return a (promise of) string. - * 2. As environment for the ECECUTE SCRIPT, For this case function can be any function. [0] - * - * Note: There are also some built-in functions available: - * @see ./JavascriptEvalExecutionTools.ts - */ - functions?: Record; + /** + * Functions to be executed in the JavaScript evaluation. + * + * This can be used in two ways: + * 1. To provide custom postprocessing functions. For this case function must receive one string and return a (promise of) string. + * 2. As environment for the ECECUTE SCRIPT, For this case function can be any function. [0] + * + * Note: There are also some built-in functions available: + * @see ./JavascriptEvalExecutionTools.ts + */ + functions?: Record< + string_postprocessing_function_name, + PostprocessingFunction + >; }; /** * Function that can be used to postprocess the output of the LLM */ export type PostprocessingFunction = - | ((value: string) => Promisable) - // Note: [0] - // eslint-disable-next-line @typescript-eslint/ban-types - | Function; + | ((value: string) => Promisable) + // Note: [0] + // eslint-disable-next-line @typescript-eslint/ban-types + | Function; /** * TODO: [🧠][💙] Distinct between options passed into ExecutionTools and to ExecutionTools.execute diff --git a/src/scripting/javascript/postprocessing-functions.ts b/src/scripting/javascript/postprocessing-functions.ts index b1beb736dc..186c158893 100644 --- a/src/scripting/javascript/postprocessing-functions.ts +++ b/src/scripting/javascript/postprocessing-functions.ts @@ -1,35 +1,35 @@ -import spaceTrim from 'spacetrim'; -import { extractBlock } from '../../postprocessing/utils/extractBlock'; -import { prettifyMarkdown } from '../../utils/markdown/prettifyMarkdown'; -import { capitalize } from '../../utils/normalization/capitalize'; -import { decapitalize } from '../../utils/normalization/decapitalize'; -import { nameToUriPart } from '../../utils/normalization/nameToUriPart'; -import { nameToUriParts } from '../../utils/normalization/nameToUriParts'; -import { normalizeToKebabCase } from '../../utils/normalization/normalize-to-kebab-case'; -import { normalizeTo_PascalCase } from '../../utils/normalization/normalizeTo_PascalCase'; -import { normalizeTo_SCREAMING_CASE } from '../../utils/normalization/normalizeTo_SCREAMING_CASE'; -import { normalizeTo_camelCase } from '../../utils/normalization/normalizeTo_camelCase'; -import { normalizeTo_snake_case } from '../../utils/normalization/normalizeTo_snake_case'; -import { normalizeWhitespaces } from '../../utils/normalization/normalizeWhitespaces'; -import { parseKeywordsFromString } from '../../utils/normalization/parseKeywordsFromString'; -import { removeDiacritics } from '../../utils/normalization/removeDiacritics'; -import { removeEmojis } from '../../utils/removeEmojis'; -import { removeQuotes } from '../../utils/removeQuotes'; -import { trimCodeBlock } from '../../utils/trimCodeBlock'; -import { trimEndOfCodeBlock } from '../../utils/trimEndOfCodeBlock'; -import { unwrapResult } from '../../utils/unwrapResult'; +import spaceTrim from "spacetrim"; +import { extractBlock } from "../../postprocessing/utils/extractBlock"; +import { prettifyMarkdown } from "../../utils/markdown/prettifyMarkdown"; +import { capitalize } from "../../utils/normalization/capitalize"; +import { decapitalize } from "../../utils/normalization/decapitalize"; +import { nameToUriPart } from "../../utils/normalization/nameToUriPart"; +import { nameToUriParts } from "../../utils/normalization/nameToUriParts"; +import { normalizeToKebabCase } from "../../utils/normalization/normalize-to-kebab-case"; +import { normalizeTo_PascalCase } from "../../utils/normalization/normalizeTo_PascalCase"; +import { normalizeTo_SCREAMING_CASE } from "../../utils/normalization/normalizeTo_SCREAMING_CASE"; +import { normalizeTo_camelCase } from "../../utils/normalization/normalizeTo_camelCase"; +import { normalizeTo_snake_case } from "../../utils/normalization/normalizeTo_snake_case"; +import { normalizeWhitespaces } from "../../utils/normalization/normalizeWhitespaces"; +import { parseKeywordsFromString } from "../../utils/normalization/parseKeywordsFromString"; +import { removeDiacritics } from "../../utils/normalization/removeDiacritics"; +import { removeEmojis } from "../../utils/removeEmojis"; +import { removeQuotes } from "../../utils/removeQuotes"; +import { trimCodeBlock } from "../../utils/trimCodeBlock"; +import { trimEndOfCodeBlock } from "../../utils/trimEndOfCodeBlock"; +import { unwrapResult } from "../../utils/unwrapResult"; const parseKeywords = (input: string) => - // TODO: DRY [🍯] - Array.from(parseKeywordsFromString(input)).join( - ', ', - ); /* <- TODO: [🧠] What is the best format comma list, bullet list,...? */ + // TODO: DRY [🍯] + Array.from(parseKeywordsFromString(input)).join( + ", ", + ); /* <- TODO: [🧠] What is the best format comma list, bullet list,...? */ // TODO: DRY [🍯] const trim = (str: string) => str.trim(); // TODO: DRY [🍯] -const reverse = (str: string) => str.split('').reverse().join(''); +const reverse = (str: string) => str.split("").reverse().join(""); /** * @@@ @@ -37,28 +37,28 @@ const reverse = (str: string) => str.split('').reverse().join(''); * @public exported from `@promptbook/execute-javascript` */ export const POSTPROCESSING_FUNCTIONS = { - spaceTrim, - removeQuotes, - unwrapResult, - trimEndOfCodeBlock, - trimCodeBlock, - trim, - reverse, - removeEmojis, - prettifyMarkdown, - capitalize, - decapitalize, - nameToUriPart, - nameToUriParts, - removeDiacritics, - normalizeWhitespaces, - normalizeToKebabCase, - normalizeTo_camelCase, - normalizeTo_snake_case, - normalizeTo_PascalCase, - parseKeywords, - normalizeTo_SCREAMING_CASE, - extractBlock, // <- [🍓] Remove balast in all other functions, use this one as example + spaceTrim, + removeQuotes, + unwrapResult, + trimEndOfCodeBlock, + trimCodeBlock, + trim, + reverse, + removeEmojis, + prettifyMarkdown, + capitalize, + decapitalize, + nameToUriPart, + nameToUriParts, + removeDiacritics, + normalizeWhitespaces, + normalizeToKebabCase, + normalizeTo_camelCase, + normalizeTo_snake_case, + normalizeTo_PascalCase, + parseKeywords, + normalizeTo_SCREAMING_CASE, + extractBlock, // <- [🍓] Remove balast in all other functions, use this one as example }; /** diff --git a/src/scripting/javascript/utils/preserve.ts b/src/scripting/javascript/utils/preserve.ts index c70b014275..b9db30de77 100644 --- a/src/scripting/javascript/utils/preserve.ts +++ b/src/scripting/javascript/utils/preserve.ts @@ -1,5 +1,5 @@ -import { forTime } from 'waitasecond'; -import type { really_any } from '../../../utils/organization/really_any'; +import { forTime } from "waitasecond"; +import type { really_any } from "../../../utils/organization/really_any"; /** * Does nothing, but preserves the function in the bundle @@ -9,20 +9,22 @@ import type { really_any } from '../../../utils/organization/really_any'; * @returns nothing * @private internal function of `JavascriptExecutionTools` and `JavascriptEvalExecutionTools` */ -export function preserve(func: (...params: ReadonlyArray) => unknown): void { - // Note: NOT calling the function +export function preserve( + func: (...params: ReadonlyArray) => unknown, +): void { + // Note: NOT calling the function - (async () => { - // TODO: [💩] Change to `await forEver` or something better - await forTime(100000000); + (async () => { + // TODO: [💩] Change to `await forEver` or something better + await forTime(100000000); - // [1] - try { - await func(); - } finally { - // do nothing - } - })(); + // [1] + try { + await func(); + } finally { + // do nothing + } + })(); } /** diff --git a/src/scripting/python/PythonExecutionTools.ts b/src/scripting/python/PythonExecutionTools.ts index 8d795e3b39..25f4d934fe 100644 --- a/src/scripting/python/PythonExecutionTools.ts +++ b/src/scripting/python/PythonExecutionTools.ts @@ -1,8 +1,8 @@ -import { spaceTrim } from 'spacetrim'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; -import type { ScriptExecutionTools } from '../../execution/ScriptExecutionTools'; -import type { ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptExecutionTools'; +import { spaceTrim } from "spacetrim"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; +import type { ScriptExecutionTools } from "../../execution/ScriptExecutionTools"; +import type { ScriptExecutionToolsExecuteOptions } from "../../execution/ScriptExecutionTools"; /** * ScriptExecutionTools for Python @@ -12,31 +12,33 @@ import type { ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptE * @private still in development */ export class PythonExecutionTools implements ScriptExecutionTools { - public constructor(protected readonly options: CommonToolsOptions = {}) {} + public constructor(protected readonly options: CommonToolsOptions = {}) {} - /** - * Executes a Python - */ - public async execute(options: ScriptExecutionToolsExecuteOptions): Promise { - const { scriptLanguage, script } = options; + /** + * Executes a Python + */ + public async execute( + options: ScriptExecutionToolsExecuteOptions, + ): Promise { + const { scriptLanguage, script } = options; - if (scriptLanguage !== 'python') { - throw new PipelineExecutionError( - `Script language ${scriptLanguage} not supported to be executed by PythonExecutionTools`, - ); - } + if (scriptLanguage !== "python") { + throw new PipelineExecutionError( + `Script language ${scriptLanguage} not supported to be executed by PythonExecutionTools`, + ); + } - if (this.options.isVerbose) { - console.info( - spaceTrim( - (block) => ` + if (this.options.isVerbose) { + console.info( + spaceTrim( + (block) => ` 🚀 NOT Evaluating ${scriptLanguage} script: ${block(script)}`, - ), - ); - } + ), + ); + } - throw new PipelineExecutionError('Not implemented'); - } + throw new PipelineExecutionError("Not implemented"); + } } diff --git a/src/scripting/typescript/TypescriptExecutionTools.ts b/src/scripting/typescript/TypescriptExecutionTools.ts index b2dc5a47da..851b1c1fea 100644 --- a/src/scripting/typescript/TypescriptExecutionTools.ts +++ b/src/scripting/typescript/TypescriptExecutionTools.ts @@ -1,8 +1,8 @@ -import { spaceTrim } from 'spacetrim'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import type { CommonToolsOptions } from '../../execution/CommonToolsOptions'; -import type { ScriptExecutionTools } from '../../execution/ScriptExecutionTools'; -import type { ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptExecutionTools'; +import { spaceTrim } from "spacetrim"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import type { CommonToolsOptions } from "../../execution/CommonToolsOptions"; +import type { ScriptExecutionTools } from "../../execution/ScriptExecutionTools"; +import type { ScriptExecutionToolsExecuteOptions } from "../../execution/ScriptExecutionTools"; /** * ScriptExecutionTools for TypeScript @@ -12,31 +12,33 @@ import type { ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptE * @private still in development */ export class TypescriptExecutionTools implements ScriptExecutionTools { - public constructor(protected readonly options: CommonToolsOptions = {}) {} + public constructor(protected readonly options: CommonToolsOptions = {}) {} - /** - * Executes a TypeScript - */ - public async execute(options: ScriptExecutionToolsExecuteOptions): Promise { - const { scriptLanguage, script } = options; + /** + * Executes a TypeScript + */ + public async execute( + options: ScriptExecutionToolsExecuteOptions, + ): Promise { + const { scriptLanguage, script } = options; - if (scriptLanguage !== 'typescript') { - throw new PipelineExecutionError( - `Script language ${scriptLanguage} not supported to be executed by TypescriptExecutionTools`, - ); - } + if (scriptLanguage !== "typescript") { + throw new PipelineExecutionError( + `Script language ${scriptLanguage} not supported to be executed by TypescriptExecutionTools`, + ); + } - if (this.options.isVerbose) { - console.info( - spaceTrim( - (block) => ` + if (this.options.isVerbose) { + console.info( + spaceTrim( + (block) => ` 🚀 NOT Evaluating ${scriptLanguage} script: ${block(script)}`, - ), - ); - } + ), + ); + } - throw new PipelineExecutionError('Not implemented'); - } + throw new PipelineExecutionError("Not implemented"); + } } diff --git a/src/storage/_common/PromptbookStorage.test-type.ts b/src/storage/_common/PromptbookStorage.test-type.ts index 5d330192f9..4316f54ad3 100644 --- a/src/storage/_common/PromptbookStorage.test-type.ts +++ b/src/storage/_common/PromptbookStorage.test-type.ts @@ -1,5 +1,5 @@ -import { keepUnused } from '../../utils/organization/keepUnused'; -import type { PromptbookStorage } from './PromptbookStorage'; +import { keepUnused } from "../../utils/organization/keepUnused"; +import type { PromptbookStorage } from "./PromptbookStorage"; // Note: Checking that `localStorage` and `sessionStorage` are implementing `PromptbookStorage` let storage: PromptbookStorage; diff --git a/src/storage/_common/PromptbookStorage.ts b/src/storage/_common/PromptbookStorage.ts index 35ad025ef0..a573839a06 100644 --- a/src/storage/_common/PromptbookStorage.ts +++ b/src/storage/_common/PromptbookStorage.ts @@ -1,4 +1,4 @@ -import type { Promisable } from 'type-fest'; +import type { Promisable } from "type-fest"; /** * Storage of objects with asynchronous API @@ -7,20 +7,20 @@ import type { Promisable } from 'type-fest'; * Note: This is simmilar to Web Storage API interface but everything is asynchronous and can store JSON objects. */ export type PromptbookStorage = { - /** - * Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object - */ - getItem(key: string): Promisable; + /** + * Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object + */ + getItem(key: string): Promisable; - /** - * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously - */ - setItem(key: string, value: TItem): Promisable; + /** + * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously + */ + setItem(key: string, value: TItem): Promisable; - /** - * Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists - */ - removeItem(key: string): Promisable; + /** + * Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists + */ + removeItem(key: string): Promisable; }; /** diff --git a/src/storage/blackhole/BlackholeStorage.ts b/src/storage/blackhole/BlackholeStorage.ts index e7eb89ac3d..028a37f82f 100644 --- a/src/storage/blackhole/BlackholeStorage.ts +++ b/src/storage/blackhole/BlackholeStorage.ts @@ -1,5 +1,5 @@ -import { keepUnused } from '../../utils/organization/keepUnused'; -import type { PromptbookStorage } from '../_common/PromptbookStorage'; +import { keepUnused } from "../../utils/organization/keepUnused"; +import type { PromptbookStorage } from "../_common/PromptbookStorage"; /** * Behaves like a storage but forgets everything you put in it @@ -7,46 +7,46 @@ import type { PromptbookStorage } from '../_common/PromptbookStorage'; * @public exported from `@promptbook/core` */ export class BlackholeStorage implements PromptbookStorage { - /** - * Returns the number of key/value pairs currently present in the list associated with the object. - */ - public get length(): number { - return 0; - } + /** + * Returns the number of key/value pairs currently present in the list associated with the object. + */ + public get length(): number { + return 0; + } - /** - * Empties the list associated with the object of all key/value pairs, if there are any. - */ - public clear(): void {} + /** + * Empties the list associated with the object of all key/value pairs, if there are any. + */ + public clear(): void {} - /** - * Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object. - * @param key - */ - public getItem(key: string): null { - keepUnused(key); - return null; - } + /** + * Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object. + * @param key + */ + public getItem(key: string): null { + keepUnused(key); + return null; + } - /** - * Returns the name of the nth key in the list, or null if n is greater than or equal to the number of key/value pairs in the object. - */ - public key(index: number): null { - keepUnused(index); - return null; - } + /** + * Returns the name of the nth key in the list, or null if n is greater than or equal to the number of key/value pairs in the object. + */ + public key(index: number): null { + keepUnused(index); + return null; + } - /** - * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously. - */ - public setItem(key: string, value: TItem): void { - keepUnused(key, value); - } + /** + * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously. + */ + public setItem(key: string, value: TItem): void { + keepUnused(key, value); + } - /** - * Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists. - */ - public removeItem(key: string): void { - keepUnused(key); - } + /** + * Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists. + */ + public removeItem(key: string): void { + keepUnused(key); + } } diff --git a/src/storage/file-cache-storage/FileCacheStorage.ts b/src/storage/file-cache-storage/FileCacheStorage.ts index 4adbcf208f..fb99157a7c 100644 --- a/src/storage/file-cache-storage/FileCacheStorage.ts +++ b/src/storage/file-cache-storage/FileCacheStorage.ts @@ -1,20 +1,20 @@ -import hexEncoder from 'crypto-js/enc-hex'; -import sha256 from 'crypto-js/sha256'; -import { mkdir, readFile, unlink, writeFile } from 'fs/promises'; -import { dirname, join } from 'path'; -import { MAX_FILENAME_LENGTH } from '../../config'; -import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { ExecutionTools } from '../../execution/ExecutionTools'; -import type { string_filename } from '../../types/typeAliases'; -import { stringifyPipelineJson } from '../../utils/editable/utils/stringifyPipelineJson'; -import { $isRunningInNode } from '../../utils/environment/$isRunningInNode'; -import { isFileExisting } from '../../utils/files/isFileExisting'; -import { titleToName } from '../../utils/normalization/titleToName'; -import { isSerializableAsJson } from '../../utils/serialization/isSerializableAsJson'; -import type { PromptbookStorage } from '../_common/PromptbookStorage'; -import type { FileCacheStorageOptions } from './FileCacheStorageOptions'; -import { nameToSubfolderPath } from './utils/nameToSubfolderPath'; +import { dirname, join } from "path"; +import hexEncoder from "crypto-js/enc-hex"; +import sha256 from "crypto-js/sha256"; +import { mkdir, readFile, unlink, writeFile } from "fs/promises"; +import { MAX_FILENAME_LENGTH } from "../../config"; +import { EnvironmentMismatchError } from "../../errors/EnvironmentMismatchError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { ExecutionTools } from "../../execution/ExecutionTools"; +import type { string_filename } from "../../types/typeAliases"; +import { stringifyPipelineJson } from "../../utils/editable/utils/stringifyPipelineJson"; +import { $isRunningInNode } from "../../utils/environment/$isRunningInNode"; +import { isFileExisting } from "../../utils/files/isFileExisting"; +import { titleToName } from "../../utils/normalization/titleToName"; +import { isSerializableAsJson } from "../../utils/serialization/isSerializableAsJson"; +import type { PromptbookStorage } from "../_common/PromptbookStorage"; +import type { FileCacheStorageOptions } from "./FileCacheStorageOptions"; +import { nameToSubfolderPath } from "./utils/nameToSubfolderPath"; /** * @@@ @@ -22,79 +22,85 @@ import { nameToSubfolderPath } from './utils/nameToSubfolderPath'; * @public exported from `@promptbook/node` */ export class FileCacheStorage implements PromptbookStorage { - constructor( - protected readonly tools: Required>, - private readonly options: FileCacheStorageOptions, - ) { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError(`FileCacheStorage works only in Node.js environment`); - } - } - - /** - * @@@ - */ - private getFilenameForKey(key: string): string_filename { - // TODO: [👬] DRY - const name = titleToName(key); - const nameStart = name.split('-', 2)[0] || 'unnamed'; - const hash = sha256(hexEncoder.parse(name)).toString(/* hex */); - // <- TODO: [🥬] Encapsulate sha256 to some private utility function - - return join( - this.options.rootFolderPath, - nameStart, // <- Note: This mechanism is to better segment the files in .promptbook cache folder - ...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), - `${name.substring(0, MAX_FILENAME_LENGTH)}.json`, - ); - } - - /** - * @@@ Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object. - */ - public async getItem(key: string): Promise { - const filename = this.getFilenameForKey(key); - - if (!(await isFileExisting(filename, this.tools.fs))) { - return null; - } - - const fileContent = await readFile(filename, 'utf-8'); - const value = JSON.parse(fileContent) as TItem; - - // TODO: [🌗] - - return value; - } - - /** - * @@@ Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously. - */ - public async setItem(key: string, value: TItem): Promise { - const filename = this.getFilenameForKey(key); - - if (!isSerializableAsJson(value)) { - throw new UnexpectedError(`The "${key}" you want to store in JSON file is not serializable as JSON`); - } - - const fileContent = stringifyPipelineJson(value); - - await mkdir(dirname(filename), { recursive: true }); // <- [0] - await writeFile(filename, fileContent, 'utf-8'); - } - - /** - * @@@ Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists. - */ - public async removeItem(key: string): Promise { - const filename = this.getFilenameForKey(key); - - // TODO: [🧠] What to use `unlink` or `rm` - await unlink(filename); - - // <- TODO: [🐿][🧠] Maybe remove empty folders - // [0] When `setItem` and `removeItem` called, the state of the file system should be the same - } + constructor( + protected readonly tools: Required>, + private readonly options: FileCacheStorageOptions, + ) { + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + `FileCacheStorage works only in Node.js environment`, + ); + } + } + + /** + * @@@ + */ + private getFilenameForKey(key: string): string_filename { + // TODO: [👬] DRY + const name = titleToName(key); + const nameStart = name.split("-", 2)[0] || "unnamed"; + const hash = sha256(hexEncoder.parse(name)).toString(/* hex */); + // <- TODO: [🥬] Encapsulate sha256 to some private utility function + + return join( + this.options.rootFolderPath, + nameStart, // <- Note: This mechanism is to better segment the files in .promptbook cache folder + ...nameToSubfolderPath( + hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */, + ), + `${name.substring(0, MAX_FILENAME_LENGTH)}.json`, + ); + } + + /** + * @@@ Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object. + */ + public async getItem(key: string): Promise { + const filename = this.getFilenameForKey(key); + + if (!(await isFileExisting(filename, this.tools.fs))) { + return null; + } + + const fileContent = await readFile(filename, "utf-8"); + const value = JSON.parse(fileContent) as TItem; + + // TODO: [🌗] + + return value; + } + + /** + * @@@ Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously. + */ + public async setItem(key: string, value: TItem): Promise { + const filename = this.getFilenameForKey(key); + + if (!isSerializableAsJson(value)) { + throw new UnexpectedError( + `The "${key}" you want to store in JSON file is not serializable as JSON`, + ); + } + + const fileContent = stringifyPipelineJson(value); + + await mkdir(dirname(filename), { recursive: true }); // <- [0] + await writeFile(filename, fileContent, "utf-8"); + } + + /** + * @@@ Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists. + */ + public async removeItem(key: string): Promise { + const filename = this.getFilenameForKey(key); + + // TODO: [🧠] What to use `unlink` or `rm` + await unlink(filename); + + // <- TODO: [🐿][🧠] Maybe remove empty folders + // [0] When `setItem` and `removeItem` called, the state of the file system should be the same + } } /** diff --git a/src/storage/file-cache-storage/FileCacheStorageOptions.ts b/src/storage/file-cache-storage/FileCacheStorageOptions.ts index 3bba6569f8..ce3efb9e53 100644 --- a/src/storage/file-cache-storage/FileCacheStorageOptions.ts +++ b/src/storage/file-cache-storage/FileCacheStorageOptions.ts @@ -1,11 +1,11 @@ -import type { string_dirname } from '../../types/typeAliases'; +import type { string_dirname } from "../../types/typeAliases"; /** * @@@ */ export type FileCacheStorageOptions = { - /** - * @@@ - */ - rootFolderPath: string_dirname; + /** + * @@@ + */ + rootFolderPath: string_dirname; }; diff --git a/src/storage/file-cache-storage/utils/nameToSubfolderPath.test.ts b/src/storage/file-cache-storage/utils/nameToSubfolderPath.test.ts index 32a99a2841..d553e69c39 100644 --- a/src/storage/file-cache-storage/utils/nameToSubfolderPath.test.ts +++ b/src/storage/file-cache-storage/utils/nameToSubfolderPath.test.ts @@ -1,11 +1,11 @@ -import { describe, expect, it } from '@jest/globals'; -import { nameToSubfolderPath } from './nameToSubfolderPath'; +import { describe, expect, it } from "@jest/globals"; +import { nameToSubfolderPath } from "./nameToSubfolderPath"; -describe('how name to subfolder path works', () => { - it('should convert name to subfolder path', () => { - expect(nameToSubfolderPath('hello')).toEqual(['h', 'e']); - expect(nameToSubfolderPath('hEllo')).toEqual(['h', 'e']); - expect(nameToSubfolderPath('foobar')).toEqual(['f', 'o']); - expect(nameToSubfolderPath('123456')).toEqual(['1', '2']); - }); +describe("how name to subfolder path works", () => { + it("should convert name to subfolder path", () => { + expect(nameToSubfolderPath("hello")).toEqual(["h", "e"]); + expect(nameToSubfolderPath("hEllo")).toEqual(["h", "e"]); + expect(nameToSubfolderPath("foobar")).toEqual(["f", "o"]); + expect(nameToSubfolderPath("123456")).toEqual(["1", "2"]); + }); }); diff --git a/src/storage/file-cache-storage/utils/nameToSubfolderPath.ts b/src/storage/file-cache-storage/utils/nameToSubfolderPath.ts index fc04368c06..4e9fbe96e0 100644 --- a/src/storage/file-cache-storage/utils/nameToSubfolderPath.ts +++ b/src/storage/file-cache-storage/utils/nameToSubfolderPath.ts @@ -1,4 +1,4 @@ -import type { string_name } from '../../../types/typeAliases'; +import type { string_name } from "../../../types/typeAliases"; /** * @@@ @@ -6,5 +6,5 @@ import type { string_name } from '../../../types/typeAliases'; * @private for `FileCacheStorage` */ export function nameToSubfolderPath(name: string_name): ReadonlyArray { - return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()]; + return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()]; } diff --git a/src/storage/local-storage/getLocalStorage.ts b/src/storage/local-storage/getLocalStorage.ts index f350f0c41f..29afca024e 100644 --- a/src/storage/local-storage/getLocalStorage.ts +++ b/src/storage/local-storage/getLocalStorage.ts @@ -1,7 +1,7 @@ -import { isRunningInBrowser } from 'openai/core'; -import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError'; -import type { PromptbookStorage } from '../_common/PromptbookStorage'; -import { makePromptbookStorageFromWebStorage } from './utils/makePromptbookStorageFromWebStorage'; +import { isRunningInBrowser } from "openai/core"; +import { EnvironmentMismatchError } from "../../errors/EnvironmentMismatchError"; +import type { PromptbookStorage } from "../_common/PromptbookStorage"; +import { makePromptbookStorageFromWebStorage } from "./utils/makePromptbookStorageFromWebStorage"; /** * Gets wrapper around `localStorage` object which can be used as `PromptbookStorage` @@ -9,11 +9,13 @@ import { makePromptbookStorageFromWebStorage } from './utils/makePromptbookStora * @public exported from `@promptbook/browser` */ export function getLocalStorage(): PromptbookStorage { - if (!isRunningInBrowser()) { - throw new EnvironmentMismatchError(`You can get localStorage works only in browser environment`); - } + if (!isRunningInBrowser()) { + throw new EnvironmentMismatchError( + `You can get localStorage works only in browser environment`, + ); + } - return makePromptbookStorageFromWebStorage(localStorage); + return makePromptbookStorageFromWebStorage(localStorage); } /** diff --git a/src/storage/local-storage/getSessionStorage.ts b/src/storage/local-storage/getSessionStorage.ts index df9505c5a3..8fb3695da2 100644 --- a/src/storage/local-storage/getSessionStorage.ts +++ b/src/storage/local-storage/getSessionStorage.ts @@ -1,7 +1,7 @@ -import { isRunningInBrowser } from 'openai/core'; -import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError'; -import type { PromptbookStorage } from '../_common/PromptbookStorage'; -import { makePromptbookStorageFromWebStorage } from './utils/makePromptbookStorageFromWebStorage'; +import { isRunningInBrowser } from "openai/core"; +import { EnvironmentMismatchError } from "../../errors/EnvironmentMismatchError"; +import type { PromptbookStorage } from "../_common/PromptbookStorage"; +import { makePromptbookStorageFromWebStorage } from "./utils/makePromptbookStorageFromWebStorage"; /** * Gets wrapper around `sessionStorage` object which can be used as `PromptbookStorage` @@ -9,11 +9,13 @@ import { makePromptbookStorageFromWebStorage } from './utils/makePromptbookStora * @public exported from `@promptbook/browser` */ export function getSessionStorage(): PromptbookStorage { - if (!isRunningInBrowser()) { - throw new EnvironmentMismatchError(`You can get sessionStorage works only in browser environment`); - } + if (!isRunningInBrowser()) { + throw new EnvironmentMismatchError( + `You can get sessionStorage works only in browser environment`, + ); + } - return makePromptbookStorageFromWebStorage(sessionStorage); + return makePromptbookStorageFromWebStorage(sessionStorage); } /** diff --git a/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.ts b/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.ts index 5ff0177bc3..5927478852 100644 --- a/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.ts +++ b/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.ts @@ -1,42 +1,46 @@ -import { UnexpectedError } from '../../../errors/UnexpectedError'; -import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson'; -import { isSerializableAsJson } from '../../../utils/serialization/isSerializableAsJson'; -import type { PromptbookStorage } from '../../_common/PromptbookStorage'; +import { UnexpectedError } from "../../../errors/UnexpectedError"; +import { stringifyPipelineJson } from "../../../utils/editable/utils/stringifyPipelineJson"; +import { isSerializableAsJson } from "../../../utils/serialization/isSerializableAsJson"; +import type { PromptbookStorage } from "../../_common/PromptbookStorage"; /** * @@@ * * @private for `getLocalStorage` and `getSessionStorage` */ -export function makePromptbookStorageFromWebStorage(webStorage: Storage): PromptbookStorage { - return { - getItem(key: string): TValue | null { - const stringValue = webStorage.getItem(key); - - if (stringValue === null) { - return null; - } - - const value = JSON.parse(stringValue) as TValue; - - // TODO: [🌗] - - return value; - }, - - setItem(key: string, value: TValue): void { - if (!isSerializableAsJson(value)) { - throw new UnexpectedError(`The "${key}" you want to store in web storage is not serializable as JSON`); - } - - const stringValue = stringifyPipelineJson(value); - webStorage.setItem(key, stringValue); - }, - - removeItem(key: string): void { - webStorage.removeItem(key); - }, - }; +export function makePromptbookStorageFromWebStorage( + webStorage: Storage, +): PromptbookStorage { + return { + getItem(key: string): TValue | null { + const stringValue = webStorage.getItem(key); + + if (stringValue === null) { + return null; + } + + const value = JSON.parse(stringValue) as TValue; + + // TODO: [🌗] + + return value; + }, + + setItem(key: string, value: TValue): void { + if (!isSerializableAsJson(value)) { + throw new UnexpectedError( + `The "${key}" you want to store in web storage is not serializable as JSON`, + ); + } + + const stringValue = stringifyPipelineJson(value); + webStorage.setItem(key, stringValue); + }, + + removeItem(key: string): void { + webStorage.removeItem(key); + }, + }; } /** diff --git a/src/storage/memory/MemoryStorage.ts b/src/storage/memory/MemoryStorage.ts index ae51e99174..b8fd7595fa 100644 --- a/src/storage/memory/MemoryStorage.ts +++ b/src/storage/memory/MemoryStorage.ts @@ -1,4 +1,4 @@ -import type { PromptbookStorage } from '../_common/PromptbookStorage'; +import type { PromptbookStorage } from "../_common/PromptbookStorage"; /** * Stores data in memory (HEAP) @@ -6,47 +6,47 @@ import type { PromptbookStorage } from '../_common/PromptbookStorage'; * @public exported from `@promptbook/core` */ export class MemoryStorage implements PromptbookStorage { - private storage: Record = {}; + private storage: Record = {}; - /** - * Returns the number of key/value pairs currently present in the list associated with the object. - */ - public get length(): number { - return Object.keys(this.storage).length; - } + /** + * Returns the number of key/value pairs currently present in the list associated with the object. + */ + public get length(): number { + return Object.keys(this.storage).length; + } - /** - * Empties the list associated with the object of all key/value pairs, if there are any. - */ - public clear(): void { - this.storage = {}; - } + /** + * Empties the list associated with the object of all key/value pairs, if there are any. + */ + public clear(): void { + this.storage = {}; + } - /** - * Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object. - */ - public getItem(key: string): TItem | null { - return this.storage[key] || null; - } + /** + * Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object. + */ + public getItem(key: string): TItem | null { + return this.storage[key] || null; + } - /** - * Returns the name of the nth key in the list, or null if n is greater than or equal to the number of key/value pairs in the object. - */ - public key(index: number): string | null { - return Object.keys(this.storage)[index] || null; - } + /** + * Returns the name of the nth key in the list, or null if n is greater than or equal to the number of key/value pairs in the object. + */ + public key(index: number): string | null { + return Object.keys(this.storage)[index] || null; + } - /** - * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously. - */ - public setItem(key: string, value: TItem): void { - this.storage[key] = value; - } + /** + * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously. + */ + public setItem(key: string, value: TItem): void { + this.storage[key] = value; + } - /** - * Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists. - */ - public removeItem(key: string): void { - delete this.storage[key]; - } + /** + * Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists. + */ + public removeItem(key: string): void { + delete this.storage[key]; + } } diff --git a/src/storage/utils/PrefixStorage.ts b/src/storage/utils/PrefixStorage.ts index e5271e888d..7a86363109 100644 --- a/src/storage/utils/PrefixStorage.ts +++ b/src/storage/utils/PrefixStorage.ts @@ -1,35 +1,38 @@ -import type { Promisable } from 'type-fest'; -import type { PromptbookStorage } from '../_common/PromptbookStorage'; +import type { Promisable } from "type-fest"; +import type { PromptbookStorage } from "../_common/PromptbookStorage"; /** * This class behaves like LocalStorage but separates keys by prefix * * @public exported from `@promptbook/core` */ export class PrefixStorage implements PromptbookStorage { - constructor( - private baseStorage: PromptbookStorage, - private keyPrefix: string, - private separator: string = '_', - ) {} + constructor( + private baseStorage: PromptbookStorage, + private keyPrefix: string, + private separator = "_", + ) {} - /** - * Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object. - */ - public getItem(key: string): Promisable { - return this.baseStorage.getItem(this.keyPrefix + this.separator + key); - } + /** + * Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object. + */ + public getItem(key: string): Promisable { + return this.baseStorage.getItem(this.keyPrefix + this.separator + key); + } - /** - * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously. - */ - public setItem(key: string, value: TItem): Promisable { - return this.baseStorage.setItem(this.keyPrefix + this.separator + key, value); - } + /** + * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously. + */ + public setItem(key: string, value: TItem): Promisable { + return this.baseStorage.setItem( + this.keyPrefix + this.separator + key, + value, + ); + } - /** - * Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists. - */ - public removeItem(key: string): void { - this.baseStorage.removeItem(this.keyPrefix + this.separator + key); - } + /** + * Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists. + */ + public removeItem(key: string): void { + this.baseStorage.removeItem(this.keyPrefix + this.separator + key); + } } diff --git a/src/types/IntermediateFilesStrategy.ts b/src/types/IntermediateFilesStrategy.ts index 3ab5305362..acad08b8a9 100644 --- a/src/types/IntermediateFilesStrategy.ts +++ b/src/types/IntermediateFilesStrategy.ts @@ -1,7 +1,9 @@ /** * @@@ */ -export type IntermediateFilesStrategy = 'HIDE_AND_CLEAN' | 'HIDE_AND_KEEP' /* | 'VISIBLE' <- TODO: [😡] Add */; +export type IntermediateFilesStrategy = + | "HIDE_AND_CLEAN" + | "HIDE_AND_KEEP" /* | 'VISIBLE' <- TODO: [😡] Add */; /** * TODO: [🎅] Maube add options for all kinds of cache, unite with `isCacheReloaded` and make `cacheStrategy` (which is not specific only for intermediate files) diff --git a/src/types/ModelRequirements.ts b/src/types/ModelRequirements.ts index 73f1d333ee..b35272fa26 100644 --- a/src/types/ModelRequirements.ts +++ b/src/types/ModelRequirements.ts @@ -1,8 +1,8 @@ -import type { ModelVariant } from './ModelVariant'; -import type { number_model_temperature } from './typeAliases'; -import type { number_seed } from './typeAliases'; -import type { string_model_name } from './typeAliases'; -import type { string_system_message } from './typeAliases'; +import type { ModelVariant } from "./ModelVariant"; +import type { number_model_temperature } from "./typeAliases"; +import type { number_seed } from "./typeAliases"; +import type { string_model_name } from "./typeAliases"; +import type { string_system_message } from "./typeAliases"; /** * Abstract way to specify the LLM. @@ -12,9 +12,9 @@ import type { string_system_message } from './typeAliases'; * @see https://github.com/webgptorg/promptbook#model-requirements */ export type ModelRequirements = - | CompletionModelRequirements - | ChatModelRequirements - | EmbeddingModelRequirements /* <- [🤖] */; + | CompletionModelRequirements + | ChatModelRequirements + | EmbeddingModelRequirements /* <- [🤖] */; /** * Model requirements for the completion variant @@ -22,10 +22,10 @@ export type ModelRequirements = * Note: [🚉] This is fully serializable as JSON */ export type CompletionModelRequirements = CommonModelRequirements & { - /** - * Completion model variant - */ - modelVariant: 'COMPLETION'; + /** + * Completion model variant + */ + modelVariant: "COMPLETION"; }; /** @@ -34,15 +34,15 @@ export type CompletionModelRequirements = CommonModelRequirements & { * Note: [🚉] This is fully serializable as JSON */ export type ChatModelRequirements = CommonModelRequirements & { - /** - * Chat model variant - */ - modelVariant: 'CHAT'; + /** + * Chat model variant + */ + modelVariant: "CHAT"; - /** - * System message to be used in the model - */ - readonly systemMessage?: string_system_message; + /** + * System message to be used in the model + */ + readonly systemMessage?: string_system_message; }; /** @@ -51,10 +51,10 @@ export type ChatModelRequirements = CommonModelRequirements & { * Note: [🚉] This is fully serializable as JSON */ export type EmbeddingModelRequirements = CommonModelRequirements & { - /** - * Embedding model variant - */ - modelVariant: 'EMBEDDING'; + /** + * Embedding model variant + */ + modelVariant: "EMBEDDING"; }; // <- Note: [🤖] Add new model variant here @@ -67,47 +67,47 @@ export type EmbeddingModelRequirements = CommonModelRequirements & { * @private just abstract the common properties of the model requirements */ export type CommonModelRequirements = { - /** - * Model variant describes the very general type of the model - * - * There are 3 variants: - * - **COMPLETION** - model that takes prompt and writes the rest of the text - * - **CHAT** - model that takes prompt and previous messages and returns response - * - **EMBEDDING** - model that takes prompt and returns embedding - * <- [🤖] - */ - readonly modelVariant: ModelVariant; + /** + * Model variant describes the very general type of the model + * + * There are 3 variants: + * - **COMPLETION** - model that takes prompt and writes the rest of the text + * - **CHAT** - model that takes prompt and previous messages and returns response + * - **EMBEDDING** - model that takes prompt and returns embedding + * <- [🤖] + */ + readonly modelVariant: ModelVariant; - /** - * The model for text prompt - * - * Note: Model must be compatible with the model variant - * Note: If not specified, the best model for the variant will be used - * - * @example 'gpt-4', 'gpt-4-32k-0314', 'gpt-3.5-turbo-instruct',... - */ - readonly modelName?: string_model_name; + /** + * The model for text prompt + * + * Note: Model must be compatible with the model variant + * Note: If not specified, the best model for the variant will be used + * + * @example 'gpt-4', 'gpt-4-32k-0314', 'gpt-3.5-turbo-instruct',... + */ + readonly modelName?: string_model_name; - /** - * The temperature of the model - * - * Note: [💱] Promptbook is using just `temperature` (not `top_k` and `top_p`) - */ - readonly temperature?: number_model_temperature; + /** + * The temperature of the model + * + * Note: [💱] Promptbook is using just `temperature` (not `top_k` and `top_p`) + */ + readonly temperature?: number_model_temperature; - /** - * Seed for the model - */ - readonly seed?: number_seed; + /** + * Seed for the model + */ + readonly seed?: number_seed; - /** - * Maximum number of tokens that can be generated by the model - * - * Note: [🌾] - */ - readonly maxTokens?: number; + /** + * Maximum number of tokens that can be generated by the model + * + * Note: [🌾] + */ + readonly maxTokens?: number; - // <- Note: [🧆] Look here when adding new properties to `ModelRequirements` + // <- Note: [🧆] Look here when adding new properties to `ModelRequirements` }; /** diff --git a/src/types/ModelVariant.ts b/src/types/ModelVariant.ts index 8ef99dd5ce..2848368d05 100644 --- a/src/types/ModelVariant.ts +++ b/src/types/ModelVariant.ts @@ -1,4 +1,4 @@ -import type { TupleToUnion } from 'type-fest'; +import type { TupleToUnion } from "type-fest"; /** * Model variant describes the very general type of the model @@ -14,4 +14,8 @@ export type ModelVariant = TupleToUnion; * * @public exported from `@promptbook/core` */ -export const MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */] as const; +export const MODEL_VARIANTS = [ + "COMPLETION", + "CHAT", + "EMBEDDING" /* <- TODO [🏳] */ /* <- [🤖] */, +] as const; diff --git a/src/types/NonEmptyArray.ts b/src/types/NonEmptyArray.ts index f8dc5b6788..6cf414de22 100644 --- a/src/types/NonEmptyArray.ts +++ b/src/types/NonEmptyArray.ts @@ -6,4 +6,5 @@ export type NonEmptyArray = [TItem, ...TItem[]]; /** * Represents an array that is guaranteed to have at least one element and is readonly */ -export type NonEmptyReadonlyArray = ReadonlyArray & NonEmptyArray; +export type NonEmptyReadonlyArray = ReadonlyArray & + NonEmptyArray; diff --git a/src/types/Prompt.ts b/src/types/Prompt.ts index 1403318c6f..fd3a83a819 100644 --- a/src/types/Prompt.ts +++ b/src/types/Prompt.ts @@ -1,15 +1,15 @@ -import type { FormatCommand } from '../commands/FORMAT/FormatCommand'; -import type { Expectations } from '../pipeline/PipelineJson/Expectations'; -import type { ChatModelRequirements } from './ModelRequirements'; -import type { CompletionModelRequirements } from './ModelRequirements'; -import type { EmbeddingModelRequirements } from './ModelRequirements'; -import type { ModelRequirements } from './ModelRequirements'; -import type { Parameters } from './typeAliases'; -import type { string_pipeline_url_with_task_hash } from './typeAliases'; -import type { string_postprocessing_function_name } from './typeAliases'; -import type { string_prompt } from './typeAliases'; -import type { string_template } from './typeAliases'; -import type { string_title } from './typeAliases'; +import type { FormatCommand } from "../commands/FORMAT/FormatCommand"; +import type { Expectations } from "../pipeline/PipelineJson/Expectations"; +import type { ChatModelRequirements } from "./ModelRequirements"; +import type { CompletionModelRequirements } from "./ModelRequirements"; +import type { EmbeddingModelRequirements } from "./ModelRequirements"; +import type { ModelRequirements } from "./ModelRequirements"; +import type { Parameters } from "./typeAliases"; +import type { string_pipeline_url_with_task_hash } from "./typeAliases"; +import type { string_postprocessing_function_name } from "./typeAliases"; +import type { string_prompt } from "./typeAliases"; +import type { string_template } from "./typeAliases"; +import type { string_title } from "./typeAliases"; /** * Prompt in a text along with model requirements, but without any execution or templating logic. @@ -17,7 +17,10 @@ import type { string_title } from './typeAliases'; * Note: [🚉] This is fully serializable as JSON * @see https://github.com/webgptorg/promptbook#prompt */ -export type Prompt = CompletionPrompt | ChatPrompt | EmbeddingPrompt /* <- [🤖] */; +export type Prompt = + | CompletionPrompt + | ChatPrompt + | EmbeddingPrompt /* <- [🤖] */; /** * Completion prompt @@ -25,10 +28,10 @@ export type Prompt = CompletionPrompt | ChatPrompt | EmbeddingPrompt /* <- [🤖 * Note: [🚉] This is fully serializable as JSON */ export type CompletionPrompt = CommonPrompt & { - /** - * Requirements for completion model - */ - modelRequirements: CompletionModelRequirements; + /** + * Requirements for completion model + */ + modelRequirements: CompletionModelRequirements; }; /** @@ -37,12 +40,12 @@ export type CompletionPrompt = CommonPrompt & { * Note: [🚉] This is fully serializable as JSON */ export type ChatPrompt = CommonPrompt & { - /** - * Requirements for chat model - */ - modelRequirements: ChatModelRequirements; + /** + * Requirements for chat model + */ + modelRequirements: ChatModelRequirements; - // <-TODO: [🗯][🧠] Figure out way how to pass thread / previous messages + // <-TODO: [🗯][🧠] Figure out way how to pass thread / previous messages }; /** @@ -51,12 +54,12 @@ export type ChatPrompt = CommonPrompt & { * Note: [🚉] This is fully serializable as JSON */ export type EmbeddingPrompt = CommonPrompt & { - /** - * Requirements for chat model - */ - modelRequirements: EmbeddingModelRequirements; + /** + * Requirements for chat model + */ + modelRequirements: EmbeddingModelRequirements; - // <-TODO: [🗯][🧠] Figure out way how to pass thread / previous messages + // <-TODO: [🗯][🧠] Figure out way how to pass thread / previous messages }; // <- Note: [🤖] Add new model variant here @@ -69,59 +72,59 @@ export type EmbeddingPrompt = CommonPrompt & { * @private just abstract the common properties of the prompts */ export type CommonPrompt = { - /** - * The title of the prompt - * - * Note: This has no effect on the model, it is just for the reporting - */ - readonly title: string_title; - - /** - * The text of the prompt with placeholders for parameters - * - * @example "What is the capital of {country}?" - */ - readonly content: string_prompt & string_template; - - /** - * Requirements for the model - */ - readonly modelRequirements: ModelRequirements; - - /** - * List of postprocessing steps that are executed after the prompt - */ - readonly postprocessingFunctionNames?: ReadonlyArray; - - /** - * Expectations for the answer - * - * For example 5 words, 3 sentences, 2 paragraphs, ... - * If not set, nothing is expected from the answer - */ - readonly expectations?: Expectations; - - /** - * Expect this format of the answer - * - * Note: Expectations are performed after all postprocessing steps - * @deprecated [💝] - */ - readonly format?: FormatCommand['format']; - - /** - * Unique identifier of the pipeline with specific task name as hash - * - * @example https://promptbook.studio/webgpt/write-website-content-cs.book#keywords - */ - readonly pipelineUrl?: string_pipeline_url_with_task_hash; - - /** - * Parameters used in the `content` - */ - readonly parameters: Parameters; - - // <- Note: [🧆] Look here when adding new properties to `Prompt` + /** + * The title of the prompt + * + * Note: This has no effect on the model, it is just for the reporting + */ + readonly title: string_title; + + /** + * The text of the prompt with placeholders for parameters + * + * @example "What is the capital of {country}?" + */ + readonly content: string_prompt & string_template; + + /** + * Requirements for the model + */ + readonly modelRequirements: ModelRequirements; + + /** + * List of postprocessing steps that are executed after the prompt + */ + readonly postprocessingFunctionNames?: ReadonlyArray; + + /** + * Expectations for the answer + * + * For example 5 words, 3 sentences, 2 paragraphs, ... + * If not set, nothing is expected from the answer + */ + readonly expectations?: Expectations; + + /** + * Expect this format of the answer + * + * Note: Expectations are performed after all postprocessing steps + * @deprecated [💝] + */ + readonly format?: FormatCommand["format"]; + + /** + * Unique identifier of the pipeline with specific task name as hash + * + * @example https://promptbook.studio/webgpt/write-website-content-cs.book#keywords + */ + readonly pipelineUrl?: string_pipeline_url_with_task_hash; + + /** + * Parameters used in the `content` + */ + readonly parameters: Parameters; + + // <- Note: [🧆] Look here when adding new properties to `Prompt` }; /** diff --git a/src/types/ScriptLanguage.ts b/src/types/ScriptLanguage.ts index 608b144b3f..994125f2b1 100644 --- a/src/types/ScriptLanguage.ts +++ b/src/types/ScriptLanguage.ts @@ -1,4 +1,4 @@ -import type { TupleToUnion } from 'type-fest'; +import type { TupleToUnion } from "type-fest"; /** * Script language @@ -12,5 +12,9 @@ export type ScriptLanguage = TupleToUnion; * * @private internal base for `ScriptLanguage` */ -export const SUPPORTED_SCRIPT_LANGUAGES = ['javascript', 'typescript', 'python'] as const; +export const SUPPORTED_SCRIPT_LANGUAGES = [ + "javascript", + "typescript", + "python", +] as const; // <- TODO: [🏥] DRY diff --git a/src/types/SectionType.ts b/src/types/SectionType.ts index 7d43d576df..c01b9fbfd1 100644 --- a/src/types/SectionType.ts +++ b/src/types/SectionType.ts @@ -1,5 +1,5 @@ -import type { TupleToUnion } from 'type-fest'; -import { TaskTypes } from './TaskType'; +import type { TupleToUnion } from "type-fest"; +import { TaskTypes } from "./TaskType"; /** * Type of the section @@ -11,7 +11,12 @@ export type SectionType = TupleToUnion; * * @public exported from `@promptbook/core` */ -export const NonTaskSectionTypes = ['EXAMPLE', 'KNOWLEDGE', 'INSTRUMENT', 'ACTION'] as const; +export const NonTaskSectionTypes = [ + "EXAMPLE", + "KNOWLEDGE", + "INSTRUMENT", + "ACTION", +] as const; /** * All available section types @@ -23,6 +28,6 @@ export const NonTaskSectionTypes = ['EXAMPLE', 'KNOWLEDGE', 'INSTRUMENT', 'ACTIO * @public exported from `@promptbook/core` */ export const SectionTypes = [ - ...TaskTypes.map((TaskType) => `${TaskType}_TASK` as const), - ...NonTaskSectionTypes, + ...TaskTypes.map((TaskType) => `${TaskType}_TASK` as const), + ...NonTaskSectionTypes, ] as const; diff --git a/src/types/TaskType.ts b/src/types/TaskType.ts index 7b59f73d3b..cf035a7e08 100644 --- a/src/types/TaskType.ts +++ b/src/types/TaskType.ts @@ -1,4 +1,4 @@ -import type { TupleToUnion } from 'type-fest'; +import type { TupleToUnion } from "type-fest"; /** * Type of the task @@ -15,9 +15,9 @@ export type TaskType = TupleToUnion; * @public exported from `@promptbook/core` */ export const TaskTypes = [ - 'PROMPT', - 'SIMPLE', - 'SCRIPT', - 'DIALOG', - // <- [🅱] + "PROMPT", + "SIMPLE", + "SCRIPT", + "DIALOG", + // <- [🅱] ] as const; diff --git a/src/types/typeAliasEmoji.ts b/src/types/typeAliasEmoji.ts index f0fd1913ee..4b3e1bdfe3 100644 --- a/src/types/typeAliasEmoji.ts +++ b/src/types/typeAliasEmoji.ts @@ -13,7 +13,7 @@ * > CollboardSdk.EMOJIS.map((e)=>`"${e}"`).join(' | ') */ export type string_char_emoji = - '"😀" | "😃" | "😄" | "😁" | "😆" | "😅" | "😂" | "🤣" | "🥲" | "☺️" | "😊" | "😇" | "🙂" | "🙃" | "😉" | "😌" | "😍" | "🥰" | "😘" | "😗" | "😙" | "😚" | "😋" | "😛" | "😝" | "😜" | "🤪" | "🤨" | "🧐" | "🤓" | "😎" | "🥸" | "🤩" | "🥳" | "😏" | "😒" | "😞" | "😔" | "😟" | "😕" | "🙁" | "☹️" | "😣" | "😖" | "😫" | "😩" | "🥺" | "😢" | "😭" | "😤" | "😠" | "😡" | "🤬" | "🤯" | "😳" | "🥵" | "🥶" | "😱" | "😨" | "😰" | "😥" | "😓" | "🤗" | "🤔" | "🤭" | "🤫" | "🤥" | "😶" | "😐" | "😑" | "😬" | "🙄" | "😯" | "😦" | "😧" | "😮" | "😲" | "🥱" | "😴" | "🤤" | "😪" | "😵" | "🤐" | "🥴" | "🤢" | "🤮" | "🤧" | "😷" | "🤒" | "🤕" | "🤑" | "🤠" | "😈" | "👿" | "👹" | "👺" | "🤡" | "💩" | "👻" | "💀" | "☠️" | "👽" | "👾" | "🤖" | "🎃" | "😺" | "😸" | "😹" | "😻" | "😼" | "😽" | "🙀" | "😿" | "😾" | "👋" | "🤚" | "🖐" | "✋" | "🖖" | "👌" | "🤌" | "🤏" | "✌️" | "🤞" | "🤟" | "🤘" | "🤙" | "👈" | "👉" | "👆" | "🖕" | "👇" | "☝️" | "👍" | "👎" | "✊" | "👊" | "🤛" | "🤜" | "👏" | "🙌" | "👐" | "🤲" | "🤝" | "🙏" | "✍️" | "💅" | "🤳" | "💪" | "🦾" | "🦵" | "🦿" | "🦶" | "👣" | "👂" | "🦻" | "👃" | "🫀" | "🫁" | "🧠" | "🦷" | "🦴" | "👀" | "👁" | "👅" | "👄" | "💋" | "🩸" | "👶" | "👧" | "🧒" | "👦" | "👩" | "🧑" | "👨" | "👩‍🦱" | "🧑‍🦱" | "👨‍🦱" | "👩‍🦰" | "🧑‍🦰" | "👨‍🦰" | "👱‍♀️" | "👱" | "👱‍♂️" | "👩‍🦳" | "🧑‍🦳" | "👨‍🦳" | "👩‍🦲" | "🧑‍🦲" | "👨‍🦲" | "🧔" | "👵" | "🧓" | "👴" | "👲" | "👳‍♀️" | "👳" | "👳‍♂️" | "🧕" | "👮‍♀️" | "👮" | "👮‍♂️" | "👷‍♀️" | "👷" | "👷‍♂️" | "💂‍♀️" | "💂" | "💂‍♂️" | "🕵️‍♀️" | "🕵️" | "🕵️‍♂️" | "👩‍⚕️" | "🧑‍⚕️" | "👨‍⚕️" | "👩‍🌾" | "🧑‍🌾" | "👨‍🌾" | "👩‍🍳" | "🧑‍🍳" | "👨‍🍳" | "👩‍🎓" | "🧑‍🎓" | "👨‍🎓" | "👩‍🎤" | "🧑‍🎤" | "👨‍🎤" | "👩‍🏫" | "🧑‍🏫" | "👨‍🏫" | "👩‍🏭" | "🧑‍🏭" | "👨‍🏭" | "👩‍💻" | "🧑‍💻" | "👨‍💻" | "👩‍💼" | "🧑‍💼" | "👨‍💼" | "👩‍🔧" | "🧑‍🔧" | "👨‍🔧" | "👩‍🔬" | "🧑‍🔬" | "👨‍🔬" | "👩‍🎨" | "🧑‍🎨" | "👨‍🎨" | "👩‍🚒" | "🧑‍🚒" | "👨‍🚒" | "👩‍✈️" | "🧑‍✈️" | "👨‍✈️" | "👩‍🚀" | "🧑‍🚀" | "👨‍🚀" | "👩‍⚖️" | "🧑‍⚖️" | "👨‍⚖️" | "👰‍♀️" | "👰" | "👰‍♂️" | "🤵‍♀️" | "🤵" | "🤵‍♂️" | "👸" | "🤴" | "🥷" | "🦸‍♀️" | "🦸" | "🦸‍♂️" | "🦹‍♀️" | "🦹" | "🦹‍♂️" | "🤶" | "🧑‍🎄" | "🎅" | "🧙‍♀️" | "🧙" | "🧙‍♂️" | "🧝‍♀️" | "🧝" | "🧝‍♂️" | "🧛‍♀️" | "🧛" | "🧛‍♂️" | "🧟‍♀️" | "🧟" | "🧟‍♂️" | "🧞‍♀️" | "🧞" | "🧞‍♂️" | "🧜‍♀️" | "🧜" | "🧜‍♂️" | "🧚‍♀️" | "🧚" | "🧚‍♂️" | "👼" | "🤰" | "🤱" | "👩‍🍼" | "🧑‍🍼" | "👨‍🍼" | "🙇‍♀️" | "🙇" | "🙇‍♂️" | "💁‍♀️" | "💁" | "💁‍♂️" | "🙅‍♀️" | "🙅" | "🙅‍♂️" | "🙆‍♀️" | "🙆" | "🙆‍♂️" | "🙋‍♀️" | "🙋" | "🙋‍♂️" | "🧏‍♀️" | "🧏" | "🧏‍♂️" | "🤦‍♀️" | "🤦" | "🤦‍♂️" | "🤷‍♀️" | "🤷" | "🤷‍♂️" | "🙎‍♀️" | "🙎" | "🙎‍♂️" | "🙍‍♀️" | "🙍" | "🙍‍♂️" | "💇‍♀️" | "💇" | "💇‍♂️" | "💆‍♀️" | "💆" | "💆‍♂️" | "🧖‍♀️" | "🧖" | "🧖‍♂️" | "💃" | "🕺" | "👯‍♀️" | "👯" | "👯‍♂️" | "🕴" | "👩‍🦽" | "🧑‍🦽" | "👨‍🦽" | "👩‍🦼" | "🧑‍🦼" | "👨‍🦼" | "🚶‍♀️" | "🚶" | "🚶‍♂️" | "👩‍🦯" | "🧑‍🦯" | "👨‍🦯" | "🧎‍♀️" | "🧎" | "🧎‍♂️" | "🏃‍♀️" | "🏃" | "🏃‍♂️" | "🧍‍♀️" | "🧍" | "🧍‍♂️" | "👭" | "🧑‍🤝‍🧑" | "👬" | "👫" | "👩‍❤️‍👩" | "💑" | "👨‍❤️‍👨" | "👩‍❤️‍👨" | "👩‍❤️‍💋‍👩" | "💏" | "👨‍❤️‍💋‍👨" | "👩‍❤️‍💋‍👨" | "👪" | "👨‍👩‍👦" | "👨‍👩‍👧" | "👨‍👩‍👧‍👦" | "👨‍👩‍👦‍👦" | "👨‍👩‍👧‍👧" | "👨‍👨‍👦" | "👨‍👨‍👧" | "👨‍👨‍👧‍👦" | "👨‍👨‍👦‍👦" | "👨‍👨‍👧‍👧" | "👩‍👩‍👦" | "👩‍👩‍👧" | "👩‍👩‍👧‍👦" | "👩‍👩‍👦‍👦" | "👩‍👩‍👧‍👧" | "👨‍👦" | "👨‍👦‍👦" | "👨‍👧" | "👨‍👧‍👦" | "👨‍👧‍👧" | "👩‍👦" | "👩‍👦‍👦" | "👩‍👧" | "👩‍👧‍👦" | "👩‍👧‍👧" | "🗣" | "👤" | "👥" | "🫂" | "🧳" | "🌂" | "☂️" | "🧵" | "🪡" | "🪢" | "🧶" | "👓" | "🕶" | "🥽" | "🥼" | "🦺" | "👔" | "👕" | "👖" | "🧣" | "🧤" | "🧥" | "🧦" | "👗" | "👘" | "🥻" | "🩴" | "🩱" | "🩲" | "🩳" | "👙" | "👚" | "👛" | "👜" | "👝" | "🎒" | "👞" | "👟" | "🥾" | "🥿" | "👠" | "👡" | "🩰" | "👢" | "👑" | "👒" | "🎩" | "🎓" | "🧢" | "⛑" | "🪖" | "💄" | "💍" | "💼" | "👋🏻" | "🤚🏻" | "🖐🏻" | "✋🏻" | "🖖🏻" | "👌🏻" | "🤌🏻" | "🤏🏻" | "✌🏻" | "🤞🏻" | "🤟🏻" | "🤘🏻" | "🤙🏻" | "👈🏻" | "👉🏻" | "👆🏻" | "🖕🏻" | "👇🏻" | "☝🏻" | "👍🏻" | "👎🏻" | "✊🏻" | "👊🏻" | "🤛🏻" | "🤜🏻" | "👏🏻" | "🙌🏻" | "👐🏻" | "🤲🏻" | "🙏🏻" | "✍🏻" | "💅🏻" | "🤳🏻" | "💪🏻" | "🦵🏻" | "🦶🏻" | "👂🏻" | "🦻🏻" | "👃🏻" | "👶🏻" | "👧🏻" | "🧒🏻" | "👦🏻" | "👩🏻" | "🧑🏻" | "👨🏻" | "👩🏻‍🦱" | "🧑🏻‍🦱" | "👨🏻‍🦱" | "👩🏻‍🦰" | "🧑🏻‍🦰" | "👨🏻‍🦰" | "👱🏻‍♀️" | "👱🏻" | "👱🏻‍♂️" | "👩🏻‍🦳" | "🧑🏻‍🦳" | "👨🏻‍🦳" | "👩🏻‍🦲" | "🧑🏻‍🦲" | "👨🏻‍🦲" | "🧔🏻" | "👵🏻" | "🧓🏻" | "👴🏻" | "👲🏻" | "👳🏻‍♀️" | "👳🏻" | "👳🏻‍♂️" | "🧕🏻" | "👮🏻‍♀️" | "👮🏻" | "👮🏻‍♂️" | "👷🏻‍♀️" | "👷🏻" | "👷🏻‍♂️" | "💂🏻‍♀️" | "💂🏻" | "💂🏻‍♂️" | "🕵🏻‍♀️" | "🕵🏻" | "🕵🏻‍♂️" | "👩🏻‍⚕️" | "🧑🏻‍⚕️" | "👨🏻‍⚕️" | "👩🏻‍🌾" | "🧑🏻‍🌾" | "👨🏻‍🌾" | "👩🏻‍🍳" | "🧑🏻‍🍳" | "👨🏻‍🍳" | "👩🏻‍🎓" | "🧑🏻‍🎓" | "👨🏻‍🎓" | "👩🏻‍🎤" | "🧑🏻‍🎤" | "👨🏻‍🎤" | "👩🏻‍🏫" | "🧑🏻‍🏫" | "👨🏻‍🏫" | "👩🏻‍🏭" | "🧑🏻‍🏭" | "👨🏻‍🏭" | "👩🏻‍💻" | "🧑🏻‍💻" | "👨🏻‍💻" | "👩🏻‍💼" | "🧑🏻‍💼" | "👨🏻‍💼" | "👩🏻‍🔧" | "🧑🏻‍🔧" | "👨🏻‍🔧" | "👩🏻‍🔬" | "🧑🏻‍🔬" | "👨🏻‍🔬" | "👩🏻‍🎨" | "🧑🏻‍🎨" | "👨🏻‍🎨" | "👩🏻‍🚒" | "🧑🏻‍🚒" | "👨🏻‍🚒" | "👩🏻‍✈️" | "🧑🏻‍✈️" | "👨🏻‍✈️" | "👩🏻‍🚀" | "🧑🏻‍🚀" | "👨🏻‍🚀" | "👩🏻‍⚖️" | "🧑🏻‍⚖️" | "👨🏻‍⚖️" | "👰🏻‍♀️" | "👰🏻" | "👰🏻‍♂️" | "🤵🏻‍♀️" | "🤵🏻" | "🤵🏻‍♂️" | "👸🏻" | "🤴🏻" | "🥷🏻" | "🦸🏻‍♀️" | "🦸🏻" | "🦸🏻‍♂️" | "🦹🏻‍♀️" | "🦹🏻" | "🦹🏻‍♂️" | "🤶🏻" | "🧑🏻‍🎄" | "🎅🏻" | "🧙🏻‍♀️" | "🧙🏻" | "🧙🏻‍♂️" | "🧝🏻‍♀️" | "🧝🏻" | "🧝🏻‍♂️" | "🧛🏻‍♀️" | "🧛🏻" | "🧛🏻‍♂️" | "🧜🏻‍♀️" | "🧜🏻" | "🧜🏻‍♂️" | "🧚🏻‍♀️" | "🧚🏻" | "🧚🏻‍♂️" | "👼🏻" | "🤰🏻" | "🤱🏻" | "👩🏻‍🍼" | "🧑🏻‍🍼" | "👨🏻‍🍼" | "🙇🏻‍♀️" | "🙇🏻" | "🙇🏻‍♂️" | "💁🏻‍♀️" | "💁🏻" | "💁🏻‍♂️" | "🙅🏻‍♀️" | "🙅🏻" | "🙅🏻‍♂️" | "🙆🏻‍♀️" | "🙆🏻" | "🙆🏻‍♂️" | "🙋🏻‍♀️" | "🙋🏻" | "🙋🏻‍♂️" | "🧏🏻‍♀️" | "🧏🏻" | "🧏🏻‍♂️" | "🤦🏻‍♀️" | "🤦🏻" | "🤦🏻‍♂️" | "🤷🏻‍♀️" | "🤷🏻" | "🤷🏻‍♂️" | "🙎🏻‍♀️" | "🙎🏻" | "🙎🏻‍♂️" | "🙍🏻‍♀️" | "🙍🏻" | "🙍🏻‍♂️" | "💇🏻‍♀️" | "💇🏻" | "💇🏻‍♂️" | "💆🏻‍♀️" | "💆🏻" | "💆🏻‍♂️" | "🧖🏻‍♀️" | "🧖🏻" | "🧖🏻‍♂️" | "💃🏻" | "🕺🏻" | "🕴🏻" | "👩🏻‍🦽" | "🧑🏻‍🦽" | "👨🏻‍🦽" | "👩🏻‍🦼" | "🧑🏻‍🦼" | "👨🏻‍🦼" | "🚶🏻‍♀️" | "🚶🏻" | "🚶🏻‍♂️" | "👩🏻‍🦯" | "🧑🏻‍🦯" | "👨🏻‍🦯" | "🧎🏻‍♀️" | "🧎🏻" | "🧎🏻‍♂️" | "🏃🏻‍♀️" | "🏃🏻" | "🏃🏻‍♂️" | "🧍🏻‍♀️" | "🧍🏻" | "🧍🏻‍♂️" | "👭🏻" | "🧑🏻‍🤝‍🧑🏻" | "👬🏻" | "👫🏻" | "🧗🏻‍♀️" | "🧗🏻" | "🧗🏻‍♂️" | "🏇🏻" | "🏂🏻" | "🏌🏻‍♀️" | "🏌🏻" | "🏌🏻‍♂️" | "🏄🏻‍♀️" | "🏄🏻" | "🏄🏻‍♂️" | "🚣🏻‍♀️" | "🚣🏻" | "🚣🏻‍♂️" | "🏊🏻‍♀️" | "🏊🏻" | "🏊🏻‍♂️" | "⛹🏻‍♀️" | "⛹🏻" | "⛹🏻‍♂️" | "🏋🏻‍♀️" | "🏋🏻" | "🏋🏻‍♂️" | "🚴🏻‍♀️" | "🚴🏻" | "🚴🏻‍♂️" | "🚵🏻‍♀️" | "🚵🏻" | "🚵🏻‍♂️" | "🤸🏻‍♀️" | "🤸🏻" | "🤸🏻‍♂️" | "🤽🏻‍♀️" | "🤽🏻" | "🤽🏻‍♂️" | "🤾🏻‍♀️" | "🤾🏻" | "🤾🏻‍♂️" | "🤹🏻‍♀️" | "🤹🏻" | "🤹🏻‍♂️" | "🧘🏻‍♀️" | "🧘🏻" | "🧘🏻‍♂️" | "🛀🏻" | "🛌" | "👋🏼" | "🤚🏼" | "🖐🏼" | "✋🏼" | "🖖🏼" | "👌🏼" | "🤌🏼" | "🤏🏼" | "✌🏼" | "🤞🏼" | "🤟🏼" | "🤘🏼" | "🤙🏼" | "👈🏼" | "👉🏼" | "👆🏼" | "🖕🏼" | "👇🏼" | "☝🏼" | "👍🏼" | "👎🏼" | "✊🏼" | "👊🏼" | "🤛🏼" | "🤜🏼" | "👏🏼" | "🙌🏼" | "👐🏼" | "🤲🏼" | "🙏🏼" | "✍🏼" | "💅🏼" | "🤳🏼" | "💪🏼" | "🦵🏼" | "🦶🏼" | "👂🏼" | "🦻🏼" | "👃🏼" | "👶🏼" | "👧🏼" | "🧒🏼" | "👦🏼" | "👩🏼" | "🧑🏼" | "👨🏼" | "👩🏼‍🦱" | "🧑🏼‍🦱" | "👨🏼‍🦱" | "👩🏼‍🦰" | "🧑🏼‍🦰" | "👨🏼‍🦰" | "👱🏼‍♀️" | "👱🏼" | "👱🏼‍♂️" | "👩🏼‍🦳" | "🧑🏼‍🦳" | "👨🏼‍🦳" | "👩🏼‍🦲" | "🧑🏼‍🦲" | "👨🏼‍🦲" | "🧔🏼" | "👵🏼" | "🧓🏼" | "👴🏼" | "👲🏼" | "👳🏼‍♀️" | "👳🏼" | "👳🏼‍♂️" | "🧕🏼" | "👮🏼‍♀️" | "👮🏼" | "👮🏼‍♂️" | "👷🏼‍♀️" | "👷🏼" | "👷🏼‍♂️" | "💂🏼‍♀️" | "💂🏼" | "💂🏼‍♂️" | "🕵🏼‍♀️" | "🕵🏼" | "🕵🏼‍♂️" | "👩🏼‍⚕️" | "🧑🏼‍⚕️" | "👨🏼‍⚕️" | "👩🏼‍🌾" | "🧑🏼‍🌾" | "👨🏼‍🌾" | "👩🏼‍🍳" | "🧑🏼‍🍳" | "👨🏼‍🍳" | "👩🏼‍🎓" | "🧑🏼‍🎓" | "👨🏼‍🎓" | "👩🏼‍🎤" | "🧑🏼‍🎤" | "👨🏼‍🎤" | "👩🏼‍🏫" | "🧑🏼‍🏫" | "👨🏼‍🏫" | "👩🏼‍🏭" | "🧑🏼‍🏭" | "👨🏼‍🏭" | "👩🏼‍💻" | "🧑🏼‍💻" | "👨🏼‍💻" | "👩🏼‍💼" | "🧑🏼‍💼" | "👨🏼‍💼" | "👩🏼‍🔧" | "🧑🏼‍🔧" | "👨🏼‍🔧" | "👩🏼‍🔬" | "🧑🏼‍🔬" | "👨🏼‍🔬" | "👩🏼‍🎨" | "🧑🏼‍🎨" | "👨🏼‍🎨" | "👩🏼‍🚒" | "🧑🏼‍🚒" | "👨🏼‍🚒" | "👩🏼‍✈️" | "🧑🏼‍✈️" | "👨🏼‍✈️" | "👩🏼‍🚀" | "🧑🏼‍🚀" | "👨🏼‍🚀" | "👩🏼‍⚖️" | "🧑🏼‍⚖️" | "👨🏼‍⚖️" | "👰🏼‍♀️" | "👰🏼" | "👰🏼‍♂️" | "🤵🏼‍♀️" | "🤵🏼" | "🤵🏼‍♂️" | "👸🏼" | "🤴🏼" | "🥷🏼" | "🦸🏼‍♀️" | "🦸🏼" | "🦸🏼‍♂️" | "🦹🏼‍♀️" | "🦹🏼" | "🦹🏼‍♂️" | "🤶🏼" | "🧑🏼‍🎄" | "🎅🏼" | "🧙🏼‍♀️" | "🧙🏼" | "🧙🏼‍♂️" | "🧝🏼‍♀️" | "🧝🏼" | "🧝🏼‍♂️" | "🧛🏼‍♀️" | "🧛🏼" | "🧛🏼‍♂️" | "🧜🏼‍♀️" | "🧜🏼" | "🧜🏼‍♂️" | "🧚🏼‍♀️" | "🧚🏼" | "🧚🏼‍♂️" | "👼🏼" | "🤰🏼" | "🤱🏼" | "👩🏼‍🍼" | "🧑🏼‍🍼" | "👨🏼‍🍼" | "🙇🏼‍♀️" | "🙇🏼" | "🙇🏼‍♂️" | "💁🏼‍♀️" | "💁🏼" | "💁🏼‍♂️" | "🙅🏼‍♀️" | "🙅🏼" | "🙅🏼‍♂️" | "🙆🏼‍♀️" | "🙆🏼" | "🙆🏼‍♂️" | "🙋🏼‍♀️" | "🙋🏼" | "🙋🏼‍♂️" | "🧏🏼‍♀️" | "🧏🏼" | "🧏🏼‍♂️" | "🤦🏼‍♀️" | "🤦🏼" | "🤦🏼‍♂️" | "🤷🏼‍♀️" | "🤷🏼" | "🤷🏼‍♂️" | "🙎🏼‍♀️" | "🙎🏼" | "🙎🏼‍♂️" | "🙍🏼‍♀️" | "🙍🏼" | "🙍🏼‍♂️" | "💇🏼‍♀️" | "💇🏼" | "💇🏼‍♂️" | "💆🏼‍♀️" | "💆🏼" | "💆🏼‍♂️" | "🧖🏼‍♀️" | "🧖🏼" | "🧖🏼‍♂️" | "💃🏼" | "🕺🏼" | "🕴🏼" | "👩🏼‍🦽" | "🧑🏼‍🦽" | "👨🏼‍🦽" | "👩🏼‍🦼" | "🧑🏼‍🦼" | "👨🏼‍🦼" | "🚶🏼‍♀️" | "🚶🏼" | "🚶🏼‍♂️" | "👩🏼‍🦯" | "🧑🏼‍🦯" | "👨🏼‍🦯" | "🧎🏼‍♀️" | "🧎🏼" | "🧎🏼‍♂️" | "🏃🏼‍♀️" | "🏃🏼" | "🏃🏼‍♂️" | "🧍🏼‍♀️" | "🧍🏼" | "🧍🏼‍♂️" | "👭🏼" | "🧑🏼‍🤝‍🧑🏼" | "👬🏼" | "👫🏼" | "🧗🏼‍♀️" | "🧗🏼" | "🧗🏼‍♂️" | "🏇🏼" | "🏂🏼" | "🏌🏼‍♀️" | "🏌🏼" | "🏌🏼‍♂️" | "🏄🏼‍♀️" | "🏄🏼" | "🏄🏼‍♂️" | "🚣🏼‍♀️" | "🚣🏼" | "🚣🏼‍♂️" | "🏊🏼‍♀️" | "🏊🏼" | "🏊🏼‍♂️" | "⛹🏼‍♀️" | "⛹🏼" | "⛹🏼‍♂️" | "🏋🏼‍♀️" | "🏋🏼" | "🏋🏼‍♂️" | "🚴🏼‍♀️" | "🚴🏼" | "🚴🏼‍♂️" | "🚵🏼‍♀️" | "🚵🏼" | "🚵🏼‍♂️" | "🤸🏼‍♀️" | "🤸🏼" | "🤸🏼‍♂️" | "🤽🏼‍♀️" | "🤽🏼" | "🤽🏼‍♂️" | "🤾🏼‍♀️" | "🤾🏼" | "🤾🏼‍♂️" | "🤹🏼‍♀️" | "🤹🏼" | "🤹🏼‍♂️" | "🧘🏼‍♀️" | "🧘🏼" | "🧘🏼‍♂️" | "🛀🏼" | "👋🏽" | "🤚🏽" | "🖐🏽" | "✋🏽" | "🖖🏽" | "👌🏽" | "🤌🏽" | "🤏🏽" | "✌🏽" | "🤞🏽" | "🤟🏽" | "🤘🏽" | "🤙🏽" | "👈🏽" | "👉🏽" | "👆🏽" | "🖕🏽" | "👇🏽" | "☝🏽" | "👍🏽" | "👎🏽" | "✊🏽" | "👊🏽" | "🤛🏽" | "🤜🏽" | "👏🏽" | "🙌🏽" | "👐🏽" | "🤲🏽" | "🙏🏽" | "✍🏽" | "💅🏽" | "🤳🏽" | "💪🏽" | "🦵🏽" | "🦶🏽" | "👂🏽" | "🦻🏽" | "👃🏽" | "👶🏽" | "👧🏽" | "🧒🏽" | "👦🏽" | "👩🏽" | "🧑🏽" | "👨🏽" | "👩🏽‍🦱" | "🧑🏽‍🦱" | "👨🏽‍🦱" | "👩🏽‍🦰" | "🧑🏽‍🦰" | "👨🏽‍🦰" | "👱🏽‍♀️" | "👱🏽" | "👱🏽‍♂️" | "👩🏽‍🦳" | "🧑🏽‍🦳" | "👨🏽‍🦳" | "👩🏽‍🦲" | "🧑🏽‍🦲" | "👨🏽‍🦲" | "🧔🏽" | "👵🏽" | "🧓🏽" | "👴🏽" | "👲🏽" | "👳🏽‍♀️" | "👳🏽" | "👳🏽‍♂️" | "🧕🏽" | "👮🏽‍♀️" | "👮🏽" | "👮🏽‍♂️" | "👷🏽‍♀️" | "👷🏽" | "👷🏽‍♂️" | "💂🏽‍♀️" | "💂🏽" | "💂🏽‍♂️" | "🕵🏽‍♀️" | "🕵🏽" | "🕵🏽‍♂️" | "👩🏽‍⚕️" | "🧑🏽‍⚕️" | "👨🏽‍⚕️" | "👩🏽‍🌾" | "🧑🏽‍🌾" | "👨🏽‍🌾" | "👩🏽‍🍳" | "🧑🏽‍🍳" | "👨🏽‍🍳" | "👩🏽‍🎓" | "🧑🏽‍🎓" | "👨🏽‍🎓" | "👩🏽‍🎤" | "🧑🏽‍🎤" | "👨🏽‍🎤" | "👩🏽‍🏫" | "🧑🏽‍🏫" | "👨🏽‍🏫" | "👩🏽‍🏭" | "🧑🏽‍🏭" | "👨🏽‍🏭" | "👩🏽‍💻" | "🧑🏽‍💻" | "👨🏽‍💻" | "👩🏽‍💼" | "🧑🏽‍💼" | "👨🏽‍💼" | "👩🏽‍🔧" | "🧑🏽‍🔧" | "👨🏽‍🔧" | "👩🏽‍🔬" | "🧑🏽‍🔬" | "👨🏽‍🔬" | "👩🏽‍🎨" | "🧑🏽‍🎨" | "👨🏽‍🎨" | "👩🏽‍🚒" | "🧑🏽‍🚒" | "👨🏽‍🚒" | "👩🏽‍✈️" | "🧑🏽‍✈️" | "👨🏽‍✈️" | "👩🏽‍🚀" | "🧑🏽‍🚀" | "👨🏽‍🚀" | "👩🏽‍⚖️" | "🧑🏽‍⚖️" | "👨🏽‍⚖️" | "👰🏽‍♀️" | "👰🏽" | "👰🏽‍♂️" | "🤵🏽‍♀️" | "🤵🏽" | "🤵🏽‍♂️" | "👸🏽" | "🤴🏽" | "🥷🏽" | "🦸🏽‍♀️" | "🦸🏽" | "🦸🏽‍♂️" | "🦹🏽‍♀️" | "🦹🏽" | "🦹🏽‍♂️" | "🤶🏽" | "🧑🏽‍🎄" | "🎅🏽" | "🧙🏽‍♀️" | "🧙🏽" | "🧙🏽‍♂️" | "🧝🏽‍♀️" | "🧝🏽" | "🧝🏽‍♂️" | "🧛🏽‍♀️" | "🧛🏽" | "🧛🏽‍♂️" | "🧜🏽‍♀️" | "🧜🏽" | "🧜🏽‍♂️" | "🧚🏽‍♀️" | "🧚🏽" | "🧚🏽‍♂️" | "👼🏽" | "🤰🏽" | "🤱🏽" | "👩🏽‍🍼" | "🧑🏽‍🍼" | "👨🏽‍🍼" | "🙇🏽‍♀️" | "🙇🏽" | "🙇🏽‍♂️" | "💁🏽‍♀️" | "💁🏽" | "💁🏽‍♂️" | "🙅🏽‍♀️" | "🙅🏽" | "🙅🏽‍♂️" | "🙆🏽‍♀️" | "🙆🏽" | "🙆🏽‍♂️" | "🙋🏽‍♀️" | "🙋🏽" | "🙋🏽‍♂️" | "🧏🏽‍♀️" | "🧏🏽" | "🧏🏽‍♂️" | "🤦🏽‍♀️" | "🤦🏽" | "🤦🏽‍♂️" | "🤷🏽‍♀️" | "🤷🏽" | "🤷🏽‍♂️" | "🙎🏽‍♀️" | "🙎🏽" | "🙎🏽‍♂️" | "🙍🏽‍♀️" | "🙍🏽" | "🙍🏽‍♂️" | "💇🏽‍♀️" | "💇🏽" | "💇🏽‍♂️" | "💆🏽‍♀️" | "💆🏽" | "💆🏽‍♂️" | "🧖🏽‍♀️" | "🧖🏽" | "🧖🏽‍♂️" | "💃🏽" | "🕺🏽" | "🕴🏽" | "👩🏽‍🦽" | "🧑🏽‍🦽" | "👨🏽‍🦽" | "👩🏽‍🦼" | "🧑🏽‍🦼" | "👨🏽‍🦼" | "🚶🏽‍♀️" | "🚶🏽" | "🚶🏽‍♂️" | "👩🏽‍🦯" | "🧑🏽‍🦯" | "👨🏽‍🦯" | "🧎🏽‍♀️" | "🧎🏽" | "🧎🏽‍♂️" | "🏃🏽‍♀️" | "🏃🏽" | "🏃🏽‍♂️" | "🧍🏽‍♀️" | "🧍🏽" | "🧍🏽‍♂️" | "👭🏽" | "🧑🏽‍🤝‍🧑🏽" | "👬🏽" | "👫🏽" | "🧗🏽‍♀️" | "🧗🏽" | "🧗🏽‍♂️" | "🏇🏽" | "🏂🏽" | "🏌🏽‍♀️" | "🏌🏽" | "🏌🏽‍♂️" | "🏄🏽‍♀️" | "🏄🏽" | "🏄🏽‍♂️" | "🚣🏽‍♀️" | "🚣🏽" | "🚣🏽‍♂️" | "🏊🏽‍♀️" | "🏊🏽" | "🏊🏽‍♂️" | "⛹🏽‍♀️" | "⛹🏽" | "⛹🏽‍♂️" | "🏋🏽‍♀️" | "🏋🏽" | "🏋🏽‍♂️" | "🚴🏽‍♀️" | "🚴🏽" | "🚴🏽‍♂️" | "🚵🏽‍♀️" | "🚵🏽" | "🚵🏽‍♂️" | "🤸🏽‍♀️" | "🤸🏽" | "🤸🏽‍♂️" | "🤽🏽‍♀️" | "🤽🏽" | "🤽🏽‍♂️" | "🤾🏽‍♀️" | "🤾🏽" | "🤾🏽‍♂️" | "🤹🏽‍♀️" | "🤹🏽" | "🤹🏽‍♂️" | "🧘🏽‍♀️" | "🧘🏽" | "🧘🏽‍♂️" | "🛀🏽" | "👋🏾" | "🤚🏾" | "🖐🏾" | "✋🏾" | "🖖🏾" | "👌🏾" | "🤌🏾" | "🤏🏾" | "✌🏾" | "🤞🏾" | "🤟🏾" | "🤘🏾" | "🤙🏾" | "👈🏾" | "👉🏾" | "👆🏾" | "🖕🏾" | "👇🏾" | "☝🏾" | "👍🏾" | "👎🏾" | "✊🏾" | "👊🏾" | "🤛🏾" | "🤜🏾" | "👏🏾" | "🙌🏾" | "👐🏾" | "🤲🏾" | "🙏🏾" | "✍🏾" | "💅🏾" | "🤳🏾" | "💪🏾" | "🦵🏾" | "🦶🏾" | "👂🏾" | "🦻🏾" | "👃🏾" | "👶🏾" | "👧🏾" | "🧒🏾" | "👦🏾" | "👩🏾" | "🧑🏾" | "👨🏾" | "👩🏾‍🦱" | "🧑🏾‍🦱" | "👨🏾‍🦱" | "👩🏾‍🦰" | "🧑🏾‍🦰" | "👨🏾‍🦰" | "👱🏾‍♀️" | "👱🏾" | "👱🏾‍♂️" | "👩🏾‍🦳" | "🧑🏾‍🦳" | "👨🏾‍🦳" | "👩🏾‍🦲" | "🧑🏾‍🦲" | "👨🏾‍🦲" | "🧔🏾" | "👵🏾" | "🧓🏾" | "👴🏾" | "👲🏾" | "👳🏾‍♀️" | "👳🏾" | "👳🏾‍♂️" | "🧕🏾" | "👮🏾‍♀️" | "👮🏾" | "👮🏾‍♂️" | "👷🏾‍♀️" | "👷🏾" | "👷🏾‍♂️" | "💂🏾‍♀️" | "💂🏾" | "💂🏾‍♂️" | "🕵🏾‍♀️" | "🕵🏾" | "🕵🏾‍♂️" | "👩🏾‍⚕️" | "🧑🏾‍⚕️" | "👨🏾‍⚕️" | "👩🏾‍🌾" | "🧑🏾‍🌾" | "👨🏾‍🌾" | "👩🏾‍🍳" | "🧑🏾‍🍳" | "👨🏾‍🍳" | "👩🏾‍🎓" | "🧑🏾‍🎓" | "👨🏾‍🎓" | "👩🏾‍🎤" | "🧑🏾‍🎤" | "👨🏾‍🎤" | "👩🏾‍🏫" | "🧑🏾‍🏫" | "👨🏾‍🏫" | "👩🏾‍🏭" | "🧑🏾‍🏭" | "👨🏾‍🏭" | "👩🏾‍💻" | "🧑🏾‍💻" | "👨🏾‍💻" | "👩🏾‍💼" | "🧑🏾‍💼" | "👨🏾‍💼" | "👩🏾‍🔧" | "🧑🏾‍🔧" | "👨🏾‍🔧" | "👩🏾‍🔬" | "🧑🏾‍🔬" | "👨🏾‍🔬" | "👩🏾‍🎨" | "🧑🏾‍🎨" | "👨🏾‍🎨" | "👩🏾‍🚒" | "🧑🏾‍🚒" | "👨🏾‍🚒" | "👩🏾‍✈️" | "🧑🏾‍✈️" | "👨🏾‍✈️" | "👩🏾‍🚀" | "🧑🏾‍🚀" | "👨🏾‍🚀" | "👩🏾‍⚖️" | "🧑🏾‍⚖️" | "👨🏾‍⚖️" | "👰🏾‍♀️" | "👰🏾" | "👰🏾‍♂️" | "🤵🏾‍♀️" | "🤵🏾" | "🤵🏾‍♂️" | "👸🏾" | "🤴🏾" | "🥷🏾" | "🦸🏾‍♀️" | "🦸🏾" | "🦸🏾‍♂️" | "🦹🏾‍♀️" | "🦹🏾" | "🦹🏾‍♂️" | "🤶🏾" | "🧑🏾‍🎄" | "🎅🏾" | "🧙🏾‍♀️" | "🧙🏾" | "🧙🏾‍♂️" | "🧝🏾‍♀️" | "🧝🏾" | "🧝🏾‍♂️" | "🧛🏾‍♀️" | "🧛🏾" | "🧛🏾‍♂️" | "🧜🏾‍♀️" | "🧜🏾" | "🧜🏾‍♂️" | "🧚🏾‍♀️" | "🧚🏾" | "🧚🏾‍♂️" | "👼🏾" | "🤰🏾" | "🤱🏾" | "👩🏾‍🍼" | "🧑🏾‍🍼" | "👨🏾‍🍼" | "🙇🏾‍♀️" | "🙇🏾" | "🙇🏾‍♂️" | "💁🏾‍♀️" | "💁🏾" | "💁🏾‍♂️" | "🙅🏾‍♀️" | "🙅🏾" | "🙅🏾‍♂️" | "🙆🏾‍♀️" | "🙆🏾" | "🙆🏾‍♂️" | "🙋🏾‍♀️" | "🙋🏾" | "🙋🏾‍♂️" | "🧏🏾‍♀️" | "🧏🏾" | "🧏🏾‍♂️" | "🤦🏾‍♀️" | "🤦🏾" | "🤦🏾‍♂️" | "🤷🏾‍♀️" | "🤷🏾" | "🤷🏾‍♂️" | "🙎🏾‍♀️" | "🙎🏾" | "🙎🏾‍♂️" | "🙍🏾‍♀️" | "🙍🏾" | "🙍🏾‍♂️" | "💇🏾‍♀️" | "💇🏾" | "💇🏾‍♂️" | "💆🏾‍♀️" | "💆🏾" | "💆🏾‍♂️" | "🧖🏾‍♀️" | "🧖🏾" | "🧖🏾‍♂️" | "💃🏾" | "🕺🏾" | "🕴🏿" | "👩🏾‍🦽" | "🧑🏾‍🦽" | "👨🏾‍🦽" | "👩🏾‍🦼" | "🧑🏾‍🦼" | "👨🏾‍🦼" | "🚶🏾‍♀️" | "🚶🏾" | "🚶🏾‍♂️" | "👩🏾‍🦯" | "🧑🏾‍🦯" | "👨🏾‍🦯" | "🧎🏾‍♀️" | "🧎🏾" | "🧎🏾‍♂️" | "🏃🏾‍♀️" | "🏃🏾" | "🏃🏾‍♂️" | "🧍🏾‍♀️" | "🧍🏾" | "🧍🏾‍♂️" | "👭🏾" | "🧑🏾‍🤝‍🧑🏾" | "👬🏾" | "👫🏾" | "🧗🏾‍♀️" | "🧗🏾" | "🧗🏾‍♂️" | "🏇🏾" | "🏂🏾" | "🏌🏾‍♀️" | "🏌🏾" | "🏌🏾‍♂️" | "🏄🏾‍♀️" | "🏄🏾" | "🏄🏾‍♂️" | "🚣🏾‍♀️" | "🚣🏾" | "🚣🏾‍♂️" | "🏊🏾‍♀️" | "🏊🏾" | "🏊🏾‍♂️" | "⛹🏾‍♀️" | "⛹🏾" | "⛹🏾‍♂️" | "🏋🏾‍♀️" | "🏋🏾" | "🏋🏾‍♂️" | "🚴🏾‍♀️" | "🚴🏾" | "🚴🏾‍♂️" | "🚵🏾‍♀️" | "🚵🏾" | "🚵🏾‍♂️" | "🤸🏾‍♀️" | "🤸🏾" | "🤸🏾‍♂️" | "🤽🏾‍♀️" | "🤽🏾" | "🤽🏾‍♂️" | "🤾🏾‍♀️" | "🤾🏾" | "🤾🏾‍♂️" | "🤹🏾‍♀️" | "🤹🏾" | "🤹🏾‍♂️" | "🧘🏾‍♀️" | "🧘🏾" | "🧘🏾‍♂️" | "🛀🏾" | "👋🏿" | "🤚🏿" | "🖐🏿" | "✋🏿" | "🖖🏿" | "👌🏿" | "🤌🏿" | "🤏🏿" | "✌🏿" | "🤞🏿" | "🤟🏿" | "🤘🏿" | "🤙🏿" | "👈🏿" | "👉🏿" | "👆🏿" | "🖕🏿" | "👇🏿" | "☝🏿" | "👍🏿" | "👎🏿" | "✊🏿" | "👊🏿" | "🤛🏿" | "🤜🏿" | "👏🏿" | "🙌🏿" | "👐🏿" | "🤲🏿" | "🙏🏿" | "✍🏿" | "💅🏿" | "🤳🏿" | "💪🏿" | "🦵🏿" | "🦶🏿" | "👂🏿" | "🦻🏿" | "👃🏿" | "👶🏿" | "👧🏿" | "🧒🏿" | "👦🏿" | "👩🏿" | "🧑🏿" | "👨🏿" | "👩🏿‍🦱" | "🧑🏿‍🦱" | "👨🏿‍🦱" | "👩🏿‍🦰" | "🧑🏿‍🦰" | "👨🏿‍🦰" | "👱🏿‍♀️" | "👱🏿" | "👱🏿‍♂️" | "👩🏿‍🦳" | "🧑🏿‍🦳" | "👨🏿‍🦳" | "👩🏿‍🦲" | "🧑🏿‍🦲" | "👨🏿‍🦲" | "🧔🏿" | "👵🏿" | "🧓🏿" | "👴🏿" | "👲🏿" | "👳🏿‍♀️" | "👳🏿" | "👳🏿‍♂️" | "🧕🏿" | "👮🏿‍♀️" | "👮🏿" | "👮🏿‍♂️" | "👷🏿‍♀️" | "👷🏿" | "👷🏿‍♂️" | "💂🏿‍♀️" | "💂🏿" | "💂🏿‍♂️" | "🕵🏿‍♀️" | "🕵🏿" | "🕵🏿‍♂️" | "👩🏿‍⚕️" | "🧑🏿‍⚕️" | "👨🏿‍⚕️" | "👩🏿‍🌾" | "🧑🏿‍🌾" | "👨🏿‍🌾" | "👩🏿‍🍳" | "🧑🏿‍🍳" | "👨🏿‍🍳" | "👩🏿‍🎓" | "🧑🏿‍🎓" | "👨🏿‍🎓" | "👩🏿‍🎤" | "🧑🏿‍🎤" | "👨🏿‍🎤" | "👩🏿‍🏫" | "🧑🏿‍🏫" | "👨🏿‍🏫" | "👩🏿‍🏭" | "🧑🏿‍🏭" | "👨🏿‍🏭" | "👩🏿‍💻" | "🧑🏿‍💻" | "👨🏿‍💻" | "👩🏿‍💼" | "🧑🏿‍💼" | "👨🏿‍💼" | "👩🏿‍🔧" | "🧑🏿‍🔧" | "👨🏿‍🔧" | "👩🏿‍🔬" | "🧑🏿‍🔬" | "👨🏿‍🔬" | "👩🏿‍🎨" | "🧑🏿‍🎨" | "👨🏿‍🎨" | "👩🏿‍🚒" | "🧑🏿‍🚒" | "👨🏿‍🚒" | "👩🏿‍✈️" | "🧑🏿‍✈️" | "👨🏿‍✈️" | "👩🏿‍🚀" | "🧑🏿‍🚀" | "👨🏿‍🚀" | "👩🏿‍⚖️" | "🧑🏿‍⚖️" | "👨🏿‍⚖️" | "👰🏿‍♀️" | "👰🏿" | "👰🏿‍♂️" | "🤵🏿‍♀️" | "🤵🏿" | "🤵🏿‍♂️" | "👸🏿" | "🤴🏿" | "🥷🏿" | "🦸🏿‍♀️" | "🦸🏿" | "🦸🏿‍♂️" | "🦹🏿‍♀️" | "🦹🏿" | "🦹🏿‍♂️" | "🤶🏿" | "🧑🏿‍🎄" | "🎅🏿" | "🧙🏿‍♀️" | "🧙🏿" | "🧙🏿‍♂️" | "🧝🏿‍♀️" | "🧝🏿" | "🧝🏿‍♂️" | "🧛🏿‍♀️" | "🧛🏿" | "🧛🏿‍♂️" | "🧜🏿‍♀️" | "🧜🏿" | "🧜🏿‍♂️" | "🧚🏿‍♀️" | "🧚🏿" | "🧚🏿‍♂️" | "👼🏿" | "🤰🏿" | "🤱🏿" | "👩🏿‍🍼" | "🧑🏿‍🍼" | "👨🏿‍🍼" | "🙇🏿‍♀️" | "🙇🏿" | "🙇🏿‍♂️" | "💁🏿‍♀️" | "💁🏿" | "💁🏿‍♂️" | "🙅🏿‍♀️" | "🙅🏿" | "🙅🏿‍♂️" | "🙆🏿‍♀️" | "🙆🏿" | "🙆🏿‍♂️" | "🙋🏿‍♀️" | "🙋🏿" | "🙋🏿‍♂️" | "🧏🏿‍♀️" | "🧏🏿" | "🧏🏿‍♂️" | "🤦🏿‍♀️" | "🤦🏿" | "🤦🏿‍♂️" | "🤷🏿‍♀️" | "🤷🏿" | "🤷🏿‍♂️" | "🙎🏿‍♀️" | "🙎🏿" | "🙎🏿‍♂️" | "🙍🏿‍♀️" | "🙍🏿" | "🙍🏿‍♂️" | "💇🏿‍♀️" | "💇🏿" | "💇🏿‍♂️" | "💆🏿‍♀️" | "💆🏿" | "💆🏿‍♂️" | "🧖🏿‍♀️" | "🧖🏿" | "🧖🏿‍♂️" | "💃🏿" | "🕺🏿" | "👩🏿‍🦽" | "🧑🏿‍🦽" | "👨🏿‍🦽" | "👩🏿‍🦼" | "🧑🏿‍🦼" | "👨🏿‍🦼" | "🚶🏿‍♀️" | "🚶🏿" | "🚶🏿‍♂️" | "👩🏿‍🦯" | "🧑🏿‍🦯" | "👨🏿‍🦯" | "🧎🏿‍♀️" | "🧎🏿" | "🧎🏿‍♂️" | "🏃🏿‍♀️" | "🏃🏿" | "🏃🏿‍♂️" | "🧍🏿‍♀️" | "🧍🏿" | "🧍🏿‍♂️" | "👭🏿" | "🧑🏿‍🤝‍🧑🏿" | "👬🏿" | "👫🏿" | "🧗🏿‍♀️" | "🧗🏿" | "🧗🏿‍♂️" | "🏇🏿" | "🏂🏿" | "🏌🏿‍♀️" | "🏌🏿" | "🏌🏿‍♂️" | "🏄🏿‍♀️" | "🏄🏿" | "🏄🏿‍♂️" | "🚣🏿‍♀️" | "🚣🏿" | "🚣🏿‍♂️" | "🏊🏿‍♀️" | "🏊🏿" | "🏊🏿‍♂️" | "⛹🏿‍♀️" | "⛹🏿" | "⛹🏿‍♂️" | "🏋🏿‍♀️" | "🏋🏿" | "🏋🏿‍♂️" | "🚴🏿‍♀️" | "🚴🏿" | "🚴🏿‍♂️" | "🚵🏿‍♀️" | "🚵🏿" | "🚵🏿‍♂️" | "🤸🏿‍♀️" | "🤸🏿" | "🤸🏿‍♂️" | "🤽🏿‍♀️" | "🤽🏿" | "🤽🏿‍♂️" | "🤾🏿‍♀️" | "🤾🏿" | "🤾🏿‍♂️" | "🤹🏿‍♀️" | "🤹🏿" | "🤹🏿‍♂️" | "🧘🏿‍♀️" | "🧘🏿" | "🧘🏿‍♂️" | "🛀🏿" | "🛌🏿" | "🐶" | "🐱" | "🐭" | "🐹" | "🐰" | "🦊" | "🐻" | "🐼" | "🐻‍❄️" | "🐨" | "🐯" | "🦁" | "🐮" | "🐷" | "🐽" | "🐸" | "🐵" | "🙈" | "🙉" | "🙊" | "🐒" | "🐔" | "🐧" | "🐦" | "🐤" | "🐣" | "🐥" | "🦆" | "🦅" | "🦉" | "🦇" | "🐺" | "🐗" | "🐴" | "🦄" | "🐝" | "🪱" | "🐛" | "🦋" | "🐌" | "🐞" | "🐜" | "🪰" | "🪲" | "🪳" | "🦟" | "🦗" | "🕷" | "🕸" | "🦂" | "🐢" | "🐍" | "🦎" | "🦖" | "🦕" | "🐙" | "🦑" | "🦐" | "🦞" | "🦀" | "🐡" | "🐠" | "🐟" | "🐬" | "🐳" | "🐋" | "🦈" | "🐊" | "🐅" | "🐆" | "🦓" | "🦍" | "🦧" | "🦣" | "🐘" | "🦛" | "🦏" | "🐪" | "🐫" | "🦒" | "🦘" | "🦬" | "🐃" | "🐂" | "🐄" | "🐎" | "🐖" | "🐏" | "🐑" | "🦙" | "🐐" | "🦌" | "🐕" | "🐩" | "🦮" | "🐕‍🦺" | "🐈" | "🐈‍⬛" | "🪶" | "🐓" | "🦃" | "🦤" | "🦚" | "🦜" | "🦢" | "🦩" | "🕊" | "🐇" | "🦝" | "🦨" | "🦡" | "🦫" | "🦦" | "🦥" | "🐁" | "🐀" | "🐿" | "🦔" | "🐾" | "🐉" | "🐲" | "🌵" | "🎄" | "🌲" | "🌳" | "🌴" | "🪵" | "🌱" | "🌿" | "☘️" | "🍀" | "🎍" | "🪴" | "🎋" | "🍃" | "🍂" | "🍁" | "🍄" | "🐚" | "🪨" | "🌾" | "💐" | "🌷" | "🌹" | "🥀" | "🌺" | "🌸" | "🌼" | "🌻" | "🌞" | "🌝" | "🌛" | "🌜" | "🌚" | "🌕" | "🌖" | "🌗" | "🌘" | "🌑" | "🌒" | "🌓" | "🌔" | "🌙" | "🌎" | "🌍" | "🌏" | "🪐" | "💫" | "⭐️" | "🌟" | "✨" | "⚡️" | "☄️" | "💥" | "🔥" | "🌪" | "🌈" | "☀️" | "🌤" | "⛅️" | "🌥" | "☁️" | "🌦" | "🌧" | "⛈" | "🌩" | "🌨" | "❄️" | "☃️" | "⛄️" | "🌬" | "💨" | "💧" | "💦" | "☔️" | "🌊" | "🌫" | "🍏" | "🍎" | "🍐" | "🍊" | "🍋" | "🍌" | "🍉" | "🍇" | "🍓" | "🫐" | "🍈" | "🍒" | "🍑" | "🥭" | "🍍" | "🥥" | "🥝" | "🍅" | "🍆" | "🥑" | "🥦" | "🥬" | "🥒" | "🌶" | "🫑" | "🌽" | "🥕" | "🫒" | "🧄" | "🧅" | "🥔" | "🍠" | "🥐" | "🥯" | "🍞" | "🥖" | "🥨" | "🧀" | "🥚" | "🍳" | "🧈" | "🥞" | "🧇" | "🥓" | "🥩" | "🍗" | "🍖" | "🌭" | "🍔" | "🍟" | "🍕" | "🫓" | "🥪" | "🥙" | "🧆" | "🌮" | "🌯" | "🫔" | "🥗" | "🥘" | "🫕" | "🥫" | "🍝" | "🍜" | "🍲" | "🍛" | "🍣" | "🍱" | "🥟" | "🦪" | "🍤" | "🍙" | "🍚" | "🍘" | "🍥" | "🥠" | "🥮" | "🍢" | "🍡" | "🍧" | "🍨" | "🍦" | "🥧" | "🧁" | "🍰" | "🎂" | "🍮" | "🍭" | "🍬" | "🍫" | "🍿" | "🍩" | "🍪" | "🌰" | "🥜" | "🍯" | "🥛" | "🍼" | "🫖" | "☕️" | "🍵" | "🧃" | "🥤" | "🧋" | "🍶" | "🍺" | "🍻" | "🥂" | "🍷" | "🥃" | "🍸" | "🍹" | "🧉" | "🍾" | "🧊" | "🥄" | "🍴" | "🍽" | "🥣" | "🥡" | "🥢" | "🧂" | "⚽️" | "🏀" | "🏈" | "⚾️" | "🥎" | "🎾" | "🏐" | "🏉" | "🥏" | "🎱" | "🪀" | "🏓" | "🏸" | "🏒" | "🏑" | "🥍" | "🏏" | "🪃" | "🥅" | "⛳️" | "🪁" | "🏹" | "🎣" | "🤿" | "🥊" | "🥋" | "🎽" | "🛹" | "🛼" | "🛷" | "⛸" | "🥌" | "🎿" | "⛷" | "🏂" | "🪂" | "🏋️‍♀️" | "🏋️" | "🏋️‍♂️" | "🤼‍♀️" | "🤼" | "🤼‍♂️" | "🤸‍♀️" | "🤸" | "🤸‍♂️" | "⛹️‍♀️" | "⛹️" | "⛹️‍♂️" | "🤺" | "🤾‍♀️" | "🤾" | "🤾‍♂️" | "🏌️‍♀️" | "🏌️" | "🏌️‍♂️" | "🏇" | "🧘‍♀️" | "🧘" | "🧘‍♂️" | "🏄‍♀️" | "🏄" | "🏄‍♂️" | "🏊‍♀️" | "🏊" | "🏊‍♂️" | "🤽‍♀️" | "🤽" | "🤽‍♂️" | "🚣‍♀️" | "🚣" | "🚣‍♂️" | "🧗‍♀️" | "🧗" | "🧗‍♂️" | "🚵‍♀️" | "🚵" | "🚵‍♂️" | "🚴‍♀️" | "🚴" | "🚴‍♂️" | "🏆" | "🥇" | "🥈" | "🥉" | "🏅" | "🎖" | "🏵" | "🎗" | "🎫" | "🎟" | "🎪" | "🤹" | "🤹‍♂️" | "🤹‍♀️" | "🎭" | "🎨" | "🎬" | "🎤" | "🎧" | "🎼" | "🎹" | "🥁" | "🪘" | "🎷" | "🎺" | "🪗" | "🎸" | "🪕" | "🎻" | "🎲" | "♟" | "🎯" | "🎳" | "🎮" | "🎰" | "🧩" | "🚗" | "🚕" | "🚙" | "🚌" | "🚎" | "🏎" | "🚓" | "🚑" | "🚒" | "🚐" | "🛻" | "🚚" | "🚛" | "🚜" | "🦯" | "🦽" | "🦼" | "🛴" | "🚲" | "🛵" | "🏍" | "🛺" | "🚨" | "🚔" | "🚍" | "🚘" | "🚖" | "🚡" | "🚠" | "🚟" | "🚃" | "🚋" | "🚞" | "🚝" | "🚄" | "🚅" | "🚈" | "🚂" | "🚆" | "🚇" | "🚊" | "🚉" | "✈️" | "🛫" | "🛬" | "🛩" | "💺" | "🛰" | "🚀" | "🛸" | "🚁" | "🛶" | "⛵️" | "🚤" | "🛥" | "🛳" | "⛴" | "🚢" | "⚓️" | "🪝" | "⛽️" | "🚧" | "🚦" | "🚥" | "🚏" | "🗺" | "🗿" | "🗽" | "🗼" | "🏰" | "🏯" | "🏟" | "🎡" | "🎢" | "🎠" | "⛲️" | "⛱" | "🏖" | "🏝" | "🏜" | "🌋" | "⛰" | "🏔" | "🗻" | "🏕" | "⛺️" | "🛖" | "🏠" | "🏡" | "🏘" | "🏚" | "🏗" | "🏭" | "🏢" | "🏬" | "🏣" | "🏤" | "🏥" | "🏦" | "🏨" | "🏪" | "🏫" | "🏩" | "💒" | "🏛" | "⛪️" | "🕌" | "🕍" | "🛕" | "🕋" | "⛩" | "🛤" | "🛣" | "🗾" | "🎑" | "🏞" | "🌅" | "🌄" | "🌠" | "🎇" | "🎆" | "🌇" | "🌆" | "🏙" | "🌃" | "🌌" | "🌉" | "🌁" | "⌚️" | "📱" | "📲" | "💻" | "⌨️" | "🖥" | "🖨" | "🖱" | "🖲" | "🕹" | "🗜" | "💽" | "💾" | "💿" | "📀" | "📼" | "📷" | "📸" | "📹" | "🎥" | "📽" | "🎞" | "📞" | "☎️" | "📟" | "📠" | "📺" | "📻" | "🎙" | "🎚" | "🎛" | "🧭" | "⏱" | "⏲" | "⏰" | "🕰" | "⌛️" | "⏳" | "📡" | "🔋" | "🔌" | "💡" | "🔦" | "🕯" | "🪔" | "🧯" | "🛢" | "💸" | "💵" | "💴" | "💶" | "💷" | "🪙" | "💰" | "💳" | "💎" | "⚖️" | "🪜" | "🧰" | "🪛" | "🔧" | "🔨" | "⚒" | "🛠" | "⛏" | "🪚" | "🔩" | "⚙️" | "🪤" | "🧱" | "⛓" | "🧲" | "🔫" | "💣" | "🧨" | "🪓" | "🔪" | "🗡" | "⚔️" | "🛡" | "🚬" | "⚰️" | "🪦" | "⚱️" | "🏺" | "🔮" | "📿" | "🧿" | "💈" | "⚗️" | "🔭" | "🔬" | "🕳" | "🩹" | "🩺" | "💊" | "💉" | "🧬" | "🦠" | "🧫" | "🧪" | "🌡" | "🧹" | "🪠" | "🧺" | "🧻" | "🚽" | "🚰" | "🚿" | "🛁" | "🛀" | "🧼" | "🪥" | "🪒" | "🧽" | "🪣" | "🧴" | "🛎" | "🔑" | "🗝" | "🚪" | "🪑" | "🛋" | "🛏" | "🧸" | "🪆" | "🖼" | "🪞" | "🪟" | "🛍" | "🛒" | "🎁" | "🎈" | "🎏" | "🎀" | "🪄" | "🪅" | "🎊" | "🎉" | "🎎" | "🏮" | "🎐" | "🧧" | "✉️" | "📩" | "📨" | "📧" | "💌" | "📥" | "📤" | "📦" | "🏷" | "🪧" | "📪" | "📫" | "📬" | "📭" | "📮" | "📯" | "📜" | "📃" | "📄" | "📑" | "🧾" | "📊" | "📈" | "📉" | "🗒" | "🗓" | "📆" | "📅" | "🗑" | "📇" | "🗃" | "🗳" | "🗄" | "📋" | "📁" | "📂" | "🗂" | "🗞" | "📰" | "📓" | "📔" | "📒" | "📕" | "📗" | "📘" | "📙" | "📚" | "📖" | "🔖" | "🧷" | "🔗" | "📎" | "🖇" | "📐" | "📏" | "🧮" | "📌" | "📍" | "✂️" | "🖊" | "🖋" | "✒️" | "🖌" | "🖍" | "📝" | "✏️" | "🔍" | "🔎" | "🔏" | "🔐" | "🔒" | "🔓" | "❤️" | "🧡" | "💛" | "💚" | "💙" | "💜" | "🖤" | "🤍" | "🤎" | "💔" | "❣️" | "💕" | "💞" | "💓" | "💗" | "💖" | "💘" | "💝" | "💟" | "☮️" | "✝️" | "☪️" | "🕉" | "☸️" | "✡️" | "🔯" | "🕎" | "☯️" | "☦️" | "🛐" | "⛎" | "♈️" | "♉️" | "♊️" | "♋️" | "♌️" | "♍️" | "♎️" | "♏️" | "♐️" | "♑️" | "♒️" | "♓️" | "🆔" | "⚛️" | "🉑" | "☢️" | "☣️" | "📴" | "📳" | "🈶" | "🈚️" | "🈸" | "🈺" | "🈷️" | "✴️" | "🆚" | "💮" | "🉐" | "㊙️" | "㊗️" | "🈴" | "🈵" | "🈹" | "🈲" | "🅰️" | "🅱️" | "🆎" | "🆑" | "🅾️" | "🆘" | "❌" | "⭕️" | "🛑" | "⛔️" | "📛" | "🚫" | "💯" | "💢" | "♨️" | "🚷" | "🚯" | "🚳" | "🚱" | "🔞" | "📵" | "🚭" | "❗️" | "❕" | "❓" | "❔" | "‼️" | "⁉️" | "🔅" | "🔆" | "〽️" | "⚠️" | "🚸" | "🔱" | "⚜️" | "🔰" | "♻️" | "✅" | "🈯️" | "💹" | "❇️" | "✳️" | "❎" | "🌐" | "💠" | "Ⓜ️" | "🌀" | "💤" | "🏧" | "🚾" | "♿️" | "🅿️" | "🛗" | "🈳" | "🈂️" | "🛂" | "🛃" | "🛄" | "🛅" | "🚹" | "🚺" | "🚼" | "⚧" | "🚻" | "🚮" | "🎦" | "📶" | "🈁" | "🔣" | "ℹ️" | "🔤" | "🔡" | "🔠" | "🆖" | "🆗" | "🆙" | "🆒" | "🆕" | "🆓" | "0️⃣" | "1️⃣" | "2️⃣" | "3️⃣" | "4️⃣" | "5️⃣" | "6️⃣" | "7️⃣" | "8️⃣" | "9️⃣" | "🔟" | "🔢" | "#️⃣" | "*️⃣" | "⏏️" | "▶️" | "⏸" | "⏯" | "⏹" | "⏺" | "⏭" | "⏮" | "⏩" | "⏪" | "⏫" | "⏬" | "◀️" | "🔼" | "🔽" | "➡️" | "⬅️" | "⬆️" | "⬇️" | "↗️" | "↘️" | "↙️" | "↖️" | "↕️" | "↔️" | "↪️" | "↩️" | "⤴️" | "⤵️" | "🔀" | "🔁" | "🔂" | "🔄" | "🔃" | "🎵" | "🎶" | "➕" | "➖" | "➗" | "✖️" | "♾" | "💲" | "💱" | "™️" | "©️" | "®️" | "〰️" | "➰" | "➿" | "🔚" | "🔙" | "🔛" | "🔝" | "🔜" | "✔️" | "☑️" | "🔘" | "🔴" | "🟠" | "🟡" | "🟢" | "🔵" | "🟣" | "⚫️" | "⚪️" | "🟤" | "🔺" | "🔻" | "🔸" | "🔹" | "🔶" | "🔷" | "🔳" | "🔲" | "▪️" | "▫️" | "◾️" | "◽️" | "◼️" | "◻️" | "🟥" | "🟧" | "🟨" | "🟩" | "🟦" | "🟪" | "⬛️" | "⬜️" | "🟫" | "🔈" | "🔇" | "🔉" | "🔊" | "🔔" | "🔕" | "📣" | "📢" | "👁‍🗨" | "💬" | "💭" | "🗯" | "♠️" | "♣️" | "♥️" | "♦️" | "🃏" | "🎴" | "🀄️" | "🕐" | "🕑" | "🕒" | "🕓" | "🕔" | "🕕" | "🕖" | "🕗" | "🕘" | "🕙" | "🕚" | "🕛" | "🕜" | "🕝" | "🕞" | "🕟" | "🕠" | "🕡" | "🕢" | "🕣" | "🕤" | "🕥" | "🕦" | "🕧" | "✢" | "✣" | "✤" | "✥" | "✦" | "✧" | "★" | "☆" | "✯" | "✡︎" | "✩" | "✪" | "✫" | "✬" | "✭" | "✮" | "✶" | "✷" | "✵" | "✸" | "✹" | "→" | "⇒" | "⟹" | "⇨" | "⇾" | "➾" | "⇢" | "☛" | "☞" | "➔" | "➜" | "➙" | "➛" | "➝" | "➞" | "♠︎" | "♣︎" | "♥︎" | "♦︎" | "♤" | "♧" | "♡" | "♢" | "♚" | "♛" | "♜" | "♝" | "♞" | "♔" | "♕" | "♖" | "♗" | "♘" | "♙" | "⚀" | "⚁" | "⚂" | "⚃" | "⚄" | "⚅" | "🂠" | "⚈" | "⚉" | "⚆" | "⚇" | "𓀀" | "𓀁" | "𓀂" | "𓀃" | "𓀄" | "𓀅" | "𓀆" | "𓀇" | "𓀈" | "𓀉" | "𓀊" | "𓀋" | "𓀌" | "𓀍" | "𓀎" | "𓀏" | "𓀐" | "𓀑" | "𓀒" | "𓀓" | "𓀔" | "𓀕" | "𓀖" | "𓀗" | "𓀘" | "𓀙" | "𓀚" | "𓀛" | "𓀜" | "𓀝" | "🏳️" | "🏴" | "🏁" | "🚩" | "🏳️‍🌈" | "🏳️‍⚧️" | "🏴‍☠️" | "🇦🇫" | "🇦🇽" | "🇦🇱" | "🇩🇿" | "🇦🇸" | "🇦🇩" | "🇦🇴" | "🇦🇮" | "🇦🇶" | "🇦🇬" | "🇦🇷" | "🇦🇲" | "🇦🇼" | "🇦🇺" | "🇦🇹" | "🇦🇿" | "🇧🇸" | "🇧🇭" | "🇧🇩" | "🇧🇧" | "🇧🇾" | "🇧🇪" | "🇧🇿" | "🇧🇯" | "🇧🇲" | "🇧🇹" | "🇧🇴" | "🇧🇦" | "🇧🇼" | "🇧🇷" | "🇮🇴" | "🇻🇬" | "🇧🇳" | "🇧🇬" | "🇧🇫" | "🇧🇮" | "🇰🇭" | "🇨🇲" | "🇨🇦" | "🇮🇨" | "🇨🇻" | "🇧🇶" | "🇰🇾" | "🇨🇫" | "🇹🇩" | "🇨🇱" | "🇨🇳" | "🇨🇽" | "🇨🇨" | "🇨🇴" | "🇰🇲" | "🇨🇬" | "🇨🇩" | "🇨🇰" | "🇨🇷" | "🇨🇮" | "🇭🇷" | "🇨🇺" | "🇨🇼" | "🇨🇾" | "🇨🇿" | "🇩🇰" | "🇩🇯" | "🇩🇲" | "🇩🇴" | "🇪🇨" | "🇪🇬" | "🇸🇻" | "🇬🇶" | "🇪🇷" | "🇪🇪" | "🇪🇹" | "🇪🇺" | "🇫🇰" | "🇫🇴" | "🇫🇯" | "🇫🇮" | "🇫🇷" | "🇬🇫" | "🇵🇫" | "🇹🇫" | "🇬🇦" | "🇬🇲" | "🇬🇪" | "🇩🇪" | "🇬🇭" | "🇬🇮" | "🇬🇷" | "🇬🇱" | "🇬🇩" | "🇬🇵" | "🇬🇺" | "🇬🇹" | "🇬🇬" | "🇬🇳" | "🇬🇼" | "🇬🇾" | "🇭🇹" | "🇭🇳" | "🇭🇰" | "🇭🇺" | "🇮🇸" | "🇮🇳" | "🇮🇩" | "🇮🇷" | "🇮🇶" | "🇮🇪" | "🇮🇲" | "🇮🇱" | "🇮🇹" | "🇯🇲" | "🇯🇵" | "🎌" | "🇯🇪" | "🇯🇴" | "🇰🇿" | "🇰🇪" | "🇰🇮" | "🇽🇰" | "🇰🇼" | "🇰🇬" | "🇱🇦" | "🇱🇻" | "🇱🇧" | "🇱🇸" | "🇱🇷" | "🇱🇾" | "🇱🇮" | "🇱🇹" | "🇱🇺" | "🇲🇴" | "🇲🇰" | "🇲🇬" | "🇲🇼" | "🇲🇾" | "🇲🇻" | "🇲🇱" | "🇲🇹" | "🇲🇭" | "🇲🇶" | "🇲🇷" | "🇲🇺" | "🇾🇹" | "🇲🇽" | "🇫🇲" | "🇲🇩" | "🇲🇨" | "🇲🇳" | "🇲🇪" | "🇲🇸" | "🇲🇦" | "🇲🇿" | "🇲🇲" | "🇳🇦" | "🇳🇷" | "🇳🇵" | "🇳🇱" | "🇳🇨" | "🇳🇿" | "🇳🇮" | "🇳🇪" | "🇳🇬" | "🇳🇺" | "🇳🇫" | "🇰🇵" | "🇲🇵" | "🇳🇴" | "🇴🇲" | "🇵🇰" | "🇵🇼" | "🇵🇸" | "🇵🇦" | "🇵🇬" | "🇵🇾" | "🇵🇪" | "🇵🇭" | "🇵🇳" | "🇵🇱" | "🇵🇹" | "🇵🇷" | "🇶🇦" | "🇷🇪" | "🇷🇴" | "🇷🇺" | "🇷🇼" | "🇼🇸" | "🇸🇲" | "🇸🇦" | "🇸🇳" | "🇷🇸" | "🇸🇨" | "🇸🇱" | "🇸🇬" | "🇸🇽" | "🇸🇰" | "🇸🇮" | "🇬🇸" | "🇸🇧" | "🇸🇴" | "🇿🇦" | "🇰🇷" | "🇸🇸" | "🇪🇸" | "🇱🇰" | "🇧🇱" | "🇸🇭" | "🇰🇳" | "🇱🇨" | "🇵🇲" | "🇻🇨" | "🇸🇩" | "🇸🇷" | "🇸🇿" | "🇸🇪" | "🇨🇭" | "🇸🇾" | "🇹🇼" | "🇹🇯" | "🇹🇿" | "🇹🇭" | "🇹🇱" | "🇹🇬" | "🇹🇰" | "🇹🇴" | "🇹🇹" | "🇹🇳" | "🇹🇷" | "🇹🇲" | "🇹🇨" | "🇹🇻" | "🇻🇮" | "🇺🇬" | "🇺🇦" | "🇦🇪" | "🇬🇧" | "🏴󠁧󠁢󠁥󠁮󠁧󠁿" | "🏴󠁧󠁢󠁳󠁣󠁴󠁿" | "🏴󠁧󠁢󠁷󠁬󠁳󠁿" | "🇺🇳" | "🇺🇸" | "🇺🇾" | "🇺🇿" | "🇻🇺" | "🇻🇦" | "🇻🇪" | "🇻🇳" | "🇼🇫" | "🇪🇭" | "🇾🇪" | "🇿🇲" | "🇿🇼" | "🦭" | "😮‍💨" | "😵‍💫" | "😶‍🌫️" | "❤️‍🔥" | "❤️‍🩹" | "🧔‍♀️" | "🧔🏻‍♀️" | "🧔🏼‍♀️" | "🧔🏽‍♀️" | "🧔🏾‍♀️" | "🧔🏿‍♀️" | "🧔‍♂️" | "🧔🏻‍♂️" | "🧔🏼‍♂️" | "🧔🏽‍♂️" | "🧔🏾‍♂️" | "🧔🏿‍♂️" | "💑🏻" | "💑🏼" | "💑🏽" | "💑🏾" | "💑🏿" | "💏🏻" | "💏🏼" | "💏🏽" | "💏🏾" | "💏🏿" | "👨🏻‍❤️‍👨🏻" | "👨🏻‍❤️‍👨🏼" | "👨🏻‍❤️‍👨🏽" | "👨🏻‍❤️‍👨🏾" | "👨🏻‍❤️‍👨🏿" | "👨🏼‍❤️‍👨🏻" | "👨🏼‍❤️‍👨🏼" | "👨🏼‍❤️‍👨🏽" | "👨🏼‍❤️‍👨🏾" | "👨🏼‍❤️‍👨🏿" | "👨🏽‍❤️‍👨🏻" | "👨🏽‍❤️‍👨🏼" | "👨🏽‍❤️‍👨🏽" | "👨🏽‍❤️‍👨🏾" | "👨🏽‍❤️‍👨🏿" | "👨🏾‍❤️‍👨🏻" | "👨🏾‍❤️‍👨🏼" | "👨🏾‍❤️‍👨🏽" | "👨🏾‍❤️‍👨🏾" | "👨🏾‍❤️‍👨🏿" | "👨🏿‍❤️‍👨🏻" | "👨🏿‍❤️‍👨🏼" | "👨🏿‍❤️‍👨🏽" | "👨🏿‍❤️‍👨🏾" | "👨🏿‍❤️‍👨🏿" | "👩🏻‍❤️‍👨🏻" | "👩🏻‍❤️‍👨🏼" | "👩🏻‍❤️‍👨🏽" | "👩🏻‍❤️‍👨🏾" | "👩🏻‍❤️‍👨🏿" | "👩🏻‍❤️‍👩🏻" | "👩🏻‍❤️‍👩🏼" | "👩🏻‍❤️‍👩🏽" | "👩🏻‍❤️‍👩🏾" | "👩🏻‍❤️‍👩🏿" | "👩🏼‍❤️‍👨🏻" | "👩🏼‍❤️‍👨🏼" | "👩🏼‍❤️‍👨🏽" | "👩🏼‍❤️‍👨🏾" | "👩🏼‍❤️‍👨🏿" | "👩🏼‍❤️‍👩🏻" | "👩🏼‍❤️‍👩🏼" | "👩🏼‍❤️‍👩🏽" | "👩🏼‍❤️‍👩🏾" | "👩🏼‍❤️‍👩🏿" | "👩🏽‍❤️‍👨🏻" | "👩🏽‍❤️‍👨🏼" | "👩🏽‍❤️‍👨🏽" | "👩🏽‍❤️‍👨🏾" | "👩🏽‍❤️‍👨🏿" | "👩🏽‍❤️‍👩🏻" | "👩🏽‍❤️‍👩🏼" | "👩🏽‍❤️‍👩🏽" | "👩🏽‍❤️‍👩🏾" | "👩🏽‍❤️‍👩🏿" | "👩🏾‍❤️‍👨🏻" | "👩🏾‍❤️‍👨🏼" | "👩🏾‍❤️‍👨🏽" | "👩🏾‍❤️‍👨🏾" | "👩🏾‍❤️‍👨🏿" | "👩🏾‍❤️‍👩🏻" | "👩🏾‍❤️‍👩🏼" | "👩🏾‍❤️‍👩🏽" | "👩🏾‍❤️‍👩🏾" | "👩🏾‍❤️‍👩🏿" | "👩🏿‍❤️‍👨🏻" | "👩🏿‍❤️‍👨🏼" | "👩🏿‍❤️‍👨🏽" | "👩🏿‍❤️‍👨🏾" | "👩🏿‍❤️‍👨🏿" | "👩🏿‍❤️‍👩🏻" | "👩🏿‍❤️‍👩🏼" | "👩🏿‍❤️‍👩🏽" | "👩🏿‍❤️‍👩🏾" | "👩🏿‍❤️‍👩🏿" | "🧑🏻‍❤️‍🧑🏼" | "🧑🏻‍❤️‍🧑🏽" | "🧑🏻‍❤️‍🧑🏾" | "🧑🏻‍❤️‍🧑🏿" | "🧑🏼‍❤️‍🧑🏻" | "🧑🏼‍❤️‍🧑🏽" | "🧑🏼‍❤️‍🧑🏾" | "🧑🏼‍❤️‍🧑🏿" | "🧑🏽‍❤️‍🧑🏻" | "🧑🏽‍❤️‍🧑🏼" | "🧑🏽‍❤️‍🧑🏾" | "🧑🏽‍❤️‍🧑🏿" | "🧑🏾‍❤️‍🧑🏻" | "🧑🏾‍❤️‍🧑🏼" | "🧑🏾‍❤️‍🧑🏽" | "🧑🏾‍❤️‍🧑🏿" | "🧑🏿‍❤️‍🧑🏻" | "🧑🏿‍❤️‍🧑🏼" | "🧑🏿‍❤️‍🧑🏽" | "🧑🏿‍❤️‍🧑🏾" | "👨🏻‍❤️‍💋‍👨🏻" | "👨🏻‍❤️‍💋‍👨🏼" | "👨🏻‍❤️‍💋‍👨🏽" | "👨🏻‍❤️‍💋‍👨🏾" | "👨🏻‍❤️‍💋‍👨🏿" | "👨🏼‍❤️‍💋‍👨🏻" | "👨🏼‍❤️‍💋‍👨🏼" | "👨🏼‍❤️‍💋‍👨🏽" | "👨🏼‍❤️‍💋‍👨🏾" | "👨🏼‍❤️‍💋‍👨🏿" | "👨🏽‍❤️‍💋‍👨🏻" | "👨🏽‍❤️‍💋‍👨🏼" | "👨🏽‍❤️‍💋‍👨🏽" | "👨🏽‍❤️‍💋‍👨🏾" | "👨🏽‍❤️‍💋‍👨🏿" | "👨🏾‍❤️‍💋‍👨🏻" | "👨🏾‍❤️‍💋‍👨🏼" | "👨🏾‍❤️‍💋‍👨🏽" | "👨🏾‍❤️‍💋‍👨🏾" | "👨🏾‍❤️‍💋‍👨🏿" | "👨🏿‍❤️‍💋‍👨🏻" | "👨🏿‍❤️‍💋‍👨🏼" | "👨🏿‍❤️‍💋‍👨🏽" | "👨🏿‍❤️‍💋‍👨🏾" | "👨🏿‍❤️‍💋‍👨🏿" | "👩🏻‍❤️‍💋‍👨🏻" | "👩🏻‍❤️‍💋‍👨🏼" | "👩🏻‍❤️‍💋‍👨🏽" | "👩🏻‍❤️‍💋‍👨🏾" | "👩🏻‍❤️‍💋‍👨🏿" | "👩🏻‍❤️‍💋‍👩🏻" | "👩🏻‍❤️‍💋‍👩🏼" | "👩🏻‍❤️‍💋‍👩🏽" | "👩🏻‍❤️‍💋‍👩🏾" | "👩🏻‍❤️‍💋‍👩🏿" | "👩🏼‍❤️‍💋‍👨🏻" | "👩🏼‍❤️‍💋‍👨🏼" | "👩🏼‍❤️‍💋‍👨🏽" | "👩🏼‍❤️‍💋‍👨🏾" | "👩🏼‍❤️‍💋‍👨🏿" | "👩🏼‍❤️‍💋‍👩🏻" | "👩🏼‍❤️‍💋‍👩🏼" | "👩🏼‍❤️‍💋‍👩🏽" | "👩🏼‍❤️‍💋‍👩🏾" | "👩🏼‍❤️‍💋‍👩🏿" | "👩🏽‍❤️‍💋‍👨🏻" | "👩🏽‍❤️‍💋‍👨🏼" | "👩🏽‍❤️‍💋‍👨🏽" | "👩🏽‍❤️‍💋‍👨🏾" | "👩🏽‍❤️‍💋‍👨🏿" | "👩🏽‍❤️‍💋‍👩🏻" | "👩🏽‍❤️‍💋‍👩🏼" | "👩🏽‍❤️‍💋‍👩🏽" | "👩🏽‍❤️‍💋‍👩🏾" | "👩🏽‍❤️‍💋‍👩🏿" | "👩🏾‍❤️‍💋‍👨🏻" | "👩🏾‍❤️‍💋‍👨🏼" | "👩🏾‍❤️‍💋‍👨🏽" | "👩🏾‍❤️‍💋‍👨🏾" | "👩🏾‍❤️‍💋‍👨🏿" | "👩🏾‍❤️‍💋‍👩🏻" | "👩🏾‍❤️‍💋‍👩🏼" | "👩🏾‍❤️‍💋‍👩🏽" | "👩🏾‍❤️‍💋‍👩🏾" | "👩🏾‍❤️‍💋‍👩🏿" | "👩🏿‍❤️‍💋‍👨🏻" | "👩🏿‍❤️‍💋‍👨🏼" | "👩🏿‍❤️‍💋‍👨🏽" | "👩🏿‍❤️‍💋‍👨🏾" | "👩🏿‍❤️‍💋‍👨🏿" | "👩🏿‍❤️‍💋‍👩🏻" | "👩🏿‍❤️‍💋‍👩🏼" | "👩🏿‍❤️‍💋‍👩🏽" | "👩🏿‍❤️‍💋‍👩🏾" | "👩🏿‍❤️‍💋‍👩🏿" | "🧑🏻‍❤️‍💋‍🧑🏼" | "🧑🏻‍❤️‍💋‍🧑🏽" | "🧑🏻‍❤️‍💋‍🧑🏾" | "🧑🏻‍❤️‍💋‍🧑🏿" | "🧑🏼‍❤️‍💋‍🧑🏻" | "🧑🏼‍❤️‍💋‍🧑🏽" | "🧑🏼‍❤️‍💋‍🧑🏾" | "🧑🏼‍❤️‍💋‍🧑🏿" | "🧑🏽‍❤️‍💋‍🧑🏻" | "🧑🏽‍❤️‍💋‍🧑🏼" | "🧑🏽‍❤️‍💋‍🧑🏾" | "🧑🏽‍❤️‍💋‍🧑🏿" | "🧑🏾‍❤️‍💋‍🧑🏻" | "🧑🏾‍❤️‍💋‍🧑🏼" | "🧑🏾‍❤️‍💋‍🧑🏽" | "🧑🏾‍❤️‍💋‍🧑🏿" | "🧑🏿‍❤️‍💋‍🧑🏻" | "🧑🏿‍❤️‍💋‍🧑🏼" | "🧑🏿‍❤️‍💋‍🧑🏽" | "🧑🏿‍❤️‍💋‍🧑🏾"'; + '"😀" | "😃" | "😄" | "😁" | "😆" | "😅" | "😂" | "🤣" | "🥲" | "☺️" | "😊" | "😇" | "🙂" | "🙃" | "😉" | "😌" | "😍" | "🥰" | "😘" | "😗" | "😙" | "😚" | "😋" | "😛" | "😝" | "😜" | "🤪" | "🤨" | "🧐" | "🤓" | "😎" | "🥸" | "🤩" | "🥳" | "😏" | "😒" | "😞" | "😔" | "😟" | "😕" | "🙁" | "☹️" | "😣" | "😖" | "😫" | "😩" | "🥺" | "😢" | "😭" | "😤" | "😠" | "😡" | "🤬" | "🤯" | "😳" | "🥵" | "🥶" | "😱" | "😨" | "😰" | "😥" | "😓" | "🤗" | "🤔" | "🤭" | "🤫" | "🤥" | "😶" | "😐" | "😑" | "😬" | "🙄" | "😯" | "😦" | "😧" | "😮" | "😲" | "🥱" | "😴" | "🤤" | "😪" | "😵" | "🤐" | "🥴" | "🤢" | "🤮" | "🤧" | "😷" | "🤒" | "🤕" | "🤑" | "🤠" | "😈" | "👿" | "👹" | "👺" | "🤡" | "💩" | "👻" | "💀" | "☠️" | "👽" | "👾" | "🤖" | "🎃" | "😺" | "😸" | "😹" | "😻" | "😼" | "😽" | "🙀" | "😿" | "😾" | "👋" | "🤚" | "🖐" | "✋" | "🖖" | "👌" | "🤌" | "🤏" | "✌️" | "🤞" | "🤟" | "🤘" | "🤙" | "👈" | "👉" | "👆" | "🖕" | "👇" | "☝️" | "👍" | "👎" | "✊" | "👊" | "🤛" | "🤜" | "👏" | "🙌" | "👐" | "🤲" | "🤝" | "🙏" | "✍️" | "💅" | "🤳" | "💪" | "🦾" | "🦵" | "🦿" | "🦶" | "👣" | "👂" | "🦻" | "👃" | "🫀" | "🫁" | "🧠" | "🦷" | "🦴" | "👀" | "👁" | "👅" | "👄" | "💋" | "🩸" | "👶" | "👧" | "🧒" | "👦" | "👩" | "🧑" | "👨" | "👩‍🦱" | "🧑‍🦱" | "👨‍🦱" | "👩‍🦰" | "🧑‍🦰" | "👨‍🦰" | "👱‍♀️" | "👱" | "👱‍♂️" | "👩‍🦳" | "🧑‍🦳" | "👨‍🦳" | "👩‍🦲" | "🧑‍🦲" | "👨‍🦲" | "🧔" | "👵" | "🧓" | "👴" | "👲" | "👳‍♀️" | "👳" | "👳‍♂️" | "🧕" | "👮‍♀️" | "👮" | "👮‍♂️" | "👷‍♀️" | "👷" | "👷‍♂️" | "💂‍♀️" | "💂" | "💂‍♂️" | "🕵️‍♀️" | "🕵️" | "🕵️‍♂️" | "👩‍⚕️" | "🧑‍⚕️" | "👨‍⚕️" | "👩‍🌾" | "🧑‍🌾" | "👨‍🌾" | "👩‍🍳" | "🧑‍🍳" | "👨‍🍳" | "👩‍🎓" | "🧑‍🎓" | "👨‍🎓" | "👩‍🎤" | "🧑‍🎤" | "👨‍🎤" | "👩‍🏫" | "🧑‍🏫" | "👨‍🏫" | "👩‍🏭" | "🧑‍🏭" | "👨‍🏭" | "👩‍💻" | "🧑‍💻" | "👨‍💻" | "👩‍💼" | "🧑‍💼" | "👨‍💼" | "👩‍🔧" | "🧑‍🔧" | "👨‍🔧" | "👩‍🔬" | "🧑‍🔬" | "👨‍🔬" | "👩‍🎨" | "🧑‍🎨" | "👨‍🎨" | "👩‍🚒" | "🧑‍🚒" | "👨‍🚒" | "👩‍✈️" | "🧑‍✈️" | "👨‍✈️" | "👩‍🚀" | "🧑‍🚀" | "👨‍🚀" | "👩‍⚖️" | "🧑‍⚖️" | "👨‍⚖️" | "👰‍♀️" | "👰" | "👰‍♂️" | "🤵‍♀️" | "🤵" | "🤵‍♂️" | "👸" | "🤴" | "🥷" | "🦸‍♀️" | "🦸" | "🦸‍♂️" | "🦹‍♀️" | "🦹" | "🦹‍♂️" | "🤶" | "🧑‍🎄" | "🎅" | "🧙‍♀️" | "🧙" | "🧙‍♂️" | "🧝‍♀️" | "🧝" | "🧝‍♂️" | "🧛‍♀️" | "🧛" | "🧛‍♂️" | "🧟‍♀️" | "🧟" | "🧟‍♂️" | "🧞‍♀️" | "🧞" | "🧞‍♂️" | "🧜‍♀️" | "🧜" | "🧜‍♂️" | "🧚‍♀️" | "🧚" | "🧚‍♂️" | "👼" | "🤰" | "🤱" | "👩‍🍼" | "🧑‍🍼" | "👨‍🍼" | "🙇‍♀️" | "🙇" | "🙇‍♂️" | "💁‍♀️" | "💁" | "💁‍♂️" | "🙅‍♀️" | "🙅" | "🙅‍♂️" | "🙆‍♀️" | "🙆" | "🙆‍♂️" | "🙋‍♀️" | "🙋" | "🙋‍♂️" | "🧏‍♀️" | "🧏" | "🧏‍♂️" | "🤦‍♀️" | "🤦" | "🤦‍♂️" | "🤷‍♀️" | "🤷" | "🤷‍♂️" | "🙎‍♀️" | "🙎" | "🙎‍♂️" | "🙍‍♀️" | "🙍" | "🙍‍♂️" | "💇‍♀️" | "💇" | "💇‍♂️" | "💆‍♀️" | "💆" | "💆‍♂️" | "🧖‍♀️" | "🧖" | "🧖‍♂️" | "💃" | "🕺" | "👯‍♀️" | "👯" | "👯‍♂️" | "🕴" | "👩‍🦽" | "🧑‍🦽" | "👨‍🦽" | "👩‍🦼" | "🧑‍🦼" | "👨‍🦼" | "🚶‍♀️" | "🚶" | "🚶‍♂️" | "👩‍🦯" | "🧑‍🦯" | "👨‍🦯" | "🧎‍♀️" | "🧎" | "🧎‍♂️" | "🏃‍♀️" | "🏃" | "🏃‍♂️" | "🧍‍♀️" | "🧍" | "🧍‍♂️" | "👭" | "🧑‍🤝‍🧑" | "👬" | "👫" | "👩‍❤️‍👩" | "💑" | "👨‍❤️‍👨" | "👩‍❤️‍👨" | "👩‍❤️‍💋‍👩" | "💏" | "👨‍❤️‍💋‍👨" | "👩‍❤️‍💋‍👨" | "👪" | "👨‍👩‍👦" | "👨‍👩‍👧" | "👨‍👩‍👧‍👦" | "👨‍👩‍👦‍👦" | "👨‍👩‍👧‍👧" | "👨‍👨‍👦" | "👨‍👨‍👧" | "👨‍👨‍👧‍👦" | "👨‍👨‍👦‍👦" | "👨‍👨‍👧‍👧" | "👩‍👩‍👦" | "👩‍👩‍👧" | "👩‍👩‍👧‍👦" | "👩‍👩‍👦‍👦" | "👩‍👩‍👧‍👧" | "👨‍👦" | "👨‍👦‍👦" | "👨‍👧" | "👨‍👧‍👦" | "👨‍👧‍👧" | "👩‍👦" | "👩‍👦‍👦" | "👩‍👧" | "👩‍👧‍👦" | "👩‍👧‍👧" | "🗣" | "👤" | "👥" | "🫂" | "🧳" | "🌂" | "☂️" | "🧵" | "🪡" | "🪢" | "🧶" | "👓" | "🕶" | "🥽" | "🥼" | "🦺" | "👔" | "👕" | "👖" | "🧣" | "🧤" | "🧥" | "🧦" | "👗" | "👘" | "🥻" | "🩴" | "🩱" | "🩲" | "🩳" | "👙" | "👚" | "👛" | "👜" | "👝" | "🎒" | "👞" | "👟" | "🥾" | "🥿" | "👠" | "👡" | "🩰" | "👢" | "👑" | "👒" | "🎩" | "🎓" | "🧢" | "⛑" | "🪖" | "💄" | "💍" | "💼" | "👋🏻" | "🤚🏻" | "🖐🏻" | "✋🏻" | "🖖🏻" | "👌🏻" | "🤌🏻" | "🤏🏻" | "✌🏻" | "🤞🏻" | "🤟🏻" | "🤘🏻" | "🤙🏻" | "👈🏻" | "👉🏻" | "👆🏻" | "🖕🏻" | "👇🏻" | "☝🏻" | "👍🏻" | "👎🏻" | "✊🏻" | "👊🏻" | "🤛🏻" | "🤜🏻" | "👏🏻" | "🙌🏻" | "👐🏻" | "🤲🏻" | "🙏🏻" | "✍🏻" | "💅🏻" | "🤳🏻" | "💪🏻" | "🦵🏻" | "🦶🏻" | "👂🏻" | "🦻🏻" | "👃🏻" | "👶🏻" | "👧🏻" | "🧒🏻" | "👦🏻" | "👩🏻" | "🧑🏻" | "👨🏻" | "👩🏻‍🦱" | "🧑🏻‍🦱" | "👨🏻‍🦱" | "👩🏻‍🦰" | "🧑🏻‍🦰" | "👨🏻‍🦰" | "👱🏻‍♀️" | "👱🏻" | "👱🏻‍♂️" | "👩🏻‍🦳" | "🧑🏻‍🦳" | "👨🏻‍🦳" | "👩🏻‍🦲" | "🧑🏻‍🦲" | "👨🏻‍🦲" | "🧔🏻" | "👵🏻" | "🧓🏻" | "👴🏻" | "👲🏻" | "👳🏻‍♀️" | "👳🏻" | "👳🏻‍♂️" | "🧕🏻" | "👮🏻‍♀️" | "👮🏻" | "👮🏻‍♂️" | "👷🏻‍♀️" | "👷🏻" | "👷🏻‍♂️" | "💂🏻‍♀️" | "💂🏻" | "💂🏻‍♂️" | "🕵🏻‍♀️" | "🕵🏻" | "🕵🏻‍♂️" | "👩🏻‍⚕️" | "🧑🏻‍⚕️" | "👨🏻‍⚕️" | "👩🏻‍🌾" | "🧑🏻‍🌾" | "👨🏻‍🌾" | "👩🏻‍🍳" | "🧑🏻‍🍳" | "👨🏻‍🍳" | "👩🏻‍🎓" | "🧑🏻‍🎓" | "👨🏻‍🎓" | "👩🏻‍🎤" | "🧑🏻‍🎤" | "👨🏻‍🎤" | "👩🏻‍🏫" | "🧑🏻‍🏫" | "👨🏻‍🏫" | "👩🏻‍🏭" | "🧑🏻‍🏭" | "👨🏻‍🏭" | "👩🏻‍💻" | "🧑🏻‍💻" | "👨🏻‍💻" | "👩🏻‍💼" | "🧑🏻‍💼" | "👨🏻‍💼" | "👩🏻‍🔧" | "🧑🏻‍🔧" | "👨🏻‍🔧" | "👩🏻‍🔬" | "🧑🏻‍🔬" | "👨🏻‍🔬" | "👩🏻‍🎨" | "🧑🏻‍🎨" | "👨🏻‍🎨" | "👩🏻‍🚒" | "🧑🏻‍🚒" | "👨🏻‍🚒" | "👩🏻‍✈️" | "🧑🏻‍✈️" | "👨🏻‍✈️" | "👩🏻‍🚀" | "🧑🏻‍🚀" | "👨🏻‍🚀" | "👩🏻‍⚖️" | "🧑🏻‍⚖️" | "👨🏻‍⚖️" | "👰🏻‍♀️" | "👰🏻" | "👰🏻‍♂️" | "🤵🏻‍♀️" | "🤵🏻" | "🤵🏻‍♂️" | "👸🏻" | "🤴🏻" | "🥷🏻" | "🦸🏻‍♀️" | "🦸🏻" | "🦸🏻‍♂️" | "🦹🏻‍♀️" | "🦹🏻" | "🦹🏻‍♂️" | "🤶🏻" | "🧑🏻‍🎄" | "🎅🏻" | "🧙🏻‍♀️" | "🧙🏻" | "🧙🏻‍♂️" | "🧝🏻‍♀️" | "🧝🏻" | "🧝🏻‍♂️" | "🧛🏻‍♀️" | "🧛🏻" | "🧛🏻‍♂️" | "🧜🏻‍♀️" | "🧜🏻" | "🧜🏻‍♂️" | "🧚🏻‍♀️" | "🧚🏻" | "🧚🏻‍♂️" | "👼🏻" | "🤰🏻" | "🤱🏻" | "👩🏻‍🍼" | "🧑🏻‍🍼" | "👨🏻‍🍼" | "🙇🏻‍♀️" | "🙇🏻" | "🙇🏻‍♂️" | "💁🏻‍♀️" | "💁🏻" | "💁🏻‍♂️" | "🙅🏻‍♀️" | "🙅🏻" | "🙅🏻‍♂️" | "🙆🏻‍♀️" | "🙆🏻" | "🙆🏻‍♂️" | "🙋🏻‍♀️" | "🙋🏻" | "🙋🏻‍♂️" | "🧏🏻‍♀️" | "🧏🏻" | "🧏🏻‍♂️" | "🤦🏻‍♀️" | "🤦🏻" | "🤦🏻‍♂️" | "🤷🏻‍♀️" | "🤷🏻" | "🤷🏻‍♂️" | "🙎🏻‍♀️" | "🙎🏻" | "🙎🏻‍♂️" | "🙍🏻‍♀️" | "🙍🏻" | "🙍🏻‍♂️" | "💇🏻‍♀️" | "💇🏻" | "💇🏻‍♂️" | "💆🏻‍♀️" | "💆🏻" | "💆🏻‍♂️" | "🧖🏻‍♀️" | "🧖🏻" | "🧖🏻‍♂️" | "💃🏻" | "🕺🏻" | "🕴🏻" | "👩🏻‍🦽" | "🧑🏻‍🦽" | "👨🏻‍🦽" | "👩🏻‍🦼" | "🧑🏻‍🦼" | "👨🏻‍🦼" | "🚶🏻‍♀️" | "🚶🏻" | "🚶🏻‍♂️" | "👩🏻‍🦯" | "🧑🏻‍🦯" | "👨🏻‍🦯" | "🧎🏻‍♀️" | "🧎🏻" | "🧎🏻‍♂️" | "🏃🏻‍♀️" | "🏃🏻" | "🏃🏻‍♂️" | "🧍🏻‍♀️" | "🧍🏻" | "🧍🏻‍♂️" | "👭🏻" | "🧑🏻‍🤝‍🧑🏻" | "👬🏻" | "👫🏻" | "🧗🏻‍♀️" | "🧗🏻" | "🧗🏻‍♂️" | "🏇🏻" | "🏂🏻" | "🏌🏻‍♀️" | "🏌🏻" | "🏌🏻‍♂️" | "🏄🏻‍♀️" | "🏄🏻" | "🏄🏻‍♂️" | "🚣🏻‍♀️" | "🚣🏻" | "🚣🏻‍♂️" | "🏊🏻‍♀️" | "🏊🏻" | "🏊🏻‍♂️" | "⛹🏻‍♀️" | "⛹🏻" | "⛹🏻‍♂️" | "🏋🏻‍♀️" | "🏋🏻" | "🏋🏻‍♂️" | "🚴🏻‍♀️" | "🚴🏻" | "🚴🏻‍♂️" | "🚵🏻‍♀️" | "🚵🏻" | "🚵🏻‍♂️" | "🤸🏻‍♀️" | "🤸🏻" | "🤸🏻‍♂️" | "🤽🏻‍♀️" | "🤽🏻" | "🤽🏻‍♂️" | "🤾🏻‍♀️" | "🤾🏻" | "🤾🏻‍♂️" | "🤹🏻‍♀️" | "🤹🏻" | "🤹🏻‍♂️" | "🧘🏻‍♀️" | "🧘🏻" | "🧘🏻‍♂️" | "🛀🏻" | "🛌" | "👋🏼" | "🤚🏼" | "🖐🏼" | "✋🏼" | "🖖🏼" | "👌🏼" | "🤌🏼" | "🤏🏼" | "✌🏼" | "🤞🏼" | "🤟🏼" | "🤘🏼" | "🤙🏼" | "👈🏼" | "👉🏼" | "👆🏼" | "🖕🏼" | "👇🏼" | "☝🏼" | "👍🏼" | "👎🏼" | "✊🏼" | "👊🏼" | "🤛🏼" | "🤜🏼" | "👏🏼" | "🙌🏼" | "👐🏼" | "🤲🏼" | "🙏🏼" | "✍🏼" | "💅🏼" | "🤳🏼" | "💪🏼" | "🦵🏼" | "🦶🏼" | "👂🏼" | "🦻🏼" | "👃🏼" | "👶🏼" | "👧🏼" | "🧒🏼" | "👦🏼" | "👩🏼" | "🧑🏼" | "👨🏼" | "👩🏼‍🦱" | "🧑🏼‍🦱" | "👨🏼‍🦱" | "👩🏼‍🦰" | "🧑🏼‍🦰" | "👨🏼‍🦰" | "👱🏼‍♀️" | "👱🏼" | "👱🏼‍♂️" | "👩🏼‍🦳" | "🧑🏼‍🦳" | "👨🏼‍🦳" | "👩🏼‍🦲" | "🧑🏼‍🦲" | "👨🏼‍🦲" | "🧔🏼" | "👵🏼" | "🧓🏼" | "👴🏼" | "👲🏼" | "👳🏼‍♀️" | "👳🏼" | "👳🏼‍♂️" | "🧕🏼" | "👮🏼‍♀️" | "👮🏼" | "👮🏼‍♂️" | "👷🏼‍♀️" | "👷🏼" | "👷🏼‍♂️" | "💂🏼‍♀️" | "💂🏼" | "💂🏼‍♂️" | "🕵🏼‍♀️" | "🕵🏼" | "🕵🏼‍♂️" | "👩🏼‍⚕️" | "🧑🏼‍⚕️" | "👨🏼‍⚕️" | "👩🏼‍🌾" | "🧑🏼‍🌾" | "👨🏼‍🌾" | "👩🏼‍🍳" | "🧑🏼‍🍳" | "👨🏼‍🍳" | "👩🏼‍🎓" | "🧑🏼‍🎓" | "👨🏼‍🎓" | "👩🏼‍🎤" | "🧑🏼‍🎤" | "👨🏼‍🎤" | "👩🏼‍🏫" | "🧑🏼‍🏫" | "👨🏼‍🏫" | "👩🏼‍🏭" | "🧑🏼‍🏭" | "👨🏼‍🏭" | "👩🏼‍💻" | "🧑🏼‍💻" | "👨🏼‍💻" | "👩🏼‍💼" | "🧑🏼‍💼" | "👨🏼‍💼" | "👩🏼‍🔧" | "🧑🏼‍🔧" | "👨🏼‍🔧" | "👩🏼‍🔬" | "🧑🏼‍🔬" | "👨🏼‍🔬" | "👩🏼‍🎨" | "🧑🏼‍🎨" | "👨🏼‍🎨" | "👩🏼‍🚒" | "🧑🏼‍🚒" | "👨🏼‍🚒" | "👩🏼‍✈️" | "🧑🏼‍✈️" | "👨🏼‍✈️" | "👩🏼‍🚀" | "🧑🏼‍🚀" | "👨🏼‍🚀" | "👩🏼‍⚖️" | "🧑🏼‍⚖️" | "👨🏼‍⚖️" | "👰🏼‍♀️" | "👰🏼" | "👰🏼‍♂️" | "🤵🏼‍♀️" | "🤵🏼" | "🤵🏼‍♂️" | "👸🏼" | "🤴🏼" | "🥷🏼" | "🦸🏼‍♀️" | "🦸🏼" | "🦸🏼‍♂️" | "🦹🏼‍♀️" | "🦹🏼" | "🦹🏼‍♂️" | "🤶🏼" | "🧑🏼‍🎄" | "🎅🏼" | "🧙🏼‍♀️" | "🧙🏼" | "🧙🏼‍♂️" | "🧝🏼‍♀️" | "🧝🏼" | "🧝🏼‍♂️" | "🧛🏼‍♀️" | "🧛🏼" | "🧛🏼‍♂️" | "🧜🏼‍♀️" | "🧜🏼" | "🧜🏼‍♂️" | "🧚🏼‍♀️" | "🧚🏼" | "🧚🏼‍♂️" | "👼🏼" | "🤰🏼" | "🤱🏼" | "👩🏼‍🍼" | "🧑🏼‍🍼" | "👨🏼‍🍼" | "🙇🏼‍♀️" | "🙇🏼" | "🙇🏼‍♂️" | "💁🏼‍♀️" | "💁🏼" | "💁🏼‍♂️" | "🙅🏼‍♀️" | "🙅🏼" | "🙅🏼‍♂️" | "🙆🏼‍♀️" | "🙆🏼" | "🙆🏼‍♂️" | "🙋🏼‍♀️" | "🙋🏼" | "🙋🏼‍♂️" | "🧏🏼‍♀️" | "🧏🏼" | "🧏🏼‍♂️" | "🤦🏼‍♀️" | "🤦🏼" | "🤦🏼‍♂️" | "🤷🏼‍♀️" | "🤷🏼" | "🤷🏼‍♂️" | "🙎🏼‍♀️" | "🙎🏼" | "🙎🏼‍♂️" | "🙍🏼‍♀️" | "🙍🏼" | "🙍🏼‍♂️" | "💇🏼‍♀️" | "💇🏼" | "💇🏼‍♂️" | "💆🏼‍♀️" | "💆🏼" | "💆🏼‍♂️" | "🧖🏼‍♀️" | "🧖🏼" | "🧖🏼‍♂️" | "💃🏼" | "🕺🏼" | "🕴🏼" | "👩🏼‍🦽" | "🧑🏼‍🦽" | "👨🏼‍🦽" | "👩🏼‍🦼" | "🧑🏼‍🦼" | "👨🏼‍🦼" | "🚶🏼‍♀️" | "🚶🏼" | "🚶🏼‍♂️" | "👩🏼‍🦯" | "🧑🏼‍🦯" | "👨🏼‍🦯" | "🧎🏼‍♀️" | "🧎🏼" | "🧎🏼‍♂️" | "🏃🏼‍♀️" | "🏃🏼" | "🏃🏼‍♂️" | "🧍🏼‍♀️" | "🧍🏼" | "🧍🏼‍♂️" | "👭🏼" | "🧑🏼‍🤝‍🧑🏼" | "👬🏼" | "👫🏼" | "🧗🏼‍♀️" | "🧗🏼" | "🧗🏼‍♂️" | "🏇🏼" | "🏂🏼" | "🏌🏼‍♀️" | "🏌🏼" | "🏌🏼‍♂️" | "🏄🏼‍♀️" | "🏄🏼" | "🏄🏼‍♂️" | "🚣🏼‍♀️" | "🚣🏼" | "🚣🏼‍♂️" | "🏊🏼‍♀️" | "🏊🏼" | "🏊🏼‍♂️" | "⛹🏼‍♀️" | "⛹🏼" | "⛹🏼‍♂️" | "🏋🏼‍♀️" | "🏋🏼" | "🏋🏼‍♂️" | "🚴🏼‍♀️" | "🚴🏼" | "🚴🏼‍♂️" | "🚵🏼‍♀️" | "🚵🏼" | "🚵🏼‍♂️" | "🤸🏼‍♀️" | "🤸🏼" | "🤸🏼‍♂️" | "🤽🏼‍♀️" | "🤽🏼" | "🤽🏼‍♂️" | "🤾🏼‍♀️" | "🤾🏼" | "🤾🏼‍♂️" | "🤹🏼‍♀️" | "🤹🏼" | "🤹🏼‍♂️" | "🧘🏼‍♀️" | "🧘🏼" | "🧘🏼‍♂️" | "🛀🏼" | "👋🏽" | "🤚🏽" | "🖐🏽" | "✋🏽" | "🖖🏽" | "👌🏽" | "🤌🏽" | "🤏🏽" | "✌🏽" | "🤞🏽" | "🤟🏽" | "🤘🏽" | "🤙🏽" | "👈🏽" | "👉🏽" | "👆🏽" | "🖕🏽" | "👇🏽" | "☝🏽" | "👍🏽" | "👎🏽" | "✊🏽" | "👊🏽" | "🤛🏽" | "🤜🏽" | "👏🏽" | "🙌🏽" | "👐🏽" | "🤲🏽" | "🙏🏽" | "✍🏽" | "💅🏽" | "🤳🏽" | "💪🏽" | "🦵🏽" | "🦶🏽" | "👂🏽" | "🦻🏽" | "👃🏽" | "👶🏽" | "👧🏽" | "🧒🏽" | "👦🏽" | "👩🏽" | "🧑🏽" | "👨🏽" | "👩🏽‍🦱" | "🧑🏽‍🦱" | "👨🏽‍🦱" | "👩🏽‍🦰" | "🧑🏽‍🦰" | "👨🏽‍🦰" | "👱🏽‍♀️" | "👱🏽" | "👱🏽‍♂️" | "👩🏽‍🦳" | "🧑🏽‍🦳" | "👨🏽‍🦳" | "👩🏽‍🦲" | "🧑🏽‍🦲" | "👨🏽‍🦲" | "🧔🏽" | "👵🏽" | "🧓🏽" | "👴🏽" | "👲🏽" | "👳🏽‍♀️" | "👳🏽" | "👳🏽‍♂️" | "🧕🏽" | "👮🏽‍♀️" | "👮🏽" | "👮🏽‍♂️" | "👷🏽‍♀️" | "👷🏽" | "👷🏽‍♂️" | "💂🏽‍♀️" | "💂🏽" | "💂🏽‍♂️" | "🕵🏽‍♀️" | "🕵🏽" | "🕵🏽‍♂️" | "👩🏽‍⚕️" | "🧑🏽‍⚕️" | "👨🏽‍⚕️" | "👩🏽‍🌾" | "🧑🏽‍🌾" | "👨🏽‍🌾" | "👩🏽‍🍳" | "🧑🏽‍🍳" | "👨🏽‍🍳" | "👩🏽‍🎓" | "🧑🏽‍🎓" | "👨🏽‍🎓" | "👩🏽‍🎤" | "🧑🏽‍🎤" | "👨🏽‍🎤" | "👩🏽‍🏫" | "🧑🏽‍🏫" | "👨🏽‍🏫" | "👩🏽‍🏭" | "🧑🏽‍🏭" | "👨🏽‍🏭" | "👩🏽‍💻" | "🧑🏽‍💻" | "👨🏽‍💻" | "👩🏽‍💼" | "🧑🏽‍💼" | "👨🏽‍💼" | "👩🏽‍🔧" | "🧑🏽‍🔧" | "👨🏽‍🔧" | "👩🏽‍🔬" | "🧑🏽‍🔬" | "👨🏽‍🔬" | "👩🏽‍🎨" | "🧑🏽‍🎨" | "👨🏽‍🎨" | "👩🏽‍🚒" | "🧑🏽‍🚒" | "👨🏽‍🚒" | "👩🏽‍✈️" | "🧑🏽‍✈️" | "👨🏽‍✈️" | "👩🏽‍🚀" | "🧑🏽‍🚀" | "👨🏽‍🚀" | "👩🏽‍⚖️" | "🧑🏽‍⚖️" | "👨🏽‍⚖️" | "👰🏽‍♀️" | "👰🏽" | "👰🏽‍♂️" | "🤵🏽‍♀️" | "🤵🏽" | "🤵🏽‍♂️" | "👸🏽" | "🤴🏽" | "🥷🏽" | "🦸🏽‍♀️" | "🦸🏽" | "🦸🏽‍♂️" | "🦹🏽‍♀️" | "🦹🏽" | "🦹🏽‍♂️" | "🤶🏽" | "🧑🏽‍🎄" | "🎅🏽" | "🧙🏽‍♀️" | "🧙🏽" | "🧙🏽‍♂️" | "🧝🏽‍♀️" | "🧝🏽" | "🧝🏽‍♂️" | "🧛🏽‍♀️" | "🧛🏽" | "🧛🏽‍♂️" | "🧜🏽‍♀️" | "🧜🏽" | "🧜🏽‍♂️" | "🧚🏽‍♀️" | "🧚🏽" | "🧚🏽‍♂️" | "👼🏽" | "🤰🏽" | "🤱🏽" | "👩🏽‍🍼" | "🧑🏽‍🍼" | "👨🏽‍🍼" | "🙇🏽‍♀️" | "🙇🏽" | "🙇🏽‍♂️" | "💁🏽‍♀️" | "💁🏽" | "💁🏽‍♂️" | "🙅🏽‍♀️" | "🙅🏽" | "🙅🏽‍♂️" | "🙆🏽‍♀️" | "🙆🏽" | "🙆🏽‍♂️" | "🙋🏽‍♀️" | "🙋🏽" | "🙋🏽‍♂️" | "🧏🏽‍♀️" | "🧏🏽" | "🧏🏽‍♂️" | "🤦🏽‍♀️" | "🤦🏽" | "🤦🏽‍♂️" | "🤷🏽‍♀️" | "🤷🏽" | "🤷🏽‍♂️" | "🙎🏽‍♀️" | "🙎🏽" | "🙎🏽‍♂️" | "🙍🏽‍♀️" | "🙍🏽" | "🙍🏽‍♂️" | "💇🏽‍♀️" | "💇🏽" | "💇🏽‍♂️" | "💆🏽‍♀️" | "💆🏽" | "💆🏽‍♂️" | "🧖🏽‍♀️" | "🧖🏽" | "🧖🏽‍♂️" | "💃🏽" | "🕺🏽" | "🕴🏽" | "👩🏽‍🦽" | "🧑🏽‍🦽" | "👨🏽‍🦽" | "👩🏽‍🦼" | "🧑🏽‍🦼" | "👨🏽‍🦼" | "🚶🏽‍♀️" | "🚶🏽" | "🚶🏽‍♂️" | "👩🏽‍🦯" | "🧑🏽‍🦯" | "👨🏽‍🦯" | "🧎🏽‍♀️" | "🧎🏽" | "🧎🏽‍♂️" | "🏃🏽‍♀️" | "🏃🏽" | "🏃🏽‍♂️" | "🧍🏽‍♀️" | "🧍🏽" | "🧍🏽‍♂️" | "👭🏽" | "🧑🏽‍🤝‍🧑🏽" | "👬🏽" | "👫🏽" | "🧗🏽‍♀️" | "🧗🏽" | "🧗🏽‍♂️" | "🏇🏽" | "🏂🏽" | "🏌🏽‍♀️" | "🏌🏽" | "🏌🏽‍♂️" | "🏄🏽‍♀️" | "🏄🏽" | "🏄🏽‍♂️" | "🚣🏽‍♀️" | "🚣🏽" | "🚣🏽‍♂️" | "🏊🏽‍♀️" | "🏊🏽" | "🏊🏽‍♂️" | "⛹🏽‍♀️" | "⛹🏽" | "⛹🏽‍♂️" | "🏋🏽‍♀️" | "🏋🏽" | "🏋🏽‍♂️" | "🚴🏽‍♀️" | "🚴🏽" | "🚴🏽‍♂️" | "🚵🏽‍♀️" | "🚵🏽" | "🚵🏽‍♂️" | "🤸🏽‍♀️" | "🤸🏽" | "🤸🏽‍♂️" | "🤽🏽‍♀️" | "🤽🏽" | "🤽🏽‍♂️" | "🤾🏽‍♀️" | "🤾🏽" | "🤾🏽‍♂️" | "🤹🏽‍♀️" | "🤹🏽" | "🤹🏽‍♂️" | "🧘🏽‍♀️" | "🧘🏽" | "🧘🏽‍♂️" | "🛀🏽" | "👋🏾" | "🤚🏾" | "🖐🏾" | "✋🏾" | "🖖🏾" | "👌🏾" | "🤌🏾" | "🤏🏾" | "✌🏾" | "🤞🏾" | "🤟🏾" | "🤘🏾" | "🤙🏾" | "👈🏾" | "👉🏾" | "👆🏾" | "🖕🏾" | "👇🏾" | "☝🏾" | "👍🏾" | "👎🏾" | "✊🏾" | "👊🏾" | "🤛🏾" | "🤜🏾" | "👏🏾" | "🙌🏾" | "👐🏾" | "🤲🏾" | "🙏🏾" | "✍🏾" | "💅🏾" | "🤳🏾" | "💪🏾" | "🦵🏾" | "🦶🏾" | "👂🏾" | "🦻🏾" | "👃🏾" | "👶🏾" | "👧🏾" | "🧒🏾" | "👦🏾" | "👩🏾" | "🧑🏾" | "👨🏾" | "👩🏾‍🦱" | "🧑🏾‍🦱" | "👨🏾‍🦱" | "👩🏾‍🦰" | "🧑🏾‍🦰" | "👨🏾‍🦰" | "👱🏾‍♀️" | "👱🏾" | "👱🏾‍♂️" | "👩🏾‍🦳" | "🧑🏾‍🦳" | "👨🏾‍🦳" | "👩🏾‍🦲" | "🧑🏾‍🦲" | "👨🏾‍🦲" | "🧔🏾" | "👵🏾" | "🧓🏾" | "👴🏾" | "👲🏾" | "👳🏾‍♀️" | "👳🏾" | "👳🏾‍♂️" | "🧕🏾" | "👮🏾‍♀️" | "👮🏾" | "👮🏾‍♂️" | "👷🏾‍♀️" | "👷🏾" | "👷🏾‍♂️" | "💂🏾‍♀️" | "💂🏾" | "💂🏾‍♂️" | "🕵🏾‍♀️" | "🕵🏾" | "🕵🏾‍♂️" | "👩🏾‍⚕️" | "🧑🏾‍⚕️" | "👨🏾‍⚕️" | "👩🏾‍🌾" | "🧑🏾‍🌾" | "👨🏾‍🌾" | "👩🏾‍🍳" | "🧑🏾‍🍳" | "👨🏾‍🍳" | "👩🏾‍🎓" | "🧑🏾‍🎓" | "👨🏾‍🎓" | "👩🏾‍🎤" | "🧑🏾‍🎤" | "👨🏾‍🎤" | "👩🏾‍🏫" | "🧑🏾‍🏫" | "👨🏾‍🏫" | "👩🏾‍🏭" | "🧑🏾‍🏭" | "👨🏾‍🏭" | "👩🏾‍💻" | "🧑🏾‍💻" | "👨🏾‍💻" | "👩🏾‍💼" | "🧑🏾‍💼" | "👨🏾‍💼" | "👩🏾‍🔧" | "🧑🏾‍🔧" | "👨🏾‍🔧" | "👩🏾‍🔬" | "🧑🏾‍🔬" | "👨🏾‍🔬" | "👩🏾‍🎨" | "🧑🏾‍🎨" | "👨🏾‍🎨" | "👩🏾‍🚒" | "🧑🏾‍🚒" | "👨🏾‍🚒" | "👩🏾‍✈️" | "🧑🏾‍✈️" | "👨🏾‍✈️" | "👩🏾‍🚀" | "🧑🏾‍🚀" | "👨🏾‍🚀" | "👩🏾‍⚖️" | "🧑🏾‍⚖️" | "👨🏾‍⚖️" | "👰🏾‍♀️" | "👰🏾" | "👰🏾‍♂️" | "🤵🏾‍♀️" | "🤵🏾" | "🤵🏾‍♂️" | "👸🏾" | "🤴🏾" | "🥷🏾" | "🦸🏾‍♀️" | "🦸🏾" | "🦸🏾‍♂️" | "🦹🏾‍♀️" | "🦹🏾" | "🦹🏾‍♂️" | "🤶🏾" | "🧑🏾‍🎄" | "🎅🏾" | "🧙🏾‍♀️" | "🧙🏾" | "🧙🏾‍♂️" | "🧝🏾‍♀️" | "🧝🏾" | "🧝🏾‍♂️" | "🧛🏾‍♀️" | "🧛🏾" | "🧛🏾‍♂️" | "🧜🏾‍♀️" | "🧜🏾" | "🧜🏾‍♂️" | "🧚🏾‍♀️" | "🧚🏾" | "🧚🏾‍♂️" | "👼🏾" | "🤰🏾" | "🤱🏾" | "👩🏾‍🍼" | "🧑🏾‍🍼" | "👨🏾‍🍼" | "🙇🏾‍♀️" | "🙇🏾" | "🙇🏾‍♂️" | "💁🏾‍♀️" | "💁🏾" | "💁🏾‍♂️" | "🙅🏾‍♀️" | "🙅🏾" | "🙅🏾‍♂️" | "🙆🏾‍♀️" | "🙆🏾" | "🙆🏾‍♂️" | "🙋🏾‍♀️" | "🙋🏾" | "🙋🏾‍♂️" | "🧏🏾‍♀️" | "🧏🏾" | "🧏🏾‍♂️" | "🤦🏾‍♀️" | "🤦🏾" | "🤦🏾‍♂️" | "🤷🏾‍♀️" | "🤷🏾" | "🤷🏾‍♂️" | "🙎🏾‍♀️" | "🙎🏾" | "🙎🏾‍♂️" | "🙍🏾‍♀️" | "🙍🏾" | "🙍🏾‍♂️" | "💇🏾‍♀️" | "💇🏾" | "💇🏾‍♂️" | "💆🏾‍♀️" | "💆🏾" | "💆🏾‍♂️" | "🧖🏾‍♀️" | "🧖🏾" | "🧖🏾‍♂️" | "💃🏾" | "🕺🏾" | "🕴🏿" | "👩🏾‍🦽" | "🧑🏾‍🦽" | "👨🏾‍🦽" | "👩🏾‍🦼" | "🧑🏾‍🦼" | "👨🏾‍🦼" | "🚶🏾‍♀️" | "🚶🏾" | "🚶🏾‍♂️" | "👩🏾‍🦯" | "🧑🏾‍🦯" | "👨🏾‍🦯" | "🧎🏾‍♀️" | "🧎🏾" | "🧎🏾‍♂️" | "🏃🏾‍♀️" | "🏃🏾" | "🏃🏾‍♂️" | "🧍🏾‍♀️" | "🧍🏾" | "🧍🏾‍♂️" | "👭🏾" | "🧑🏾‍🤝‍🧑🏾" | "👬🏾" | "👫🏾" | "🧗🏾‍♀️" | "🧗🏾" | "🧗🏾‍♂️" | "🏇🏾" | "🏂🏾" | "🏌🏾‍♀️" | "🏌🏾" | "🏌🏾‍♂️" | "🏄🏾‍♀️" | "🏄🏾" | "🏄🏾‍♂️" | "🚣🏾‍♀️" | "🚣🏾" | "🚣🏾‍♂️" | "🏊🏾‍♀️" | "🏊🏾" | "🏊🏾‍♂️" | "⛹🏾‍♀️" | "⛹🏾" | "⛹🏾‍♂️" | "🏋🏾‍♀️" | "🏋🏾" | "🏋🏾‍♂️" | "🚴🏾‍♀️" | "🚴🏾" | "🚴🏾‍♂️" | "🚵🏾‍♀️" | "🚵🏾" | "🚵🏾‍♂️" | "🤸🏾‍♀️" | "🤸🏾" | "🤸🏾‍♂️" | "🤽🏾‍♀️" | "🤽🏾" | "🤽🏾‍♂️" | "🤾🏾‍♀️" | "🤾🏾" | "🤾🏾‍♂️" | "🤹🏾‍♀️" | "🤹🏾" | "🤹🏾‍♂️" | "🧘🏾‍♀️" | "🧘🏾" | "🧘🏾‍♂️" | "🛀🏾" | "👋🏿" | "🤚🏿" | "🖐🏿" | "✋🏿" | "🖖🏿" | "👌🏿" | "🤌🏿" | "🤏🏿" | "✌🏿" | "🤞🏿" | "🤟🏿" | "🤘🏿" | "🤙🏿" | "👈🏿" | "👉🏿" | "👆🏿" | "🖕🏿" | "👇🏿" | "☝🏿" | "👍🏿" | "👎🏿" | "✊🏿" | "👊🏿" | "🤛🏿" | "🤜🏿" | "👏🏿" | "🙌🏿" | "👐🏿" | "🤲🏿" | "🙏🏿" | "✍🏿" | "💅🏿" | "🤳🏿" | "💪🏿" | "🦵🏿" | "🦶🏿" | "👂🏿" | "🦻🏿" | "👃🏿" | "👶🏿" | "👧🏿" | "🧒🏿" | "👦🏿" | "👩🏿" | "🧑🏿" | "👨🏿" | "👩🏿‍🦱" | "🧑🏿‍🦱" | "👨🏿‍🦱" | "👩🏿‍🦰" | "🧑🏿‍🦰" | "👨🏿‍🦰" | "👱🏿‍♀️" | "👱🏿" | "👱🏿‍♂️" | "👩🏿‍🦳" | "🧑🏿‍🦳" | "👨🏿‍🦳" | "👩🏿‍🦲" | "🧑🏿‍🦲" | "👨🏿‍🦲" | "🧔🏿" | "👵🏿" | "🧓🏿" | "👴🏿" | "👲🏿" | "👳🏿‍♀️" | "👳🏿" | "👳🏿‍♂️" | "🧕🏿" | "👮🏿‍♀️" | "👮🏿" | "👮🏿‍♂️" | "👷🏿‍♀️" | "👷🏿" | "👷🏿‍♂️" | "💂🏿‍♀️" | "💂🏿" | "💂🏿‍♂️" | "🕵🏿‍♀️" | "🕵🏿" | "🕵🏿‍♂️" | "👩🏿‍⚕️" | "🧑🏿‍⚕️" | "👨🏿‍⚕️" | "👩🏿‍🌾" | "🧑🏿‍🌾" | "👨🏿‍🌾" | "👩🏿‍🍳" | "🧑🏿‍🍳" | "👨🏿‍🍳" | "👩🏿‍🎓" | "🧑🏿‍🎓" | "👨🏿‍🎓" | "👩🏿‍🎤" | "🧑🏿‍🎤" | "👨🏿‍🎤" | "👩🏿‍🏫" | "🧑🏿‍🏫" | "👨🏿‍🏫" | "👩🏿‍🏭" | "🧑🏿‍🏭" | "👨🏿‍🏭" | "👩🏿‍💻" | "🧑🏿‍💻" | "👨🏿‍💻" | "👩🏿‍💼" | "🧑🏿‍💼" | "👨🏿‍💼" | "👩🏿‍🔧" | "🧑🏿‍🔧" | "👨🏿‍🔧" | "👩🏿‍🔬" | "🧑🏿‍🔬" | "👨🏿‍🔬" | "👩🏿‍🎨" | "🧑🏿‍🎨" | "👨🏿‍🎨" | "👩🏿‍🚒" | "🧑🏿‍🚒" | "👨🏿‍🚒" | "👩🏿‍✈️" | "🧑🏿‍✈️" | "👨🏿‍✈️" | "👩🏿‍🚀" | "🧑🏿‍🚀" | "👨🏿‍🚀" | "👩🏿‍⚖️" | "🧑🏿‍⚖️" | "👨🏿‍⚖️" | "👰🏿‍♀️" | "👰🏿" | "👰🏿‍♂️" | "🤵🏿‍♀️" | "🤵🏿" | "🤵🏿‍♂️" | "👸🏿" | "🤴🏿" | "🥷🏿" | "🦸🏿‍♀️" | "🦸🏿" | "🦸🏿‍♂️" | "🦹🏿‍♀️" | "🦹🏿" | "🦹🏿‍♂️" | "🤶🏿" | "🧑🏿‍🎄" | "🎅🏿" | "🧙🏿‍♀️" | "🧙🏿" | "🧙🏿‍♂️" | "🧝🏿‍♀️" | "🧝🏿" | "🧝🏿‍♂️" | "🧛🏿‍♀️" | "🧛🏿" | "🧛🏿‍♂️" | "🧜🏿‍♀️" | "🧜🏿" | "🧜🏿‍♂️" | "🧚🏿‍♀️" | "🧚🏿" | "🧚🏿‍♂️" | "👼🏿" | "🤰🏿" | "🤱🏿" | "👩🏿‍🍼" | "🧑🏿‍🍼" | "👨🏿‍🍼" | "🙇🏿‍♀️" | "🙇🏿" | "🙇🏿‍♂️" | "💁🏿‍♀️" | "💁🏿" | "💁🏿‍♂️" | "🙅🏿‍♀️" | "🙅🏿" | "🙅🏿‍♂️" | "🙆🏿‍♀️" | "🙆🏿" | "🙆🏿‍♂️" | "🙋🏿‍♀️" | "🙋🏿" | "🙋🏿‍♂️" | "🧏🏿‍♀️" | "🧏🏿" | "🧏🏿‍♂️" | "🤦🏿‍♀️" | "🤦🏿" | "🤦🏿‍♂️" | "🤷🏿‍♀️" | "🤷🏿" | "🤷🏿‍♂️" | "🙎🏿‍♀️" | "🙎🏿" | "🙎🏿‍♂️" | "🙍🏿‍♀️" | "🙍🏿" | "🙍🏿‍♂️" | "💇🏿‍♀️" | "💇🏿" | "💇🏿‍♂️" | "💆🏿‍♀️" | "💆🏿" | "💆🏿‍♂️" | "🧖🏿‍♀️" | "🧖🏿" | "🧖🏿‍♂️" | "💃🏿" | "🕺🏿" | "👩🏿‍🦽" | "🧑🏿‍🦽" | "👨🏿‍🦽" | "👩🏿‍🦼" | "🧑🏿‍🦼" | "👨🏿‍🦼" | "🚶🏿‍♀️" | "🚶🏿" | "🚶🏿‍♂️" | "👩🏿‍🦯" | "🧑🏿‍🦯" | "👨🏿‍🦯" | "🧎🏿‍♀️" | "🧎🏿" | "🧎🏿‍♂️" | "🏃🏿‍♀️" | "🏃🏿" | "🏃🏿‍♂️" | "🧍🏿‍♀️" | "🧍🏿" | "🧍🏿‍♂️" | "👭🏿" | "🧑🏿‍🤝‍🧑🏿" | "👬🏿" | "👫🏿" | "🧗🏿‍♀️" | "🧗🏿" | "🧗🏿‍♂️" | "🏇🏿" | "🏂🏿" | "🏌🏿‍♀️" | "🏌🏿" | "🏌🏿‍♂️" | "🏄🏿‍♀️" | "🏄🏿" | "🏄🏿‍♂️" | "🚣🏿‍♀️" | "🚣🏿" | "🚣🏿‍♂️" | "🏊🏿‍♀️" | "🏊🏿" | "🏊🏿‍♂️" | "⛹🏿‍♀️" | "⛹🏿" | "⛹🏿‍♂️" | "🏋🏿‍♀️" | "🏋🏿" | "🏋🏿‍♂️" | "🚴🏿‍♀️" | "🚴🏿" | "🚴🏿‍♂️" | "🚵🏿‍♀️" | "🚵🏿" | "🚵🏿‍♂️" | "🤸🏿‍♀️" | "🤸🏿" | "🤸🏿‍♂️" | "🤽🏿‍♀️" | "🤽🏿" | "🤽🏿‍♂️" | "🤾🏿‍♀️" | "🤾🏿" | "🤾🏿‍♂️" | "🤹🏿‍♀️" | "🤹🏿" | "🤹🏿‍♂️" | "🧘🏿‍♀️" | "🧘🏿" | "🧘🏿‍♂️" | "🛀🏿" | "🛌🏿" | "🐶" | "🐱" | "🐭" | "🐹" | "🐰" | "🦊" | "🐻" | "🐼" | "🐻‍❄️" | "🐨" | "🐯" | "🦁" | "🐮" | "🐷" | "🐽" | "🐸" | "🐵" | "🙈" | "🙉" | "🙊" | "🐒" | "🐔" | "🐧" | "🐦" | "🐤" | "🐣" | "🐥" | "🦆" | "🦅" | "🦉" | "🦇" | "🐺" | "🐗" | "🐴" | "🦄" | "🐝" | "🪱" | "🐛" | "🦋" | "🐌" | "🐞" | "🐜" | "🪰" | "🪲" | "🪳" | "🦟" | "🦗" | "🕷" | "🕸" | "🦂" | "🐢" | "🐍" | "🦎" | "🦖" | "🦕" | "🐙" | "🦑" | "🦐" | "🦞" | "🦀" | "🐡" | "🐠" | "🐟" | "🐬" | "🐳" | "🐋" | "🦈" | "🐊" | "🐅" | "🐆" | "🦓" | "🦍" | "🦧" | "🦣" | "🐘" | "🦛" | "🦏" | "🐪" | "🐫" | "🦒" | "🦘" | "🦬" | "🐃" | "🐂" | "🐄" | "🐎" | "🐖" | "🐏" | "🐑" | "🦙" | "🐐" | "🦌" | "🐕" | "🐩" | "🦮" | "🐕‍🦺" | "🐈" | "🐈‍⬛" | "🪶" | "🐓" | "🦃" | "🦤" | "🦚" | "🦜" | "🦢" | "🦩" | "🕊" | "🐇" | "🦝" | "🦨" | "🦡" | "🦫" | "🦦" | "🦥" | "🐁" | "🐀" | "🐿" | "🦔" | "🐾" | "🐉" | "🐲" | "🌵" | "🎄" | "🌲" | "🌳" | "🌴" | "🪵" | "🌱" | "🌿" | "☘️" | "🍀" | "🎍" | "🪴" | "🎋" | "🍃" | "🍂" | "🍁" | "🍄" | "🐚" | "🪨" | "🌾" | "💐" | "🌷" | "🌹" | "🥀" | "🌺" | "🌸" | "🌼" | "🌻" | "🌞" | "🌝" | "🌛" | "🌜" | "🌚" | "🌕" | "🌖" | "🌗" | "🌘" | "🌑" | "🌒" | "🌓" | "🌔" | "🌙" | "🌎" | "🌍" | "🌏" | "🪐" | "💫" | "⭐️" | "🌟" | "✨" | "⚡️" | "☄️" | "💥" | "🔥" | "🌪" | "🌈" | "☀️" | "🌤" | "⛅️" | "🌥" | "☁️" | "🌦" | "🌧" | "⛈" | "🌩" | "🌨" | "❄️" | "☃️" | "⛄️" | "🌬" | "💨" | "💧" | "💦" | "☔️" | "🌊" | "🌫" | "🍏" | "🍎" | "🍐" | "🍊" | "🍋" | "🍌" | "🍉" | "🍇" | "🍓" | "🫐" | "🍈" | "🍒" | "🍑" | "🥭" | "🍍" | "🥥" | "🥝" | "🍅" | "🍆" | "🥑" | "🥦" | "🥬" | "🥒" | "🌶" | "🫑" | "🌽" | "🥕" | "🫒" | "🧄" | "🧅" | "🥔" | "🍠" | "🥐" | "🥯" | "🍞" | "🥖" | "🥨" | "🧀" | "🥚" | "🍳" | "🧈" | "🥞" | "🧇" | "🥓" | "🥩" | "🍗" | "🍖" | "🌭" | "🍔" | "🍟" | "🍕" | "🫓" | "🥪" | "🥙" | "🧆" | "🌮" | "🌯" | "🫔" | "🥗" | "🥘" | "🫕" | "🥫" | "🍝" | "🍜" | "🍲" | "🍛" | "🍣" | "🍱" | "🥟" | "🦪" | "🍤" | "🍙" | "🍚" | "🍘" | "🍥" | "🥠" | "🥮" | "🍢" | "🍡" | "🍧" | "🍨" | "🍦" | "🥧" | "🧁" | "🍰" | "🎂" | "🍮" | "🍭" | "🍬" | "🍫" | "🍿" | "🍩" | "🍪" | "🌰" | "🥜" | "🍯" | "🥛" | "🍼" | "🫖" | "☕️" | "🍵" | "🧃" | "🥤" | "🧋" | "🍶" | "🍺" | "🍻" | "🥂" | "🍷" | "🥃" | "🍸" | "🍹" | "🧉" | "🍾" | "🧊" | "🥄" | "🍴" | "🍽" | "🥣" | "🥡" | "🥢" | "🧂" | "⚽️" | "🏀" | "🏈" | "⚾️" | "🥎" | "🎾" | "🏐" | "🏉" | "🥏" | "🎱" | "🪀" | "🏓" | "🏸" | "🏒" | "🏑" | "🥍" | "🏏" | "🪃" | "🥅" | "⛳️" | "🪁" | "🏹" | "🎣" | "🤿" | "🥊" | "🥋" | "🎽" | "🛹" | "🛼" | "🛷" | "⛸" | "🥌" | "🎿" | "⛷" | "🏂" | "🪂" | "🏋️‍♀️" | "🏋️" | "🏋️‍♂️" | "🤼‍♀️" | "🤼" | "🤼‍♂️" | "🤸‍♀️" | "🤸" | "🤸‍♂️" | "⛹️‍♀️" | "⛹️" | "⛹️‍♂️" | "🤺" | "🤾‍♀️" | "🤾" | "🤾‍♂️" | "🏌️‍♀️" | "🏌️" | "🏌️‍♂️" | "🏇" | "🧘‍♀️" | "🧘" | "🧘‍♂️" | "🏄‍♀️" | "🏄" | "🏄‍♂️" | "🏊‍♀️" | "🏊" | "🏊‍♂️" | "🤽‍♀️" | "🤽" | "🤽‍♂️" | "🚣‍♀️" | "🚣" | "🚣‍♂️" | "🧗‍♀️" | "🧗" | "🧗‍♂️" | "🚵‍♀️" | "🚵" | "🚵‍♂️" | "🚴‍♀️" | "🚴" | "🚴‍♂️" | "🏆" | "🥇" | "🥈" | "🥉" | "🏅" | "🎖" | "🏵" | "🎗" | "🎫" | "🎟" | "🎪" | "🤹" | "🤹‍♂️" | "🤹‍♀️" | "🎭" | "🎨" | "🎬" | "🎤" | "🎧" | "🎼" | "🎹" | "🥁" | "🪘" | "🎷" | "🎺" | "🪗" | "🎸" | "🪕" | "🎻" | "🎲" | "♟" | "🎯" | "🎳" | "🎮" | "🎰" | "🧩" | "🚗" | "🚕" | "🚙" | "🚌" | "🚎" | "🏎" | "🚓" | "🚑" | "🚒" | "🚐" | "🛻" | "🚚" | "🚛" | "🚜" | "🦯" | "🦽" | "🦼" | "🛴" | "🚲" | "🛵" | "🏍" | "🛺" | "🚨" | "🚔" | "🚍" | "🚘" | "🚖" | "🚡" | "🚠" | "🚟" | "🚃" | "🚋" | "🚞" | "🚝" | "🚄" | "🚅" | "🚈" | "🚂" | "🚆" | "🚇" | "🚊" | "🚉" | "✈️" | "🛫" | "🛬" | "🛩" | "💺" | "🛰" | "🚀" | "🛸" | "🚁" | "🛶" | "⛵️" | "🚤" | "🛥" | "🛳" | "⛴" | "🚢" | "⚓️" | "🪝" | "⛽️" | "🚧" | "🚦" | "🚥" | "🚏" | "🗺" | "🗿" | "🗽" | "🗼" | "🏰" | "🏯" | "🏟" | "🎡" | "🎢" | "🎠" | "⛲️" | "⛱" | "🏖" | "🏝" | "🏜" | "🌋" | "⛰" | "🏔" | "🗻" | "🏕" | "⛺️" | "🛖" | "🏠" | "🏡" | "🏘" | "🏚" | "🏗" | "🏭" | "🏢" | "🏬" | "🏣" | "🏤" | "🏥" | "🏦" | "🏨" | "🏪" | "🏫" | "🏩" | "💒" | "🏛" | "⛪️" | "🕌" | "🕍" | "🛕" | "🕋" | "⛩" | "🛤" | "🛣" | "🗾" | "🎑" | "🏞" | "🌅" | "🌄" | "🌠" | "🎇" | "🎆" | "🌇" | "🌆" | "🏙" | "🌃" | "🌌" | "🌉" | "🌁" | "⌚️" | "📱" | "📲" | "💻" | "⌨️" | "🖥" | "🖨" | "🖱" | "🖲" | "🕹" | "🗜" | "💽" | "💾" | "💿" | "📀" | "📼" | "📷" | "📸" | "📹" | "🎥" | "📽" | "🎞" | "📞" | "☎️" | "📟" | "📠" | "📺" | "📻" | "🎙" | "🎚" | "🎛" | "🧭" | "⏱" | "⏲" | "⏰" | "🕰" | "⌛️" | "⏳" | "📡" | "🔋" | "🔌" | "💡" | "🔦" | "🕯" | "🪔" | "🧯" | "🛢" | "💸" | "💵" | "💴" | "💶" | "💷" | "🪙" | "💰" | "💳" | "💎" | "⚖️" | "🪜" | "🧰" | "🪛" | "🔧" | "🔨" | "⚒" | "🛠" | "⛏" | "🪚" | "🔩" | "⚙️" | "🪤" | "🧱" | "⛓" | "🧲" | "🔫" | "💣" | "🧨" | "🪓" | "🔪" | "🗡" | "⚔️" | "🛡" | "🚬" | "⚰️" | "🪦" | "⚱️" | "🏺" | "🔮" | "📿" | "🧿" | "💈" | "⚗️" | "🔭" | "🔬" | "🕳" | "🩹" | "🩺" | "💊" | "💉" | "🧬" | "🦠" | "🧫" | "🧪" | "🌡" | "🧹" | "🪠" | "🧺" | "🧻" | "🚽" | "🚰" | "🚿" | "🛁" | "🛀" | "🧼" | "🪥" | "🪒" | "🧽" | "🪣" | "🧴" | "🛎" | "🔑" | "🗝" | "🚪" | "🪑" | "🛋" | "🛏" | "🧸" | "🪆" | "🖼" | "🪞" | "🪟" | "🛍" | "🛒" | "🎁" | "🎈" | "🎏" | "🎀" | "🪄" | "🪅" | "🎊" | "🎉" | "🎎" | "🏮" | "🎐" | "🧧" | "✉️" | "📩" | "📨" | "📧" | "💌" | "📥" | "📤" | "📦" | "🏷" | "🪧" | "📪" | "📫" | "📬" | "📭" | "📮" | "📯" | "📜" | "📃" | "📄" | "📑" | "🧾" | "📊" | "📈" | "📉" | "🗒" | "🗓" | "📆" | "📅" | "🗑" | "📇" | "🗃" | "🗳" | "🗄" | "📋" | "📁" | "📂" | "🗂" | "🗞" | "📰" | "📓" | "📔" | "📒" | "📕" | "📗" | "📘" | "📙" | "📚" | "📖" | "🔖" | "🧷" | "🔗" | "📎" | "🖇" | "📐" | "📏" | "🧮" | "📌" | "📍" | "✂️" | "🖊" | "🖋" | "✒️" | "🖌" | "🖍" | "📝" | "✏️" | "🔍" | "🔎" | "🔏" | "🔐" | "🔒" | "🔓" | "❤️" | "🧡" | "💛" | "💚" | "💙" | "💜" | "🖤" | "🤍" | "🤎" | "💔" | "❣️" | "💕" | "💞" | "💓" | "💗" | "💖" | "💘" | "💝" | "💟" | "☮️" | "✝️" | "☪️" | "🕉" | "☸️" | "✡️" | "🔯" | "🕎" | "☯️" | "☦️" | "🛐" | "⛎" | "♈️" | "♉️" | "♊️" | "♋️" | "♌️" | "♍️" | "♎️" | "♏️" | "♐️" | "♑️" | "♒️" | "♓️" | "🆔" | "⚛️" | "🉑" | "☢️" | "☣️" | "📴" | "📳" | "🈶" | "🈚️" | "🈸" | "🈺" | "🈷️" | "✴️" | "🆚" | "💮" | "🉐" | "㊙️" | "㊗️" | "🈴" | "🈵" | "🈹" | "🈲" | "🅰️" | "🅱️" | "🆎" | "🆑" | "🅾️" | "🆘" | "❌" | "⭕️" | "🛑" | "⛔️" | "📛" | "🚫" | "💯" | "💢" | "♨️" | "🚷" | "🚯" | "🚳" | "🚱" | "🔞" | "📵" | "🚭" | "❗️" | "❕" | "❓" | "❔" | "‼️" | "⁉️" | "🔅" | "🔆" | "〽️" | "⚠️" | "🚸" | "🔱" | "⚜️" | "🔰" | "♻️" | "✅" | "🈯️" | "💹" | "❇️" | "✳️" | "❎" | "🌐" | "💠" | "Ⓜ️" | "🌀" | "💤" | "🏧" | "🚾" | "♿️" | "🅿️" | "🛗" | "🈳" | "🈂️" | "🛂" | "🛃" | "🛄" | "🛅" | "🚹" | "🚺" | "🚼" | "⚧" | "🚻" | "🚮" | "🎦" | "📶" | "🈁" | "🔣" | "ℹ️" | "🔤" | "🔡" | "🔠" | "🆖" | "🆗" | "🆙" | "🆒" | "🆕" | "🆓" | "0️⃣" | "1️⃣" | "2️⃣" | "3️⃣" | "4️⃣" | "5️⃣" | "6️⃣" | "7️⃣" | "8️⃣" | "9️⃣" | "🔟" | "🔢" | "#️⃣" | "*️⃣" | "⏏️" | "▶️" | "⏸" | "⏯" | "⏹" | "⏺" | "⏭" | "⏮" | "⏩" | "⏪" | "⏫" | "⏬" | "◀️" | "🔼" | "🔽" | "➡️" | "⬅️" | "⬆️" | "⬇️" | "↗️" | "↘️" | "↙️" | "↖️" | "↕️" | "↔️" | "↪️" | "↩️" | "⤴️" | "⤵️" | "🔀" | "🔁" | "🔂" | "🔄" | "🔃" | "🎵" | "🎶" | "➕" | "➖" | "➗" | "✖️" | "♾" | "💲" | "💱" | "™️" | "©️" | "®️" | "〰️" | "➰" | "➿" | "🔚" | "🔙" | "🔛" | "🔝" | "🔜" | "✔️" | "☑️" | "🔘" | "🔴" | "🟠" | "🟡" | "🟢" | "🔵" | "🟣" | "⚫️" | "⚪️" | "🟤" | "🔺" | "🔻" | "🔸" | "🔹" | "🔶" | "🔷" | "🔳" | "🔲" | "▪️" | "▫️" | "◾️" | "◽️" | "◼️" | "◻️" | "🟥" | "🟧" | "🟨" | "🟩" | "🟦" | "🟪" | "⬛️" | "⬜️" | "🟫" | "🔈" | "🔇" | "🔉" | "🔊" | "🔔" | "🔕" | "📣" | "📢" | "👁‍🗨" | "💬" | "💭" | "🗯" | "♠️" | "♣️" | "♥️" | "♦️" | "🃏" | "🎴" | "🀄️" | "🕐" | "🕑" | "🕒" | "🕓" | "🕔" | "🕕" | "🕖" | "🕗" | "🕘" | "🕙" | "🕚" | "🕛" | "🕜" | "🕝" | "🕞" | "🕟" | "🕠" | "🕡" | "🕢" | "🕣" | "🕤" | "🕥" | "🕦" | "🕧" | "✢" | "✣" | "✤" | "✥" | "✦" | "✧" | "★" | "☆" | "✯" | "✡︎" | "✩" | "✪" | "✫" | "✬" | "✭" | "✮" | "✶" | "✷" | "✵" | "✸" | "✹" | "→" | "⇒" | "⟹" | "⇨" | "⇾" | "➾" | "⇢" | "☛" | "☞" | "➔" | "➜" | "➙" | "➛" | "➝" | "➞" | "♠︎" | "♣︎" | "♥︎" | "♦︎" | "♤" | "♧" | "♡" | "♢" | "♚" | "♛" | "♜" | "♝" | "♞" | "♔" | "♕" | "♖" | "♗" | "♘" | "♙" | "⚀" | "⚁" | "⚂" | "⚃" | "⚄" | "⚅" | "🂠" | "⚈" | "⚉" | "⚆" | "⚇" | "𓀀" | "𓀁" | "𓀂" | "𓀃" | "𓀄" | "𓀅" | "𓀆" | "𓀇" | "𓀈" | "𓀉" | "𓀊" | "𓀋" | "𓀌" | "𓀍" | "𓀎" | "𓀏" | "𓀐" | "𓀑" | "𓀒" | "𓀓" | "𓀔" | "𓀕" | "𓀖" | "𓀗" | "𓀘" | "𓀙" | "𓀚" | "𓀛" | "𓀜" | "𓀝" | "🏳️" | "🏴" | "🏁" | "🚩" | "🏳️‍🌈" | "🏳️‍⚧️" | "🏴‍☠️" | "🇦🇫" | "🇦🇽" | "🇦🇱" | "🇩🇿" | "🇦🇸" | "🇦🇩" | "🇦🇴" | "🇦🇮" | "🇦🇶" | "🇦🇬" | "🇦🇷" | "🇦🇲" | "🇦🇼" | "🇦🇺" | "🇦🇹" | "🇦🇿" | "🇧🇸" | "🇧🇭" | "🇧🇩" | "🇧🇧" | "🇧🇾" | "🇧🇪" | "🇧🇿" | "🇧🇯" | "🇧🇲" | "🇧🇹" | "🇧🇴" | "🇧🇦" | "🇧🇼" | "🇧🇷" | "🇮🇴" | "🇻🇬" | "🇧🇳" | "🇧🇬" | "🇧🇫" | "🇧🇮" | "🇰🇭" | "🇨🇲" | "🇨🇦" | "🇮🇨" | "🇨🇻" | "🇧🇶" | "🇰🇾" | "🇨🇫" | "🇹🇩" | "🇨🇱" | "🇨🇳" | "🇨🇽" | "🇨🇨" | "🇨🇴" | "🇰🇲" | "🇨🇬" | "🇨🇩" | "🇨🇰" | "🇨🇷" | "🇨🇮" | "🇭🇷" | "🇨🇺" | "🇨🇼" | "🇨🇾" | "🇨🇿" | "🇩🇰" | "🇩🇯" | "🇩🇲" | "🇩🇴" | "🇪🇨" | "🇪🇬" | "🇸🇻" | "🇬🇶" | "🇪🇷" | "🇪🇪" | "🇪🇹" | "🇪🇺" | "🇫🇰" | "🇫🇴" | "🇫🇯" | "🇫🇮" | "🇫🇷" | "🇬🇫" | "🇵🇫" | "🇹🇫" | "🇬🇦" | "🇬🇲" | "🇬🇪" | "🇩🇪" | "🇬🇭" | "🇬🇮" | "🇬🇷" | "🇬🇱" | "🇬🇩" | "🇬🇵" | "🇬🇺" | "🇬🇹" | "🇬🇬" | "🇬🇳" | "🇬🇼" | "🇬🇾" | "🇭🇹" | "🇭🇳" | "🇭🇰" | "🇭🇺" | "🇮🇸" | "🇮🇳" | "🇮🇩" | "🇮🇷" | "🇮🇶" | "🇮🇪" | "🇮🇲" | "🇮🇱" | "🇮🇹" | "🇯🇲" | "🇯🇵" | "🎌" | "🇯🇪" | "🇯🇴" | "🇰🇿" | "🇰🇪" | "🇰🇮" | "🇽🇰" | "🇰🇼" | "🇰🇬" | "🇱🇦" | "🇱🇻" | "🇱🇧" | "🇱🇸" | "🇱🇷" | "🇱🇾" | "🇱🇮" | "🇱🇹" | "🇱🇺" | "🇲🇴" | "🇲🇰" | "🇲🇬" | "🇲🇼" | "🇲🇾" | "🇲🇻" | "🇲🇱" | "🇲🇹" | "🇲🇭" | "🇲🇶" | "🇲🇷" | "🇲🇺" | "🇾🇹" | "🇲🇽" | "🇫🇲" | "🇲🇩" | "🇲🇨" | "🇲🇳" | "🇲🇪" | "🇲🇸" | "🇲🇦" | "🇲🇿" | "🇲🇲" | "🇳🇦" | "🇳🇷" | "🇳🇵" | "🇳🇱" | "🇳🇨" | "🇳🇿" | "🇳🇮" | "🇳🇪" | "🇳🇬" | "🇳🇺" | "🇳🇫" | "🇰🇵" | "🇲🇵" | "🇳🇴" | "🇴🇲" | "🇵🇰" | "🇵🇼" | "🇵🇸" | "🇵🇦" | "🇵🇬" | "🇵🇾" | "🇵🇪" | "🇵🇭" | "🇵🇳" | "🇵🇱" | "🇵🇹" | "🇵🇷" | "🇶🇦" | "🇷🇪" | "🇷🇴" | "🇷🇺" | "🇷🇼" | "🇼🇸" | "🇸🇲" | "🇸🇦" | "🇸🇳" | "🇷🇸" | "🇸🇨" | "🇸🇱" | "🇸🇬" | "🇸🇽" | "🇸🇰" | "🇸🇮" | "🇬🇸" | "🇸🇧" | "🇸🇴" | "🇿🇦" | "🇰🇷" | "🇸🇸" | "🇪🇸" | "🇱🇰" | "🇧🇱" | "🇸🇭" | "🇰🇳" | "🇱🇨" | "🇵🇲" | "🇻🇨" | "🇸🇩" | "🇸🇷" | "🇸🇿" | "🇸🇪" | "🇨🇭" | "🇸🇾" | "🇹🇼" | "🇹🇯" | "🇹🇿" | "🇹🇭" | "🇹🇱" | "🇹🇬" | "🇹🇰" | "🇹🇴" | "🇹🇹" | "🇹🇳" | "🇹🇷" | "🇹🇲" | "🇹🇨" | "🇹🇻" | "🇻🇮" | "🇺🇬" | "🇺🇦" | "🇦🇪" | "🇬🇧" | "🏴󠁧󠁢󠁥󠁮󠁧󠁿" | "🏴󠁧󠁢󠁳󠁣󠁴󠁿" | "🏴󠁧󠁢󠁷󠁬󠁳󠁿" | "🇺🇳" | "🇺🇸" | "🇺🇾" | "🇺🇿" | "🇻🇺" | "🇻🇦" | "🇻🇪" | "🇻🇳" | "🇼🇫" | "🇪🇭" | "🇾🇪" | "🇿🇲" | "🇿🇼" | "🦭" | "😮‍💨" | "😵‍💫" | "😶‍🌫️" | "❤️‍🔥" | "❤️‍🩹" | "🧔‍♀️" | "🧔🏻‍♀️" | "🧔🏼‍♀️" | "🧔🏽‍♀️" | "🧔🏾‍♀️" | "🧔🏿‍♀️" | "🧔‍♂️" | "🧔🏻‍♂️" | "🧔🏼‍♂️" | "🧔🏽‍♂️" | "🧔🏾‍♂️" | "🧔🏿‍♂️" | "💑🏻" | "💑🏼" | "💑🏽" | "💑🏾" | "💑🏿" | "💏🏻" | "💏🏼" | "💏🏽" | "💏🏾" | "💏🏿" | "👨🏻‍❤️‍👨🏻" | "👨🏻‍❤️‍👨🏼" | "👨🏻‍❤️‍👨🏽" | "👨🏻‍❤️‍👨🏾" | "👨🏻‍❤️‍👨🏿" | "👨🏼‍❤️‍👨🏻" | "👨🏼‍❤️‍👨🏼" | "👨🏼‍❤️‍👨🏽" | "👨🏼‍❤️‍👨🏾" | "👨🏼‍❤️‍👨🏿" | "👨🏽‍❤️‍👨🏻" | "👨🏽‍❤️‍👨🏼" | "👨🏽‍❤️‍👨🏽" | "👨🏽‍❤️‍👨🏾" | "👨🏽‍❤️‍👨🏿" | "👨🏾‍❤️‍👨🏻" | "👨🏾‍❤️‍👨🏼" | "👨🏾‍❤️‍👨🏽" | "👨🏾‍❤️‍👨🏾" | "👨🏾‍❤️‍👨🏿" | "👨🏿‍❤️‍👨🏻" | "👨🏿‍❤️‍👨🏼" | "👨🏿‍❤️‍👨🏽" | "👨🏿‍❤️‍👨🏾" | "👨🏿‍❤️‍👨🏿" | "👩🏻‍❤️‍👨🏻" | "👩🏻‍❤️‍👨🏼" | "👩🏻‍❤️‍👨🏽" | "👩🏻‍❤️‍👨🏾" | "👩🏻‍❤️‍👨🏿" | "👩🏻‍❤️‍👩🏻" | "👩🏻‍❤️‍👩🏼" | "👩🏻‍❤️‍👩🏽" | "👩🏻‍❤️‍👩🏾" | "👩🏻‍❤️‍👩🏿" | "👩🏼‍❤️‍👨🏻" | "👩🏼‍❤️‍👨🏼" | "👩🏼‍❤️‍👨🏽" | "👩🏼‍❤️‍👨🏾" | "👩🏼‍❤️‍👨🏿" | "👩🏼‍❤️‍👩🏻" | "👩🏼‍❤️‍👩🏼" | "👩🏼‍❤️‍👩🏽" | "👩🏼‍❤️‍👩🏾" | "👩🏼‍❤️‍👩🏿" | "👩🏽‍❤️‍👨🏻" | "👩🏽‍❤️‍👨🏼" | "👩🏽‍❤️‍👨🏽" | "👩🏽‍❤️‍👨🏾" | "👩🏽‍❤️‍👨🏿" | "👩🏽‍❤️‍👩🏻" | "👩🏽‍❤️‍👩🏼" | "👩🏽‍❤️‍👩🏽" | "👩🏽‍❤️‍👩🏾" | "👩🏽‍❤️‍👩🏿" | "👩🏾‍❤️‍👨🏻" | "👩🏾‍❤️‍👨🏼" | "👩🏾‍❤️‍👨🏽" | "👩🏾‍❤️‍👨🏾" | "👩🏾‍❤️‍👨🏿" | "👩🏾‍❤️‍👩🏻" | "👩🏾‍❤️‍👩🏼" | "👩🏾‍❤️‍👩🏽" | "👩🏾‍❤️‍👩🏾" | "👩🏾‍❤️‍👩🏿" | "👩🏿‍❤️‍👨🏻" | "👩🏿‍❤️‍👨🏼" | "👩🏿‍❤️‍👨🏽" | "👩🏿‍❤️‍👨🏾" | "👩🏿‍❤️‍👨🏿" | "👩🏿‍❤️‍👩🏻" | "👩🏿‍❤️‍👩🏼" | "👩🏿‍❤️‍👩🏽" | "👩🏿‍❤️‍👩🏾" | "👩🏿‍❤️‍👩🏿" | "🧑🏻‍❤️‍🧑🏼" | "🧑🏻‍❤️‍🧑🏽" | "🧑🏻‍❤️‍🧑🏾" | "🧑🏻‍❤️‍🧑🏿" | "🧑🏼‍❤️‍🧑🏻" | "🧑🏼‍❤️‍🧑🏽" | "🧑🏼‍❤️‍🧑🏾" | "🧑🏼‍❤️‍🧑🏿" | "🧑🏽‍❤️‍🧑🏻" | "🧑🏽‍❤️‍🧑🏼" | "🧑🏽‍❤️‍🧑🏾" | "🧑🏽‍❤️‍🧑🏿" | "🧑🏾‍❤️‍🧑🏻" | "🧑🏾‍❤️‍🧑🏼" | "🧑🏾‍❤️‍🧑🏽" | "🧑🏾‍❤️‍🧑🏿" | "🧑🏿‍❤️‍🧑🏻" | "🧑🏿‍❤️‍🧑🏼" | "🧑🏿‍❤️‍🧑🏽" | "🧑🏿‍❤️‍🧑🏾" | "👨🏻‍❤️‍💋‍👨🏻" | "👨🏻‍❤️‍💋‍👨🏼" | "👨🏻‍❤️‍💋‍👨🏽" | "👨🏻‍❤️‍💋‍👨🏾" | "👨🏻‍❤️‍💋‍👨🏿" | "👨🏼‍❤️‍💋‍👨🏻" | "👨🏼‍❤️‍💋‍👨🏼" | "👨🏼‍❤️‍💋‍👨🏽" | "👨🏼‍❤️‍💋‍👨🏾" | "👨🏼‍❤️‍💋‍👨🏿" | "👨🏽‍❤️‍💋‍👨🏻" | "👨🏽‍❤️‍💋‍👨🏼" | "👨🏽‍❤️‍💋‍👨🏽" | "👨🏽‍❤️‍💋‍👨🏾" | "👨🏽‍❤️‍💋‍👨🏿" | "👨🏾‍❤️‍💋‍👨🏻" | "👨🏾‍❤️‍💋‍👨🏼" | "👨🏾‍❤️‍💋‍👨🏽" | "👨🏾‍❤️‍💋‍👨🏾" | "👨🏾‍❤️‍💋‍👨🏿" | "👨🏿‍❤️‍💋‍👨🏻" | "👨🏿‍❤️‍💋‍👨🏼" | "👨🏿‍❤️‍💋‍👨🏽" | "👨🏿‍❤️‍💋‍👨🏾" | "👨🏿‍❤️‍💋‍👨🏿" | "👩🏻‍❤️‍💋‍👨🏻" | "👩🏻‍❤️‍💋‍👨🏼" | "👩🏻‍❤️‍💋‍👨🏽" | "👩🏻‍❤️‍💋‍👨🏾" | "👩🏻‍❤️‍💋‍👨🏿" | "👩🏻‍❤️‍💋‍👩🏻" | "👩🏻‍❤️‍💋‍👩🏼" | "👩🏻‍❤️‍💋‍👩🏽" | "👩🏻‍❤️‍💋‍👩🏾" | "👩🏻‍❤️‍💋‍👩🏿" | "👩🏼‍❤️‍💋‍👨🏻" | "👩🏼‍❤️‍💋‍👨🏼" | "👩🏼‍❤️‍💋‍👨🏽" | "👩🏼‍❤️‍💋‍👨🏾" | "👩🏼‍❤️‍💋‍👨🏿" | "👩🏼‍❤️‍💋‍👩🏻" | "👩🏼‍❤️‍💋‍👩🏼" | "👩🏼‍❤️‍💋‍👩🏽" | "👩🏼‍❤️‍💋‍👩🏾" | "👩🏼‍❤️‍💋‍👩🏿" | "👩🏽‍❤️‍💋‍👨🏻" | "👩🏽‍❤️‍💋‍👨🏼" | "👩🏽‍❤️‍💋‍👨🏽" | "👩🏽‍❤️‍💋‍👨🏾" | "👩🏽‍❤️‍💋‍👨🏿" | "👩🏽‍❤️‍💋‍👩🏻" | "👩🏽‍❤️‍💋‍👩🏼" | "👩🏽‍❤️‍💋‍👩🏽" | "👩🏽‍❤️‍💋‍👩🏾" | "👩🏽‍❤️‍💋‍👩🏿" | "👩🏾‍❤️‍💋‍👨🏻" | "👩🏾‍❤️‍💋‍👨🏼" | "👩🏾‍❤️‍💋‍👨🏽" | "👩🏾‍❤️‍💋‍👨🏾" | "👩🏾‍❤️‍💋‍👨🏿" | "👩🏾‍❤️‍💋‍👩🏻" | "👩🏾‍❤️‍💋‍👩🏼" | "👩🏾‍❤️‍💋‍👩🏽" | "👩🏾‍❤️‍💋‍👩🏾" | "👩🏾‍❤️‍💋‍👩🏿" | "👩🏿‍❤️‍💋‍👨🏻" | "👩🏿‍❤️‍💋‍👨🏼" | "👩🏿‍❤️‍💋‍👨🏽" | "👩🏿‍❤️‍💋‍👨🏾" | "👩🏿‍❤️‍💋‍👨🏿" | "👩🏿‍❤️‍💋‍👩🏻" | "👩🏿‍❤️‍💋‍👩🏼" | "👩🏿‍❤️‍💋‍👩🏽" | "👩🏿‍❤️‍💋‍👩🏾" | "👩🏿‍❤️‍💋‍👩🏿" | "🧑🏻‍❤️‍💋‍🧑🏼" | "🧑🏻‍❤️‍💋‍🧑🏽" | "🧑🏻‍❤️‍💋‍🧑🏾" | "🧑🏻‍❤️‍💋‍🧑🏿" | "🧑🏼‍❤️‍💋‍🧑🏻" | "🧑🏼‍❤️‍💋‍🧑🏽" | "🧑🏼‍❤️‍💋‍🧑🏾" | "🧑🏼‍❤️‍💋‍🧑🏿" | "🧑🏽‍❤️‍💋‍🧑🏻" | "🧑🏽‍❤️‍💋‍🧑🏼" | "🧑🏽‍❤️‍💋‍🧑🏾" | "🧑🏽‍❤️‍💋‍🧑🏿" | "🧑🏾‍❤️‍💋‍🧑🏻" | "🧑🏾‍❤️‍💋‍🧑🏼" | "🧑🏾‍❤️‍💋‍🧑🏽" | "🧑🏾‍❤️‍💋‍🧑🏿" | "🧑🏿‍❤️‍💋‍🧑🏻" | "🧑🏿‍❤️‍💋‍🧑🏼" | "🧑🏿‍❤️‍💋‍🧑🏽" | "🧑🏿‍❤️‍💋‍🧑🏾"'; /** * Note: [💞] Ignore a discrepancy between file name and entity name diff --git a/src/types/typeAliases.ts b/src/types/typeAliases.ts index 2ba156a407..3b424bfb0d 100644 --- a/src/types/typeAliases.ts +++ b/src/types/typeAliases.ts @@ -4,14 +4,19 @@ '--------------------------------------------------' */ -import type { TupleToUnion } from 'type-fest'; -import { RESERVED_PARAMETER_NAMES } from '../constants'; -import type { really_unknown } from '../utils/organization/really_unknown'; +import type { TupleToUnion } from "type-fest"; +import type { RESERVED_PARAMETER_NAMES } from "../constants"; +import type { really_unknown } from "../utils/organization/really_unknown"; /** * Semantic helper */ -export type string_business_category_name = 'restaurant' | 'grocery' | 'person' | 'conference' | string; +export type string_business_category_name = + | "restaurant" + | "grocery" + | "person" + | "conference" + | string; /** * Semantic helper @@ -19,18 +24,18 @@ export type string_business_category_name = 'restaurant' | 'grocery' | 'person' * For example `"gpt-4"` */ export type string_model_name = - | 'gpt-4' - | 'gpt-4-0314' - | 'gpt-4-0613' - | 'gpt-4-32k' - | 'gpt-4-32k-0314' - | 'gpt-4-32k-0613' - | 'gpt-3.5-turbo' - | 'gpt-3.5-turbo-16k' - | 'gpt-3.5-turbo-0301' - | 'gpt-3.5-turbo-0613' - | 'gpt-3.5-turbo-16k-0613' - | string /* <- TODO: Import from 'openai' package */; + | "gpt-4" + | "gpt-4-0314" + | "gpt-4-0613" + | "gpt-4-32k" + | "gpt-4-32k-0314" + | "gpt-4-32k-0613" + | "gpt-3.5-turbo" + | "gpt-3.5-turbo-16k" + | "gpt-3.5-turbo-0301" + | "gpt-3.5-turbo-0613" + | "gpt-3.5-turbo-16k-0613" + | string /* <- TODO: Import from 'openai' package */; /** * Semantic helper @@ -80,7 +85,7 @@ export type string_completion_prompt = string_text_prompt; * For example `"index"` or `"explanation"` * Always in kebab-case */ -export type string_page = 'index' | string; +export type string_page = "index" | string; /** * Semantic helper @@ -141,7 +146,10 @@ export type string_parameter_value = string; * Note: [🚉] This is fully serializable as JSON * @see https://ptbk.io/parameters */ -export type Parameters = Exclude, ReservedParameters>; +export type Parameters = Exclude< + Record, + ReservedParameters +>; /** * Parameters to pass to execution of the pipeline @@ -149,7 +157,10 @@ export type Parameters = Exclude, ReservedParameters>; +export type InputParameters = Exclude< + Record, + ReservedParameters +>; // <- TODO: [🧠] Maybe rename `Parameters` because it is already defined in global scope and also it is used more generally [👩🏾‍🤝‍🧑🏽] @@ -159,14 +170,19 @@ export type InputParameters = Exclude; +export type string_reserved_parameter_name = TupleToUnion< + typeof RESERVED_PARAMETER_NAMES +>; /** * @@@ * * Note: [🚉] This is fully serializable as JSON */ -export type ReservedParameters = Record; +export type ReservedParameters = Record< + string_reserved_parameter_name, + string_parameter_value +>; /** * Semantic helper @@ -202,7 +218,9 @@ export type string_model_description = string; * * @@@ string_knowledge_source vs string_knowledge_source_link */ -export type string_knowledge_source_content = string_knowledge_source_link | string_markdown; +export type string_knowledge_source_content = + | string_knowledge_source_link + | string_markdown; /** * One link to knowledge source @@ -282,15 +300,22 @@ export type string_markdown_text = string; * * @public exported from `@promptbook/markdown-utils` */ -export type string_markdown_codeblock_language = 'book' | 'markdown' | 'text' | 'javascript' | 'css' | 'json'; +export type string_markdown_codeblock_language = + | "book" + | "markdown" + | "text" + | "javascript" + | "css" + | "json"; // <- TODO: [🏥] DRY /** * @@@ */ -export type string_promptbook_documentation_url = `https://github.com/webgptorg/promptbook/discussions/${ - | number - | `@@${string}`}`; +export type string_promptbook_documentation_url = + `https://github.com/webgptorg/promptbook/discussions/${ + | number + | `@@${string}`}`; /** * Semantic helper @@ -341,7 +366,10 @@ export type string_javascript = string; * * For example `{"foo": "bar"}` */ -export type string_json = string & { _type: 'string_json'; scheme: TType }; +export type string_json = string & { + _type: "string_json"; + scheme: TType; +}; /** * Semantic helper @@ -411,7 +439,8 @@ export type string_pipeline_url_with_task_hash = string; * * For example `"data:text/plain;base64,SGVsbG8sIFdvcmxkIQ=="` */ -export type string_data_url = `data:${string_mime_type};base64,${string_base64}`; +export type string_data_url = + `data:${string_mime_type};base64,${string_base64}`; /** * Semantic helper @@ -473,7 +502,7 @@ export type string_host = string; /** * Semantic helper */ -export type string_protocol = 'http:' | 'https:'; +export type string_protocol = "http:" | "https:"; /** * Semantic helper @@ -501,7 +530,7 @@ export type string_emails = string; * TODO: [🥬] Make some system for hashes and ids of promptbook */ export type string_uuid = string & { - readonly _type: 'uuid' /* <- TODO: [🏟] What is the best shape of the additional object in branded types */; + readonly _type: "uuid" /* <- TODO: [🏟] What is the best shape of the additional object in branded types */; }; /** @@ -563,7 +592,9 @@ export type string_relative_filename = string; /** * Semantic helper */ -export type string_filename = string_absolute_filename | string_relative_filename; +export type string_filename = + | string_absolute_filename + | string_relative_filename; // TODO: Do not use universal string_filename/string_dirname but specific ones likestring_relative_filename @@ -650,7 +681,7 @@ export type string_translate_name_not_normalized = string; * * TODO: Probably use enum */ -export type string_translate_language = 'en' | 'cs'; +export type string_translate_language = "en" | "cs"; /** * Semantic helper; For example "callbackName" or "renderMe" @@ -677,7 +708,8 @@ export type string_pgp_key = string; * @see https://en.wikipedia.org/wiki/ISO_8601 * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString */ -export type string_date_iso8601 = `${number}-${number}-${number}${string}${number}:${number}:${number}${string}`; +export type string_date_iso8601 = + `${number}-${number}-${number}${string}${number}:${number}:${number}${string}`; //=========================[ Numbers ]========================= diff --git a/src/utils/$Register.ts b/src/utils/$Register.ts index d46e420131..ef704ea47a 100644 --- a/src/utils/$Register.ts +++ b/src/utils/$Register.ts @@ -1,37 +1,37 @@ -import { type IDestroyable } from 'destroyable'; -import { DEFAULT_IS_VERBOSE } from '../config'; -import { NotYetImplementedError } from '../errors/NotYetImplementedError'; -import { UnexpectedError } from '../errors/UnexpectedError'; -import type { string_name } from '../types/typeAliases'; -import { $getGlobalScope } from './environment/$getGlobalScope'; -import { normalizeTo_snake_case } from './normalization/normalizeTo_snake_case'; -import type { TODO_string } from './organization/TODO_string'; +import type { IDestroyable } from "destroyable"; +import { DEFAULT_IS_VERBOSE } from "../config"; +import { NotYetImplementedError } from "../errors/NotYetImplementedError"; +import { UnexpectedError } from "../errors/UnexpectedError"; +import type { string_name } from "../types/typeAliases"; +import { $getGlobalScope } from "./environment/$getGlobalScope"; +import { normalizeTo_snake_case } from "./normalization/normalizeTo_snake_case"; +import type { TODO_string } from "./organization/TODO_string"; /** * @@@ */ export type Registered = { - /** - * @@@ - */ - readonly packageName: TODO_string; + /** + * @@@ + */ + readonly packageName: TODO_string; - /** - * @@@ - */ - readonly className: TODO_string; + /** + * @@@ + */ + readonly className: TODO_string; }; /** * @@@ */ export type Registration = Registered & - IDestroyable & { - /** - * @@@ - */ - readonly registerName: string_name; - }; + IDestroyable & { + /** + * @@@ + */ + readonly registerName: string_name; + }; /** * Register is @@@ @@ -41,61 +41,66 @@ export type Registration = Registered & * @private internal utility, exported are only signleton instances of this class */ export class $Register { - private readonly storage: Array; + private readonly storage: Array; - constructor(private readonly registerName: string_name) { - const storageName = `_promptbook_${normalizeTo_snake_case(registerName)}`; + constructor(private readonly registerName: string_name) { + const storageName = `_promptbook_${normalizeTo_snake_case(registerName)}`; - const globalScope = $getGlobalScope(); + const globalScope = $getGlobalScope(); - if (globalScope[storageName] === undefined) { - globalScope[storageName] = []; - } else if (!Array.isArray(globalScope[storageName])) { - throw new UnexpectedError( - `Expected (global) ${storageName} to be an array, but got ${typeof globalScope[storageName]}`, - ); - } + if (globalScope[storageName] === undefined) { + globalScope[storageName] = []; + } else if (!Array.isArray(globalScope[storageName])) { + throw new UnexpectedError( + `Expected (global) ${storageName} to be an array, but got ${typeof globalScope[storageName]}`, + ); + } - this.storage = globalScope[storageName]; - } + this.storage = globalScope[storageName]; + } - public list(): ReadonlyArray { - // <- TODO: ReadonlyDeep> - return this.storage; - } + public list(): ReadonlyArray { + // <- TODO: ReadonlyDeep> + return this.storage; + } - public register(registered: TRegistered): Registration { - const { packageName, className } = registered; + public register(registered: TRegistered): Registration { + const { packageName, className } = registered; - const existingRegistrationIndex = this.storage.findIndex( - (item) => item.packageName === packageName && item.className === className, - ); - const existingRegistration = this.storage[existingRegistrationIndex]; + const existingRegistrationIndex = this.storage.findIndex( + (item) => + item.packageName === packageName && item.className === className, + ); + const existingRegistration = this.storage[existingRegistrationIndex]; - if (!existingRegistration) { - if (DEFAULT_IS_VERBOSE) { - console.warn(`[📦] Registering \`${packageName}.${className}\` to \`${this.registerName}\``); - } - this.storage.push(registered); - } else { - if (DEFAULT_IS_VERBOSE) { - console.warn(`[📦] Re-registering \`${packageName}.${className}\` to \`${this.registerName}\``); - } - this.storage[existingRegistrationIndex] = registered; - } + if (!existingRegistration) { + if (DEFAULT_IS_VERBOSE) { + console.warn( + `[📦] Registering \`${packageName}.${className}\` to \`${this.registerName}\``, + ); + } + this.storage.push(registered); + } else { + if (DEFAULT_IS_VERBOSE) { + console.warn( + `[📦] Re-registering \`${packageName}.${className}\` to \`${this.registerName}\``, + ); + } + this.storage[existingRegistrationIndex] = registered; + } - return { - registerName: this.registerName, - packageName, - className, - get isDestroyed() { - return false; - }, - destroy() { - throw new NotYetImplementedError( - `Registration to ${this.registerName} is permanent in this version of Promptbook`, - ); - }, - }; - } + return { + registerName: this.registerName, + packageName, + className, + get isDestroyed() { + return false; + }, + destroy() { + throw new NotYetImplementedError( + `Registration to ${this.registerName} is permanent in this version of Promptbook`, + ); + }, + }; + } } diff --git a/src/utils/$getCurrentDate.ts b/src/utils/$getCurrentDate.ts index a4720c110e..ed5bdabfe0 100644 --- a/src/utils/$getCurrentDate.ts +++ b/src/utils/$getCurrentDate.ts @@ -1,4 +1,4 @@ -import type { string_date_iso8601 } from '../types/typeAliases'; +import type { string_date_iso8601 } from "../types/typeAliases"; /** * Simple wrapper `new Date().toISOString()` @@ -9,5 +9,5 @@ import type { string_date_iso8601 } from '../types/typeAliases'; * @public exported from `@promptbook/utils` */ export function $getCurrentDate(): string_date_iso8601 { - return new Date().toISOString() as string_date_iso8601; + return new Date().toISOString() as string_date_iso8601; } diff --git a/src/utils/FromtoItems.ts b/src/utils/FromtoItems.ts index 1997aea12a..b1131aab1a 100644 --- a/src/utils/FromtoItems.ts +++ b/src/utils/FromtoItems.ts @@ -2,20 +2,20 @@ * Represents items which are compared to each other */ export type FromtoItems = ReadonlyArray<{ - /** - * The title of the item - */ - readonly title: string; + /** + * The title of the item + */ + readonly title: string; - /** - * The start of the item - * - * This can mean anything, like seconds, milliseconds, dollars, etc. - */ - readonly from: number; + /** + * The start of the item + * + * This can mean anything, like seconds, milliseconds, dollars, etc. + */ + readonly from: number; - /** - * The end of the item - */ - readonly to: number; + /** + * The end of the item + */ + readonly to: number; }>; diff --git a/src/utils/arrayableToArray.test.ts b/src/utils/arrayableToArray.test.ts index 7be072782d..5fc2fa5922 100644 --- a/src/utils/arrayableToArray.test.ts +++ b/src/utils/arrayableToArray.test.ts @@ -1,27 +1,27 @@ -import { arrayableToArray } from './arrayableToArray'; +import { arrayableToArray } from "./arrayableToArray"; -describe('how arrayableToArray works', () => { - it('will create array from nothing', () => { - expect(arrayableToArray(undefined)).toEqual([]); - expect(arrayableToArray()).toEqual([]); - expect(arrayableToArray([])).toEqual([]); - }); +describe("how arrayableToArray works", () => { + it("will create array from nothing", () => { + expect(arrayableToArray(undefined)).toEqual([]); + expect(arrayableToArray()).toEqual([]); + expect(arrayableToArray([])).toEqual([]); + }); - it('will create array from one item', () => { - expect(arrayableToArray(1)).toEqual([1]); - expect(arrayableToArray(null)).toEqual([null]); - expect(arrayableToArray([undefined])).toEqual([undefined]); - expect(arrayableToArray('a')).toEqual(['a']); - expect(arrayableToArray([1])).toEqual([1]); - expect(arrayableToArray(['a'])).toEqual(['a']); - expect(arrayableToArray({ a: 1 })).toEqual([{ a: 1 }]); - expect(arrayableToArray([{ a: 1 }])).toEqual([{ a: 1 }]); - }); + it("will create array from one item", () => { + expect(arrayableToArray(1)).toEqual([1]); + expect(arrayableToArray(null)).toEqual([null]); + expect(arrayableToArray([undefined])).toEqual([undefined]); + expect(arrayableToArray("a")).toEqual(["a"]); + expect(arrayableToArray([1])).toEqual([1]); + expect(arrayableToArray(["a"])).toEqual(["a"]); + expect(arrayableToArray({ a: 1 })).toEqual([{ a: 1 }]); + expect(arrayableToArray([{ a: 1 }])).toEqual([{ a: 1 }]); + }); - it('will create array from multiple items', () => { - expect(arrayableToArray([])).toEqual([]); - expect(arrayableToArray([[]])).toEqual([[]]); - expect(arrayableToArray([1, 2, 3])).toEqual([1, 2, 3]); - expect(arrayableToArray(['a', 2, 3])).toEqual(['a', 2, 3]); - }); + it("will create array from multiple items", () => { + expect(arrayableToArray([])).toEqual([]); + expect(arrayableToArray([[]])).toEqual([[]]); + expect(arrayableToArray([1, 2, 3])).toEqual([1, 2, 3]); + expect(arrayableToArray(["a", 2, 3])).toEqual(["a", 2, 3]); + }); }); diff --git a/src/utils/arrayableToArray.ts b/src/utils/arrayableToArray.ts index 6bac3f45be..54a17a7583 100644 --- a/src/utils/arrayableToArray.ts +++ b/src/utils/arrayableToArray.ts @@ -1,4 +1,4 @@ -import type { Arrayable } from '../types/Arrayable'; +import type { Arrayable } from "../types/Arrayable"; /** * Takes an item or an array of items and returns an array of items @@ -9,14 +9,16 @@ import type { Arrayable } from '../types/Arrayable'; * * @private internal utility */ -export function arrayableToArray(input?: Arrayable): ReadonlyArray { - if (input === undefined) { - return []; - } +export function arrayableToArray( + input?: Arrayable, +): ReadonlyArray { + if (input === undefined) { + return []; + } - if (input instanceof Array) { - return input; - } + if (input instanceof Array) { + return input; + } - return [input]; + return [input]; } diff --git a/src/utils/editable/edit-pipeline-string/addPipelineCommand.test.ts b/src/utils/editable/edit-pipeline-string/addPipelineCommand.test.ts index 8dfd77220f..25b36e633e 100644 --- a/src/utils/editable/edit-pipeline-string/addPipelineCommand.test.ts +++ b/src/utils/editable/edit-pipeline-string/addPipelineCommand.test.ts @@ -1,44 +1,44 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { DEFAULT_BOOK_TITLE } from '../../../config'; -import type { PipelineString } from '../../../pipeline/PipelineString'; -import { just } from '../../organization/just'; -import { addPipelineCommand } from './addPipelineCommand'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { DEFAULT_BOOK_TITLE } from "../../../config"; +import type { PipelineString } from "../../../pipeline/PipelineString"; +import { just } from "../../organization/just"; +import { addPipelineCommand } from "./addPipelineCommand"; -describe('how `addPipelineCommand` works', () => { - it('should add HEAD command', () => - expect( - addPipelineCommand({ - commandString: 'KNOWLEDGE https://pavolhejny.com/', - pipelineString: spaceTrim(` +describe("how `addPipelineCommand` works", () => { + it("should add HEAD command", () => + expect( + addPipelineCommand({ + commandString: "KNOWLEDGE https://pavolhejny.com/", + pipelineString: spaceTrim(` # Book `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - }), - ).toBe( - just( - spaceTrim(` + // <- TODO: [📼] Use`book\`` string literal notation + }), + ).toBe( + just( + spaceTrim(` # Book - KNOWLEDGE https://pavolhejny.com/ `), - ), - )); + ), + )); - it('should add command to flat pipeline', () => - expect( - addPipelineCommand({ - commandString: 'KNOWLEDGE https://pavolhejny.com/', - pipelineString: spaceTrim(` + it("should add command to flat pipeline", () => + expect( + addPipelineCommand({ + commandString: "KNOWLEDGE https://pavolhejny.com/", + pipelineString: spaceTrim(` Hello, how are you? -> {answer} `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - }), - ).toBe( - just( - spaceTrim(` + // <- TODO: [📼] Use`book\`` string literal notation + }), + ).toBe( + just( + spaceTrim(` # ${DEFAULT_BOOK_TITLE} - KNOWLEDGE https://pavolhejny.com/ @@ -49,31 +49,31 @@ describe('how `addPipelineCommand` works', () => { -> {answer} `), - // <- TODO: [📼] Use`book\`` string literal notation - ), - )); + // <- TODO: [📼] Use`book\`` string literal notation + ), + )); - it('should preserve existing commands', () => - expect( - addPipelineCommand({ - commandString: 'KNOWLEDGE https://example.com/', - pipelineString: spaceTrim(` + it("should preserve existing commands", () => + expect( + addPipelineCommand({ + commandString: "KNOWLEDGE https://example.com/", + pipelineString: spaceTrim(` # Book - PERSONA Paul, developer - KNOWLEDGE https://pavolhejny.com/ `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - }), - ).toBe( - just( - spaceTrim(` + // <- TODO: [📼] Use`book\`` string literal notation + }), + ).toBe( + just( + spaceTrim(` # Book - PERSONA Paul, developer - KNOWLEDGE https://pavolhejny.com/ - KNOWLEDGE https://example.com/ `), - ), - )); + ), + )); }); diff --git a/src/utils/editable/edit-pipeline-string/addPipelineCommand.ts b/src/utils/editable/edit-pipeline-string/addPipelineCommand.ts index 81b2074573..acaaf64e7d 100644 --- a/src/utils/editable/edit-pipeline-string/addPipelineCommand.ts +++ b/src/utils/editable/edit-pipeline-string/addPipelineCommand.ts @@ -1,11 +1,11 @@ -import spaceTrim from 'spacetrim'; -import type { PipelineString } from '../../../pipeline/PipelineString'; -import type { string_markdown_text } from '../../../types/typeAliases'; -import { deflatePipeline } from './deflatePipeline'; +import spaceTrim from "spacetrim"; +import type { PipelineString } from "../../../pipeline/PipelineString"; +import type { string_markdown_text } from "../../../types/typeAliases"; +import { deflatePipeline } from "./deflatePipeline"; type AddPipelineCommandOptions = { - commandString: string_markdown_text; - pipelineString: PipelineString; + commandString: string_markdown_text; + pipelineString: PipelineString; }; /** @@ -13,35 +13,37 @@ type AddPipelineCommandOptions = { * * @public exported from `@promptbook/editable` */ -export function addPipelineCommand(options: AddPipelineCommandOptions): PipelineString { - const { commandString, pipelineString } = options; +export function addPipelineCommand( + options: AddPipelineCommandOptions, +): PipelineString { + const { commandString, pipelineString } = options; - const deflatedPipelineString = deflatePipeline(pipelineString); + const deflatedPipelineString = deflatePipeline(pipelineString); - const lines = deflatedPipelineString.split('\n'); - const newLines: Array = []; - let isCommandAdded = false; + const lines = deflatedPipelineString.split("\n"); + const newLines: Array = []; + let isCommandAdded = false; - for (const line of lines) { - // Add command before second (or more) heading - if (!isCommandAdded && line.startsWith('##')) { - newLines.push(`- ${commandString}`); - newLines.push(''); - isCommandAdded = true; - } + for (const line of lines) { + // Add command before second (or more) heading + if (!isCommandAdded && line.startsWith("##")) { + newLines.push(`- ${commandString}`); + newLines.push(""); + isCommandAdded = true; + } - newLines.push(line); - } + newLines.push(line); + } - if (!isCommandAdded) { - // Note: Only situation when this should happen is when pipeline has no tasks + if (!isCommandAdded) { + // Note: Only situation when this should happen is when pipeline has no tasks - if ((newLines[newLines.length - 1] || '').startsWith('#')) { - newLines.push(''); - } - newLines.push(`- ${commandString}`); + if ((newLines[newLines.length - 1] || "").startsWith("#")) { + newLines.push(""); + } + newLines.push(`- ${commandString}`); - /* + /* TODO: [🧠] Is this error relevant: throw new UnexpectedError( spaceTrim( @@ -66,9 +68,9 @@ export function addPipelineCommand(options: AddPipelineCommandOptions): Pipeline ), ); */ - } + } - return spaceTrim(newLines.join('\n')) as PipelineString; + return spaceTrim(newLines.join("\n")) as PipelineString; } /** diff --git a/src/utils/editable/edit-pipeline-string/deflatePipeline.test.ts b/src/utils/editable/edit-pipeline-string/deflatePipeline.test.ts index 76b4b715ac..9fa670e8aa 100644 --- a/src/utils/editable/edit-pipeline-string/deflatePipeline.test.ts +++ b/src/utils/editable/edit-pipeline-string/deflatePipeline.test.ts @@ -1,13 +1,13 @@ -import { describe, expect, it } from '@jest/globals'; -import { DEFAULT_BOOK_OUTPUT_PARAMETER_NAME } from '../../../config'; -import { DEFAULT_BOOK_TITLE } from '../../../config'; -import { DEFAULT_PROMPT_TASK_TITLE } from '../../../config'; -import { book } from '../../../pipeline/book-notation'; -import { deflatePipeline } from './deflatePipeline'; - -describe('deflatePipeline', () => { - it('should not modify non-flat pipeline', () => { - const pipeline = book` +import { describe, expect, it } from "@jest/globals"; +import { DEFAULT_BOOK_OUTPUT_PARAMETER_NAME } from "../../../config"; +import { DEFAULT_BOOK_TITLE } from "../../../config"; +import { DEFAULT_PROMPT_TASK_TITLE } from "../../../config"; +import { book } from "../../../pipeline/book-notation"; +import { deflatePipeline } from "./deflatePipeline"; + +describe("deflatePipeline", () => { + it("should not modify non-flat pipeline", () => { + const pipeline = book` # Some Title ## Greeting @@ -17,15 +17,15 @@ describe('deflatePipeline', () => { -> {response} `; - expect(deflatePipeline(pipeline)).toBe(pipeline); - }); + expect(deflatePipeline(pipeline)).toBe(pipeline); + }); - it('should convert flat pipeline to structured format', () => { - const flatPipeline = book` + it("should convert flat pipeline to structured format", () => { + const flatPipeline = book` Hello `; - const expected = book` + const expected = book` # ${DEFAULT_BOOK_TITLE} ## ${DEFAULT_PROMPT_TASK_TITLE} @@ -35,17 +35,17 @@ describe('deflatePipeline', () => { -> {${DEFAULT_BOOK_OUTPUT_PARAMETER_NAME}} `; - expect(deflatePipeline(flatPipeline)).toBe(expected); - }); + expect(deflatePipeline(flatPipeline)).toBe(expected); + }); - it('should convert flat pipeline with return statement to structured format', () => { - const flatPipeline = book` + it("should convert flat pipeline with return statement to structured format", () => { + const flatPipeline = book` Hello -> {response} `; - const expected = book` + const expected = book` # ${DEFAULT_BOOK_TITLE} ## Prompt @@ -55,11 +55,11 @@ describe('deflatePipeline', () => { -> {response} `; - expect(deflatePipeline(flatPipeline)).toBe(expected); - }); + expect(deflatePipeline(flatPipeline)).toBe(expected); + }); - it('should handle multi-line prompts', () => { - const flatPipeline = book` + it("should handle multi-line prompts", () => { + const flatPipeline = book` Line 1 Line 2 Line 3 @@ -67,7 +67,7 @@ describe('deflatePipeline', () => { -> {response} `; - const expected = book` + const expected = book` # ${DEFAULT_BOOK_TITLE} ## ${DEFAULT_PROMPT_TASK_TITLE} @@ -81,6 +81,6 @@ describe('deflatePipeline', () => { -> {response} `; - expect(deflatePipeline(flatPipeline)).toBe(expected); - }); + expect(deflatePipeline(flatPipeline)).toBe(expected); + }); }); diff --git a/src/utils/editable/edit-pipeline-string/deflatePipeline.ts b/src/utils/editable/edit-pipeline-string/deflatePipeline.ts index b7fe183635..65396ce65f 100644 --- a/src/utils/editable/edit-pipeline-string/deflatePipeline.ts +++ b/src/utils/editable/edit-pipeline-string/deflatePipeline.ts @@ -1,53 +1,55 @@ -import spaceTrim from 'spacetrim'; -import { DEFAULT_BOOK_OUTPUT_PARAMETER_NAME } from '../../../config'; -import { DEFAULT_BOOK_TITLE } from '../../../config'; -import type { PipelineString } from '../../../pipeline/PipelineString'; -import { validatePipelineString } from '../../../pipeline/validatePipelineString'; -import type { string_prompt } from '../../../types/typeAliases'; -import { isFlatPipeline } from '../utils/isFlatPipeline'; +import spaceTrim from "spacetrim"; +import { DEFAULT_BOOK_OUTPUT_PARAMETER_NAME } from "../../../config"; +import { DEFAULT_BOOK_TITLE } from "../../../config"; +import type { PipelineString } from "../../../pipeline/PipelineString"; +import { validatePipelineString } from "../../../pipeline/validatePipelineString"; +import type { string_prompt } from "../../../types/typeAliases"; +import { isFlatPipeline } from "../utils/isFlatPipeline"; /** * @@@ * * @public exported from `@promptbook/editable` */ -export function deflatePipeline(pipelineString: PipelineString): PipelineString { - if (!isFlatPipeline(pipelineString)) { - return pipelineString; - } +export function deflatePipeline( + pipelineString: PipelineString, +): PipelineString { + if (!isFlatPipeline(pipelineString)) { + return pipelineString; + } - const pipelineStringLines = pipelineString.split('\n'); - const potentialReturnStatement = pipelineStringLines.pop()!; - let returnStatement: string; + const pipelineStringLines = pipelineString.split("\n"); + const potentialReturnStatement = pipelineStringLines.pop()!; + let returnStatement: string; - if (/(-|=)>\s*\{.*\}/.test(potentialReturnStatement)) { - // Note: Last line is return statement - returnStatement = potentialReturnStatement; - } else { - // Note: Last line is not a return statement - returnStatement = `-> {${DEFAULT_BOOK_OUTPUT_PARAMETER_NAME}}`; - pipelineStringLines.push(potentialReturnStatement); - } + if (/(-|=)>\s*\{.*\}/.test(potentialReturnStatement)) { + // Note: Last line is return statement + returnStatement = potentialReturnStatement; + } else { + // Note: Last line is not a return statement + returnStatement = `-> {${DEFAULT_BOOK_OUTPUT_PARAMETER_NAME}}`; + pipelineStringLines.push(potentialReturnStatement); + } - const prompt = spaceTrim(pipelineStringLines.join('\n')); + const prompt = spaceTrim(pipelineStringLines.join("\n")); - let quotedPrompt: string_prompt; + let quotedPrompt: string_prompt; - if (prompt.split('\n').length <= 1) { - quotedPrompt = `> ${prompt}`; - } else { - quotedPrompt = spaceTrim( - (block) => ` + if (prompt.split("\n").length <= 1) { + quotedPrompt = `> ${prompt}`; + } else { + quotedPrompt = spaceTrim( + (block) => ` \`\`\` - ${block(prompt.split('`').join('\\`'))} + ${block(prompt.split("`").join("\\`"))} \`\`\` `, - ); - } + ); + } - pipelineString = validatePipelineString( - spaceTrim( - (block) => ` + pipelineString = validatePipelineString( + spaceTrim( + (block) => ` # ${DEFAULT_BOOK_TITLE} ## Prompt @@ -56,11 +58,11 @@ export function deflatePipeline(pipelineString: PipelineString): PipelineString ${returnStatement} `, - ), - ); - // <- TODO: Maybe use book` notation + ), + ); + // <- TODO: Maybe use book` notation - return pipelineString; + return pipelineString; } /** diff --git a/src/utils/editable/edit-pipeline-string/removePipelineCommand.test.ts b/src/utils/editable/edit-pipeline-string/removePipelineCommand.test.ts index 0f620894a1..589b5f8827 100644 --- a/src/utils/editable/edit-pipeline-string/removePipelineCommand.test.ts +++ b/src/utils/editable/edit-pipeline-string/removePipelineCommand.test.ts @@ -1,56 +1,56 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import type { PipelineString } from '../../../pipeline/PipelineString'; -import { just } from '../../organization/just'; -import { removePipelineCommand } from './removePipelineCommand'; - -describe('how `removePipelineCommand` works', () => { - it('should keep the pipeline when command is not there', () => - expect( - removePipelineCommand({ - command: 'PERSONA', - pipelineString: spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import type { PipelineString } from "../../../pipeline/PipelineString"; +import { just } from "../../organization/just"; +import { removePipelineCommand } from "./removePipelineCommand"; + +describe("how `removePipelineCommand` works", () => { + it("should keep the pipeline when command is not there", () => + expect( + removePipelineCommand({ + command: "PERSONA", + pipelineString: spaceTrim(` # Book - KNOWLEDGE https://pavolhejny.com/ `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - }), - ).toBe( - just( - spaceTrim(` + // <- TODO: [📼] Use`book\`` string literal notation + }), + ).toBe( + just( + spaceTrim(` # Book - KNOWLEDGE https://pavolhejny.com/ `), - ), - )); - - it('should remove the command', () => - expect( - removePipelineCommand({ - command: 'KNOWLEDGE', - pipelineString: spaceTrim(` + ), + )); + + it("should remove the command", () => + expect( + removePipelineCommand({ + command: "KNOWLEDGE", + pipelineString: spaceTrim(` # Book - KNOWLEDGE https://pavolhejny.com/ `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - }), - ).toBe( - just( - spaceTrim(` + // <- TODO: [📼] Use`book\`` string literal notation + }), + ).toBe( + just( + spaceTrim(` # Book `), - ), - )); - - it('should work in advanced case', () => - expect( - removePipelineCommand({ - command: 'KNOWLEDGE', - pipelineString: spaceTrim(` + ), + )); + + it("should work in advanced case", () => + expect( + removePipelineCommand({ + command: "KNOWLEDGE", + pipelineString: spaceTrim(` # Book - KNOWLEDGE https://pavolhejny.com/ @@ -70,11 +70,11 @@ describe('how `removePipelineCommand` works', () => { - KNOWLEDGE https://pavolhejny.com/ \`\`\` `) as PipelineString, - // <- TODO: [📼] Use`book\`` string literal notation - }), - ).toBe( - just( - spaceTrim(` + // <- TODO: [📼] Use`book\`` string literal notation + }), + ).toBe( + just( + spaceTrim(` # Book - PERSONA Paul, developer of the Promptbook @@ -92,6 +92,6 @@ describe('how `removePipelineCommand` works', () => { - KNOWLEDGE https://pavolhejny.com/ \`\`\` `), - ), - )); + ), + )); }); diff --git a/src/utils/editable/edit-pipeline-string/removePipelineCommand.ts b/src/utils/editable/edit-pipeline-string/removePipelineCommand.ts index 2d7c9989cc..d25395bb88 100644 --- a/src/utils/editable/edit-pipeline-string/removePipelineCommand.ts +++ b/src/utils/editable/edit-pipeline-string/removePipelineCommand.ts @@ -1,21 +1,21 @@ -import spaceTrim from 'spacetrim'; -import type { CommandType } from '../../../commands/_common/types/CommandType'; -import type { PipelineString } from '../../../pipeline/PipelineString'; +import spaceTrim from "spacetrim"; +import type { CommandType } from "../../../commands/_common/types/CommandType"; +import type { PipelineString } from "../../../pipeline/PipelineString"; /** * Options for `removePipelineCommand` */ type RemovePipelineCommandOptions = { - /** - * The command you want to remove - */ - command: CommandType; - // <- TODO: [🧘] Use here `CommandTypeOrAlias` - - /** - * Pipeline you want to remove command from - */ - pipelineString: PipelineString; + /** + * The command you want to remove + */ + command: CommandType; + // <- TODO: [🧘] Use here `CommandTypeOrAlias` + + /** + * Pipeline you want to remove command from + */ + pipelineString: PipelineString; }; /** @@ -23,41 +23,47 @@ type RemovePipelineCommandOptions = { * * @public exported from `@promptbook/editable` */ -export function removePipelineCommand(options: RemovePipelineCommandOptions): PipelineString { - const { command, pipelineString } = options; - - const lines = pipelineString.split('\n'); - - // TODO: [🧽] DRY - let currentType: 'MARKDOWN' | 'CODE_BLOCK' | 'COMMENT' = 'MARKDOWN'; - - const newLines: Array = []; - - for (const line of lines) { - if (currentType === 'MARKDOWN') { - if (line.startsWith('```')) { - currentType = 'CODE_BLOCK'; - } else if (line.includes('')) { - currentType = 'MARKDOWN'; - } - } - - if (currentType === 'MARKDOWN' && /^(-|\d\))/m.test(line) && line.toUpperCase().includes(command)) { - continue; - } - - newLines.push(line); - } - - const newPipeline = spaceTrim(newLines.join('\n')); - - return newPipeline as PipelineString; +export function removePipelineCommand( + options: RemovePipelineCommandOptions, +): PipelineString { + const { command, pipelineString } = options; + + const lines = pipelineString.split("\n"); + + // TODO: [🧽] DRY + let currentType: "MARKDOWN" | "CODE_BLOCK" | "COMMENT" = "MARKDOWN"; + + const newLines: Array = []; + + for (const line of lines) { + if (currentType === "MARKDOWN") { + if (line.startsWith("```")) { + currentType = "CODE_BLOCK"; + } else if (line.includes("")) { + currentType = "MARKDOWN"; + } + } + + if ( + currentType === "MARKDOWN" && + /^(-|\d\))/m.test(line) && + line.toUpperCase().includes(command) + ) { + continue; + } + + newLines.push(line); + } + + const newPipeline = spaceTrim(newLines.join("\n")); + + return newPipeline as PipelineString; } diff --git a/src/utils/editable/types/PipelineEditableSerialized.ts b/src/utils/editable/types/PipelineEditableSerialized.ts index 4153cfe5e0..5713f29810 100644 --- a/src/utils/editable/types/PipelineEditableSerialized.ts +++ b/src/utils/editable/types/PipelineEditableSerialized.ts @@ -1,6 +1,6 @@ -import type { ErrorJson } from '../../../errors/utils/ErrorJson'; -import type { PipelineJson } from '../../../pipeline/PipelineJson/PipelineJson'; -import type { string_date_iso8601 } from '../../../types/typeAliases'; +import type { ErrorJson } from "../../../errors/utils/ErrorJson"; +import type { PipelineJson } from "../../../pipeline/PipelineJson/PipelineJson"; +import type { string_date_iso8601 } from "../../../types/typeAliases"; /** * Represents a single pipeline in PromptbookStudio @@ -13,17 +13,17 @@ import type { string_date_iso8601 } from '../../../types/typeAliases'; * @public exported from `@promptbook/editable` */ export type PipelineEditableSerialized = PipelineJson & { - /** - * When was the pipeline created - */ - readonly createdAt: string_date_iso8601; + /** + * When was the pipeline created + */ + readonly createdAt: string_date_iso8601; - /** - * When was the pipeline last modified - */ - readonly updatedAt: string_date_iso8601 | null; + /** + * When was the pipeline last modified + */ + readonly updatedAt: string_date_iso8601 | null; - /* + /* TODO: [▫] /** * Unique identifier of the pipeline @@ -33,8 +33,8 @@ export type PipelineEditableSerialized = PipelineJson & { readonly pipelineUrl: string_pipeline_url; */ - /** - * Known errors to transfer to new PipelineEditable - */ - readonly knownErrors: Array; + /** + * Known errors to transfer to new PipelineEditable + */ + readonly knownErrors: Array; }; diff --git a/src/utils/editable/utils/isFlatPipeline.test.ts b/src/utils/editable/utils/isFlatPipeline.test.ts index a5de488b80..797b06fdb2 100644 --- a/src/utils/editable/utils/isFlatPipeline.test.ts +++ b/src/utils/editable/utils/isFlatPipeline.test.ts @@ -1,40 +1,40 @@ -import { describe, expect, it } from '@jest/globals'; -import { book } from '../../../pipeline/book-notation'; -import { isFlatPipeline } from './isFlatPipeline'; +import { describe, expect, it } from "@jest/globals"; +import { book } from "../../../pipeline/book-notation"; +import { isFlatPipeline } from "./isFlatPipeline"; -describe('isFlatPipeline', () => { - it('should detect simple flat pipeline', () => { - const pipeline = book` +describe("isFlatPipeline", () => { + it("should detect simple flat pipeline", () => { + const pipeline = book` Hello `; - expect(isFlatPipeline(pipeline)).toBe(true); - }); + expect(isFlatPipeline(pipeline)).toBe(true); + }); - it('should detect multiline flat pipeline', () => { - const pipeline = book` + it("should detect multiline flat pipeline", () => { + const pipeline = book` Hello How Are You `; - expect(isFlatPipeline(pipeline)).toBe(true); - }); + expect(isFlatPipeline(pipeline)).toBe(true); + }); - it('should detect flat pipeline with return statement', () => { - const pipeline = book` + it("should detect flat pipeline with return statement", () => { + const pipeline = book` Hello how are you? -> {response} `; - expect(isFlatPipeline(pipeline)).toBe(true); - }); + expect(isFlatPipeline(pipeline)).toBe(true); + }); - it('should detect non-flat pipeline because of ``` quoted block', () => { - // Note: This is not a valid pipeline, its not flat because of the quoted block and also lacks things mandatory for non-flat pipeline - const pipeline = book` + it("should detect non-flat pipeline because of ``` quoted block", () => { + // Note: This is not a valid pipeline, its not flat because of the quoted block and also lacks things mandatory for non-flat pipeline + const pipeline = book` Hello how are you? \`\`\` @@ -44,12 +44,12 @@ describe('isFlatPipeline', () => { -> {response} `; - expect(isFlatPipeline(pipeline)).toBe(false); - }); + expect(isFlatPipeline(pipeline)).toBe(false); + }); - it('should detect non-flat pipeline because of > quoted block', () => { - // Note: This is not a valid pipeline, its not flat because of the quoted block and also lacks things mandatory for non-flat pipeline - const pipeline = book` + it("should detect non-flat pipeline because of > quoted block", () => { + // Note: This is not a valid pipeline, its not flat because of the quoted block and also lacks things mandatory for non-flat pipeline + const pipeline = book` Hello how are you? > I am file! @@ -57,13 +57,13 @@ describe('isFlatPipeline', () => { -> {response} `; - expect(isFlatPipeline(pipeline)).toBe(false); - }); + expect(isFlatPipeline(pipeline)).toBe(false); + }); - // TODO: [🧉] + // TODO: [🧉] - it('should detect non-flat pipeline', () => { - const pipeline = book` + it("should detect non-flat pipeline", () => { + const pipeline = book` # Some Title ## Greeting @@ -73,6 +73,6 @@ describe('isFlatPipeline', () => { -> {response} `; - expect(isFlatPipeline(pipeline)).toBe(false); - }); + expect(isFlatPipeline(pipeline)).toBe(false); + }); }); diff --git a/src/utils/editable/utils/isFlatPipeline.ts b/src/utils/editable/utils/isFlatPipeline.ts index e8f66e601b..f7fc3b92c1 100644 --- a/src/utils/editable/utils/isFlatPipeline.ts +++ b/src/utils/editable/utils/isFlatPipeline.ts @@ -1,6 +1,6 @@ -import spaceTrim from 'spacetrim'; -import type { PipelineString } from '../../../pipeline/PipelineString'; -import { removeMarkdownComments } from '../../markdown/removeMarkdownComments'; +import spaceTrim from "spacetrim"; +import type { PipelineString } from "../../../pipeline/PipelineString"; +import { removeMarkdownComments } from "../../markdown/removeMarkdownComments"; /** * @@@ @@ -8,22 +8,24 @@ import { removeMarkdownComments } from '../../markdown/removeMarkdownComments'; * @public exported from `@promptbook/editable` */ export function isFlatPipeline(pipelineString: PipelineString): boolean { - pipelineString = removeMarkdownComments(pipelineString); - pipelineString = spaceTrim(pipelineString) as PipelineString; + pipelineString = removeMarkdownComments(pipelineString); + pipelineString = spaceTrim(pipelineString) as PipelineString; - const isMarkdownBeginningWithHeadline = pipelineString.startsWith('# '); - //const isLastLineReturnStatement = pipelineString.split('\n').pop()!.split('`').join('').startsWith('->'); + const isMarkdownBeginningWithHeadline = pipelineString.startsWith("# "); + //const isLastLineReturnStatement = pipelineString.split('\n').pop()!.split('`').join('').startsWith('->'); - const isBacktickBlockUsed = pipelineString.includes('```'); - const isQuoteBlocksUsed = /^>\s+/m.test(pipelineString); - const isBlocksUsed = isBacktickBlockUsed || isQuoteBlocksUsed; + const isBacktickBlockUsed = pipelineString.includes("```"); + const isQuoteBlocksUsed = /^>\s+/m.test(pipelineString); + const isBlocksUsed = isBacktickBlockUsed || isQuoteBlocksUsed; - // TODO: [🧉] Also (double)check - // > const usedCommands - // > const isBlocksUsed - // > const returnStatementCount + // TODO: [🧉] Also (double)check + // > const usedCommands + // > const isBlocksUsed + // > const returnStatementCount - const isFlat = !isMarkdownBeginningWithHeadline && !isBlocksUsed; /* && isLastLineReturnStatement */ + const isFlat = + !isMarkdownBeginningWithHeadline && + !isBlocksUsed; /* && isLastLineReturnStatement */ - return isFlat; + return isFlat; } diff --git a/src/utils/editable/utils/renamePipelineParameter.test.ts b/src/utils/editable/utils/renamePipelineParameter.test.ts index 98dce27dd6..11026d15a6 100644 --- a/src/utils/editable/utils/renamePipelineParameter.test.ts +++ b/src/utils/editable/utils/renamePipelineParameter.test.ts @@ -1,271 +1,275 @@ -import { describe, expect, it } from '@jest/globals'; -import { validatePipeline } from '../../../conversion/validation/validatePipeline'; -import { renamePipelineParameter } from './renamePipelineParameter'; -describe('how renamePipelineParameter works', () => { - it('should rename parameter in simple promptbook', () => { - expect( - renamePipelineParameter({ - pipeline: validatePipeline({ - title: '✨ Example prompt', - bookVersion: '1.0.0', - description: 'Show how to use a simple prompt with no parameters.', - formfactorName: 'GENERIC', - parameters: [ - { - name: 'greet', - isInput: false, - isOutput: true, - }, - ], - tasks: [ - { - name: 'prompt', - title: '💬 Prompt', - dependentParameterNames: [], - taskType: 'PROMPT_TASK', - modelRequirements: { - modelVariant: 'CHAT', - modelName: 'gpt-3.5-turbo', - }, - content: 'Hello', - resultingParameterName: 'greet', - }, - ], - knowledgeSources: [], - knowledgePieces: [], - personas: [], - preparations: [], - sources: [], - }), - oldParameterName: 'greet', - newParameterName: 'greeting', - }), - ).toEqual( - validatePipeline({ - title: '✨ Example prompt', - bookVersion: '1.0.0', - description: 'Show how to use a simple prompt with no parameters.', - formfactorName: 'GENERIC', - parameters: [ - { - name: 'greeting', - isInput: false, - isOutput: true, - }, - ], - tasks: [ - { - name: 'prompt', - title: '💬 Prompt', - dependentParameterNames: [], - taskType: 'PROMPT_TASK', - modelRequirements: { - modelVariant: 'CHAT', - modelName: 'gpt-3.5-turbo', - }, - content: 'Hello', - resultingParameterName: 'greeting', - }, - ], - knowledgeSources: [], - knowledgePieces: [], - personas: [], - preparations: [], - sources: [], - }), - ); - }); - it('should rename parameter in normal promptbook', () => { - expect( - renamePipelineParameter({ - pipeline: validatePipeline({ - title: '✨ Example prompt with two consecutive prompts', - pipelineUrl: 'https://promptbook.studio/examples/two.book', - bookVersion: '1.0.0', - description: 'Show how to use two consecutive prompts with one parameter each.', - formfactorName: 'GENERIC', - parameters: [ - { - name: 'word', - description: 'Any single word', - isInput: true, - isOutput: false, - }, - { - name: 'sentenceWithTwoSynonyms', - isInput: false, - isOutput: true, - }, - { - name: 'wordSynonym', - isInput: false, - isOutput: false, - }, - ], - tasks: [ - { - name: 'synonym', - title: '💬 Synonym', - description: 'Synonym for word', - dependentParameterNames: ['word'], - taskType: 'PROMPT_TASK', - modelRequirements: { - modelVariant: 'CHAT', - modelName: 'gpt-3.5-turbo', - }, - content: 'Write synonym for "{word}"', - resultingParameterName: 'wordSynonym', - }, - { - name: 'sentence', - title: '💬 Sentence', - description: 'Sentence with word and wordSynonym', - dependentParameterNames: ['word', 'wordSynonym'], - taskType: 'PROMPT_TASK', - modelRequirements: { - modelVariant: 'CHAT', - modelName: 'gpt-3.5-turbo', - }, - content: 'Write sentence with "{word}" and "{wordSynonym}" in it', - resultingParameterName: 'sentenceWithTwoSynonyms', - }, - ], - knowledgeSources: [], - knowledgePieces: [], - personas: [], - preparations: [], - sources: [], - }), - oldParameterName: 'word', - newParameterName: 'originalWord', - }), - ).toEqual( - validatePipeline({ - title: '✨ Example prompt with two consecutive prompts', - pipelineUrl: 'https://promptbook.studio/examples/two.book', - bookVersion: '1.0.0', - description: 'Show how to use two consecutive prompts with one parameter each.', - formfactorName: 'GENERIC', - parameters: [ - { - name: 'originalWord', - description: 'Any single word', - isInput: true, - isOutput: false, - }, - { - name: 'sentenceWithTwoSynonyms', - isInput: false, - isOutput: true, - }, - { - name: 'wordSynonym', - isInput: false, - isOutput: false, - }, - ], - tasks: [ - { - name: 'synonym', - title: '💬 Synonym', - description: 'Synonym for word', - dependentParameterNames: ['originalWord'], - taskType: 'PROMPT_TASK', - modelRequirements: { - modelVariant: 'CHAT', - modelName: 'gpt-3.5-turbo', - }, - content: 'Write synonym for "{originalWord}"', - resultingParameterName: 'wordSynonym', - }, - { - name: 'sentence', - title: '💬 Sentence', - description: 'Sentence with word and wordSynonym', - dependentParameterNames: ['originalWord', 'wordSynonym'], - taskType: 'PROMPT_TASK', - modelRequirements: { - modelVariant: 'CHAT', - modelName: 'gpt-3.5-turbo', - }, - content: 'Write sentence with "{originalWord}" and "{wordSynonym}" in it', - resultingParameterName: 'sentenceWithTwoSynonyms', - }, - ], - knowledgeSources: [], - knowledgePieces: [], - personas: [], - preparations: [], - sources: [], - }), - ); - }); - it('should detect name collision', () => { - expect(() => - renamePipelineParameter({ - pipeline: validatePipeline({ - title: '✨ Example prompt with two consecutive prompts', - pipelineUrl: 'https://promptbook.studio/examples/two.book', - bookVersion: '1.0.0', - description: 'Show how to use two consecutive prompts with one parameter each.', - formfactorName: 'GENERIC', - parameters: [ - { - name: 'word', - description: 'Any single word', - isInput: true, - isOutput: false, - }, - { - name: 'sentenceWithTwoSynonyms', - isInput: false, - isOutput: true, - }, - { - name: 'wordSynonym', - isInput: false, - isOutput: false, - }, - ], - tasks: [ - { - name: 'synonym', - title: '💬 Synonym', - description: 'Synonym for word', - dependentParameterNames: ['word'], - taskType: 'PROMPT_TASK', - modelRequirements: { - modelVariant: 'CHAT', - modelName: 'gpt-3.5-turbo', - }, - content: 'Write synonym for "{word}"', - resultingParameterName: 'wordSynonym', - }, - { - name: 'sentence', - title: '💬 Sentence', - description: 'Sentence with word and wordSynonym', - dependentParameterNames: ['word', 'wordSynonym'], - taskType: 'PROMPT_TASK', - modelRequirements: { - modelVariant: 'CHAT', - modelName: 'gpt-3.5-turbo', - }, - content: 'Write sentence with "{word}" and "{wordSynonym}" in it', - resultingParameterName: 'sentenceWithTwoSynonyms', - }, - ], - knowledgeSources: [], - knowledgePieces: [], - personas: [], - preparations: [], - sources: [], - }), - oldParameterName: 'word', - newParameterName: 'wordSynonym', - }), - ).toThrowError( - /Can not replace {word} to {wordSynonym} because {wordSynonym} is already used in the pipeline/i, - ); - }); +import { describe, expect, it } from "@jest/globals"; +import { validatePipeline } from "../../../conversion/validation/validatePipeline"; +import { renamePipelineParameter } from "./renamePipelineParameter"; +describe("how renamePipelineParameter works", () => { + it("should rename parameter in simple promptbook", () => { + expect( + renamePipelineParameter({ + pipeline: validatePipeline({ + title: "✨ Example prompt", + bookVersion: "1.0.0", + description: "Show how to use a simple prompt with no parameters.", + formfactorName: "GENERIC", + parameters: [ + { + name: "greet", + isInput: false, + isOutput: true, + }, + ], + tasks: [ + { + name: "prompt", + title: "💬 Prompt", + dependentParameterNames: [], + taskType: "PROMPT_TASK", + modelRequirements: { + modelVariant: "CHAT", + modelName: "gpt-3.5-turbo", + }, + content: "Hello", + resultingParameterName: "greet", + }, + ], + knowledgeSources: [], + knowledgePieces: [], + personas: [], + preparations: [], + sources: [], + }), + oldParameterName: "greet", + newParameterName: "greeting", + }), + ).toEqual( + validatePipeline({ + title: "✨ Example prompt", + bookVersion: "1.0.0", + description: "Show how to use a simple prompt with no parameters.", + formfactorName: "GENERIC", + parameters: [ + { + name: "greeting", + isInput: false, + isOutput: true, + }, + ], + tasks: [ + { + name: "prompt", + title: "💬 Prompt", + dependentParameterNames: [], + taskType: "PROMPT_TASK", + modelRequirements: { + modelVariant: "CHAT", + modelName: "gpt-3.5-turbo", + }, + content: "Hello", + resultingParameterName: "greeting", + }, + ], + knowledgeSources: [], + knowledgePieces: [], + personas: [], + preparations: [], + sources: [], + }), + ); + }); + it("should rename parameter in normal promptbook", () => { + expect( + renamePipelineParameter({ + pipeline: validatePipeline({ + title: "✨ Example prompt with two consecutive prompts", + pipelineUrl: "https://promptbook.studio/examples/two.book", + bookVersion: "1.0.0", + description: + "Show how to use two consecutive prompts with one parameter each.", + formfactorName: "GENERIC", + parameters: [ + { + name: "word", + description: "Any single word", + isInput: true, + isOutput: false, + }, + { + name: "sentenceWithTwoSynonyms", + isInput: false, + isOutput: true, + }, + { + name: "wordSynonym", + isInput: false, + isOutput: false, + }, + ], + tasks: [ + { + name: "synonym", + title: "💬 Synonym", + description: "Synonym for word", + dependentParameterNames: ["word"], + taskType: "PROMPT_TASK", + modelRequirements: { + modelVariant: "CHAT", + modelName: "gpt-3.5-turbo", + }, + content: 'Write synonym for "{word}"', + resultingParameterName: "wordSynonym", + }, + { + name: "sentence", + title: "💬 Sentence", + description: "Sentence with word and wordSynonym", + dependentParameterNames: ["word", "wordSynonym"], + taskType: "PROMPT_TASK", + modelRequirements: { + modelVariant: "CHAT", + modelName: "gpt-3.5-turbo", + }, + content: 'Write sentence with "{word}" and "{wordSynonym}" in it', + resultingParameterName: "sentenceWithTwoSynonyms", + }, + ], + knowledgeSources: [], + knowledgePieces: [], + personas: [], + preparations: [], + sources: [], + }), + oldParameterName: "word", + newParameterName: "originalWord", + }), + ).toEqual( + validatePipeline({ + title: "✨ Example prompt with two consecutive prompts", + pipelineUrl: "https://promptbook.studio/examples/two.book", + bookVersion: "1.0.0", + description: + "Show how to use two consecutive prompts with one parameter each.", + formfactorName: "GENERIC", + parameters: [ + { + name: "originalWord", + description: "Any single word", + isInput: true, + isOutput: false, + }, + { + name: "sentenceWithTwoSynonyms", + isInput: false, + isOutput: true, + }, + { + name: "wordSynonym", + isInput: false, + isOutput: false, + }, + ], + tasks: [ + { + name: "synonym", + title: "💬 Synonym", + description: "Synonym for word", + dependentParameterNames: ["originalWord"], + taskType: "PROMPT_TASK", + modelRequirements: { + modelVariant: "CHAT", + modelName: "gpt-3.5-turbo", + }, + content: 'Write synonym for "{originalWord}"', + resultingParameterName: "wordSynonym", + }, + { + name: "sentence", + title: "💬 Sentence", + description: "Sentence with word and wordSynonym", + dependentParameterNames: ["originalWord", "wordSynonym"], + taskType: "PROMPT_TASK", + modelRequirements: { + modelVariant: "CHAT", + modelName: "gpt-3.5-turbo", + }, + content: + 'Write sentence with "{originalWord}" and "{wordSynonym}" in it', + resultingParameterName: "sentenceWithTwoSynonyms", + }, + ], + knowledgeSources: [], + knowledgePieces: [], + personas: [], + preparations: [], + sources: [], + }), + ); + }); + it("should detect name collision", () => { + expect(() => + renamePipelineParameter({ + pipeline: validatePipeline({ + title: "✨ Example prompt with two consecutive prompts", + pipelineUrl: "https://promptbook.studio/examples/two.book", + bookVersion: "1.0.0", + description: + "Show how to use two consecutive prompts with one parameter each.", + formfactorName: "GENERIC", + parameters: [ + { + name: "word", + description: "Any single word", + isInput: true, + isOutput: false, + }, + { + name: "sentenceWithTwoSynonyms", + isInput: false, + isOutput: true, + }, + { + name: "wordSynonym", + isInput: false, + isOutput: false, + }, + ], + tasks: [ + { + name: "synonym", + title: "💬 Synonym", + description: "Synonym for word", + dependentParameterNames: ["word"], + taskType: "PROMPT_TASK", + modelRequirements: { + modelVariant: "CHAT", + modelName: "gpt-3.5-turbo", + }, + content: 'Write synonym for "{word}"', + resultingParameterName: "wordSynonym", + }, + { + name: "sentence", + title: "💬 Sentence", + description: "Sentence with word and wordSynonym", + dependentParameterNames: ["word", "wordSynonym"], + taskType: "PROMPT_TASK", + modelRequirements: { + modelVariant: "CHAT", + modelName: "gpt-3.5-turbo", + }, + content: 'Write sentence with "{word}" and "{wordSynonym}" in it', + resultingParameterName: "sentenceWithTwoSynonyms", + }, + ], + knowledgeSources: [], + knowledgePieces: [], + personas: [], + preparations: [], + sources: [], + }), + oldParameterName: "word", + newParameterName: "wordSynonym", + }), + ).toThrowError( + /Can not replace {word} to {wordSynonym} because {wordSynonym} is already used in the pipeline/i, + ); + }); }); diff --git a/src/utils/editable/utils/renamePipelineParameter.ts b/src/utils/editable/utils/renamePipelineParameter.ts index 4904dc40e6..6c2165bae3 100644 --- a/src/utils/editable/utils/renamePipelineParameter.ts +++ b/src/utils/editable/utils/renamePipelineParameter.ts @@ -1,25 +1,25 @@ -import type { $PipelineJson } from '../../../commands/_common/types/CommandParser'; -import { PipelineLogicError } from '../../../errors/PipelineLogicError'; -import type { PipelineJson } from '../../../pipeline/PipelineJson/PipelineJson'; -import type { string_name } from '../../../types/typeAliases'; -import type { TODO_remove_as } from '../../organization/TODO_remove_as'; +import type { $PipelineJson } from "../../../commands/_common/types/CommandParser"; +import { PipelineLogicError } from "../../../errors/PipelineLogicError"; +import type { PipelineJson } from "../../../pipeline/PipelineJson/PipelineJson"; +import type { string_name } from "../../../types/typeAliases"; +import type { TODO_remove_as } from "../../organization/TODO_remove_as"; type RenameParameterOptions = { - /** - * Pipeline to search and replace for parameters - * This pipeline is returned as copy with replaced parameters - */ - readonly pipeline: PipelineJson; + /** + * Pipeline to search and replace for parameters + * This pipeline is returned as copy with replaced parameters + */ + readonly pipeline: PipelineJson; - /** - * Original parameter name that should be replaced - */ - readonly oldParameterName: string_name; + /** + * Original parameter name that should be replaced + */ + readonly oldParameterName: string_name; - /** - * New parameter name that should replace the original parameter name - */ - readonly newParameterName: string_name; + /** + * New parameter name that should replace the original parameter name + */ + readonly newParameterName: string_name; }; /** @@ -29,48 +29,64 @@ type RenameParameterOptions = { * @throws {PipelineLogicError} If the new parameter name is already used in the pipeline * @public exported from `@promptbook/editable` */ -export function renamePipelineParameter(options: RenameParameterOptions): PipelineJson { - const { pipeline: pipeline, oldParameterName, newParameterName } = options; +export function renamePipelineParameter( + options: RenameParameterOptions, +): PipelineJson { + const { pipeline, oldParameterName, newParameterName } = options; - if (pipeline.parameters.some((parameter) => parameter.name === newParameterName)) { - throw new PipelineLogicError( - `Can not replace {${oldParameterName}} to {${newParameterName}} because {${newParameterName}} is already used in the pipeline`, - ); - } + if ( + pipeline.parameters.some((parameter) => parameter.name === newParameterName) + ) { + throw new PipelineLogicError( + `Can not replace {${oldParameterName}} to {${newParameterName}} because {${newParameterName}} is already used in the pipeline`, + ); + } - const renamedPipeline: $PipelineJson = { - ...(pipeline as TODO_remove_as<$PipelineJson>), - // <- TODO: [🪓] This should be without `as $PipelineJson` - parameters: [...pipeline.parameters], - tasks: [...pipeline.tasks], - }; + const renamedPipeline: $PipelineJson = { + ...(pipeline as TODO_remove_as<$PipelineJson>), + // <- TODO: [🪓] This should be without `as $PipelineJson` + parameters: [...pipeline.parameters], + tasks: [...pipeline.tasks], + }; - for (const parameter of renamedPipeline.parameters) { - if (parameter.name !== oldParameterName) { - continue; - } - parameter.name = newParameterName; - } + for (const parameter of renamedPipeline.parameters) { + if (parameter.name !== oldParameterName) { + continue; + } + parameter.name = newParameterName; + } - for (const task of renamedPipeline.tasks) { - if (task.resultingParameterName === oldParameterName) { - task.resultingParameterName = newParameterName; - } - task.dependentParameterNames = task.dependentParameterNames.map((dependentParameterName) => - dependentParameterName === oldParameterName ? newParameterName : dependentParameterName, - ); + for (const task of renamedPipeline.tasks) { + if (task.resultingParameterName === oldParameterName) { + task.resultingParameterName = newParameterName; + } + task.dependentParameterNames = task.dependentParameterNames.map( + (dependentParameterName) => + dependentParameterName === oldParameterName + ? newParameterName + : dependentParameterName, + ); - task.content = task.content.replace(new RegExp(`{${oldParameterName}}`, 'g'), `{${newParameterName}}`); + task.content = task.content.replace( + new RegExp(`{${oldParameterName}}`, "g"), + `{${newParameterName}}`, + ); - task.title = task.title.replace(new RegExp(`{${oldParameterName}}`, 'g'), `{${newParameterName}}`); + task.title = task.title.replace( + new RegExp(`{${oldParameterName}}`, "g"), + `{${newParameterName}}`, + ); - task.description = - task.description === undefined - ? undefined - : task.description.replace(new RegExp(`{${oldParameterName}}`, 'g'), `{${newParameterName}}`); - } + task.description = + task.description === undefined + ? undefined + : task.description.replace( + new RegExp(`{${oldParameterName}}`, "g"), + `{${newParameterName}}`, + ); + } - return renamedPipeline as TODO_remove_as; + return renamedPipeline as TODO_remove_as; } /** diff --git a/src/utils/editable/utils/stringifyPipelineJson.ts b/src/utils/editable/utils/stringifyPipelineJson.ts index d3e37c4415..afb26dca17 100644 --- a/src/utils/editable/utils/stringifyPipelineJson.ts +++ b/src/utils/editable/utils/stringifyPipelineJson.ts @@ -1,9 +1,9 @@ -import spaceTrim from 'spacetrim'; -import { LOOP_LIMIT } from '../../../config'; -import { REPLACING_NONCE } from '../../../constants'; -import { UnexpectedError } from '../../../errors/UnexpectedError'; -import type { string_json } from '../../../types/typeAliases'; -import { isSerializableAsJson } from '../../serialization/isSerializableAsJson'; +import spaceTrim from "spacetrim"; +import { LOOP_LIMIT } from "../../../config"; +import { REPLACING_NONCE } from "../../../constants"; +import { UnexpectedError } from "../../../errors/UnexpectedError"; +import type { string_json } from "../../../types/typeAliases"; +import { isSerializableAsJson } from "../../serialization/isSerializableAsJson"; /** * Stringify the PipelineJson with proper formatting @@ -13,33 +13,37 @@ import { isSerializableAsJson } from '../../serialization/isSerializableAsJson'; * * @public exported from `@promptbook/editable` */ -export function stringifyPipelineJson(pipeline: TType): string_json { - if (!isSerializableAsJson(pipeline)) { - throw new UnexpectedError( - spaceTrim(` +export function stringifyPipelineJson( + pipeline: TType, +): string_json { + if (!isSerializableAsJson(pipeline)) { + throw new UnexpectedError( + spaceTrim(` Cannot stringify the pipeline, because it is not serializable as JSON There can be multiple reasons: 1) The pipeline contains circular references 2) It is not a valid PipelineJson `), - ); - } + ); + } - let pipelineJsonStringified = JSON.stringify(pipeline, null, 4); + let pipelineJsonStringified = JSON.stringify(pipeline, null, 4); - for (let i = 0; i < LOOP_LIMIT; i++) { - pipelineJsonStringified = pipelineJsonStringified.replace( - /(-?0\.\d+),[\n\s]+(-?0\.\d+)/gms, - `$1${REPLACING_NONCE}$2`, - ); - } + for (let i = 0; i < LOOP_LIMIT; i++) { + pipelineJsonStringified = pipelineJsonStringified.replace( + /(-?0\.\d+),[\n\s]+(-?0\.\d+)/gms, + `$1${REPLACING_NONCE}$2`, + ); + } - pipelineJsonStringified = pipelineJsonStringified.split(REPLACING_NONCE).join(', '); + pipelineJsonStringified = pipelineJsonStringified + .split(REPLACING_NONCE) + .join(", "); - pipelineJsonStringified += '\n'; + pipelineJsonStringified += "\n"; - return pipelineJsonStringified as string_json; + return pipelineJsonStringified as string_json; } /** diff --git a/src/utils/emojis.ts b/src/utils/emojis.ts index 495545e86c..03802790e3 100644 --- a/src/utils/emojis.ts +++ b/src/utils/emojis.ts @@ -1,5 +1,5 @@ -import type { string_char_emoji } from '../types/typeAliasEmoji'; -import type { TODO_any } from './organization/TODO_any'; +import type { string_char_emoji } from "../types/typeAliasEmoji"; +import type { TODO_any } from "./organization/TODO_any"; /** * All possible emoji chars like "🍆", "🍡", "🍤"... @@ -10,107 +10,115 @@ import type { TODO_any } from './organization/TODO_any'; * @private within the repository * @deprecated Use /\p{Extended_Pictographic}/ instead */ -export const EMOJIS_IN_CATEGORIES: Record> = { - Smileys: - '😀 😃 😄 😁 😆 😅 😂 🤣 🥲 ☺️ 😊 😇 🙂 🙃 😉 😌 😍 🥰 😘 😗 😙 😚 😋 😛 😝 😜 🤪 🤨 🧐 🤓 😎 🥸 🤩 🥳 😏 😒 😞 😔 😟 😕 🙁 ☹️ 😣 😖 😫 😩 🥺 😢 😭 😤 😠 😡 🤬 🤯 😳 🥵 🥶 😱 😨 😰 😥 😓 🤗 🤔 🤭 🤫 🤥 😶 😐 😑 😬 🙄 😯 😦 😧 😮 😲 🥱 😴 🤤 😪 😵 🤐 🥴 🤢 🤮 🤧 😷 🤒 🤕 🤑 🤠 😈 👿 👹 👺 🤡 💩 👻 💀 ☠️ 👽 👾 🤖 🎃 😺 😸 😹 😻 😼 😽 🙀 😿 😾'.split( - ' ', - ) as ReadonlyArray, - - 'Gestures and Body Parts': - '👋 🤚 🖐 ✋ 🖖 👌 🤌 🤏 ✌️ 🤞 🤟 🤘 🤙 👈 👉 👆 🖕 👇 ☝️ 👍 👎 ✊ 👊 🤛 🤜 👏 🙌 👐 🤲 🤝 🙏 ✍️ 💅 🤳 💪 🦾 🦵 🦿 🦶 👣 👂 🦻 👃 🫀 🫁 🧠 🦷 🦴 👀 👁 👅 👄 💋 🩸'.split( - ' ', - ) as ReadonlyArray, - - 'People and Fantasy': - '👶 👧 🧒 👦 👩 🧑 👨 👩‍🦱 🧑‍🦱 👨‍🦱 👩‍🦰 🧑‍🦰 👨‍🦰 👱‍♀️ 👱 👱‍♂️ 👩‍🦳 🧑‍🦳 👨‍🦳 👩‍🦲 🧑‍🦲 👨‍🦲 🧔 👵 🧓 👴 👲 👳‍♀️ 👳 👳‍♂️ 🧕 👮‍♀️ 👮 👮‍♂️ 👷‍♀️ 👷 👷‍♂️ 💂‍♀️ 💂 💂‍♂️ 🕵️‍♀️ 🕵️ 🕵️‍♂️ 👩‍⚕️ 🧑‍⚕️ 👨‍⚕️ 👩‍🌾 🧑‍🌾 👨‍🌾 👩‍🍳 🧑‍🍳 👨‍🍳 👩‍🎓 🧑‍🎓 👨‍🎓 👩‍🎤 🧑‍🎤 👨‍🎤 👩‍🏫 🧑‍🏫 👨‍🏫 👩‍🏭 🧑‍🏭 👨‍🏭 👩‍💻 🧑‍💻 👨‍💻 👩‍💼 🧑‍💼 👨‍💼 👩‍🔧 🧑‍🔧 👨‍🔧 👩‍🔬 🧑‍🔬 👨‍🔬 👩‍🎨 🧑‍🎨 👨‍🎨 👩‍🚒 🧑‍🚒 👨‍🚒 👩‍✈️ 🧑‍✈️ 👨‍✈️ 👩‍🚀 🧑‍🚀 👨‍🚀 👩‍⚖️ 🧑‍⚖️ 👨‍⚖️ 👰‍♀️ 👰 👰‍♂️ 🤵‍♀️ 🤵 🤵‍♂️ 👸 🤴 🥷 🦸‍♀️ 🦸 🦸‍♂️ 🦹‍♀️ 🦹 🦹‍♂️ 🤶 🧑‍🎄 🎅 🧙‍♀️ 🧙 🧙‍♂️ 🧝‍♀️ 🧝 🧝‍♂️ 🧛‍♀️ 🧛 🧛‍♂️ 🧟‍♀️ 🧟 🧟‍♂️ 🧞‍♀️ 🧞 🧞‍♂️ 🧜‍♀️ 🧜 🧜‍♂️ 🧚‍♀️ 🧚 🧚‍♂️ 👼 🤰 🤱 👩‍🍼 🧑‍🍼 👨‍🍼 🙇‍♀️ 🙇 🙇‍♂️ 💁‍♀️ 💁 💁‍♂️ 🙅‍♀️ 🙅 🙅‍♂️ 🙆‍♀️ 🙆 🙆‍♂️ 🙋‍♀️ 🙋 🙋‍♂️ 🧏‍♀️ 🧏 🧏‍♂️ 🤦‍♀️ 🤦 🤦‍♂️ 🤷‍♀️ 🤷 🤷‍♂️ 🙎‍♀️ 🙎 🙎‍♂️ 🙍‍♀️ 🙍 🙍‍♂️ 💇‍♀️ 💇 💇‍♂️ 💆‍♀️ 💆 💆‍♂️ 🧖‍♀️ 🧖 🧖‍♂️ 💅 🤳 💃 🕺 👯‍♀️ 👯 👯‍♂️ 🕴 👩‍🦽 🧑‍🦽 👨‍🦽 👩‍🦼 🧑‍🦼 👨‍🦼 🚶‍♀️ 🚶 🚶‍♂️ 👩‍🦯 🧑‍🦯 👨‍🦯 🧎‍♀️ 🧎 🧎‍♂️ 🏃‍♀️ 🏃 🏃‍♂️ 🧍‍♀️ 🧍 🧍‍♂️ 👭 🧑‍🤝‍🧑 👬 👫 👩‍❤️‍👩 💑 👨‍❤️‍👨 👩‍❤️‍👨 👩‍❤️‍💋‍👩 💏 👨‍❤️‍💋‍👨 👩‍❤️‍💋‍👨 👪 👨‍👩‍👦 👨‍👩‍👧 👨‍👩‍👧‍👦 👨‍👩‍👦‍👦 👨‍👩‍👧‍👧 👨‍👨‍👦 👨‍👨‍👧 👨‍👨‍👧‍👦 👨‍👨‍👦‍👦 👨‍👨‍👧‍👧 👩‍👩‍👦 👩‍👩‍👧 👩‍👩‍👧‍👦 👩‍👩‍👦‍👦 👩‍👩‍👧‍👧 👨‍👦 👨‍👦‍👦 👨‍👧 👨‍👧‍👦 👨‍👧‍👧 👩‍👦 👩‍👦‍👦 👩‍👧 👩‍👧‍👦 👩‍👧‍👧 🗣 👤 👥 🫂'.split( - ' ', - ) as ReadonlyArray, - - 'Clothing and Accessories': - '🧳 🌂 ☂️ 🧵 🪡 🪢 🧶 👓 🕶 🥽 🥼 🦺 👔 👕 👖 🧣 🧤 🧥 🧦 👗 👘 🥻 🩴 🩱 🩲 🩳 👙 👚 👛 👜 👝 🎒 👞 👟 🥾 🥿 👠 👡 🩰 👢 👑 👒 🎩 🎓 🧢 ⛑ 🪖 💄 💍 💼'.split( - ' ', - ) as ReadonlyArray, - - 'Pale Emojis': - '👋🏻 🤚🏻 🖐🏻 ✋🏻 🖖🏻 👌🏻 🤌🏻 🤏🏻 ✌🏻 🤞🏻 🤟🏻 🤘🏻 🤙🏻 👈🏻 👉🏻 👆🏻 🖕🏻 👇🏻 ☝🏻 👍🏻 👎🏻 ✊🏻 👊🏻 🤛🏻 🤜🏻 👏🏻 🙌🏻 👐🏻 🤲🏻 🙏🏻 ✍🏻 💅🏻 🤳🏻 💪🏻 🦵🏻 🦶🏻 👂🏻 🦻🏻 👃🏻 👶🏻 👧🏻 🧒🏻 👦🏻 👩🏻 🧑🏻 👨🏻 👩🏻‍🦱 🧑🏻‍🦱 👨🏻‍🦱 👩🏻‍🦰 🧑🏻‍🦰 👨🏻‍🦰 👱🏻‍♀️ 👱🏻 👱🏻‍♂️ 👩🏻‍🦳 🧑🏻‍🦳 👨🏻‍🦳 👩🏻‍🦲 🧑🏻‍🦲 👨🏻‍🦲 🧔🏻 👵🏻 🧓🏻 👴🏻 👲🏻 👳🏻‍♀️ 👳🏻 👳🏻‍♂️ 🧕🏻 👮🏻‍♀️ 👮🏻 👮🏻‍♂️ 👷🏻‍♀️ 👷🏻 👷🏻‍♂️ 💂🏻‍♀️ 💂🏻 💂🏻‍♂️ 🕵🏻‍♀️ 🕵🏻 🕵🏻‍♂️ 👩🏻‍⚕️ 🧑🏻‍⚕️ 👨🏻‍⚕️ 👩🏻‍🌾 🧑🏻‍🌾 👨🏻‍🌾 👩🏻‍🍳 🧑🏻‍🍳 👨🏻‍🍳 👩🏻‍🎓 🧑🏻‍🎓 👨🏻‍🎓 👩🏻‍🎤 🧑🏻‍🎤 👨🏻‍🎤 👩🏻‍🏫 🧑🏻‍🏫 👨🏻‍🏫 👩🏻‍🏭 🧑🏻‍🏭 👨🏻‍🏭 👩🏻‍💻 🧑🏻‍💻 👨🏻‍💻 👩🏻‍💼 🧑🏻‍💼 👨🏻‍💼 👩🏻‍🔧 🧑🏻‍🔧 👨🏻‍🔧 👩🏻‍🔬 🧑🏻‍🔬 👨🏻‍🔬 👩🏻‍🎨 🧑🏻‍🎨 👨🏻‍🎨 👩🏻‍🚒 🧑🏻‍🚒 👨🏻‍🚒 👩🏻‍✈️ 🧑🏻‍✈️ 👨🏻‍✈️ 👩🏻‍🚀 🧑🏻‍🚀 👨🏻‍🚀 👩🏻‍⚖️ 🧑🏻‍⚖️ 👨🏻‍⚖️ 👰🏻‍♀️ 👰🏻 👰🏻‍♂️ 🤵🏻‍♀️ 🤵🏻 🤵🏻‍♂️ 👸🏻 🤴🏻 🥷🏻 🦸🏻‍♀️ 🦸🏻 🦸🏻‍♂️ 🦹🏻‍♀️ 🦹🏻 🦹🏻‍♂️ 🤶🏻 🧑🏻‍🎄 🎅🏻 🧙🏻‍♀️ 🧙🏻 🧙🏻‍♂️ 🧝🏻‍♀️ 🧝🏻 🧝🏻‍♂️ 🧛🏻‍♀️ 🧛🏻 🧛🏻‍♂️ 🧜🏻‍♀️ 🧜🏻 🧜🏻‍♂️ 🧚🏻‍♀️ 🧚🏻 🧚🏻‍♂️ 👼🏻 🤰🏻 🤱🏻 👩🏻‍🍼 🧑🏻‍🍼 👨🏻‍🍼 🙇🏻‍♀️ 🙇🏻 🙇🏻‍♂️ 💁🏻‍♀️ 💁🏻 💁🏻‍♂️ 🙅🏻‍♀️ 🙅🏻 🙅🏻‍♂️ 🙆🏻‍♀️ 🙆🏻 🙆🏻‍♂️ 🙋🏻‍♀️ 🙋🏻 🙋🏻‍♂️ 🧏🏻‍♀️ 🧏🏻 🧏🏻‍♂️ 🤦🏻‍♀️ 🤦🏻 🤦🏻‍♂️ 🤷🏻‍♀️ 🤷🏻 🤷🏻‍♂️ 🙎🏻‍♀️ 🙎🏻 🙎🏻‍♂️ 🙍🏻‍♀️ 🙍🏻 🙍🏻‍♂️ 💇🏻‍♀️ 💇🏻 💇🏻‍♂️ 💆🏻‍♀️ 💆🏻 💆🏻‍♂️ 🧖🏻‍♀️ 🧖🏻 🧖🏻‍♂️ 💃🏻 🕺🏻 🕴🏻 👩🏻‍🦽 🧑🏻‍🦽 👨🏻‍🦽 👩🏻‍🦼 🧑🏻‍🦼 👨🏻‍🦼 🚶🏻‍♀️ 🚶🏻 🚶🏻‍♂️ 👩🏻‍🦯 🧑🏻‍🦯 👨🏻‍🦯 🧎🏻‍♀️ 🧎🏻 🧎🏻‍♂️ 🏃🏻‍♀️ 🏃🏻 🏃🏻‍♂️ 🧍🏻‍♀️ 🧍🏻 🧍🏻‍♂️ 👭🏻 🧑🏻‍🤝‍🧑🏻 👬🏻 👫🏻 🧗🏻‍♀️ 🧗🏻 🧗🏻‍♂️ 🏇🏻 🏂🏻 🏌🏻‍♀️ 🏌🏻 🏌🏻‍♂️ 🏄🏻‍♀️ 🏄🏻 🏄🏻‍♂️ 🚣🏻‍♀️ 🚣🏻 🚣🏻‍♂️ 🏊🏻‍♀️ 🏊🏻 🏊🏻‍♂️ ⛹🏻‍♀️ ⛹🏻 ⛹🏻‍♂️ 🏋🏻‍♀️ 🏋🏻 🏋🏻‍♂️ 🚴🏻‍♀️ 🚴🏻 🚴🏻‍♂️ 🚵🏻‍♀️ 🚵🏻 🚵🏻‍♂️ 🤸🏻‍♀️ 🤸🏻 🤸🏻‍♂️ 🤽🏻‍♀️ 🤽🏻 🤽🏻‍♂️ 🤾🏻‍♀️ 🤾🏻 🤾🏻‍♂️ 🤹🏻‍♀️ 🤹🏻 🤹🏻‍♂️ 🧘🏻‍♀️ 🧘🏻 🧘🏻‍♂️ 🛀🏻 🛌'.split( - ' ', - ) as ReadonlyArray, - - 'Cream White Emojis': - '👋🏼 🤚🏼 🖐🏼 ✋🏼 🖖🏼 👌🏼 🤌🏼 🤏🏼 ✌🏼 🤞🏼 🤟🏼 🤘🏼 🤙🏼 👈🏼 👉🏼 👆🏼 🖕🏼 👇🏼 ☝🏼 👍🏼 👎🏼 ✊🏼 👊🏼 🤛🏼 🤜🏼 👏🏼 🙌🏼 👐🏼 🤲🏼 🙏🏼 ✍🏼 💅🏼 🤳🏼 💪🏼 🦵🏼 🦶🏼 👂🏼 🦻🏼 👃🏼 👶🏼 👧🏼 🧒🏼 👦🏼 👩🏼 🧑🏼 👨🏼 👩🏼‍🦱 🧑🏼‍🦱 👨🏼‍🦱 👩🏼‍🦰 🧑🏼‍🦰 👨🏼‍🦰 👱🏼‍♀️ 👱🏼 👱🏼‍♂️ 👩🏼‍🦳 🧑🏼‍🦳 👨🏼‍🦳 👩🏼‍🦲 🧑🏼‍🦲 👨🏼‍🦲 🧔🏼 👵🏼 🧓🏼 👴🏼 👲🏼 👳🏼‍♀️ 👳🏼 👳🏼‍♂️ 🧕🏼 👮🏼‍♀️ 👮🏼 👮🏼‍♂️ 👷🏼‍♀️ 👷🏼 👷🏼‍♂️ 💂🏼‍♀️ 💂🏼 💂🏼‍♂️ 🕵🏼‍♀️ 🕵🏼 🕵🏼‍♂️ 👩🏼‍⚕️ 🧑🏼‍⚕️ 👨🏼‍⚕️ 👩🏼‍🌾 🧑🏼‍🌾 👨🏼‍🌾 👩🏼‍🍳 🧑🏼‍🍳 👨🏼‍🍳 👩🏼‍🎓 🧑🏼‍🎓 👨🏼‍🎓 👩🏼‍🎤 🧑🏼‍🎤 👨🏼‍🎤 👩🏼‍🏫 🧑🏼‍🏫 👨🏼‍🏫 👩🏼‍🏭 🧑🏼‍🏭 👨🏼‍🏭 👩🏼‍💻 🧑🏼‍💻 👨🏼‍💻 👩🏼‍💼 🧑🏼‍💼 👨🏼‍💼 👩🏼‍🔧 🧑🏼‍🔧 👨🏼‍🔧 👩🏼‍🔬 🧑🏼‍🔬 👨🏼‍🔬 👩🏼‍🎨 🧑🏼‍🎨 👨🏼‍🎨 👩🏼‍🚒 🧑🏼‍🚒 👨🏼‍🚒 👩🏼‍✈️ 🧑🏼‍✈️ 👨🏼‍✈️ 👩🏼‍🚀 🧑🏼‍🚀 👨🏼‍🚀 👩🏼‍⚖️ 🧑🏼‍⚖️ 👨🏼‍⚖️ 👰🏼‍♀️ 👰🏼 👰🏼‍♂️ 🤵🏼‍♀️ 🤵🏼 🤵🏼‍♂️ 👸🏼 🤴🏼 🥷🏼 🦸🏼‍♀️ 🦸🏼 🦸🏼‍♂️ 🦹🏼‍♀️ 🦹🏼 🦹🏼‍♂️ 🤶🏼 🧑🏼‍🎄 🎅🏼 🧙🏼‍♀️ 🧙🏼 🧙🏼‍♂️ 🧝🏼‍♀️ 🧝🏼 🧝🏼‍♂️ 🧛🏼‍♀️ 🧛🏼 🧛🏼‍♂️ 🧜🏼‍♀️ 🧜🏼 🧜🏼‍♂️ 🧚🏼‍♀️ 🧚🏼 🧚🏼‍♂️ 👼🏼 🤰🏼 🤱🏼 👩🏼‍🍼 🧑🏼‍🍼 👨🏼‍🍼 🙇🏼‍♀️ 🙇🏼 🙇🏼‍♂️ 💁🏼‍♀️ 💁🏼 💁🏼‍♂️ 🙅🏼‍♀️ 🙅🏼 🙅🏼‍♂️ 🙆🏼‍♀️ 🙆🏼 🙆🏼‍♂️ 🙋🏼‍♀️ 🙋🏼 🙋🏼‍♂️ 🧏🏼‍♀️ 🧏🏼 🧏🏼‍♂️ 🤦🏼‍♀️ 🤦🏼 🤦🏼‍♂️ 🤷🏼‍♀️ 🤷🏼 🤷🏼‍♂️ 🙎🏼‍♀️ 🙎🏼 🙎🏼‍♂️ 🙍🏼‍♀️ 🙍🏼 🙍🏼‍♂️ 💇🏼‍♀️ 💇🏼 💇🏼‍♂️ 💆🏼‍♀️ 💆🏼 💆🏼‍♂️ 🧖🏼‍♀️ 🧖🏼 🧖🏼‍♂️ 💃🏼 🕺🏼 🕴🏼 👩🏼‍🦽 🧑🏼‍🦽 👨🏼‍🦽 👩🏼‍🦼 🧑🏼‍🦼 👨🏼‍🦼 🚶🏼‍♀️ 🚶🏼 🚶🏼‍♂️ 👩🏼‍🦯 🧑🏼‍🦯 👨🏼‍🦯 🧎🏼‍♀️ 🧎🏼 🧎🏼‍♂️ 🏃🏼‍♀️ 🏃🏼 🏃🏼‍♂️ 🧍🏼‍♀️ 🧍🏼 🧍🏼‍♂️ 👭🏼 🧑🏼‍🤝‍🧑🏼 👬🏼 👫🏼 🧗🏼‍♀️ 🧗🏼 🧗🏼‍♂️ 🏇🏼 🏂🏼 🏌🏼‍♀️ 🏌🏼 🏌🏼‍♂️ 🏄🏼‍♀️ 🏄🏼 🏄🏼‍♂️ 🚣🏼‍♀️ 🚣🏼 🚣🏼‍♂️ 🏊🏼‍♀️ 🏊🏼 🏊🏼‍♂️ ⛹🏼‍♀️ ⛹🏼 ⛹🏼‍♂️ 🏋🏼‍♀️ 🏋🏼 🏋🏼‍♂️ 🚴🏼‍♀️ 🚴🏼 🚴🏼‍♂️ 🚵🏼‍♀️ 🚵🏼 🚵🏼‍♂️ 🤸🏼‍♀️ 🤸🏼 🤸🏼‍♂️ 🤽🏼‍♀️ 🤽🏼 🤽🏼‍♂️ 🤾🏼‍♀️ 🤾🏼 🤾🏼‍♂️ 🤹🏼‍♀️ 🤹🏼 🤹🏼‍♂️ 🧘🏼‍♀️ 🧘🏼 🧘🏼‍♂️ 🛀🏼 🛌'.split( - ' ', - ) as ReadonlyArray, - - 'Brown Emojis': - '👋🏽 🤚🏽 🖐🏽 ✋🏽 🖖🏽 👌🏽 🤌🏽 🤏🏽 ✌🏽 🤞🏽 🤟🏽 🤘🏽 🤙🏽 👈🏽 👉🏽 👆🏽 🖕🏽 👇🏽 ☝🏽 👍🏽 👎🏽 ✊🏽 👊🏽 🤛🏽 🤜🏽 👏🏽 🙌🏽 👐🏽 🤲🏽 🙏🏽 ✍🏽 💅🏽 🤳🏽 💪🏽 🦵🏽 🦶🏽 👂🏽 🦻🏽 👃🏽 👶🏽 👧🏽 🧒🏽 👦🏽 👩🏽 🧑🏽 👨🏽 👩🏽‍🦱 🧑🏽‍🦱 👨🏽‍🦱 👩🏽‍🦰 🧑🏽‍🦰 👨🏽‍🦰 👱🏽‍♀️ 👱🏽 👱🏽‍♂️ 👩🏽‍🦳 🧑🏽‍🦳 👨🏽‍🦳 👩🏽‍🦲 🧑🏽‍🦲 👨🏽‍🦲 🧔🏽 👵🏽 🧓🏽 👴🏽 👲🏽 👳🏽‍♀️ 👳🏽 👳🏽‍♂️ 🧕🏽 👮🏽‍♀️ 👮🏽 👮🏽‍♂️ 👷🏽‍♀️ 👷🏽 👷🏽‍♂️ 💂🏽‍♀️ 💂🏽 💂🏽‍♂️ 🕵🏽‍♀️ 🕵🏽 🕵🏽‍♂️ 👩🏽‍⚕️ 🧑🏽‍⚕️ 👨🏽‍⚕️ 👩🏽‍🌾 🧑🏽‍🌾 👨🏽‍🌾 👩🏽‍🍳 🧑🏽‍🍳 👨🏽‍🍳 👩🏽‍🎓 🧑🏽‍🎓 👨🏽‍🎓 👩🏽‍🎤 🧑🏽‍🎤 👨🏽‍🎤 👩🏽‍🏫 🧑🏽‍🏫 👨🏽‍🏫 👩🏽‍🏭 🧑🏽‍🏭 👨🏽‍🏭 👩🏽‍💻 🧑🏽‍💻 👨🏽‍💻 👩🏽‍💼 🧑🏽‍💼 👨🏽‍💼 👩🏽‍🔧 🧑🏽‍🔧 👨🏽‍🔧 👩🏽‍🔬 🧑🏽‍🔬 👨🏽‍🔬 👩🏽‍🎨 🧑🏽‍🎨 👨🏽‍🎨 👩🏽‍🚒 🧑🏽‍🚒 👨🏽‍🚒 👩🏽‍✈️ 🧑🏽‍✈️ 👨🏽‍✈️ 👩🏽‍🚀 🧑🏽‍🚀 👨🏽‍🚀 👩🏽‍⚖️ 🧑🏽‍⚖️ 👨🏽‍⚖️ 👰🏽‍♀️ 👰🏽 👰🏽‍♂️ 🤵🏽‍♀️ 🤵🏽 🤵🏽‍♂️ 👸🏽 🤴🏽 🥷🏽 🦸🏽‍♀️ 🦸🏽 🦸🏽‍♂️ 🦹🏽‍♀️ 🦹🏽 🦹🏽‍♂️ 🤶🏽 🧑🏽‍🎄 🎅🏽 🧙🏽‍♀️ 🧙🏽 🧙🏽‍♂️ 🧝🏽‍♀️ 🧝🏽 🧝🏽‍♂️ 🧛🏽‍♀️ 🧛🏽 🧛🏽‍♂️ 🧜🏽‍♀️ 🧜🏽 🧜🏽‍♂️ 🧚🏽‍♀️ 🧚🏽 🧚🏽‍♂️ 👼🏽 🤰🏽 🤱🏽 👩🏽‍🍼 🧑🏽‍🍼 👨🏽‍🍼 🙇🏽‍♀️ 🙇🏽 🙇🏽‍♂️ 💁🏽‍♀️ 💁🏽 💁🏽‍♂️ 🙅🏽‍♀️ 🙅🏽 🙅🏽‍♂️ 🙆🏽‍♀️ 🙆🏽 🙆🏽‍♂️ 🙋🏽‍♀️ 🙋🏽 🙋🏽‍♂️ 🧏🏽‍♀️ 🧏🏽 🧏🏽‍♂️ 🤦🏽‍♀️ 🤦🏽 🤦🏽‍♂️ 🤷🏽‍♀️ 🤷🏽 🤷🏽‍♂️ 🙎🏽‍♀️ 🙎🏽 🙎🏽‍♂️ 🙍🏽‍♀️ 🙍🏽 🙍🏽‍♂️ 💇🏽‍♀️ 💇🏽 💇🏽‍♂️ 💆🏽‍♀️ 💆🏽 💆🏽‍♂️ 🧖🏽‍♀️ 🧖🏽 🧖🏽‍♂️ 💃🏽 🕺🏽 🕴🏽 👩🏽‍🦽 🧑🏽‍🦽 👨🏽‍🦽 👩🏽‍🦼 🧑🏽‍🦼 👨🏽‍🦼 🚶🏽‍♀️ 🚶🏽 🚶🏽‍♂️ 👩🏽‍🦯 🧑🏽‍🦯 👨🏽‍🦯 🧎🏽‍♀️ 🧎🏽 🧎🏽‍♂️ 🏃🏽‍♀️ 🏃🏽 🏃🏽‍♂️ 🧍🏽‍♀️ 🧍🏽 🧍🏽‍♂️ 👭🏽 🧑🏽‍🤝‍🧑🏽 👬🏽 👫🏽 🧗🏽‍♀️ 🧗🏽 🧗🏽‍♂️ 🏇🏽 🏂🏽 🏌🏽‍♀️ 🏌🏽 🏌🏽‍♂️ 🏄🏽‍♀️ 🏄🏽 🏄🏽‍♂️ 🚣🏽‍♀️ 🚣🏽 🚣🏽‍♂️ 🏊🏽‍♀️ 🏊🏽 🏊🏽‍♂️ ⛹🏽‍♀️ ⛹🏽 ⛹🏽‍♂️ 🏋🏽‍♀️ 🏋🏽 🏋🏽‍♂️ 🚴🏽‍♀️ 🚴🏽 🚴🏽‍♂️ 🚵🏽‍♀️ 🚵🏽 🚵🏽‍♂️ 🤸🏽‍♀️ 🤸🏽 🤸🏽‍♂️ 🤽🏽‍♀️ 🤽🏽 🤽🏽‍♂️ 🤾🏽‍♀️ 🤾🏽 🤾🏽‍♂️ 🤹🏽‍♀️ 🤹🏽 🤹🏽‍♂️ 🧘🏽‍♀️ 🧘🏽 🧘🏽‍♂️ 🛀🏽 🛌'.split( - ' ', - ) as ReadonlyArray, - - 'Dark Brown Emojis': - '👋🏾 🤚🏾 🖐🏾 ✋🏾 🖖🏾 👌🏾 🤌🏾 🤏🏾 ✌🏾 🤞🏾 🤟🏾 🤘🏾 🤙🏾 👈🏾 👉🏾 👆🏾 🖕🏾 👇🏾 ☝🏾 👍🏾 👎🏾 ✊🏾 👊🏾 🤛🏾 🤜🏾 👏🏾 🙌🏾 👐🏾 🤲🏾 🙏🏾 ✍🏾 💅🏾 🤳🏾 💪🏾 🦵🏾 🦶🏾 👂🏾 🦻🏾 👃🏾 👶🏾 👧🏾 🧒🏾 👦🏾 👩🏾 🧑🏾 👨🏾 👩🏾‍🦱 🧑🏾‍🦱 👨🏾‍🦱 👩🏾‍🦰 🧑🏾‍🦰 👨🏾‍🦰 👱🏾‍♀️ 👱🏾 👱🏾‍♂️ 👩🏾‍🦳 🧑🏾‍🦳 👨🏾‍🦳 👩🏾‍🦲 🧑🏾‍🦲 👨🏾‍🦲 🧔🏾 👵🏾 🧓🏾 👴🏾 👲🏾 👳🏾‍♀️ 👳🏾 👳🏾‍♂️ 🧕🏾 👮🏾‍♀️ 👮🏾 👮🏾‍♂️ 👷🏾‍♀️ 👷🏾 👷🏾‍♂️ 💂🏾‍♀️ 💂🏾 💂🏾‍♂️ 🕵🏾‍♀️ 🕵🏾 🕵🏾‍♂️ 👩🏾‍⚕️ 🧑🏾‍⚕️ 👨🏾‍⚕️ 👩🏾‍🌾 🧑🏾‍🌾 👨🏾‍🌾 👩🏾‍🍳 🧑🏾‍🍳 👨🏾‍🍳 👩🏾‍🎓 🧑🏾‍🎓 👨🏾‍🎓 👩🏾‍🎤 🧑🏾‍🎤 👨🏾‍🎤 👩🏾‍🏫 🧑🏾‍🏫 👨🏾‍🏫 👩🏾‍🏭 🧑🏾‍🏭 👨🏾‍🏭 👩🏾‍💻 🧑🏾‍💻 👨🏾‍💻 👩🏾‍💼 🧑🏾‍💼 👨🏾‍💼 👩🏾‍🔧 🧑🏾‍🔧 👨🏾‍🔧 👩🏾‍🔬 🧑🏾‍🔬 👨🏾‍🔬 👩🏾‍🎨 🧑🏾‍🎨 👨🏾‍🎨 👩🏾‍🚒 🧑🏾‍🚒 👨🏾‍🚒 👩🏾‍✈️ 🧑🏾‍✈️ 👨🏾‍✈️ 👩🏾‍🚀 🧑🏾‍🚀 👨🏾‍🚀 👩🏾‍⚖️ 🧑🏾‍⚖️ 👨🏾‍⚖️ 👰🏾‍♀️ 👰🏾 👰🏾‍♂️ 🤵🏾‍♀️ 🤵🏾 🤵🏾‍♂️ 👸🏾 🤴🏾 🥷🏾 🦸🏾‍♀️ 🦸🏾 🦸🏾‍♂️ 🦹🏾‍♀️ 🦹🏾 🦹🏾‍♂️ 🤶🏾 🧑🏾‍🎄 🎅🏾 🧙🏾‍♀️ 🧙🏾 🧙🏾‍♂️ 🧝🏾‍♀️ 🧝🏾 🧝🏾‍♂️ 🧛🏾‍♀️ 🧛🏾 🧛🏾‍♂️ 🧜🏾‍♀️ 🧜🏾 🧜🏾‍♂️ 🧚🏾‍♀️ 🧚🏾 🧚🏾‍♂️ 👼🏾 🤰🏾 🤱🏾 👩🏾‍🍼 🧑🏾‍🍼 👨🏾‍🍼 🙇🏾‍♀️ 🙇🏾 🙇🏾‍♂️ 💁🏾‍♀️ 💁🏾 💁🏾‍♂️ 🙅🏾‍♀️ 🙅🏾 🙅🏾‍♂️ 🙆🏾‍♀️ 🙆🏾 🙆🏾‍♂️ 🙋🏾‍♀️ 🙋🏾 🙋🏾‍♂️ 🧏🏾‍♀️ 🧏🏾 🧏🏾‍♂️ 🤦🏾‍♀️ 🤦🏾 🤦🏾‍♂️ 🤷🏾‍♀️ 🤷🏾 🤷🏾‍♂️ 🙎🏾‍♀️ 🙎🏾 🙎🏾‍♂️ 🙍🏾‍♀️ 🙍🏾 🙍🏾‍♂️ 💇🏾‍♀️ 💇🏾 💇🏾‍♂️ 💆🏾‍♀️ 💆🏾 💆🏾‍♂️ 🧖🏾‍♀️ 🧖🏾 🧖🏾‍♂️ 💃🏾 🕺🏾 🕴🏿 👩🏾‍🦽 🧑🏾‍🦽 👨🏾‍🦽 👩🏾‍🦼 🧑🏾‍🦼 👨🏾‍🦼 🚶🏾‍♀️ 🚶🏾 🚶🏾‍♂️ 👩🏾‍🦯 🧑🏾‍🦯 👨🏾‍🦯 🧎🏾‍♀️ 🧎🏾 🧎🏾‍♂️ 🏃🏾‍♀️ 🏃🏾 🏃🏾‍♂️ 🧍🏾‍♀️ 🧍🏾 🧍🏾‍♂️ 👭🏾 🧑🏾‍🤝‍🧑🏾 👬🏾 👫🏾 🧗🏾‍♀️ 🧗🏾 🧗🏾‍♂️ 🏇🏾 🏂🏾 🏌🏾‍♀️ 🏌🏾 🏌🏾‍♂️ 🏄🏾‍♀️ 🏄🏾 🏄🏾‍♂️ 🚣🏾‍♀️ 🚣🏾 🚣🏾‍♂️ 🏊🏾‍♀️ 🏊🏾 🏊🏾‍♂️ ⛹🏾‍♀️ ⛹🏾 ⛹🏾‍♂️ 🏋🏾‍♀️ 🏋🏾 🏋🏾‍♂️ 🚴🏾‍♀️ 🚴🏾 🚴🏾‍♂️ 🚵🏾‍♀️ 🚵🏾 🚵🏾‍♂️ 🤸🏾‍♀️ 🤸🏾 🤸🏾‍♂️ 🤽🏾‍♀️ 🤽🏾 🤽🏾‍♂️ 🤾🏾‍♀️ 🤾🏾 🤾🏾‍♂️ 🤹🏾‍♀️ 🤹🏾 🤹🏾‍♂️ 🧘🏾‍♀️ 🧘🏾 🧘🏾‍♂️ 🛀🏾 🛌'.split( - ' ', - ) as ReadonlyArray, - - 'Black Emojis': - '👋🏿 🤚🏿 🖐🏿 ✋🏿 🖖🏿 👌🏿 🤌🏿 🤏🏿 ✌🏿 🤞🏿 🤟🏿 🤘🏿 🤙🏿 👈🏿 👉🏿 👆🏿 🖕🏿 👇🏿 ☝🏿 👍🏿 👎🏿 ✊🏿 👊🏿 🤛🏿 🤜🏿 👏🏿 🙌🏿 👐🏿 🤲🏿 🙏🏿 ✍🏿 💅🏿 🤳🏿 💪🏿 🦵🏿 🦶🏿 👂🏿 🦻🏿 👃🏿 👶🏿 👧🏿 🧒🏿 👦🏿 👩🏿 🧑🏿 👨🏿 👩🏿‍🦱 🧑🏿‍🦱 👨🏿‍🦱 👩🏿‍🦰 🧑🏿‍🦰 👨🏿‍🦰 👱🏿‍♀️ 👱🏿 👱🏿‍♂️ 👩🏿‍🦳 🧑🏿‍🦳 👨🏿‍🦳 👩🏿‍🦲 🧑🏿‍🦲 👨🏿‍🦲 🧔🏿 👵🏿 🧓🏿 👴🏿 👲🏿 👳🏿‍♀️ 👳🏿 👳🏿‍♂️ 🧕🏿 👮🏿‍♀️ 👮🏿 👮🏿‍♂️ 👷🏿‍♀️ 👷🏿 👷🏿‍♂️ 💂🏿‍♀️ 💂🏿 💂🏿‍♂️ 🕵🏿‍♀️ 🕵🏿 🕵🏿‍♂️ 👩🏿‍⚕️ 🧑🏿‍⚕️ 👨🏿‍⚕️ 👩🏿‍🌾 🧑🏿‍🌾 👨🏿‍🌾 👩🏿‍🍳 🧑🏿‍🍳 👨🏿‍🍳 👩🏿‍🎓 🧑🏿‍🎓 👨🏿‍🎓 👩🏿‍🎤 🧑🏿‍🎤 👨🏿‍🎤 👩🏿‍🏫 🧑🏿‍🏫 👨🏿‍🏫 👩🏿‍🏭 🧑🏿‍🏭 👨🏿‍🏭 👩🏿‍💻 🧑🏿‍💻 👨🏿‍💻 👩🏿‍💼 🧑🏿‍💼 👨🏿‍💼 👩🏿‍🔧 🧑🏿‍🔧 👨🏿‍🔧 👩🏿‍🔬 🧑🏿‍🔬 👨🏿‍🔬 👩🏿‍🎨 🧑🏿‍🎨 👨🏿‍🎨 👩🏿‍🚒 🧑🏿‍🚒 👨🏿‍🚒 👩🏿‍✈️ 🧑🏿‍✈️ 👨🏿‍✈️ 👩🏿‍🚀 🧑🏿‍🚀 👨🏿‍🚀 👩🏿‍⚖️ 🧑🏿‍⚖️ 👨🏿‍⚖️ 👰🏿‍♀️ 👰🏿 👰🏿‍♂️ 🤵🏿‍♀️ 🤵🏿 🤵🏿‍♂️ 👸🏿 🤴🏿 🥷🏿 🦸🏿‍♀️ 🦸🏿 🦸🏿‍♂️ 🦹🏿‍♀️ 🦹🏿 🦹🏿‍♂️ 🤶🏿 🧑🏿‍🎄 🎅🏿 🧙🏿‍♀️ 🧙🏿 🧙🏿‍♂️ 🧝🏿‍♀️ 🧝🏿 🧝🏿‍♂️ 🧛🏿‍♀️ 🧛🏿 🧛🏿‍♂️ 🧜🏿‍♀️ 🧜🏿 🧜🏿‍♂️ 🧚🏿‍♀️ 🧚🏿 🧚🏿‍♂️ 👼🏿 🤰🏿 🤱🏿 👩🏿‍🍼 🧑🏿‍🍼 👨🏿‍🍼 🙇🏿‍♀️ 🙇🏿 🙇🏿‍♂️ 💁🏿‍♀️ 💁🏿 💁🏿‍♂️ 🙅🏿‍♀️ 🙅🏿 🙅🏿‍♂️ 🙆🏿‍♀️ 🙆🏿 🙆🏿‍♂️ 🙋🏿‍♀️ 🙋🏿 🙋🏿‍♂️ 🧏🏿‍♀️ 🧏🏿 🧏🏿‍♂️ 🤦🏿‍♀️ 🤦🏿 🤦🏿‍♂️ 🤷🏿‍♀️ 🤷🏿 🤷🏿‍♂️ 🙎🏿‍♀️ 🙎🏿 🙎🏿‍♂️ 🙍🏿‍♀️ 🙍🏿 🙍🏿‍♂️ 💇🏿‍♀️ 💇🏿 💇🏿‍♂️ 💆🏿‍♀️ 💆🏿 💆🏿‍♂️ 🧖🏿‍♀️ 🧖🏿 🧖🏿‍♂️ 💃🏿 🕺🏿 🕴🏿 👩🏿‍🦽 🧑🏿‍🦽 👨🏿‍🦽 👩🏿‍🦼 🧑🏿‍🦼 👨🏿‍🦼 🚶🏿‍♀️ 🚶🏿 🚶🏿‍♂️ 👩🏿‍🦯 🧑🏿‍🦯 👨🏿‍🦯 🧎🏿‍♀️ 🧎🏿 🧎🏿‍♂️ 🏃🏿‍♀️ 🏃🏿 🏃🏿‍♂️ 🧍🏿‍♀️ 🧍🏿 🧍🏿‍♂️ 👭🏿 🧑🏿‍🤝‍🧑🏿 👬🏿 👫🏿 🧗🏿‍♀️ 🧗🏿 🧗🏿‍♂️ 🏇🏿 🏂🏿 🏌🏿‍♀️ 🏌🏿 🏌🏿‍♂️ 🏄🏿‍♀️ 🏄🏿 🏄🏿‍♂️ 🚣🏿‍♀️ 🚣🏿 🚣🏿‍♂️ 🏊🏿‍♀️ 🏊🏿 🏊🏿‍♂️ ⛹🏿‍♀️ ⛹🏿 ⛹🏿‍♂️ 🏋🏿‍♀️ 🏋🏿 🏋🏿‍♂️ 🚴🏿‍♀️ 🚴🏿 🚴🏿‍♂️ 🚵🏿‍♀️ 🚵🏿 🚵🏿‍♂️ 🤸🏿‍♀️ 🤸🏿 🤸🏿‍♂️ 🤽🏿‍♀️ 🤽🏿 🤽🏿‍♂️ 🤾🏿‍♀️ 🤾🏿 🤾🏿‍♂️ 🤹🏿‍♀️ 🤹🏿 🤹🏿‍♂️ 🧘🏿‍♀️ 🧘🏿 🧘🏿‍♂️ 🛀🏿 🛌🏿'.split( - ' ', - ) as ReadonlyArray, - - 'Animals & Nature': - '🐶 🐱 🐭 🐹 🐰 🦊 🐻 🐼 🐻‍❄️ 🐨 🐯 🦁 🐮 🐷 🐽 🐸 🐵 🙈 🙉 🙊 🐒 🐔 🐧 🐦 🐤 🐣 🐥 🦆 🦅 🦉 🦇 🐺 🐗 🐴 🦄 🐝 🪱 🐛 🦋 🐌 🐞 🐜 🪰 🪲 🪳 🦟 🦗 🕷 🕸 🦂 🐢 🐍 🦎 🦖 🦕 🐙 🦑 🦐 🦞 🦀 🐡 🐠 🐟 🐬 🐳 🐋 🦈 🐊 🐅 🐆 🦓 🦍 🦧 🦣 🐘 🦛 🦏 🐪 🐫 🦒 🦘 🦬 🐃 🐂 🐄 🐎 🐖 🐏 🐑 🦙 🐐 🦌 🐕 🐩 🦮 🐕‍🦺 🐈 🐈‍⬛ 🪶 🐓 🦃 🦤 🦚 🦜 🦢 🦩 🕊 🐇 🦝 🦨 🦡 🦫 🦦 🦥 🐁 🐀 🐿 🦔 🐾 🐉 🐲 🌵 🎄 🌲 🌳 🌴 🪵 🌱 🌿 ☘️ 🍀 🎍 🪴 🎋 🍃 🍂 🍁 🍄 🐚 🪨 🌾 💐 🌷 🌹 🥀 🌺 🌸 🌼 🌻 🌞 🌝 🌛 🌜 🌚 🌕 🌖 🌗 🌘 🌑 🌒 🌓 🌔 🌙 🌎 🌍 🌏 🪐 💫 ⭐️ 🌟 ✨ ⚡️ ☄️ 💥 🔥 🌪 🌈 ☀️ 🌤 ⛅️ 🌥 ☁️ 🌦 🌧 ⛈ 🌩 🌨 ❄️ ☃️ ⛄️ 🌬 💨 💧 💦 ☔️ ☂️ 🌊 🌫'.split( - ' ', - ) as ReadonlyArray, - - 'Food & Drink': - '🍏 🍎 🍐 🍊 🍋 🍌 🍉 🍇 🍓 🫐 🍈 🍒 🍑 🥭 🍍 🥥 🥝 🍅 🍆 🥑 🥦 🥬 🥒 🌶 🫑 🌽 🥕 🫒 🧄 🧅 🥔 🍠 🥐 🥯 🍞 🥖 🥨 🧀 🥚 🍳 🧈 🥞 🧇 🥓 🥩 🍗 🍖 🦴 🌭 🍔 🍟 🍕 🫓 🥪 🥙 🧆 🌮 🌯 🫔 🥗 🥘 🫕 🥫 🍝 🍜 🍲 🍛 🍣 🍱 🥟 🦪 🍤 🍙 🍚 🍘 🍥 🥠 🥮 🍢 🍡 🍧 🍨 🍦 🥧 🧁 🍰 🎂 🍮 🍭 🍬 🍫 🍿 🍩 🍪 🌰 🥜 🍯 🥛 🍼 🫖 ☕️ 🍵 🧃 🥤 🧋 🍶 🍺 🍻 🥂 🍷 🥃 🍸 🍹 🧉 🍾 🧊 🥄 🍴 🍽 🥣 🥡 🥢 🧂'.split( - ' ', - ) as ReadonlyArray, - - 'Activity and Sports': - '⚽️ 🏀 🏈 ⚾️ 🥎 🎾 🏐 🏉 🥏 🎱 🪀 🏓 🏸 🏒 🏑 🥍 🏏 🪃 🥅 ⛳️ 🪁 🏹 🎣 🤿 🥊 🥋 🎽 🛹 🛼 🛷 ⛸ 🥌 🎿 ⛷ 🏂 🪂 🏋️‍♀️ 🏋️ 🏋️‍♂️ 🤼‍♀️ 🤼 🤼‍♂️ 🤸‍♀️ 🤸 🤸‍♂️ ⛹️‍♀️ ⛹️ ⛹️‍♂️ 🤺 🤾‍♀️ 🤾 🤾‍♂️ 🏌️‍♀️ 🏌️ 🏌️‍♂️ 🏇 🧘‍♀️ 🧘 🧘‍♂️ 🏄‍♀️ 🏄 🏄‍♂️ 🏊‍♀️ 🏊 🏊‍♂️ 🤽‍♀️ 🤽 🤽‍♂️ 🚣‍♀️ 🚣 🚣‍♂️ 🧗‍♀️ 🧗 🧗‍♂️ 🚵‍♀️ 🚵 🚵‍♂️ 🚴‍♀️ 🚴 🚴‍♂️ 🏆 🥇 🥈 🥉 🏅 🎖 🏵 🎗 🎫 🎟 🎪 🤹 🤹‍♂️ 🤹‍♀️ 🎭 🩰 🎨 🎬 🎤 🎧 🎼 🎹 🥁 🪘 🎷 🎺 🪗 🎸 🪕 🎻 🎲 ♟ 🎯 🎳 🎮 🎰 🧩'.split( - ' ', - ) as ReadonlyArray, - - 'Travel & Places': - '🚗 🚕 🚙 🚌 🚎 🏎 🚓 🚑 🚒 🚐 🛻 🚚 🚛 🚜 🦯 🦽 🦼 🛴 🚲 🛵 🏍 🛺 🚨 🚔 🚍 🚘 🚖 🚡 🚠 🚟 🚃 🚋 🚞 🚝 🚄 🚅 🚈 🚂 🚆 🚇 🚊 🚉 ✈️ 🛫 🛬 🛩 💺 🛰 🚀 🛸 🚁 🛶 ⛵️ 🚤 🛥 🛳 ⛴ 🚢 ⚓️ 🪝 ⛽️ 🚧 🚦 🚥 🚏 🗺 🗿 🗽 🗼 🏰 🏯 🏟 🎡 🎢 🎠 ⛲️ ⛱ 🏖 🏝 🏜 🌋 ⛰ 🏔 🗻 🏕 ⛺️ 🛖 🏠 🏡 🏘 🏚 🏗 🏭 🏢 🏬 🏣 🏤 🏥 🏦 🏨 🏪 🏫 🏩 💒 🏛 ⛪️ 🕌 🕍 🛕 🕋 ⛩ 🛤 🛣 🗾 🎑 🏞 🌅 🌄 🌠 🎇 🎆 🌇 🌆 🏙 🌃 🌌 🌉 🌁'.split( - ' ', - ) as ReadonlyArray, - - Objects: - '⌚️ 📱 📲 💻 ⌨️ 🖥 🖨 🖱 🖲 🕹 🗜 💽 💾 💿 📀 📼 📷 📸 📹 🎥 📽 🎞 📞 ☎️ 📟 📠 📺 📻 🎙 🎚 🎛 🧭 ⏱ ⏲ ⏰ 🕰 ⌛️ ⏳ 📡 🔋 🔌 💡 🔦 🕯 🪔 🧯 🛢 💸 💵 💴 💶 💷 🪙 💰 💳 💎 ⚖️ 🪜 🧰 🪛 🔧 🔨 ⚒ 🛠 ⛏ 🪚 🔩 ⚙️ 🪤 🧱 ⛓ 🧲 🔫 💣 🧨 🪓 🔪 🗡 ⚔️ 🛡 🚬 ⚰️ 🪦 ⚱️ 🏺 🔮 📿 🧿 💈 ⚗️ 🔭 🔬 🕳 🩹 🩺 💊 💉 🩸 🧬 🦠 🧫 🧪 🌡 🧹 🪠 🧺 🧻 🚽 🚰 🚿 🛁 🛀 🧼 🪥 🪒 🧽 🪣 🧴 🛎 🔑 🗝 🚪 🪑 🛋 🛏 🛌 🧸 🪆 🖼 🪞 🪟 🛍 🛒 🎁 🎈 🎏 🎀 🪄 🪅 🎊 🎉 🎎 🏮 🎐 🧧 ✉️ 📩 📨 📧 💌 📥 📤 📦 🏷 🪧 📪 📫 📬 📭 📮 📯 📜 📃 📄 📑 🧾 📊 📈 📉 🗒 🗓 📆 📅 🗑 📇 🗃 🗳 🗄 📋 📁 📂 🗂 🗞 📰 📓 📔 📒 📕 📗 📘 📙 📚 📖 🔖 🧷 🔗 📎 🖇 📐 📏 🧮 📌 📍 ✂️ 🖊 🖋 ✒️ 🖌 🖍 📝 ✏️ 🔍 🔎 🔏 🔐 🔒 🔓'.split( - ' ', - ) as ReadonlyArray, - - Symbols: - '❤️ 🧡 💛 💚 💙 💜 🖤 🤍 🤎 💔 ❣️ 💕 💞 💓 💗 💖 💘 💝 💟 ☮️ ✝️ ☪️ 🕉 ☸️ ✡️ 🔯 🕎 ☯️ ☦️ 🛐 ⛎ ♈️ ♉️ ♊️ ♋️ ♌️ ♍️ ♎️ ♏️ ♐️ ♑️ ♒️ ♓️ 🆔 ⚛️ 🉑 ☢️ ☣️ 📴 📳 🈶 🈚️ 🈸 🈺 🈷️ ✴️ 🆚 💮 🉐 ㊙️ ㊗️ 🈴 🈵 🈹 🈲 🅰️ 🅱️ 🆎 🆑 🅾️ 🆘 ❌ ⭕️ 🛑 ⛔️ 📛 🚫 💯 💢 ♨️ 🚷 🚯 🚳 🚱 🔞 📵 🚭 ❗️ ❕ ❓ ❔ ‼️ ⁉️ 🔅 🔆 〽️ ⚠️ 🚸 🔱 ⚜️ 🔰 ♻️ ✅ 🈯️ 💹 ❇️ ✳️ ❎ 🌐 💠 Ⓜ️ 🌀 💤 🏧 🚾 ♿️ 🅿️ 🛗 🈳 🈂️ 🛂 🛃 🛄 🛅 🚹 🚺 🚼 ⚧ 🚻 🚮 🎦 📶 🈁 🔣 ℹ️ 🔤 🔡 🔠 🆖 🆗 🆙 🆒 🆕 🆓 0️⃣ 1️⃣ 2️⃣ 3️⃣ 4️⃣ 5️⃣ 6️⃣ 7️⃣ 8️⃣ 9️⃣ 🔟 🔢 #️⃣ *️⃣ ⏏️ ▶️ ⏸ ⏯ ⏹ ⏺ ⏭ ⏮ ⏩ ⏪ ⏫ ⏬ ◀️ 🔼 🔽 ➡️ ⬅️ ⬆️ ⬇️ ↗️ ↘️ ↙️ ↖️ ↕️ ↔️ ↪️ ↩️ ⤴️ ⤵️ 🔀 🔁 🔂 🔄 🔃 🎵 🎶 ➕ ➖ ➗ ✖️ ♾ 💲 💱 ™️ ©️ ®️ 〰️ ➰ ➿ 🔚 🔙 🔛 🔝 🔜 ✔️ ☑️ 🔘 🔴 🟠 🟡 🟢 🔵 🟣 ⚫️ ⚪️ 🟤 🔺 🔻 🔸 🔹 🔶 🔷 🔳 🔲 ▪️ ▫️ ◾️ ◽️ ◼️ ◻️ 🟥 🟧 🟨 🟩 🟦 🟪 ⬛️ ⬜️ 🟫 🔈 🔇 🔉 🔊 🔔 🔕 📣 📢 👁‍🗨 💬 💭 🗯 ♠️ ♣️ ♥️ ♦️ 🃏 🎴 🀄️ 🕐 🕑 🕒 🕓 🕔 🕕 🕖 🕗 🕘 🕙 🕚 🕛 🕜 🕝 🕞 🕟 🕠 🕡 🕢 🕣 🕤 🕥 🕦 🕧'.split( - ' ', - ) as ReadonlyArray, - - 'Non-Emoji Symbols': - '✢ ✣ ✤ ✥ ✦ ✧ ★ ☆ ✯ ✡︎ ✩ ✪ ✫ ✬ ✭ ✮ ✶ ✷ ✵ ✸ ✹ → ⇒ ⟹ ⇨ ⇾ ➾ ⇢ ☛ ☞ ➔ ➜ ➙ ➛ ➝ ➞ ♠︎ ♣︎ ♥︎ ♦︎ ♤ ♧ ♡ ♢ ♚ ♛ ♜ ♝ ♞ ♟ ♔ ♕ ♖ ♗ ♘ ♙ ⚀ ⚁ ⚂ ⚃ ⚄ ⚅ 🂠 ⚈ ⚉ ⚆ ⚇ 𓀀 𓀁 𓀂 𓀃 𓀄 𓀅 𓀆 𓀇 𓀈 𓀉 𓀊 𓀋 𓀌 𓀍 𓀎 𓀏 𓀐 𓀑 𓀒 𓀓 𓀔 𓀕 𓀖 𓀗 𓀘 𓀙 𓀚 𓀛 𓀜 𓀝'.split( - ' ', - ) as ReadonlyArray, - - Flags: '🏳️ 🏴 🏁 🚩 🏳️‍🌈 🏳️‍⚧️ 🏴‍☠️'.split(' ') as ReadonlyArray, - - Countries: - '🇦🇫 🇦🇽 🇦🇱 🇩🇿 🇦🇸 🇦🇩 🇦🇴 🇦🇮 🇦🇶 🇦🇬 🇦🇷 🇦🇲 🇦🇼 🇦🇺 🇦🇹 🇦🇿 🇧🇸 🇧🇭 🇧🇩 🇧🇧 🇧🇾 🇧🇪 🇧🇿 🇧🇯 🇧🇲 🇧🇹 🇧🇴 🇧🇦 🇧🇼 🇧🇷 🇮🇴 🇻🇬 🇧🇳 🇧🇬 🇧🇫 🇧🇮 🇰🇭 🇨🇲 🇨🇦 🇮🇨 🇨🇻 🇧🇶 🇰🇾 🇨🇫 🇹🇩 🇨🇱 🇨🇳 🇨🇽 🇨🇨 🇨🇴 🇰🇲 🇨🇬 🇨🇩 🇨🇰 🇨🇷 🇨🇮 🇭🇷 🇨🇺 🇨🇼 🇨🇾 🇨🇿 🇩🇰 🇩🇯 🇩🇲 🇩🇴 🇪🇨 🇪🇬 🇸🇻 🇬🇶 🇪🇷 🇪🇪 🇪🇹 🇪🇺 🇫🇰 🇫🇴 🇫🇯 🇫🇮 🇫🇷 🇬🇫 🇵🇫 🇹🇫 🇬🇦 🇬🇲 🇬🇪 🇩🇪 🇬🇭 🇬🇮 🇬🇷 🇬🇱 🇬🇩 🇬🇵 🇬🇺 🇬🇹 🇬🇬 🇬🇳 🇬🇼 🇬🇾 🇭🇹 🇭🇳 🇭🇰 🇭🇺 🇮🇸 🇮🇳 🇮🇩 🇮🇷 🇮🇶 🇮🇪 🇮🇲 🇮🇱 🇮🇹 🇯🇲 🇯🇵 🎌 🇯🇪 🇯🇴 🇰🇿 🇰🇪 🇰🇮 🇽🇰 🇰🇼 🇰🇬 🇱🇦 🇱🇻 🇱🇧 🇱🇸 🇱🇷 🇱🇾 🇱🇮 🇱🇹 🇱🇺 🇲🇴 🇲🇰 🇲🇬 🇲🇼 🇲🇾 🇲🇻 🇲🇱 🇲🇹 🇲🇭 🇲🇶 🇲🇷 🇲🇺 🇾🇹 🇲🇽 🇫🇲 🇲🇩 🇲🇨 🇲🇳 🇲🇪 🇲🇸 🇲🇦 🇲🇿 🇲🇲 🇳🇦 🇳🇷 🇳🇵 🇳🇱 🇳🇨 🇳🇿 🇳🇮 🇳🇪 🇳🇬 🇳🇺 🇳🇫 🇰🇵 🇲🇵 🇳🇴 🇴🇲 🇵🇰 🇵🇼 🇵🇸 🇵🇦 🇵🇬 🇵🇾 🇵🇪 🇵🇭 🇵🇳 🇵🇱 🇵🇹 🇵🇷 🇶🇦 🇷🇪 🇷🇴 🇷🇺 🇷🇼 🇼🇸 🇸🇲 🇸🇦 🇸🇳 🇷🇸 🇸🇨 🇸🇱 🇸🇬 🇸🇽 🇸🇰 🇸🇮 🇬🇸 🇸🇧 🇸🇴 🇿🇦 🇰🇷 🇸🇸 🇪🇸 🇱🇰 🇧🇱 🇸🇭 🇰🇳 🇱🇨 🇵🇲 🇻🇨 🇸🇩 🇸🇷 🇸🇿 🇸🇪 🇨🇭 🇸🇾 🇹🇼 🇹🇯 🇹🇿 🇹🇭 🇹🇱 🇹🇬 🇹🇰 🇹🇴 🇹🇹 🇹🇳 🇹🇷 🇹🇲 🇹🇨 🇹🇻 🇻🇮 🇺🇬 🇺🇦 🇦🇪 🇬🇧 🇺🇳 🇺🇸 🇺🇾 🇺🇿 🇻🇺 🇻🇦 🇻🇪 🇻🇳 🇼🇫 🇪🇭 🇾🇪 🇿🇲 🇿🇼'.split( - ' ', - ) as ReadonlyArray, - - 'New Emojis 2020': - '🥲 🥸 🤌 🤌🏻 🤌🏼 🤌🏽 🤌🏾 🤌🏿 🫀 🫁 🥷 🤵‍♀️ 🤵🏻‍♀️ 🤵🏼‍♀️ 🤵🏽‍♀️ 🤵🏾‍♀️ 🤵🏿‍♀️ 🤵‍♂️ 🤵🏻‍♂️ 🤵🏼‍♂️ 🤵🏽‍♂️ 🤵🏾‍♂️ 🤵🏿‍♂️ 👰‍♀️ 👰🏻‍♀️ 👰🏼‍♀️ 👰🏽‍♀️ 👰🏾‍♀️ 👰🏿‍♀️ 👰‍♂️ 👰🏻‍♂️ 👰🏼‍♂️ 👰🏽‍♂️ 👰🏾‍♂️ 👰🏿‍♂️ 👩‍🍼 👩🏻‍🍼 👩🏼‍🍼 👩🏽‍🍼 👩🏾‍🍼 👩🏿‍🍼 🧑‍🍼 🧑🏻‍🍼 🧑🏼‍🍼 🧑🏽‍🍼 🧑🏾‍🍼 🧑🏿‍🍼 👨‍🍼 👨🏻‍🍼 👨🏼‍🍼 👨🏽‍🍼 👨🏾‍🍼 👨🏿‍🍼 🧑‍🎄 🧑🏻‍🎄 🧑🏼‍🎄 🧑🏽‍🎄 🧑🏾‍🎄 🧑🏿‍🎄 🫂 🐈‍⬛ 🦬 🦣 🦫 🐻‍❄️ 🦤 🪶 🦭 🪲 🪳 🪰 🪱 🪴 🫐 🫒 🫑 🫓 🫔 🫕 🫖 🧋 🪨 🪵 🛖 🛻 🛼 🪄 🪅 🪆 🪡 🪢 🩴 🪖 🪗 🪘 🪙 🪃 🪚 🪛 🪝 🪜 🛗 🪞 🪟 🪠 🪤 🪣 🪥 🪦 🪧 🏳️‍⚧️'.split( - ' ', - ) as ReadonlyArray, - - 'New Emoji 2021': - '😮‍💨 😵‍💫 😶‍🌫️ ❤️‍🔥 ❤️‍🩹 🧔‍♀️ 🧔🏻‍♀️ 🧔🏼‍♀️ 🧔🏽‍♀️ 🧔🏾‍♀️ 🧔🏿‍♀️ 🧔‍♂️ 🧔🏻‍♂️ 🧔🏼‍♂️ 🧔🏽‍♂️ 🧔🏾‍♂️ 🧔🏿‍♂️ 💑🏻 💑🏼 💑🏽 💑🏾 💑🏿 💏🏻 💏🏼 💏🏽 💏🏾 💏🏿 👨🏻‍❤️‍👨🏻 👨🏻‍❤️‍👨🏼 👨🏻‍❤️‍👨🏽 👨🏻‍❤️‍👨🏾 👨🏻‍❤️‍👨🏿 👨🏼‍❤️‍👨🏻 👨🏼‍❤️‍👨🏼 👨🏼‍❤️‍👨🏽 👨🏼‍❤️‍👨🏾 👨🏼‍❤️‍👨🏿 👨🏽‍❤️‍👨🏻 👨🏽‍❤️‍👨🏼 👨🏽‍❤️‍👨🏽 👨🏽‍❤️‍👨🏾 👨🏽‍❤️‍👨🏿 👨🏾‍❤️‍👨🏻 👨🏾‍❤️‍👨🏼 👨🏾‍❤️‍👨🏽 👨🏾‍❤️‍👨🏾 👨🏾‍❤️‍👨🏿 👨🏿‍❤️‍👨🏻 👨🏿‍❤️‍👨🏼 👨🏿‍❤️‍👨🏽 👨🏿‍❤️‍👨🏾 👨🏿‍❤️‍👨🏿 👩🏻‍❤️‍👨🏻 👩🏻‍❤️‍👨🏼 👩🏻‍❤️‍👨🏽 👩🏻‍❤️‍👨🏾 👩🏻‍❤️‍👨🏿 👩🏻‍❤️‍👩🏻 👩🏻‍❤️‍👩🏼 👩🏻‍❤️‍👩🏽 👩🏻‍❤️‍👩🏾 👩🏻‍❤️‍👩🏿 👩🏼‍❤️‍👨🏻 👩🏼‍❤️‍👨🏼 👩🏼‍❤️‍👨🏽 👩🏼‍❤️‍👨🏾 👩🏼‍❤️‍👨🏿 👩🏼‍❤️‍👩🏻 👩🏼‍❤️‍👩🏼 👩🏼‍❤️‍👩🏽 👩🏼‍❤️‍👩🏾 👩🏼‍❤️‍👩🏿 👩🏽‍❤️‍👨🏻 👩🏽‍❤️‍👨🏼 👩🏽‍❤️‍👨🏽 👩🏽‍❤️‍👨🏾 👩🏽‍❤️‍👨🏿 👩🏽‍❤️‍👩🏻 👩🏽‍❤️‍👩🏼 👩🏽‍❤️‍👩🏽 👩🏽‍❤️‍👩🏾 👩🏽‍❤️‍👩🏿 👩🏾‍❤️‍👨🏻 👩🏾‍❤️‍👨🏼 👩🏾‍❤️‍👨🏽 👩🏾‍❤️‍👨🏾 👩🏾‍❤️‍👨🏿 👩🏾‍❤️‍👩🏻 👩🏾‍❤️‍👩🏼 👩🏾‍❤️‍👩🏽 👩🏾‍❤️‍👩🏾 👩🏾‍❤️‍👩🏿 👩🏿‍❤️‍👨🏻 👩🏿‍❤️‍👨🏼 👩🏿‍❤️‍👨🏽 👩🏿‍❤️‍👨🏾 👩🏿‍❤️‍👨🏿 👩🏿‍❤️‍👩🏻 👩🏿‍❤️‍👩🏼 👩🏿‍❤️‍👩🏽 👩🏿‍❤️‍👩🏾 👩🏿‍❤️‍👩🏿 🧑🏻‍❤️‍🧑🏼 🧑🏻‍❤️‍🧑🏽 🧑🏻‍❤️‍🧑🏾 🧑🏻‍❤️‍🧑🏿 🧑🏼‍❤️‍🧑🏻 🧑🏼‍❤️‍🧑🏽 🧑🏼‍❤️‍🧑🏾 🧑🏼‍❤️‍🧑🏿 🧑🏽‍❤️‍🧑🏻 🧑🏽‍❤️‍🧑🏼 🧑🏽‍❤️‍🧑🏾 🧑🏽‍❤️‍🧑🏿 🧑🏾‍❤️‍🧑🏻 🧑🏾‍❤️‍🧑🏼 🧑🏾‍❤️‍🧑🏽 🧑🏾‍❤️‍🧑🏿 🧑🏿‍❤️‍🧑🏻 🧑🏿‍❤️‍🧑🏼 🧑🏿‍❤️‍🧑🏽 🧑🏿‍❤️‍🧑🏾 👨🏻‍❤️‍💋‍👨🏻 👨🏻‍❤️‍💋‍👨🏼 👨🏻‍❤️‍💋‍👨🏽 👨🏻‍❤️‍💋‍👨🏾 👨🏻‍❤️‍💋‍👨🏿 👨🏼‍❤️‍💋‍👨🏻 👨🏼‍❤️‍💋‍👨🏼 👨🏼‍❤️‍💋‍👨🏽 👨🏼‍❤️‍💋‍👨🏾 👨🏼‍❤️‍💋‍👨🏿 👨🏽‍❤️‍💋‍👨🏻 👨🏽‍❤️‍💋‍👨🏼 👨🏽‍❤️‍💋‍👨🏽 👨🏽‍❤️‍💋‍👨🏾 👨🏽‍❤️‍💋‍👨🏿 👨🏾‍❤️‍💋‍👨🏻 👨🏾‍❤️‍💋‍👨🏼 👨🏾‍❤️‍💋‍👨🏽 👨🏾‍❤️‍💋‍👨🏾 👨🏾‍❤️‍💋‍👨🏿 👨🏿‍❤️‍💋‍👨🏻 👨🏿‍❤️‍💋‍👨🏼 👨🏿‍❤️‍💋‍👨🏽 👨🏿‍❤️‍💋‍👨🏾 👨🏿‍❤️‍💋‍👨🏿 👩🏻‍❤️‍💋‍👨🏻 👩🏻‍❤️‍💋‍👨🏼 👩🏻‍❤️‍💋‍👨🏽 👩🏻‍❤️‍💋‍👨🏾 👩🏻‍❤️‍💋‍👨🏿 👩🏻‍❤️‍💋‍👩🏻 👩🏻‍❤️‍💋‍👩🏼 👩🏻‍❤️‍💋‍👩🏽 👩🏻‍❤️‍💋‍👩🏾 👩🏻‍❤️‍💋‍👩🏿 👩🏼‍❤️‍💋‍👨🏻 👩🏼‍❤️‍💋‍👨🏼 👩🏼‍❤️‍💋‍👨🏽 👩🏼‍❤️‍💋‍👨🏾 👩🏼‍❤️‍💋‍👨🏿 👩🏼‍❤️‍💋‍👩🏻 👩🏼‍❤️‍💋‍👩🏼 👩🏼‍❤️‍💋‍👩🏽 👩🏼‍❤️‍💋‍👩🏾 👩🏼‍❤️‍💋‍👩🏿 👩🏽‍❤️‍💋‍👨🏻 👩🏽‍❤️‍💋‍👨🏼 👩🏽‍❤️‍💋‍👨🏽 👩🏽‍❤️‍💋‍👨🏾 👩🏽‍❤️‍💋‍👨🏿 👩🏽‍❤️‍💋‍👩🏻 👩🏽‍❤️‍💋‍👩🏼 👩🏽‍❤️‍💋‍👩🏽 👩🏽‍❤️‍💋‍👩🏾 👩🏽‍❤️‍💋‍👩🏿 👩🏾‍❤️‍💋‍👨🏻 👩🏾‍❤️‍💋‍👨🏼 👩🏾‍❤️‍💋‍👨🏽 👩🏾‍❤️‍💋‍👨🏾 👩🏾‍❤️‍💋‍👨🏿 👩🏾‍❤️‍💋‍👩🏻 👩🏾‍❤️‍💋‍👩🏼 👩🏾‍❤️‍💋‍👩🏽 👩🏾‍❤️‍💋‍👩🏾 👩🏾‍❤️‍💋‍👩🏿 👩🏿‍❤️‍💋‍👨🏻 👩🏿‍❤️‍💋‍👨🏼 👩🏿‍❤️‍💋‍👨🏽 👩🏿‍❤️‍💋‍👨🏾 👩🏿‍❤️‍💋‍👨🏿 👩🏿‍❤️‍💋‍👩🏻 👩🏿‍❤️‍💋‍👩🏼 👩🏿‍❤️‍💋‍👩🏽 👩🏿‍❤️‍💋‍👩🏾 👩🏿‍❤️‍💋‍👩🏿 🧑🏻‍❤️‍💋‍🧑🏼 🧑🏻‍❤️‍💋‍🧑🏽 🧑🏻‍❤️‍💋‍🧑🏾 🧑🏻‍❤️‍💋‍🧑🏿 🧑🏼‍❤️‍💋‍🧑🏻 🧑🏼‍❤️‍💋‍🧑🏽 🧑🏼‍❤️‍💋‍🧑🏾 🧑🏼‍❤️‍💋‍🧑🏿 🧑🏽‍❤️‍💋‍🧑🏻 🧑🏽‍❤️‍💋‍🧑🏼 🧑🏽‍❤️‍💋‍🧑🏾 🧑🏽‍❤️‍💋‍🧑🏿 🧑🏾‍❤️‍💋‍🧑🏻 🧑🏾‍❤️‍💋‍🧑🏼 🧑🏾‍❤️‍💋‍🧑🏽 🧑🏾‍❤️‍💋‍🧑🏿 🧑🏿‍❤️‍💋‍🧑🏻 🧑🏿‍❤️‍💋‍🧑🏼 🧑🏿‍❤️‍💋‍🧑🏽 🧑🏿‍❤️‍💋‍🧑🏾'.split( - ' ', - ) as ReadonlyArray, - - 'New Emojis (Discorered randomly)': '🖌️'.split(' ') as ReadonlyArray, - - 'Emojis used in WebGPT': ['▶', '❤'] as TODO_any as ReadonlyArray, +export const EMOJIS_IN_CATEGORIES: Record< + string, + ReadonlyArray +> = { + Smileys: + "😀 😃 😄 😁 😆 😅 😂 🤣 🥲 ☺️ 😊 😇 🙂 🙃 😉 😌 😍 🥰 😘 😗 😙 😚 😋 😛 😝 😜 🤪 🤨 🧐 🤓 😎 🥸 🤩 🥳 😏 😒 😞 😔 😟 😕 🙁 ☹️ 😣 😖 😫 😩 🥺 😢 😭 😤 😠 😡 🤬 🤯 😳 🥵 🥶 😱 😨 😰 😥 😓 🤗 🤔 🤭 🤫 🤥 😶 😐 😑 😬 🙄 😯 😦 😧 😮 😲 🥱 😴 🤤 😪 😵 🤐 🥴 🤢 🤮 🤧 😷 🤒 🤕 🤑 🤠 😈 👿 👹 👺 🤡 💩 👻 💀 ☠️ 👽 👾 🤖 🎃 😺 😸 😹 😻 😼 😽 🙀 😿 😾".split( + " ", + ) as ReadonlyArray, + + "Gestures and Body Parts": + "👋 🤚 🖐 ✋ 🖖 👌 🤌 🤏 ✌️ 🤞 🤟 🤘 🤙 👈 👉 👆 🖕 👇 ☝️ 👍 👎 ✊ 👊 🤛 🤜 👏 🙌 👐 🤲 🤝 🙏 ✍️ 💅 🤳 💪 🦾 🦵 🦿 🦶 👣 👂 🦻 👃 🫀 🫁 🧠 🦷 🦴 👀 👁 👅 👄 💋 🩸".split( + " ", + ) as ReadonlyArray, + + "People and Fantasy": + "👶 👧 🧒 👦 👩 🧑 👨 👩‍🦱 🧑‍🦱 👨‍🦱 👩‍🦰 🧑‍🦰 👨‍🦰 👱‍♀️ 👱 👱‍♂️ 👩‍🦳 🧑‍🦳 👨‍🦳 👩‍🦲 🧑‍🦲 👨‍🦲 🧔 👵 🧓 👴 👲 👳‍♀️ 👳 👳‍♂️ 🧕 👮‍♀️ 👮 👮‍♂️ 👷‍♀️ 👷 👷‍♂️ 💂‍♀️ 💂 💂‍♂️ 🕵️‍♀️ 🕵️ 🕵️‍♂️ 👩‍⚕️ 🧑‍⚕️ 👨‍⚕️ 👩‍🌾 🧑‍🌾 👨‍🌾 👩‍🍳 🧑‍🍳 👨‍🍳 👩‍🎓 🧑‍🎓 👨‍🎓 👩‍🎤 🧑‍🎤 👨‍🎤 👩‍🏫 🧑‍🏫 👨‍🏫 👩‍🏭 🧑‍🏭 👨‍🏭 👩‍💻 🧑‍💻 👨‍💻 👩‍💼 🧑‍💼 👨‍💼 👩‍🔧 🧑‍🔧 👨‍🔧 👩‍🔬 🧑‍🔬 👨‍🔬 👩‍🎨 🧑‍🎨 👨‍🎨 👩‍🚒 🧑‍🚒 👨‍🚒 👩‍✈️ 🧑‍✈️ 👨‍✈️ 👩‍🚀 🧑‍🚀 👨‍🚀 👩‍⚖️ 🧑‍⚖️ 👨‍⚖️ 👰‍♀️ 👰 👰‍♂️ 🤵‍♀️ 🤵 🤵‍♂️ 👸 🤴 🥷 🦸‍♀️ 🦸 🦸‍♂️ 🦹‍♀️ 🦹 🦹‍♂️ 🤶 🧑‍🎄 🎅 🧙‍♀️ 🧙 🧙‍♂️ 🧝‍♀️ 🧝 🧝‍♂️ 🧛‍♀️ 🧛 🧛‍♂️ 🧟‍♀️ 🧟 🧟‍♂️ 🧞‍♀️ 🧞 🧞‍♂️ 🧜‍♀️ 🧜 🧜‍♂️ 🧚‍♀️ 🧚 🧚‍♂️ 👼 🤰 🤱 👩‍🍼 🧑‍🍼 👨‍🍼 🙇‍♀️ 🙇 🙇‍♂️ 💁‍♀️ 💁 💁‍♂️ 🙅‍♀️ 🙅 🙅‍♂️ 🙆‍♀️ 🙆 🙆‍♂️ 🙋‍♀️ 🙋 🙋‍♂️ 🧏‍♀️ 🧏 🧏‍♂️ 🤦‍♀️ 🤦 🤦‍♂️ 🤷‍♀️ 🤷 🤷‍♂️ 🙎‍♀️ 🙎 🙎‍♂️ 🙍‍♀️ 🙍 🙍‍♂️ 💇‍♀️ 💇 💇‍♂️ 💆‍♀️ 💆 💆‍♂️ 🧖‍♀️ 🧖 🧖‍♂️ 💅 🤳 💃 🕺 👯‍♀️ 👯 👯‍♂️ 🕴 👩‍🦽 🧑‍🦽 👨‍🦽 👩‍🦼 🧑‍🦼 👨‍🦼 🚶‍♀️ 🚶 🚶‍♂️ 👩‍🦯 🧑‍🦯 👨‍🦯 🧎‍♀️ 🧎 🧎‍♂️ 🏃‍♀️ 🏃 🏃‍♂️ 🧍‍♀️ 🧍 🧍‍♂️ 👭 🧑‍🤝‍🧑 👬 👫 👩‍❤️‍👩 💑 👨‍❤️‍👨 👩‍❤️‍👨 👩‍❤️‍💋‍👩 💏 👨‍❤️‍💋‍👨 👩‍❤️‍💋‍👨 👪 👨‍👩‍👦 👨‍👩‍👧 👨‍👩‍👧‍👦 👨‍👩‍👦‍👦 👨‍👩‍👧‍👧 👨‍👨‍👦 👨‍👨‍👧 👨‍👨‍👧‍👦 👨‍👨‍👦‍👦 👨‍👨‍👧‍👧 👩‍👩‍👦 👩‍👩‍👧 👩‍👩‍👧‍👦 👩‍👩‍👦‍👦 👩‍👩‍👧‍👧 👨‍👦 👨‍👦‍👦 👨‍👧 👨‍👧‍👦 👨‍👧‍👧 👩‍👦 👩‍👦‍👦 👩‍👧 👩‍👧‍👦 👩‍👧‍👧 🗣 👤 👥 🫂".split( + " ", + ) as ReadonlyArray, + + "Clothing and Accessories": + "🧳 🌂 ☂️ 🧵 🪡 🪢 🧶 👓 🕶 🥽 🥼 🦺 👔 👕 👖 🧣 🧤 🧥 🧦 👗 👘 🥻 🩴 🩱 🩲 🩳 👙 👚 👛 👜 👝 🎒 👞 👟 🥾 🥿 👠 👡 🩰 👢 👑 👒 🎩 🎓 🧢 ⛑ 🪖 💄 💍 💼".split( + " ", + ) as ReadonlyArray, + + "Pale Emojis": + "👋🏻 🤚🏻 🖐🏻 ✋🏻 🖖🏻 👌🏻 🤌🏻 🤏🏻 ✌🏻 🤞🏻 🤟🏻 🤘🏻 🤙🏻 👈🏻 👉🏻 👆🏻 🖕🏻 👇🏻 ☝🏻 👍🏻 👎🏻 ✊🏻 👊🏻 🤛🏻 🤜🏻 👏🏻 🙌🏻 👐🏻 🤲🏻 🙏🏻 ✍🏻 💅🏻 🤳🏻 💪🏻 🦵🏻 🦶🏻 👂🏻 🦻🏻 👃🏻 👶🏻 👧🏻 🧒🏻 👦🏻 👩🏻 🧑🏻 👨🏻 👩🏻‍🦱 🧑🏻‍🦱 👨🏻‍🦱 👩🏻‍🦰 🧑🏻‍🦰 👨🏻‍🦰 👱🏻‍♀️ 👱🏻 👱🏻‍♂️ 👩🏻‍🦳 🧑🏻‍🦳 👨🏻‍🦳 👩🏻‍🦲 🧑🏻‍🦲 👨🏻‍🦲 🧔🏻 👵🏻 🧓🏻 👴🏻 👲🏻 👳🏻‍♀️ 👳🏻 👳🏻‍♂️ 🧕🏻 👮🏻‍♀️ 👮🏻 👮🏻‍♂️ 👷🏻‍♀️ 👷🏻 👷🏻‍♂️ 💂🏻‍♀️ 💂🏻 💂🏻‍♂️ 🕵🏻‍♀️ 🕵🏻 🕵🏻‍♂️ 👩🏻‍⚕️ 🧑🏻‍⚕️ 👨🏻‍⚕️ 👩🏻‍🌾 🧑🏻‍🌾 👨🏻‍🌾 👩🏻‍🍳 🧑🏻‍🍳 👨🏻‍🍳 👩🏻‍🎓 🧑🏻‍🎓 👨🏻‍🎓 👩🏻‍🎤 🧑🏻‍🎤 👨🏻‍🎤 👩🏻‍🏫 🧑🏻‍🏫 👨🏻‍🏫 👩🏻‍🏭 🧑🏻‍🏭 👨🏻‍🏭 👩🏻‍💻 🧑🏻‍💻 👨🏻‍💻 👩🏻‍💼 🧑🏻‍💼 👨🏻‍💼 👩🏻‍🔧 🧑🏻‍🔧 👨🏻‍🔧 👩🏻‍🔬 🧑🏻‍🔬 👨🏻‍🔬 👩🏻‍🎨 🧑🏻‍🎨 👨🏻‍🎨 👩🏻‍🚒 🧑🏻‍🚒 👨🏻‍🚒 👩🏻‍✈️ 🧑🏻‍✈️ 👨🏻‍✈️ 👩🏻‍🚀 🧑🏻‍🚀 👨🏻‍🚀 👩🏻‍⚖️ 🧑🏻‍⚖️ 👨🏻‍⚖️ 👰🏻‍♀️ 👰🏻 👰🏻‍♂️ 🤵🏻‍♀️ 🤵🏻 🤵🏻‍♂️ 👸🏻 🤴🏻 🥷🏻 🦸🏻‍♀️ 🦸🏻 🦸🏻‍♂️ 🦹🏻‍♀️ 🦹🏻 🦹🏻‍♂️ 🤶🏻 🧑🏻‍🎄 🎅🏻 🧙🏻‍♀️ 🧙🏻 🧙🏻‍♂️ 🧝🏻‍♀️ 🧝🏻 🧝🏻‍♂️ 🧛🏻‍♀️ 🧛🏻 🧛🏻‍♂️ 🧜🏻‍♀️ 🧜🏻 🧜🏻‍♂️ 🧚🏻‍♀️ 🧚🏻 🧚🏻‍♂️ 👼🏻 🤰🏻 🤱🏻 👩🏻‍🍼 🧑🏻‍🍼 👨🏻‍🍼 🙇🏻‍♀️ 🙇🏻 🙇🏻‍♂️ 💁🏻‍♀️ 💁🏻 💁🏻‍♂️ 🙅🏻‍♀️ 🙅🏻 🙅🏻‍♂️ 🙆🏻‍♀️ 🙆🏻 🙆🏻‍♂️ 🙋🏻‍♀️ 🙋🏻 🙋🏻‍♂️ 🧏🏻‍♀️ 🧏🏻 🧏🏻‍♂️ 🤦🏻‍♀️ 🤦🏻 🤦🏻‍♂️ 🤷🏻‍♀️ 🤷🏻 🤷🏻‍♂️ 🙎🏻‍♀️ 🙎🏻 🙎🏻‍♂️ 🙍🏻‍♀️ 🙍🏻 🙍🏻‍♂️ 💇🏻‍♀️ 💇🏻 💇🏻‍♂️ 💆🏻‍♀️ 💆🏻 💆🏻‍♂️ 🧖🏻‍♀️ 🧖🏻 🧖🏻‍♂️ 💃🏻 🕺🏻 🕴🏻 👩🏻‍🦽 🧑🏻‍🦽 👨🏻‍🦽 👩🏻‍🦼 🧑🏻‍🦼 👨🏻‍🦼 🚶🏻‍♀️ 🚶🏻 🚶🏻‍♂️ 👩🏻‍🦯 🧑🏻‍🦯 👨🏻‍🦯 🧎🏻‍♀️ 🧎🏻 🧎🏻‍♂️ 🏃🏻‍♀️ 🏃🏻 🏃🏻‍♂️ 🧍🏻‍♀️ 🧍🏻 🧍🏻‍♂️ 👭🏻 🧑🏻‍🤝‍🧑🏻 👬🏻 👫🏻 🧗🏻‍♀️ 🧗🏻 🧗🏻‍♂️ 🏇🏻 🏂🏻 🏌🏻‍♀️ 🏌🏻 🏌🏻‍♂️ 🏄🏻‍♀️ 🏄🏻 🏄🏻‍♂️ 🚣🏻‍♀️ 🚣🏻 🚣🏻‍♂️ 🏊🏻‍♀️ 🏊🏻 🏊🏻‍♂️ ⛹🏻‍♀️ ⛹🏻 ⛹🏻‍♂️ 🏋🏻‍♀️ 🏋🏻 🏋🏻‍♂️ 🚴🏻‍♀️ 🚴🏻 🚴🏻‍♂️ 🚵🏻‍♀️ 🚵🏻 🚵🏻‍♂️ 🤸🏻‍♀️ 🤸🏻 🤸🏻‍♂️ 🤽🏻‍♀️ 🤽🏻 🤽🏻‍♂️ 🤾🏻‍♀️ 🤾🏻 🤾🏻‍♂️ 🤹🏻‍♀️ 🤹🏻 🤹🏻‍♂️ 🧘🏻‍♀️ 🧘🏻 🧘🏻‍♂️ 🛀🏻 🛌".split( + " ", + ) as ReadonlyArray, + + "Cream White Emojis": + "👋🏼 🤚🏼 🖐🏼 ✋🏼 🖖🏼 👌🏼 🤌🏼 🤏🏼 ✌🏼 🤞🏼 🤟🏼 🤘🏼 🤙🏼 👈🏼 👉🏼 👆🏼 🖕🏼 👇🏼 ☝🏼 👍🏼 👎🏼 ✊🏼 👊🏼 🤛🏼 🤜🏼 👏🏼 🙌🏼 👐🏼 🤲🏼 🙏🏼 ✍🏼 💅🏼 🤳🏼 💪🏼 🦵🏼 🦶🏼 👂🏼 🦻🏼 👃🏼 👶🏼 👧🏼 🧒🏼 👦🏼 👩🏼 🧑🏼 👨🏼 👩🏼‍🦱 🧑🏼‍🦱 👨🏼‍🦱 👩🏼‍🦰 🧑🏼‍🦰 👨🏼‍🦰 👱🏼‍♀️ 👱🏼 👱🏼‍♂️ 👩🏼‍🦳 🧑🏼‍🦳 👨🏼‍🦳 👩🏼‍🦲 🧑🏼‍🦲 👨🏼‍🦲 🧔🏼 👵🏼 🧓🏼 👴🏼 👲🏼 👳🏼‍♀️ 👳🏼 👳🏼‍♂️ 🧕🏼 👮🏼‍♀️ 👮🏼 👮🏼‍♂️ 👷🏼‍♀️ 👷🏼 👷🏼‍♂️ 💂🏼‍♀️ 💂🏼 💂🏼‍♂️ 🕵🏼‍♀️ 🕵🏼 🕵🏼‍♂️ 👩🏼‍⚕️ 🧑🏼‍⚕️ 👨🏼‍⚕️ 👩🏼‍🌾 🧑🏼‍🌾 👨🏼‍🌾 👩🏼‍🍳 🧑🏼‍🍳 👨🏼‍🍳 👩🏼‍🎓 🧑🏼‍🎓 👨🏼‍🎓 👩🏼‍🎤 🧑🏼‍🎤 👨🏼‍🎤 👩🏼‍🏫 🧑🏼‍🏫 👨🏼‍🏫 👩🏼‍🏭 🧑🏼‍🏭 👨🏼‍🏭 👩🏼‍💻 🧑🏼‍💻 👨🏼‍💻 👩🏼‍💼 🧑🏼‍💼 👨🏼‍💼 👩🏼‍🔧 🧑🏼‍🔧 👨🏼‍🔧 👩🏼‍🔬 🧑🏼‍🔬 👨🏼‍🔬 👩🏼‍🎨 🧑🏼‍🎨 👨🏼‍🎨 👩🏼‍🚒 🧑🏼‍🚒 👨🏼‍🚒 👩🏼‍✈️ 🧑🏼‍✈️ 👨🏼‍✈️ 👩🏼‍🚀 🧑🏼‍🚀 👨🏼‍🚀 👩🏼‍⚖️ 🧑🏼‍⚖️ 👨🏼‍⚖️ 👰🏼‍♀️ 👰🏼 👰🏼‍♂️ 🤵🏼‍♀️ 🤵🏼 🤵🏼‍♂️ 👸🏼 🤴🏼 🥷🏼 🦸🏼‍♀️ 🦸🏼 🦸🏼‍♂️ 🦹🏼‍♀️ 🦹🏼 🦹🏼‍♂️ 🤶🏼 🧑🏼‍🎄 🎅🏼 🧙🏼‍♀️ 🧙🏼 🧙🏼‍♂️ 🧝🏼‍♀️ 🧝🏼 🧝🏼‍♂️ 🧛🏼‍♀️ 🧛🏼 🧛🏼‍♂️ 🧜🏼‍♀️ 🧜🏼 🧜🏼‍♂️ 🧚🏼‍♀️ 🧚🏼 🧚🏼‍♂️ 👼🏼 🤰🏼 🤱🏼 👩🏼‍🍼 🧑🏼‍🍼 👨🏼‍🍼 🙇🏼‍♀️ 🙇🏼 🙇🏼‍♂️ 💁🏼‍♀️ 💁🏼 💁🏼‍♂️ 🙅🏼‍♀️ 🙅🏼 🙅🏼‍♂️ 🙆🏼‍♀️ 🙆🏼 🙆🏼‍♂️ 🙋🏼‍♀️ 🙋🏼 🙋🏼‍♂️ 🧏🏼‍♀️ 🧏🏼 🧏🏼‍♂️ 🤦🏼‍♀️ 🤦🏼 🤦🏼‍♂️ 🤷🏼‍♀️ 🤷🏼 🤷🏼‍♂️ 🙎🏼‍♀️ 🙎🏼 🙎🏼‍♂️ 🙍🏼‍♀️ 🙍🏼 🙍🏼‍♂️ 💇🏼‍♀️ 💇🏼 💇🏼‍♂️ 💆🏼‍♀️ 💆🏼 💆🏼‍♂️ 🧖🏼‍♀️ 🧖🏼 🧖🏼‍♂️ 💃🏼 🕺🏼 🕴🏼 👩🏼‍🦽 🧑🏼‍🦽 👨🏼‍🦽 👩🏼‍🦼 🧑🏼‍🦼 👨🏼‍🦼 🚶🏼‍♀️ 🚶🏼 🚶🏼‍♂️ 👩🏼‍🦯 🧑🏼‍🦯 👨🏼‍🦯 🧎🏼‍♀️ 🧎🏼 🧎🏼‍♂️ 🏃🏼‍♀️ 🏃🏼 🏃🏼‍♂️ 🧍🏼‍♀️ 🧍🏼 🧍🏼‍♂️ 👭🏼 🧑🏼‍🤝‍🧑🏼 👬🏼 👫🏼 🧗🏼‍♀️ 🧗🏼 🧗🏼‍♂️ 🏇🏼 🏂🏼 🏌🏼‍♀️ 🏌🏼 🏌🏼‍♂️ 🏄🏼‍♀️ 🏄🏼 🏄🏼‍♂️ 🚣🏼‍♀️ 🚣🏼 🚣🏼‍♂️ 🏊🏼‍♀️ 🏊🏼 🏊🏼‍♂️ ⛹🏼‍♀️ ⛹🏼 ⛹🏼‍♂️ 🏋🏼‍♀️ 🏋🏼 🏋🏼‍♂️ 🚴🏼‍♀️ 🚴🏼 🚴🏼‍♂️ 🚵🏼‍♀️ 🚵🏼 🚵🏼‍♂️ 🤸🏼‍♀️ 🤸🏼 🤸🏼‍♂️ 🤽🏼‍♀️ 🤽🏼 🤽🏼‍♂️ 🤾🏼‍♀️ 🤾🏼 🤾🏼‍♂️ 🤹🏼‍♀️ 🤹🏼 🤹🏼‍♂️ 🧘🏼‍♀️ 🧘🏼 🧘🏼‍♂️ 🛀🏼 🛌".split( + " ", + ) as ReadonlyArray, + + "Brown Emojis": + "👋🏽 🤚🏽 🖐🏽 ✋🏽 🖖🏽 👌🏽 🤌🏽 🤏🏽 ✌🏽 🤞🏽 🤟🏽 🤘🏽 🤙🏽 👈🏽 👉🏽 👆🏽 🖕🏽 👇🏽 ☝🏽 👍🏽 👎🏽 ✊🏽 👊🏽 🤛🏽 🤜🏽 👏🏽 🙌🏽 👐🏽 🤲🏽 🙏🏽 ✍🏽 💅🏽 🤳🏽 💪🏽 🦵🏽 🦶🏽 👂🏽 🦻🏽 👃🏽 👶🏽 👧🏽 🧒🏽 👦🏽 👩🏽 🧑🏽 👨🏽 👩🏽‍🦱 🧑🏽‍🦱 👨🏽‍🦱 👩🏽‍🦰 🧑🏽‍🦰 👨🏽‍🦰 👱🏽‍♀️ 👱🏽 👱🏽‍♂️ 👩🏽‍🦳 🧑🏽‍🦳 👨🏽‍🦳 👩🏽‍🦲 🧑🏽‍🦲 👨🏽‍🦲 🧔🏽 👵🏽 🧓🏽 👴🏽 👲🏽 👳🏽‍♀️ 👳🏽 👳🏽‍♂️ 🧕🏽 👮🏽‍♀️ 👮🏽 👮🏽‍♂️ 👷🏽‍♀️ 👷🏽 👷🏽‍♂️ 💂🏽‍♀️ 💂🏽 💂🏽‍♂️ 🕵🏽‍♀️ 🕵🏽 🕵🏽‍♂️ 👩🏽‍⚕️ 🧑🏽‍⚕️ 👨🏽‍⚕️ 👩🏽‍🌾 🧑🏽‍🌾 👨🏽‍🌾 👩🏽‍🍳 🧑🏽‍🍳 👨🏽‍🍳 👩🏽‍🎓 🧑🏽‍🎓 👨🏽‍🎓 👩🏽‍🎤 🧑🏽‍🎤 👨🏽‍🎤 👩🏽‍🏫 🧑🏽‍🏫 👨🏽‍🏫 👩🏽‍🏭 🧑🏽‍🏭 👨🏽‍🏭 👩🏽‍💻 🧑🏽‍💻 👨🏽‍💻 👩🏽‍💼 🧑🏽‍💼 👨🏽‍💼 👩🏽‍🔧 🧑🏽‍🔧 👨🏽‍🔧 👩🏽‍🔬 🧑🏽‍🔬 👨🏽‍🔬 👩🏽‍🎨 🧑🏽‍🎨 👨🏽‍🎨 👩🏽‍🚒 🧑🏽‍🚒 👨🏽‍🚒 👩🏽‍✈️ 🧑🏽‍✈️ 👨🏽‍✈️ 👩🏽‍🚀 🧑🏽‍🚀 👨🏽‍🚀 👩🏽‍⚖️ 🧑🏽‍⚖️ 👨🏽‍⚖️ 👰🏽‍♀️ 👰🏽 👰🏽‍♂️ 🤵🏽‍♀️ 🤵🏽 🤵🏽‍♂️ 👸🏽 🤴🏽 🥷🏽 🦸🏽‍♀️ 🦸🏽 🦸🏽‍♂️ 🦹🏽‍♀️ 🦹🏽 🦹🏽‍♂️ 🤶🏽 🧑🏽‍🎄 🎅🏽 🧙🏽‍♀️ 🧙🏽 🧙🏽‍♂️ 🧝🏽‍♀️ 🧝🏽 🧝🏽‍♂️ 🧛🏽‍♀️ 🧛🏽 🧛🏽‍♂️ 🧜🏽‍♀️ 🧜🏽 🧜🏽‍♂️ 🧚🏽‍♀️ 🧚🏽 🧚🏽‍♂️ 👼🏽 🤰🏽 🤱🏽 👩🏽‍🍼 🧑🏽‍🍼 👨🏽‍🍼 🙇🏽‍♀️ 🙇🏽 🙇🏽‍♂️ 💁🏽‍♀️ 💁🏽 💁🏽‍♂️ 🙅🏽‍♀️ 🙅🏽 🙅🏽‍♂️ 🙆🏽‍♀️ 🙆🏽 🙆🏽‍♂️ 🙋🏽‍♀️ 🙋🏽 🙋🏽‍♂️ 🧏🏽‍♀️ 🧏🏽 🧏🏽‍♂️ 🤦🏽‍♀️ 🤦🏽 🤦🏽‍♂️ 🤷🏽‍♀️ 🤷🏽 🤷🏽‍♂️ 🙎🏽‍♀️ 🙎🏽 🙎🏽‍♂️ 🙍🏽‍♀️ 🙍🏽 🙍🏽‍♂️ 💇🏽‍♀️ 💇🏽 💇🏽‍♂️ 💆🏽‍♀️ 💆🏽 💆🏽‍♂️ 🧖🏽‍♀️ 🧖🏽 🧖🏽‍♂️ 💃🏽 🕺🏽 🕴🏽 👩🏽‍🦽 🧑🏽‍🦽 👨🏽‍🦽 👩🏽‍🦼 🧑🏽‍🦼 👨🏽‍🦼 🚶🏽‍♀️ 🚶🏽 🚶🏽‍♂️ 👩🏽‍🦯 🧑🏽‍🦯 👨🏽‍🦯 🧎🏽‍♀️ 🧎🏽 🧎🏽‍♂️ 🏃🏽‍♀️ 🏃🏽 🏃🏽‍♂️ 🧍🏽‍♀️ 🧍🏽 🧍🏽‍♂️ 👭🏽 🧑🏽‍🤝‍🧑🏽 👬🏽 👫🏽 🧗🏽‍♀️ 🧗🏽 🧗🏽‍♂️ 🏇🏽 🏂🏽 🏌🏽‍♀️ 🏌🏽 🏌🏽‍♂️ 🏄🏽‍♀️ 🏄🏽 🏄🏽‍♂️ 🚣🏽‍♀️ 🚣🏽 🚣🏽‍♂️ 🏊🏽‍♀️ 🏊🏽 🏊🏽‍♂️ ⛹🏽‍♀️ ⛹🏽 ⛹🏽‍♂️ 🏋🏽‍♀️ 🏋🏽 🏋🏽‍♂️ 🚴🏽‍♀️ 🚴🏽 🚴🏽‍♂️ 🚵🏽‍♀️ 🚵🏽 🚵🏽‍♂️ 🤸🏽‍♀️ 🤸🏽 🤸🏽‍♂️ 🤽🏽‍♀️ 🤽🏽 🤽🏽‍♂️ 🤾🏽‍♀️ 🤾🏽 🤾🏽‍♂️ 🤹🏽‍♀️ 🤹🏽 🤹🏽‍♂️ 🧘🏽‍♀️ 🧘🏽 🧘🏽‍♂️ 🛀🏽 🛌".split( + " ", + ) as ReadonlyArray, + + "Dark Brown Emojis": + "👋🏾 🤚🏾 🖐🏾 ✋🏾 🖖🏾 👌🏾 🤌🏾 🤏🏾 ✌🏾 🤞🏾 🤟🏾 🤘🏾 🤙🏾 👈🏾 👉🏾 👆🏾 🖕🏾 👇🏾 ☝🏾 👍🏾 👎🏾 ✊🏾 👊🏾 🤛🏾 🤜🏾 👏🏾 🙌🏾 👐🏾 🤲🏾 🙏🏾 ✍🏾 💅🏾 🤳🏾 💪🏾 🦵🏾 🦶🏾 👂🏾 🦻🏾 👃🏾 👶🏾 👧🏾 🧒🏾 👦🏾 👩🏾 🧑🏾 👨🏾 👩🏾‍🦱 🧑🏾‍🦱 👨🏾‍🦱 👩🏾‍🦰 🧑🏾‍🦰 👨🏾‍🦰 👱🏾‍♀️ 👱🏾 👱🏾‍♂️ 👩🏾‍🦳 🧑🏾‍🦳 👨🏾‍🦳 👩🏾‍🦲 🧑🏾‍🦲 👨🏾‍🦲 🧔🏾 👵🏾 🧓🏾 👴🏾 👲🏾 👳🏾‍♀️ 👳🏾 👳🏾‍♂️ 🧕🏾 👮🏾‍♀️ 👮🏾 👮🏾‍♂️ 👷🏾‍♀️ 👷🏾 👷🏾‍♂️ 💂🏾‍♀️ 💂🏾 💂🏾‍♂️ 🕵🏾‍♀️ 🕵🏾 🕵🏾‍♂️ 👩🏾‍⚕️ 🧑🏾‍⚕️ 👨🏾‍⚕️ 👩🏾‍🌾 🧑🏾‍🌾 👨🏾‍🌾 👩🏾‍🍳 🧑🏾‍🍳 👨🏾‍🍳 👩🏾‍🎓 🧑🏾‍🎓 👨🏾‍🎓 👩🏾‍🎤 🧑🏾‍🎤 👨🏾‍🎤 👩🏾‍🏫 🧑🏾‍🏫 👨🏾‍🏫 👩🏾‍🏭 🧑🏾‍🏭 👨🏾‍🏭 👩🏾‍💻 🧑🏾‍💻 👨🏾‍💻 👩🏾‍💼 🧑🏾‍💼 👨🏾‍💼 👩🏾‍🔧 🧑🏾‍🔧 👨🏾‍🔧 👩🏾‍🔬 🧑🏾‍🔬 👨🏾‍🔬 👩🏾‍🎨 🧑🏾‍🎨 👨🏾‍🎨 👩🏾‍🚒 🧑🏾‍🚒 👨🏾‍🚒 👩🏾‍✈️ 🧑🏾‍✈️ 👨🏾‍✈️ 👩🏾‍🚀 🧑🏾‍🚀 👨🏾‍🚀 👩🏾‍⚖️ 🧑🏾‍⚖️ 👨🏾‍⚖️ 👰🏾‍♀️ 👰🏾 👰🏾‍♂️ 🤵🏾‍♀️ 🤵🏾 🤵🏾‍♂️ 👸🏾 🤴🏾 🥷🏾 🦸🏾‍♀️ 🦸🏾 🦸🏾‍♂️ 🦹🏾‍♀️ 🦹🏾 🦹🏾‍♂️ 🤶🏾 🧑🏾‍🎄 🎅🏾 🧙🏾‍♀️ 🧙🏾 🧙🏾‍♂️ 🧝🏾‍♀️ 🧝🏾 🧝🏾‍♂️ 🧛🏾‍♀️ 🧛🏾 🧛🏾‍♂️ 🧜🏾‍♀️ 🧜🏾 🧜🏾‍♂️ 🧚🏾‍♀️ 🧚🏾 🧚🏾‍♂️ 👼🏾 🤰🏾 🤱🏾 👩🏾‍🍼 🧑🏾‍🍼 👨🏾‍🍼 🙇🏾‍♀️ 🙇🏾 🙇🏾‍♂️ 💁🏾‍♀️ 💁🏾 💁🏾‍♂️ 🙅🏾‍♀️ 🙅🏾 🙅🏾‍♂️ 🙆🏾‍♀️ 🙆🏾 🙆🏾‍♂️ 🙋🏾‍♀️ 🙋🏾 🙋🏾‍♂️ 🧏🏾‍♀️ 🧏🏾 🧏🏾‍♂️ 🤦🏾‍♀️ 🤦🏾 🤦🏾‍♂️ 🤷🏾‍♀️ 🤷🏾 🤷🏾‍♂️ 🙎🏾‍♀️ 🙎🏾 🙎🏾‍♂️ 🙍🏾‍♀️ 🙍🏾 🙍🏾‍♂️ 💇🏾‍♀️ 💇🏾 💇🏾‍♂️ 💆🏾‍♀️ 💆🏾 💆🏾‍♂️ 🧖🏾‍♀️ 🧖🏾 🧖🏾‍♂️ 💃🏾 🕺🏾 🕴🏿 👩🏾‍🦽 🧑🏾‍🦽 👨🏾‍🦽 👩🏾‍🦼 🧑🏾‍🦼 👨🏾‍🦼 🚶🏾‍♀️ 🚶🏾 🚶🏾‍♂️ 👩🏾‍🦯 🧑🏾‍🦯 👨🏾‍🦯 🧎🏾‍♀️ 🧎🏾 🧎🏾‍♂️ 🏃🏾‍♀️ 🏃🏾 🏃🏾‍♂️ 🧍🏾‍♀️ 🧍🏾 🧍🏾‍♂️ 👭🏾 🧑🏾‍🤝‍🧑🏾 👬🏾 👫🏾 🧗🏾‍♀️ 🧗🏾 🧗🏾‍♂️ 🏇🏾 🏂🏾 🏌🏾‍♀️ 🏌🏾 🏌🏾‍♂️ 🏄🏾‍♀️ 🏄🏾 🏄🏾‍♂️ 🚣🏾‍♀️ 🚣🏾 🚣🏾‍♂️ 🏊🏾‍♀️ 🏊🏾 🏊🏾‍♂️ ⛹🏾‍♀️ ⛹🏾 ⛹🏾‍♂️ 🏋🏾‍♀️ 🏋🏾 🏋🏾‍♂️ 🚴🏾‍♀️ 🚴🏾 🚴🏾‍♂️ 🚵🏾‍♀️ 🚵🏾 🚵🏾‍♂️ 🤸🏾‍♀️ 🤸🏾 🤸🏾‍♂️ 🤽🏾‍♀️ 🤽🏾 🤽🏾‍♂️ 🤾🏾‍♀️ 🤾🏾 🤾🏾‍♂️ 🤹🏾‍♀️ 🤹🏾 🤹🏾‍♂️ 🧘🏾‍♀️ 🧘🏾 🧘🏾‍♂️ 🛀🏾 🛌".split( + " ", + ) as ReadonlyArray, + + "Black Emojis": + "👋🏿 🤚🏿 🖐🏿 ✋🏿 🖖🏿 👌🏿 🤌🏿 🤏🏿 ✌🏿 🤞🏿 🤟🏿 🤘🏿 🤙🏿 👈🏿 👉🏿 👆🏿 🖕🏿 👇🏿 ☝🏿 👍🏿 👎🏿 ✊🏿 👊🏿 🤛🏿 🤜🏿 👏🏿 🙌🏿 👐🏿 🤲🏿 🙏🏿 ✍🏿 💅🏿 🤳🏿 💪🏿 🦵🏿 🦶🏿 👂🏿 🦻🏿 👃🏿 👶🏿 👧🏿 🧒🏿 👦🏿 👩🏿 🧑🏿 👨🏿 👩🏿‍🦱 🧑🏿‍🦱 👨🏿‍🦱 👩🏿‍🦰 🧑🏿‍🦰 👨🏿‍🦰 👱🏿‍♀️ 👱🏿 👱🏿‍♂️ 👩🏿‍🦳 🧑🏿‍🦳 👨🏿‍🦳 👩🏿‍🦲 🧑🏿‍🦲 👨🏿‍🦲 🧔🏿 👵🏿 🧓🏿 👴🏿 👲🏿 👳🏿‍♀️ 👳🏿 👳🏿‍♂️ 🧕🏿 👮🏿‍♀️ 👮🏿 👮🏿‍♂️ 👷🏿‍♀️ 👷🏿 👷🏿‍♂️ 💂🏿‍♀️ 💂🏿 💂🏿‍♂️ 🕵🏿‍♀️ 🕵🏿 🕵🏿‍♂️ 👩🏿‍⚕️ 🧑🏿‍⚕️ 👨🏿‍⚕️ 👩🏿‍🌾 🧑🏿‍🌾 👨🏿‍🌾 👩🏿‍🍳 🧑🏿‍🍳 👨🏿‍🍳 👩🏿‍🎓 🧑🏿‍🎓 👨🏿‍🎓 👩🏿‍🎤 🧑🏿‍🎤 👨🏿‍🎤 👩🏿‍🏫 🧑🏿‍🏫 👨🏿‍🏫 👩🏿‍🏭 🧑🏿‍🏭 👨🏿‍🏭 👩🏿‍💻 🧑🏿‍💻 👨🏿‍💻 👩🏿‍💼 🧑🏿‍💼 👨🏿‍💼 👩🏿‍🔧 🧑🏿‍🔧 👨🏿‍🔧 👩🏿‍🔬 🧑🏿‍🔬 👨🏿‍🔬 👩🏿‍🎨 🧑🏿‍🎨 👨🏿‍🎨 👩🏿‍🚒 🧑🏿‍🚒 👨🏿‍🚒 👩🏿‍✈️ 🧑🏿‍✈️ 👨🏿‍✈️ 👩🏿‍🚀 🧑🏿‍🚀 👨🏿‍🚀 👩🏿‍⚖️ 🧑🏿‍⚖️ 👨🏿‍⚖️ 👰🏿‍♀️ 👰🏿 👰🏿‍♂️ 🤵🏿‍♀️ 🤵🏿 🤵🏿‍♂️ 👸🏿 🤴🏿 🥷🏿 🦸🏿‍♀️ 🦸🏿 🦸🏿‍♂️ 🦹🏿‍♀️ 🦹🏿 🦹🏿‍♂️ 🤶🏿 🧑🏿‍🎄 🎅🏿 🧙🏿‍♀️ 🧙🏿 🧙🏿‍♂️ 🧝🏿‍♀️ 🧝🏿 🧝🏿‍♂️ 🧛🏿‍♀️ 🧛🏿 🧛🏿‍♂️ 🧜🏿‍♀️ 🧜🏿 🧜🏿‍♂️ 🧚🏿‍♀️ 🧚🏿 🧚🏿‍♂️ 👼🏿 🤰🏿 🤱🏿 👩🏿‍🍼 🧑🏿‍🍼 👨🏿‍🍼 🙇🏿‍♀️ 🙇🏿 🙇🏿‍♂️ 💁🏿‍♀️ 💁🏿 💁🏿‍♂️ 🙅🏿‍♀️ 🙅🏿 🙅🏿‍♂️ 🙆🏿‍♀️ 🙆🏿 🙆🏿‍♂️ 🙋🏿‍♀️ 🙋🏿 🙋🏿‍♂️ 🧏🏿‍♀️ 🧏🏿 🧏🏿‍♂️ 🤦🏿‍♀️ 🤦🏿 🤦🏿‍♂️ 🤷🏿‍♀️ 🤷🏿 🤷🏿‍♂️ 🙎🏿‍♀️ 🙎🏿 🙎🏿‍♂️ 🙍🏿‍♀️ 🙍🏿 🙍🏿‍♂️ 💇🏿‍♀️ 💇🏿 💇🏿‍♂️ 💆🏿‍♀️ 💆🏿 💆🏿‍♂️ 🧖🏿‍♀️ 🧖🏿 🧖🏿‍♂️ 💃🏿 🕺🏿 🕴🏿 👩🏿‍🦽 🧑🏿‍🦽 👨🏿‍🦽 👩🏿‍🦼 🧑🏿‍🦼 👨🏿‍🦼 🚶🏿‍♀️ 🚶🏿 🚶🏿‍♂️ 👩🏿‍🦯 🧑🏿‍🦯 👨🏿‍🦯 🧎🏿‍♀️ 🧎🏿 🧎🏿‍♂️ 🏃🏿‍♀️ 🏃🏿 🏃🏿‍♂️ 🧍🏿‍♀️ 🧍🏿 🧍🏿‍♂️ 👭🏿 🧑🏿‍🤝‍🧑🏿 👬🏿 👫🏿 🧗🏿‍♀️ 🧗🏿 🧗🏿‍♂️ 🏇🏿 🏂🏿 🏌🏿‍♀️ 🏌🏿 🏌🏿‍♂️ 🏄🏿‍♀️ 🏄🏿 🏄🏿‍♂️ 🚣🏿‍♀️ 🚣🏿 🚣🏿‍♂️ 🏊🏿‍♀️ 🏊🏿 🏊🏿‍♂️ ⛹🏿‍♀️ ⛹🏿 ⛹🏿‍♂️ 🏋🏿‍♀️ 🏋🏿 🏋🏿‍♂️ 🚴🏿‍♀️ 🚴🏿 🚴🏿‍♂️ 🚵🏿‍♀️ 🚵🏿 🚵🏿‍♂️ 🤸🏿‍♀️ 🤸🏿 🤸🏿‍♂️ 🤽🏿‍♀️ 🤽🏿 🤽🏿‍♂️ 🤾🏿‍♀️ 🤾🏿 🤾🏿‍♂️ 🤹🏿‍♀️ 🤹🏿 🤹🏿‍♂️ 🧘🏿‍♀️ 🧘🏿 🧘🏿‍♂️ 🛀🏿 🛌🏿".split( + " ", + ) as ReadonlyArray, + + "Animals & Nature": + "🐶 🐱 🐭 🐹 🐰 🦊 🐻 🐼 🐻‍❄️ 🐨 🐯 🦁 🐮 🐷 🐽 🐸 🐵 🙈 🙉 🙊 🐒 🐔 🐧 🐦 🐤 🐣 🐥 🦆 🦅 🦉 🦇 🐺 🐗 🐴 🦄 🐝 🪱 🐛 🦋 🐌 🐞 🐜 🪰 🪲 🪳 🦟 🦗 🕷 🕸 🦂 🐢 🐍 🦎 🦖 🦕 🐙 🦑 🦐 🦞 🦀 🐡 🐠 🐟 🐬 🐳 🐋 🦈 🐊 🐅 🐆 🦓 🦍 🦧 🦣 🐘 🦛 🦏 🐪 🐫 🦒 🦘 🦬 🐃 🐂 🐄 🐎 🐖 🐏 🐑 🦙 🐐 🦌 🐕 🐩 🦮 🐕‍🦺 🐈 🐈‍⬛ 🪶 🐓 🦃 🦤 🦚 🦜 🦢 🦩 🕊 🐇 🦝 🦨 🦡 🦫 🦦 🦥 🐁 🐀 🐿 🦔 🐾 🐉 🐲 🌵 🎄 🌲 🌳 🌴 🪵 🌱 🌿 ☘️ 🍀 🎍 🪴 🎋 🍃 🍂 🍁 🍄 🐚 🪨 🌾 💐 🌷 🌹 🥀 🌺 🌸 🌼 🌻 🌞 🌝 🌛 🌜 🌚 🌕 🌖 🌗 🌘 🌑 🌒 🌓 🌔 🌙 🌎 🌍 🌏 🪐 💫 ⭐️ 🌟 ✨ ⚡️ ☄️ 💥 🔥 🌪 🌈 ☀️ 🌤 ⛅️ 🌥 ☁️ 🌦 🌧 ⛈ 🌩 🌨 ❄️ ☃️ ⛄️ 🌬 💨 💧 💦 ☔️ ☂️ 🌊 🌫".split( + " ", + ) as ReadonlyArray, + + "Food & Drink": + "🍏 🍎 🍐 🍊 🍋 🍌 🍉 🍇 🍓 🫐 🍈 🍒 🍑 🥭 🍍 🥥 🥝 🍅 🍆 🥑 🥦 🥬 🥒 🌶 🫑 🌽 🥕 🫒 🧄 🧅 🥔 🍠 🥐 🥯 🍞 🥖 🥨 🧀 🥚 🍳 🧈 🥞 🧇 🥓 🥩 🍗 🍖 🦴 🌭 🍔 🍟 🍕 🫓 🥪 🥙 🧆 🌮 🌯 🫔 🥗 🥘 🫕 🥫 🍝 🍜 🍲 🍛 🍣 🍱 🥟 🦪 🍤 🍙 🍚 🍘 🍥 🥠 🥮 🍢 🍡 🍧 🍨 🍦 🥧 🧁 🍰 🎂 🍮 🍭 🍬 🍫 🍿 🍩 🍪 🌰 🥜 🍯 🥛 🍼 🫖 ☕️ 🍵 🧃 🥤 🧋 🍶 🍺 🍻 🥂 🍷 🥃 🍸 🍹 🧉 🍾 🧊 🥄 🍴 🍽 🥣 🥡 🥢 🧂".split( + " ", + ) as ReadonlyArray, + + "Activity and Sports": + "⚽️ 🏀 🏈 ⚾️ 🥎 🎾 🏐 🏉 🥏 🎱 🪀 🏓 🏸 🏒 🏑 🥍 🏏 🪃 🥅 ⛳️ 🪁 🏹 🎣 🤿 🥊 🥋 🎽 🛹 🛼 🛷 ⛸ 🥌 🎿 ⛷ 🏂 🪂 🏋️‍♀️ 🏋️ 🏋️‍♂️ 🤼‍♀️ 🤼 🤼‍♂️ 🤸‍♀️ 🤸 🤸‍♂️ ⛹️‍♀️ ⛹️ ⛹️‍♂️ 🤺 🤾‍♀️ 🤾 🤾‍♂️ 🏌️‍♀️ 🏌️ 🏌️‍♂️ 🏇 🧘‍♀️ 🧘 🧘‍♂️ 🏄‍♀️ 🏄 🏄‍♂️ 🏊‍♀️ 🏊 🏊‍♂️ 🤽‍♀️ 🤽 🤽‍♂️ 🚣‍♀️ 🚣 🚣‍♂️ 🧗‍♀️ 🧗 🧗‍♂️ 🚵‍♀️ 🚵 🚵‍♂️ 🚴‍♀️ 🚴 🚴‍♂️ 🏆 🥇 🥈 🥉 🏅 🎖 🏵 🎗 🎫 🎟 🎪 🤹 🤹‍♂️ 🤹‍♀️ 🎭 🩰 🎨 🎬 🎤 🎧 🎼 🎹 🥁 🪘 🎷 🎺 🪗 🎸 🪕 🎻 🎲 ♟ 🎯 🎳 🎮 🎰 🧩".split( + " ", + ) as ReadonlyArray, + + "Travel & Places": + "🚗 🚕 🚙 🚌 🚎 🏎 🚓 🚑 🚒 🚐 🛻 🚚 🚛 🚜 🦯 🦽 🦼 🛴 🚲 🛵 🏍 🛺 🚨 🚔 🚍 🚘 🚖 🚡 🚠 🚟 🚃 🚋 🚞 🚝 🚄 🚅 🚈 🚂 🚆 🚇 🚊 🚉 ✈️ 🛫 🛬 🛩 💺 🛰 🚀 🛸 🚁 🛶 ⛵️ 🚤 🛥 🛳 ⛴ 🚢 ⚓️ 🪝 ⛽️ 🚧 🚦 🚥 🚏 🗺 🗿 🗽 🗼 🏰 🏯 🏟 🎡 🎢 🎠 ⛲️ ⛱ 🏖 🏝 🏜 🌋 ⛰ 🏔 🗻 🏕 ⛺️ 🛖 🏠 🏡 🏘 🏚 🏗 🏭 🏢 🏬 🏣 🏤 🏥 🏦 🏨 🏪 🏫 🏩 💒 🏛 ⛪️ 🕌 🕍 🛕 🕋 ⛩ 🛤 🛣 🗾 🎑 🏞 🌅 🌄 🌠 🎇 🎆 🌇 🌆 🏙 🌃 🌌 🌉 🌁".split( + " ", + ) as ReadonlyArray, + + Objects: + "⌚️ 📱 📲 💻 ⌨️ 🖥 🖨 🖱 🖲 🕹 🗜 💽 💾 💿 📀 📼 📷 📸 📹 🎥 📽 🎞 📞 ☎️ 📟 📠 📺 📻 🎙 🎚 🎛 🧭 ⏱ ⏲ ⏰ 🕰 ⌛️ ⏳ 📡 🔋 🔌 💡 🔦 🕯 🪔 🧯 🛢 💸 💵 💴 💶 💷 🪙 💰 💳 💎 ⚖️ 🪜 🧰 🪛 🔧 🔨 ⚒ 🛠 ⛏ 🪚 🔩 ⚙️ 🪤 🧱 ⛓ 🧲 🔫 💣 🧨 🪓 🔪 🗡 ⚔️ 🛡 🚬 ⚰️ 🪦 ⚱️ 🏺 🔮 📿 🧿 💈 ⚗️ 🔭 🔬 🕳 🩹 🩺 💊 💉 🩸 🧬 🦠 🧫 🧪 🌡 🧹 🪠 🧺 🧻 🚽 🚰 🚿 🛁 🛀 🧼 🪥 🪒 🧽 🪣 🧴 🛎 🔑 🗝 🚪 🪑 🛋 🛏 🛌 🧸 🪆 🖼 🪞 🪟 🛍 🛒 🎁 🎈 🎏 🎀 🪄 🪅 🎊 🎉 🎎 🏮 🎐 🧧 ✉️ 📩 📨 📧 💌 📥 📤 📦 🏷 🪧 📪 📫 📬 📭 📮 📯 📜 📃 📄 📑 🧾 📊 📈 📉 🗒 🗓 📆 📅 🗑 📇 🗃 🗳 🗄 📋 📁 📂 🗂 🗞 📰 📓 📔 📒 📕 📗 📘 📙 📚 📖 🔖 🧷 🔗 📎 🖇 📐 📏 🧮 📌 📍 ✂️ 🖊 🖋 ✒️ 🖌 🖍 📝 ✏️ 🔍 🔎 🔏 🔐 🔒 🔓".split( + " ", + ) as ReadonlyArray, + + Symbols: + "❤️ 🧡 💛 💚 💙 💜 🖤 🤍 🤎 💔 ❣️ 💕 💞 💓 💗 💖 💘 💝 💟 ☮️ ✝️ ☪️ 🕉 ☸️ ✡️ 🔯 🕎 ☯️ ☦️ 🛐 ⛎ ♈️ ♉️ ♊️ ♋️ ♌️ ♍️ ♎️ ♏️ ♐️ ♑️ ♒️ ♓️ 🆔 ⚛️ 🉑 ☢️ ☣️ 📴 📳 🈶 🈚️ 🈸 🈺 🈷️ ✴️ 🆚 💮 🉐 ㊙️ ㊗️ 🈴 🈵 🈹 🈲 🅰️ 🅱️ 🆎 🆑 🅾️ 🆘 ❌ ⭕️ 🛑 ⛔️ 📛 🚫 💯 💢 ♨️ 🚷 🚯 🚳 🚱 🔞 📵 🚭 ❗️ ❕ ❓ ❔ ‼️ ⁉️ 🔅 🔆 〽️ ⚠️ 🚸 🔱 ⚜️ 🔰 ♻️ ✅ 🈯️ 💹 ❇️ ✳️ ❎ 🌐 💠 Ⓜ️ 🌀 💤 🏧 🚾 ♿️ 🅿️ 🛗 🈳 🈂️ 🛂 🛃 🛄 🛅 🚹 🚺 🚼 ⚧ 🚻 🚮 🎦 📶 🈁 🔣 ℹ️ 🔤 🔡 🔠 🆖 🆗 🆙 🆒 🆕 🆓 0️⃣ 1️⃣ 2️⃣ 3️⃣ 4️⃣ 5️⃣ 6️⃣ 7️⃣ 8️⃣ 9️⃣ 🔟 🔢 #️⃣ *️⃣ ⏏️ ▶️ ⏸ ⏯ ⏹ ⏺ ⏭ ⏮ ⏩ ⏪ ⏫ ⏬ ◀️ 🔼 🔽 ➡️ ⬅️ ⬆️ ⬇️ ↗️ ↘️ ↙️ ↖️ ↕️ ↔️ ↪️ ↩️ ⤴️ ⤵️ 🔀 🔁 🔂 🔄 🔃 🎵 🎶 ➕ ➖ ➗ ✖️ ♾ 💲 💱 ™️ ©️ ®️ 〰️ ➰ ➿ 🔚 🔙 🔛 🔝 🔜 ✔️ ☑️ 🔘 🔴 🟠 🟡 🟢 🔵 🟣 ⚫️ ⚪️ 🟤 🔺 🔻 🔸 🔹 🔶 🔷 🔳 🔲 ▪️ ▫️ ◾️ ◽️ ◼️ ◻️ 🟥 🟧 🟨 🟩 🟦 🟪 ⬛️ ⬜️ 🟫 🔈 🔇 🔉 🔊 🔔 🔕 📣 📢 👁‍🗨 💬 💭 🗯 ♠️ ♣️ ♥️ ♦️ 🃏 🎴 🀄️ 🕐 🕑 🕒 🕓 🕔 🕕 🕖 🕗 🕘 🕙 🕚 🕛 🕜 🕝 🕞 🕟 🕠 🕡 🕢 🕣 🕤 🕥 🕦 🕧".split( + " ", + ) as ReadonlyArray, + + "Non-Emoji Symbols": + "✢ ✣ ✤ ✥ ✦ ✧ ★ ☆ ✯ ✡︎ ✩ ✪ ✫ ✬ ✭ ✮ ✶ ✷ ✵ ✸ ✹ → ⇒ ⟹ ⇨ ⇾ ➾ ⇢ ☛ ☞ ➔ ➜ ➙ ➛ ➝ ➞ ♠︎ ♣︎ ♥︎ ♦︎ ♤ ♧ ♡ ♢ ♚ ♛ ♜ ♝ ♞ ♟ ♔ ♕ ♖ ♗ ♘ ♙ ⚀ ⚁ ⚂ ⚃ ⚄ ⚅ 🂠 ⚈ ⚉ ⚆ ⚇ 𓀀 𓀁 𓀂 𓀃 𓀄 𓀅 𓀆 𓀇 𓀈 𓀉 𓀊 𓀋 𓀌 𓀍 𓀎 𓀏 𓀐 𓀑 𓀒 𓀓 𓀔 𓀕 𓀖 𓀗 𓀘 𓀙 𓀚 𓀛 𓀜 𓀝".split( + " ", + ) as ReadonlyArray, + + Flags: "🏳️ 🏴 🏁 🚩 🏳️‍🌈 🏳️‍⚧️ 🏴‍☠️".split(" ") as ReadonlyArray, + + Countries: + "🇦🇫 🇦🇽 🇦🇱 🇩🇿 🇦🇸 🇦🇩 🇦🇴 🇦🇮 🇦🇶 🇦🇬 🇦🇷 🇦🇲 🇦🇼 🇦🇺 🇦🇹 🇦🇿 🇧🇸 🇧🇭 🇧🇩 🇧🇧 🇧🇾 🇧🇪 🇧🇿 🇧🇯 🇧🇲 🇧🇹 🇧🇴 🇧🇦 🇧🇼 🇧🇷 🇮🇴 🇻🇬 🇧🇳 🇧🇬 🇧🇫 🇧🇮 🇰🇭 🇨🇲 🇨🇦 🇮🇨 🇨🇻 🇧🇶 🇰🇾 🇨🇫 🇹🇩 🇨🇱 🇨🇳 🇨🇽 🇨🇨 🇨🇴 🇰🇲 🇨🇬 🇨🇩 🇨🇰 🇨🇷 🇨🇮 🇭🇷 🇨🇺 🇨🇼 🇨🇾 🇨🇿 🇩🇰 🇩🇯 🇩🇲 🇩🇴 🇪🇨 🇪🇬 🇸🇻 🇬🇶 🇪🇷 🇪🇪 🇪🇹 🇪🇺 🇫🇰 🇫🇴 🇫🇯 🇫🇮 🇫🇷 🇬🇫 🇵🇫 🇹🇫 🇬🇦 🇬🇲 🇬🇪 🇩🇪 🇬🇭 🇬🇮 🇬🇷 🇬🇱 🇬🇩 🇬🇵 🇬🇺 🇬🇹 🇬🇬 🇬🇳 🇬🇼 🇬🇾 🇭🇹 🇭🇳 🇭🇰 🇭🇺 🇮🇸 🇮🇳 🇮🇩 🇮🇷 🇮🇶 🇮🇪 🇮🇲 🇮🇱 🇮🇹 🇯🇲 🇯🇵 🎌 🇯🇪 🇯🇴 🇰🇿 🇰🇪 🇰🇮 🇽🇰 🇰🇼 🇰🇬 🇱🇦 🇱🇻 🇱🇧 🇱🇸 🇱🇷 🇱🇾 🇱🇮 🇱🇹 🇱🇺 🇲🇴 🇲🇰 🇲🇬 🇲🇼 🇲🇾 🇲🇻 🇲🇱 🇲🇹 🇲🇭 🇲🇶 🇲🇷 🇲🇺 🇾🇹 🇲🇽 🇫🇲 🇲🇩 🇲🇨 🇲🇳 🇲🇪 🇲🇸 🇲🇦 🇲🇿 🇲🇲 🇳🇦 🇳🇷 🇳🇵 🇳🇱 🇳🇨 🇳🇿 🇳🇮 🇳🇪 🇳🇬 🇳🇺 🇳🇫 🇰🇵 🇲🇵 🇳🇴 🇴🇲 🇵🇰 🇵🇼 🇵🇸 🇵🇦 🇵🇬 🇵🇾 🇵🇪 🇵🇭 🇵🇳 🇵🇱 🇵🇹 🇵🇷 🇶🇦 🇷🇪 🇷🇴 🇷🇺 🇷🇼 🇼🇸 🇸🇲 🇸🇦 🇸🇳 🇷🇸 🇸🇨 🇸🇱 🇸🇬 🇸🇽 🇸🇰 🇸🇮 🇬🇸 🇸🇧 🇸🇴 🇿🇦 🇰🇷 🇸🇸 🇪🇸 🇱🇰 🇧🇱 🇸🇭 🇰🇳 🇱🇨 🇵🇲 🇻🇨 🇸🇩 🇸🇷 🇸🇿 🇸🇪 🇨🇭 🇸🇾 🇹🇼 🇹🇯 🇹🇿 🇹🇭 🇹🇱 🇹🇬 🇹🇰 🇹🇴 🇹🇹 🇹🇳 🇹🇷 🇹🇲 🇹🇨 🇹🇻 🇻🇮 🇺🇬 🇺🇦 🇦🇪 🇬🇧 🇺🇳 🇺🇸 🇺🇾 🇺🇿 🇻🇺 🇻🇦 🇻🇪 🇻🇳 🇼🇫 🇪🇭 🇾🇪 🇿🇲 🇿🇼".split( + " ", + ) as ReadonlyArray, + + "New Emojis 2020": + "🥲 🥸 🤌 🤌🏻 🤌🏼 🤌🏽 🤌🏾 🤌🏿 🫀 🫁 🥷 🤵‍♀️ 🤵🏻‍♀️ 🤵🏼‍♀️ 🤵🏽‍♀️ 🤵🏾‍♀️ 🤵🏿‍♀️ 🤵‍♂️ 🤵🏻‍♂️ 🤵🏼‍♂️ 🤵🏽‍♂️ 🤵🏾‍♂️ 🤵🏿‍♂️ 👰‍♀️ 👰🏻‍♀️ 👰🏼‍♀️ 👰🏽‍♀️ 👰🏾‍♀️ 👰🏿‍♀️ 👰‍♂️ 👰🏻‍♂️ 👰🏼‍♂️ 👰🏽‍♂️ 👰🏾‍♂️ 👰🏿‍♂️ 👩‍🍼 👩🏻‍🍼 👩🏼‍🍼 👩🏽‍🍼 👩🏾‍🍼 👩🏿‍🍼 🧑‍🍼 🧑🏻‍🍼 🧑🏼‍🍼 🧑🏽‍🍼 🧑🏾‍🍼 🧑🏿‍🍼 👨‍🍼 👨🏻‍🍼 👨🏼‍🍼 👨🏽‍🍼 👨🏾‍🍼 👨🏿‍🍼 🧑‍🎄 🧑🏻‍🎄 🧑🏼‍🎄 🧑🏽‍🎄 🧑🏾‍🎄 🧑🏿‍🎄 🫂 🐈‍⬛ 🦬 🦣 🦫 🐻‍❄️ 🦤 🪶 🦭 🪲 🪳 🪰 🪱 🪴 🫐 🫒 🫑 🫓 🫔 🫕 🫖 🧋 🪨 🪵 🛖 🛻 🛼 🪄 🪅 🪆 🪡 🪢 🩴 🪖 🪗 🪘 🪙 🪃 🪚 🪛 🪝 🪜 🛗 🪞 🪟 🪠 🪤 🪣 🪥 🪦 🪧 🏳️‍⚧️".split( + " ", + ) as ReadonlyArray, + + "New Emoji 2021": + "😮‍💨 😵‍💫 😶‍🌫️ ❤️‍🔥 ❤️‍🩹 🧔‍♀️ 🧔🏻‍♀️ 🧔🏼‍♀️ 🧔🏽‍♀️ 🧔🏾‍♀️ 🧔🏿‍♀️ 🧔‍♂️ 🧔🏻‍♂️ 🧔🏼‍♂️ 🧔🏽‍♂️ 🧔🏾‍♂️ 🧔🏿‍♂️ 💑🏻 💑🏼 💑🏽 💑🏾 💑🏿 💏🏻 💏🏼 💏🏽 💏🏾 💏🏿 👨🏻‍❤️‍👨🏻 👨🏻‍❤️‍👨🏼 👨🏻‍❤️‍👨🏽 👨🏻‍❤️‍👨🏾 👨🏻‍❤️‍👨🏿 👨🏼‍❤️‍👨🏻 👨🏼‍❤️‍👨🏼 👨🏼‍❤️‍👨🏽 👨🏼‍❤️‍👨🏾 👨🏼‍❤️‍👨🏿 👨🏽‍❤️‍👨🏻 👨🏽‍❤️‍👨🏼 👨🏽‍❤️‍👨🏽 👨🏽‍❤️‍👨🏾 👨🏽‍❤️‍👨🏿 👨🏾‍❤️‍👨🏻 👨🏾‍❤️‍👨🏼 👨🏾‍❤️‍👨🏽 👨🏾‍❤️‍👨🏾 👨🏾‍❤️‍👨🏿 👨🏿‍❤️‍👨🏻 👨🏿‍❤️‍👨🏼 👨🏿‍❤️‍👨🏽 👨🏿‍❤️‍👨🏾 👨🏿‍❤️‍👨🏿 👩🏻‍❤️‍👨🏻 👩🏻‍❤️‍👨🏼 👩🏻‍❤️‍👨🏽 👩🏻‍❤️‍👨🏾 👩🏻‍❤️‍👨🏿 👩🏻‍❤️‍👩🏻 👩🏻‍❤️‍👩🏼 👩🏻‍❤️‍👩🏽 👩🏻‍❤️‍👩🏾 👩🏻‍❤️‍👩🏿 👩🏼‍❤️‍👨🏻 👩🏼‍❤️‍👨🏼 👩🏼‍❤️‍👨🏽 👩🏼‍❤️‍👨🏾 👩🏼‍❤️‍👨🏿 👩🏼‍❤️‍👩🏻 👩🏼‍❤️‍👩🏼 👩🏼‍❤️‍👩🏽 👩🏼‍❤️‍👩🏾 👩🏼‍❤️‍👩🏿 👩🏽‍❤️‍👨🏻 👩🏽‍❤️‍👨🏼 👩🏽‍❤️‍👨🏽 👩🏽‍❤️‍👨🏾 👩🏽‍❤️‍👨🏿 👩🏽‍❤️‍👩🏻 👩🏽‍❤️‍👩🏼 👩🏽‍❤️‍👩🏽 👩🏽‍❤️‍👩🏾 👩🏽‍❤️‍👩🏿 👩🏾‍❤️‍👨🏻 👩🏾‍❤️‍👨🏼 👩🏾‍❤️‍👨🏽 👩🏾‍❤️‍👨🏾 👩🏾‍❤️‍👨🏿 👩🏾‍❤️‍👩🏻 👩🏾‍❤️‍👩🏼 👩🏾‍❤️‍👩🏽 👩🏾‍❤️‍👩🏾 👩🏾‍❤️‍👩🏿 👩🏿‍❤️‍👨🏻 👩🏿‍❤️‍👨🏼 👩🏿‍❤️‍👨🏽 👩🏿‍❤️‍👨🏾 👩🏿‍❤️‍👨🏿 👩🏿‍❤️‍👩🏻 👩🏿‍❤️‍👩🏼 👩🏿‍❤️‍👩🏽 👩🏿‍❤️‍👩🏾 👩🏿‍❤️‍👩🏿 🧑🏻‍❤️‍🧑🏼 🧑🏻‍❤️‍🧑🏽 🧑🏻‍❤️‍🧑🏾 🧑🏻‍❤️‍🧑🏿 🧑🏼‍❤️‍🧑🏻 🧑🏼‍❤️‍🧑🏽 🧑🏼‍❤️‍🧑🏾 🧑🏼‍❤️‍🧑🏿 🧑🏽‍❤️‍🧑🏻 🧑🏽‍❤️‍🧑🏼 🧑🏽‍❤️‍🧑🏾 🧑🏽‍❤️‍🧑🏿 🧑🏾‍❤️‍🧑🏻 🧑🏾‍❤️‍🧑🏼 🧑🏾‍❤️‍🧑🏽 🧑🏾‍❤️‍🧑🏿 🧑🏿‍❤️‍🧑🏻 🧑🏿‍❤️‍🧑🏼 🧑🏿‍❤️‍🧑🏽 🧑🏿‍❤️‍🧑🏾 👨🏻‍❤️‍💋‍👨🏻 👨🏻‍❤️‍💋‍👨🏼 👨🏻‍❤️‍💋‍👨🏽 👨🏻‍❤️‍💋‍👨🏾 👨🏻‍❤️‍💋‍👨🏿 👨🏼‍❤️‍💋‍👨🏻 👨🏼‍❤️‍💋‍👨🏼 👨🏼‍❤️‍💋‍👨🏽 👨🏼‍❤️‍💋‍👨🏾 👨🏼‍❤️‍💋‍👨🏿 👨🏽‍❤️‍💋‍👨🏻 👨🏽‍❤️‍💋‍👨🏼 👨🏽‍❤️‍💋‍👨🏽 👨🏽‍❤️‍💋‍👨🏾 👨🏽‍❤️‍💋‍👨🏿 👨🏾‍❤️‍💋‍👨🏻 👨🏾‍❤️‍💋‍👨🏼 👨🏾‍❤️‍💋‍👨🏽 👨🏾‍❤️‍💋‍👨🏾 👨🏾‍❤️‍💋‍👨🏿 👨🏿‍❤️‍💋‍👨🏻 👨🏿‍❤️‍💋‍👨🏼 👨🏿‍❤️‍💋‍👨🏽 👨🏿‍❤️‍💋‍👨🏾 👨🏿‍❤️‍💋‍👨🏿 👩🏻‍❤️‍💋‍👨🏻 👩🏻‍❤️‍💋‍👨🏼 👩🏻‍❤️‍💋‍👨🏽 👩🏻‍❤️‍💋‍👨🏾 👩🏻‍❤️‍💋‍👨🏿 👩🏻‍❤️‍💋‍👩🏻 👩🏻‍❤️‍💋‍👩🏼 👩🏻‍❤️‍💋‍👩🏽 👩🏻‍❤️‍💋‍👩🏾 👩🏻‍❤️‍💋‍👩🏿 👩🏼‍❤️‍💋‍👨🏻 👩🏼‍❤️‍💋‍👨🏼 👩🏼‍❤️‍💋‍👨🏽 👩🏼‍❤️‍💋‍👨🏾 👩🏼‍❤️‍💋‍👨🏿 👩🏼‍❤️‍💋‍👩🏻 👩🏼‍❤️‍💋‍👩🏼 👩🏼‍❤️‍💋‍👩🏽 👩🏼‍❤️‍💋‍👩🏾 👩🏼‍❤️‍💋‍👩🏿 👩🏽‍❤️‍💋‍👨🏻 👩🏽‍❤️‍💋‍👨🏼 👩🏽‍❤️‍💋‍👨🏽 👩🏽‍❤️‍💋‍👨🏾 👩🏽‍❤️‍💋‍👨🏿 👩🏽‍❤️‍💋‍👩🏻 👩🏽‍❤️‍💋‍👩🏼 👩🏽‍❤️‍💋‍👩🏽 👩🏽‍❤️‍💋‍👩🏾 👩🏽‍❤️‍💋‍👩🏿 👩🏾‍❤️‍💋‍👨🏻 👩🏾‍❤️‍💋‍👨🏼 👩🏾‍❤️‍💋‍👨🏽 👩🏾‍❤️‍💋‍👨🏾 👩🏾‍❤️‍💋‍👨🏿 👩🏾‍❤️‍💋‍👩🏻 👩🏾‍❤️‍💋‍👩🏼 👩🏾‍❤️‍💋‍👩🏽 👩🏾‍❤️‍💋‍👩🏾 👩🏾‍❤️‍💋‍👩🏿 👩🏿‍❤️‍💋‍👨🏻 👩🏿‍❤️‍💋‍👨🏼 👩🏿‍❤️‍💋‍👨🏽 👩🏿‍❤️‍💋‍👨🏾 👩🏿‍❤️‍💋‍👨🏿 👩🏿‍❤️‍💋‍👩🏻 👩🏿‍❤️‍💋‍👩🏼 👩🏿‍❤️‍💋‍👩🏽 👩🏿‍❤️‍💋‍👩🏾 👩🏿‍❤️‍💋‍👩🏿 🧑🏻‍❤️‍💋‍🧑🏼 🧑🏻‍❤️‍💋‍🧑🏽 🧑🏻‍❤️‍💋‍🧑🏾 🧑🏻‍❤️‍💋‍🧑🏿 🧑🏼‍❤️‍💋‍🧑🏻 🧑🏼‍❤️‍💋‍🧑🏽 🧑🏼‍❤️‍💋‍🧑🏾 🧑🏼‍❤️‍💋‍🧑🏿 🧑🏽‍❤️‍💋‍🧑🏻 🧑🏽‍❤️‍💋‍🧑🏼 🧑🏽‍❤️‍💋‍🧑🏾 🧑🏽‍❤️‍💋‍🧑🏿 🧑🏾‍❤️‍💋‍🧑🏻 🧑🏾‍❤️‍💋‍🧑🏼 🧑🏾‍❤️‍💋‍🧑🏽 🧑🏾‍❤️‍💋‍🧑🏿 🧑🏿‍❤️‍💋‍🧑🏻 🧑🏿‍❤️‍💋‍🧑🏼 🧑🏿‍❤️‍💋‍🧑🏽 🧑🏿‍❤️‍💋‍🧑🏾".split( + " ", + ) as ReadonlyArray, + + "New Emojis (Discorered randomly)": "🖌️".split( + " ", + ) as ReadonlyArray, + + "Emojis used in WebGPT": [ + "▶", + "❤", + ] as TODO_any as ReadonlyArray, }; /** @@ -120,7 +128,9 @@ export const EMOJIS_IN_CATEGORIES: Record = new Set(Object.values(EMOJIS_IN_CATEGORIES).flat()); +export const EMOJIS: Set = new Set( + Object.values(EMOJIS_IN_CATEGORIES).flat(), +); /** * TODO: [💴] DRY - just one version of emojis.ts diff --git a/src/utils/environment/$getGlobalScope.ts b/src/utils/environment/$getGlobalScope.ts index 071069c75b..1adeffc66c 100644 --- a/src/utils/environment/$getGlobalScope.ts +++ b/src/utils/environment/$getGlobalScope.ts @@ -1,4 +1,4 @@ -import type { really_any } from '../organization/really_any'; +import type { really_any } from "../organization/really_any"; /** * @@@ @@ -8,5 +8,5 @@ import type { really_any } from '../organization/really_any'; * @private internal function of `$Register` */ export function $getGlobalScope(): really_any { - return Function('return this')(); + return Function("return this")(); } diff --git a/src/utils/execCommand/$execCommand.ts b/src/utils/execCommand/$execCommand.ts index a3c49ab798..b484fefc73 100644 --- a/src/utils/execCommand/$execCommand.ts +++ b/src/utils/execCommand/$execCommand.ts @@ -1,12 +1,12 @@ -import { spawn } from 'child_process'; -import colors from 'colors'; -import { spaceTrim } from 'spacetrim'; -import { forTime } from 'waitasecond'; -import { DEFAULT_IS_VERBOSE } from '../../config'; -import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError'; -import { $isRunningInNode } from '../environment/$isRunningInNode'; -import { $execCommandNormalizeOptions } from './$execCommandNormalizeOptions'; -import type { ExecCommandOptions } from './ExecCommandOptions'; +import { spawn } from "child_process"; +import colors from "colors"; +import { spaceTrim } from "spacetrim"; +import { forTime } from "waitasecond"; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import { EnvironmentMismatchError } from "../../errors/EnvironmentMismatchError"; +import { $isRunningInNode } from "../environment/$isRunningInNode"; +import { $execCommandNormalizeOptions } from "./$execCommandNormalizeOptions"; +import type { ExecCommandOptions } from "./ExecCommandOptions"; /** * Run one command in a shell @@ -20,106 +20,124 @@ import type { ExecCommandOptions } from './ExecCommandOptions'; * @public exported from `@promptbook/node` */ export function $execCommand(options: ExecCommandOptions): Promise { - if (!$isRunningInNode()) { - throw new EnvironmentMismatchError('Function `$execCommand` can run only in Node environment.js'); - } + if (!$isRunningInNode()) { + throw new EnvironmentMismatchError( + "Function `$execCommand` can run only in Node environment.js", + ); + } - return new Promise((resolve, reject) => { - // eslint-disable-next-line prefer-const - const { - command, - humanReadableCommand, - args, - cwd, - crashOnError, - timeout, - isVerbose = DEFAULT_IS_VERBOSE, - } = $execCommandNormalizeOptions(options); + return new Promise((resolve, reject) => { + // eslint-disable-next-line prefer-const + const { + command, + humanReadableCommand, + args, + cwd, + crashOnError, + timeout, + isVerbose = DEFAULT_IS_VERBOSE, + } = $execCommandNormalizeOptions(options); - if (timeout !== Infinity) { - // TODO: In waitasecond forTime(Infinity) should be equivalent to forEver() - forTime(timeout).then(() => { - if (crashOnError) { - reject(new Error(`Command "${humanReadableCommand}" exceeded time limit of ${timeout}ms`)); - } else { - console.warn( - `Command "${humanReadableCommand}" exceeded time limit of ${timeout}ms but continues running`, - ); - resolve('Command exceeded time limit'); - } - }); - } + if (timeout !== Number.POSITIVE_INFINITY) { + // TODO: In waitasecond forTime(Infinity) should be equivalent to forEver() + forTime(timeout).then(() => { + if (crashOnError) { + reject( + new Error( + `Command "${humanReadableCommand}" exceeded time limit of ${timeout}ms`, + ), + ); + } else { + console.warn( + `Command "${humanReadableCommand}" exceeded time limit of ${timeout}ms but continues running`, + ); + resolve("Command exceeded time limit"); + } + }); + } - if (isVerbose) { - console.info(colors.yellow(cwd) + ' ' + colors.green(command) + ' ' + colors.blue(args.join(' '))); - } + if (isVerbose) { + console.info( + colors.yellow(cwd) + + " " + + colors.green(command) + + " " + + colors.blue(args.join(" ")), + ); + } - try { - const commandProcess = spawn(command, args, { cwd, shell: true }); + try { + const commandProcess = spawn(command, args, { cwd, shell: true }); - if (isVerbose) { - commandProcess.on('message', (message) => { - console.info({ message }); - }); - } + if (isVerbose) { + commandProcess.on("message", (message) => { + console.info({ message }); + }); + } - const output: string[] = []; + const output: string[] = []; - commandProcess.stdout.on('data', (stdout) => { - output.push(stdout.toString()); - if (isVerbose) { - console.info(stdout.toString()); - } - }); + commandProcess.stdout.on("data", (stdout) => { + output.push(stdout.toString()); + if (isVerbose) { + console.info(stdout.toString()); + } + }); - commandProcess.stderr.on('data', (stderr) => { - output.push(stderr.toString()); - if (isVerbose && stderr.toString().trim()) { - console.warn(stderr.toString()); - } - }); + commandProcess.stderr.on("data", (stderr) => { + output.push(stderr.toString()); + if (isVerbose && stderr.toString().trim()) { + console.warn(stderr.toString()); + } + }); - const finishWithCode = (code: number) => { - if (code !== 0) { - if (crashOnError) { - reject( - new Error( - output.join('\n').trim() || - `Command "${humanReadableCommand}" exited with code ${code}`, - ), - ); - } else { - if (isVerbose) { - console.warn(`Command "${humanReadableCommand}" exited with code ${code}`); - } - resolve(spaceTrim(output.join('\n'))); - } - } else { - resolve(spaceTrim(output.join('\n'))); - } - }; + const finishWithCode = (code: number) => { + if (code !== 0) { + if (crashOnError) { + reject( + new Error( + output.join("\n").trim() || + `Command "${humanReadableCommand}" exited with code ${code}`, + ), + ); + } else { + if (isVerbose) { + console.warn( + `Command "${humanReadableCommand}" exited with code ${code}`, + ); + } + resolve(spaceTrim(output.join("\n"))); + } + } else { + resolve(spaceTrim(output.join("\n"))); + } + }; - commandProcess.on('close', finishWithCode); - commandProcess.on('exit', finishWithCode); - commandProcess.on('disconnect', () => { - // Note: Unexpected disconnection should always result in rejection - reject(new Error(`Command "${humanReadableCommand}" disconnected`)); - }); - commandProcess.on('error', (error) => { - if (crashOnError) { - reject(new Error(`Command "${humanReadableCommand}" failed: \n${error.message}`)); - } else { - if (isVerbose) { - console.warn(error); - } - resolve(spaceTrim(output.join('\n'))); - } - }); - } catch (error) { - // Note: Unexpected error in sync code should always result in rejection - reject(error); - } - }); + commandProcess.on("close", finishWithCode); + commandProcess.on("exit", finishWithCode); + commandProcess.on("disconnect", () => { + // Note: Unexpected disconnection should always result in rejection + reject(new Error(`Command "${humanReadableCommand}" disconnected`)); + }); + commandProcess.on("error", (error) => { + if (crashOnError) { + reject( + new Error( + `Command "${humanReadableCommand}" failed: \n${error.message}`, + ), + ); + } else { + if (isVerbose) { + console.warn(error); + } + resolve(spaceTrim(output.join("\n"))); + } + }); + } catch (error) { + // Note: Unexpected error in sync code should always result in rejection + reject(error); + } + }); } /** diff --git a/src/utils/execCommand/$execCommandNormalizeOptions.ts b/src/utils/execCommand/$execCommandNormalizeOptions.ts index 5dc8c9312e..de97ec4641 100644 --- a/src/utils/execCommand/$execCommandNormalizeOptions.ts +++ b/src/utils/execCommand/$execCommandNormalizeOptions.ts @@ -1,7 +1,7 @@ -import { DEFAULT_IS_VERBOSE } from '../../config'; -import type { TODO_any } from '../organization/TODO_any'; -import type { ExecCommandOptions } from './ExecCommandOptions'; -import type { ExecCommandOptionsAdvanced } from './ExecCommandOptions'; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import type { TODO_any } from "../organization/TODO_any"; +import type { ExecCommandOptions } from "./ExecCommandOptions"; +import type { ExecCommandOptionsAdvanced } from "./ExecCommandOptions"; /** * Normalize options for `execCommand` and `execCommands` @@ -11,27 +11,27 @@ import type { ExecCommandOptionsAdvanced } from './ExecCommandOptions'; * @private internal utility of `execCommand` and `execCommands` */ export function $execCommandNormalizeOptions(options: ExecCommandOptions): Pick< - ExecCommandOptionsAdvanced, - 'command' | 'args' | 'cwd' | 'crashOnError' | 'timeout' | 'isVerbose' + ExecCommandOptionsAdvanced, + "command" | "args" | "cwd" | "crashOnError" | "timeout" | "isVerbose" > & { - humanReadableCommand: string; + humanReadableCommand: string; } { - let command: string; - let cwd: string; - let crashOnError: boolean; - let args: string[] = []; - let timeout: number; - let isVerbose: boolean; + let command: string; + let cwd: string; + let crashOnError: boolean; + let args: string[] = []; + let timeout: number; + let isVerbose: boolean; - if (typeof options === 'string') { - // TODO: [1] DRY default values - command = options; - cwd = process.cwd(); - crashOnError = true; - timeout = Infinity; // <- TODO: [⏳] - isVerbose = DEFAULT_IS_VERBOSE; - } else { - /* + if (typeof options === "string") { + // TODO: [1] DRY default values + command = options; + cwd = process.cwd(); + crashOnError = true; + timeout = Number.POSITIVE_INFINITY; // <- TODO: [⏳] + isVerbose = DEFAULT_IS_VERBOSE; + } else { + /* TODO: if ((options as any).commands !== undefined) { commands = (options as any).commands; @@ -40,37 +40,47 @@ export function $execCommandNormalizeOptions(options: ExecCommandOptions): Pick< } */ - // TODO: [1] DRY default values - command = options.command; - cwd = options.cwd ?? process.cwd(); - crashOnError = options.crashOnError ?? true; - timeout = options.timeout ?? Infinity; - isVerbose = options.isVerbose ?? DEFAULT_IS_VERBOSE; - } + // TODO: [1] DRY default values + command = options.command; + cwd = options.cwd ?? process.cwd(); + crashOnError = options.crashOnError ?? true; + timeout = options.timeout ?? Number.POSITIVE_INFINITY; + isVerbose = options.isVerbose ?? DEFAULT_IS_VERBOSE; + } - // TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g - const _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g)) - .map(([match]) => match) - .filter((arg) => arg !== ''); + // TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g + const _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g)) + .map(([match]) => match) + .filter((arg) => arg !== ""); - if (_.length > 1) { - [command, ...args] = _ as TODO_any; - } + if (_.length > 1) { + [command, ...args] = _ as TODO_any; + } - if ((options as TODO_any).args) { - args = [...args, ...(options as TODO_any).args]; - } + if ((options as TODO_any).args) { + args = [...args, ...(options as TODO_any).args]; + } - let humanReadableCommand = !['npx', 'npm'].includes(command) ? command : args[0]!; - if (['ts-node'].includes(humanReadableCommand)) { - humanReadableCommand += ` ${args[1]}`; - } + let humanReadableCommand = !["npx", "npm"].includes(command) + ? command + : args[0]!; + if (["ts-node"].includes(humanReadableCommand)) { + humanReadableCommand += ` ${args[1]}`; + } - if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) { - command = `${command}.cmd`; - } + if (/^win/.test(process.platform) && ["npm", "npx"].includes(command)) { + command = `${command}.cmd`; + } - return { command, humanReadableCommand, args, cwd, crashOnError, timeout, isVerbose }; + return { + command, + humanReadableCommand, + args, + cwd, + crashOnError, + timeout, + isVerbose, + }; } // TODO: This should show type error> execCommandNormalizeOptions({ command: '', commands: [''] }); diff --git a/src/utils/execCommand/$execCommands.ts b/src/utils/execCommand/$execCommands.ts index 6f36240c1f..f1b8f1171f 100644 --- a/src/utils/execCommand/$execCommands.ts +++ b/src/utils/execCommand/$execCommands.ts @@ -1,4 +1,4 @@ -import { $execCommand } from './$execCommand'; +import { $execCommand } from "./$execCommand"; /** * Run multiple commands in a shell in sequence @@ -11,17 +11,17 @@ import { $execCommand } from './$execCommand'; * @public exported from `@promptbook/node` */ export async function $execCommands({ - commands, - cwd, - crashOnError, + commands, + cwd, + crashOnError, }: { - readonly commands: string[]; - readonly cwd: string; - readonly crashOnError?: boolean; + readonly commands: string[]; + readonly cwd: string; + readonly crashOnError?: boolean; }) { - for (const command of commands) { - await $execCommand({ command, cwd, crashOnError }); - } + for (const command of commands) { + await $execCommand({ command, cwd, crashOnError }); + } } /** diff --git a/src/utils/execCommand/ExecCommandOptions.ts b/src/utils/execCommand/ExecCommandOptions.ts index 92f39e4dc5..b8d06d3061 100644 --- a/src/utils/execCommand/ExecCommandOptions.ts +++ b/src/utils/execCommand/ExecCommandOptions.ts @@ -1,58 +1,61 @@ /** * @deprecated Use sth from type-fest or move this to type helpers */ -type RequiredAndOptional = Pick & - Partial>; +type RequiredAndOptional< + TBase, + TRequired extends keyof TBase, + TOptional extends keyof TBase, +> = Pick & Partial>; /** * Simple options for `execCommand` */ export type ExecCommandOptions = - | string - | RequiredAndOptional< - ExecCommandOptionsAdvanced, - 'command', - 'args' | 'cwd' | 'crashOnError' | 'timeout' | 'isVerbose' - >; + | string + | RequiredAndOptional< + ExecCommandOptionsAdvanced, + "command", + "args" | "cwd" | "crashOnError" | "timeout" | "isVerbose" + >; // TODO: | RequiredAndOptional; /** * Advanced options for `execCommand` */ export type ExecCommandOptionsAdvanced = { - /** - * Command to run - */ - readonly command: string; - - /** - * Arguments for the command - */ - readonly args: string[]; - - /** - * Current working directory - * - * @default process.cwd() - */ - readonly cwd: string; - - /** - * If `true` then the command will throw an error if the return code is not `0` - */ - readonly crashOnError: boolean; - - /** - * Timeout in milliseconds - */ - readonly timeout: number; - - /** - * If `true` then the command and entire CLI output will be logged to the console - * - * @default false - */ - readonly isVerbose?: boolean; + /** + * Command to run + */ + readonly command: string; + + /** + * Arguments for the command + */ + readonly args: string[]; + + /** + * Current working directory + * + * @default process.cwd() + */ + readonly cwd: string; + + /** + * If `true` then the command will throw an error if the return code is not `0` + */ + readonly crashOnError: boolean; + + /** + * Timeout in milliseconds + */ + readonly timeout: number; + + /** + * If `true` then the command and entire CLI output will be logged to the console + * + * @default false + */ + readonly isVerbose?: boolean; }; /** diff --git a/src/utils/execCommand/execCommand.test.ts b/src/utils/execCommand/execCommand.test.ts index a97b9b0827..5586d8970a 100644 --- a/src/utils/execCommand/execCommand.test.ts +++ b/src/utils/execCommand/execCommand.test.ts @@ -1,25 +1,25 @@ -import { describe, expect, it } from '@jest/globals'; -import { $execCommand } from './$execCommand'; +import { describe, expect, it } from "@jest/globals"; +import { $execCommand } from "./$execCommand"; -describe('basic usage of execCommand', () => { - it(`should pass on simple command`, () => - expect( - $execCommand({ - command: `whoami`, - }), - ).resolves.not.toThrowError()); +describe("basic usage of execCommand", () => { + it(`should pass on simple command`, () => + expect( + $execCommand({ + command: `whoami`, + }), + ).resolves.not.toThrowError()); - it(`should crash on unknown command`, () => - expect( - $execCommand({ - command: `unknown-command`, - }), - ).rejects.toThrowError( - /unknown-command/i, - /* + it(`should crash on unknown command`, () => + expect( + $execCommand({ + command: `unknown-command`, + }), + ).rejects.toThrowError( + /unknown-command/i, + /* <- Note: There is a difference in the error message: - On Linux: Command "unknown-command" failed - On Windows: 'unknown-command' is not recognized as an internal or external command, operable program or batch file. */ - )); + )); }); diff --git a/src/utils/execCommand/execCommandNormalizeOptions.test.ts b/src/utils/execCommand/execCommandNormalizeOptions.test.ts index b7315e4380..d6441c1f6e 100644 --- a/src/utils/execCommand/execCommandNormalizeOptions.test.ts +++ b/src/utils/execCommand/execCommandNormalizeOptions.test.ts @@ -1,105 +1,116 @@ -import { describe, expect, it } from '@jest/globals'; -import { DEFAULT_IS_VERBOSE } from '../../config'; -import { $execCommandNormalizeOptions } from './$execCommandNormalizeOptions'; +import { describe, expect, it } from "@jest/globals"; +import { DEFAULT_IS_VERBOSE } from "../../config"; +import { $execCommandNormalizeOptions } from "./$execCommandNormalizeOptions"; -describe('how normalizing exec options works', () => { - it('can normalize just string command', () => { - expect($execCommandNormalizeOptions('ls')).toEqual({ - command: 'ls', - args: [], - cwd: process.cwd(), - crashOnError: true, - humanReadableCommand: 'ls', - isVerbose: DEFAULT_IS_VERBOSE, - timeout: Infinity, // <- TODO: [⏳] - }); - }); +describe("how normalizing exec options works", () => { + it("can normalize just string command", () => { + expect($execCommandNormalizeOptions("ls")).toEqual({ + command: "ls", + args: [], + cwd: process.cwd(), + crashOnError: true, + humanReadableCommand: "ls", + isVerbose: DEFAULT_IS_VERBOSE, + timeout: Number.POSITIVE_INFINITY, // <- TODO: [⏳] + }); + }); - it('can normalize single command', () => { - expect($execCommandNormalizeOptions({ command: 'ls' })).toEqual({ - command: 'ls', - args: [], - cwd: process.cwd(), - crashOnError: true, - humanReadableCommand: 'ls', - isVerbose: DEFAULT_IS_VERBOSE, - timeout: Infinity, // <- TODO: [⏳] - }); - }); - it('can normalize single command and cwd', () => { - expect($execCommandNormalizeOptions({ command: 'ls', cwd: './' })).toEqual({ - command: 'ls', - args: [], - cwd: './', - crashOnError: true, - humanReadableCommand: 'ls', - isVerbose: DEFAULT_IS_VERBOSE, - timeout: Infinity, // <- TODO: [⏳] - }); - }); - it('can normalize single command and crashOnError', () => { - expect($execCommandNormalizeOptions({ command: 'ls', crashOnError: false })).toEqual({ - command: 'ls', - args: [], - cwd: process.cwd(), - crashOnError: false, - humanReadableCommand: 'ls', - isVerbose: DEFAULT_IS_VERBOSE, - timeout: Infinity, // <- TODO: [⏳] - }); - }); + it("can normalize single command", () => { + expect($execCommandNormalizeOptions({ command: "ls" })).toEqual({ + command: "ls", + args: [], + cwd: process.cwd(), + crashOnError: true, + humanReadableCommand: "ls", + isVerbose: DEFAULT_IS_VERBOSE, + timeout: Number.POSITIVE_INFINITY, // <- TODO: [⏳] + }); + }); + it("can normalize single command and cwd", () => { + expect($execCommandNormalizeOptions({ command: "ls", cwd: "./" })).toEqual({ + command: "ls", + args: [], + cwd: "./", + crashOnError: true, + humanReadableCommand: "ls", + isVerbose: DEFAULT_IS_VERBOSE, + timeout: Number.POSITIVE_INFINITY, // <- TODO: [⏳] + }); + }); + it("can normalize single command and crashOnError", () => { + expect( + $execCommandNormalizeOptions({ command: "ls", crashOnError: false }), + ).toEqual({ + command: "ls", + args: [], + cwd: process.cwd(), + crashOnError: false, + humanReadableCommand: "ls", + isVerbose: DEFAULT_IS_VERBOSE, + timeout: Number.POSITIVE_INFINITY, // <- TODO: [⏳] + }); + }); - it('can normalize single command and array args', () => { - expect($execCommandNormalizeOptions({ command: 'npm', args: ['run', 'test'] })).toEqual({ - command: expect.stringMatching(/npm(\.cmd)?/), - args: ['run', 'test'], - cwd: process.cwd(), - crashOnError: true, - humanReadableCommand: 'run', - isVerbose: DEFAULT_IS_VERBOSE, - timeout: Infinity, // <- TODO: [⏳] - }); - }); + it("can normalize single command and array args", () => { + expect( + $execCommandNormalizeOptions({ command: "npm", args: ["run", "test"] }), + ).toEqual({ + command: expect.stringMatching(/npm(\.cmd)?/), + args: ["run", "test"], + cwd: process.cwd(), + crashOnError: true, + humanReadableCommand: "run", + isVerbose: DEFAULT_IS_VERBOSE, + timeout: Number.POSITIVE_INFINITY, // <- TODO: [⏳] + }); + }); - it('can normalize single command and string args', () => { - expect($execCommandNormalizeOptions({ command: 'npm run test' })).toEqual({ - command: expect.stringMatching(/npm(\.cmd)?/), - args: ['run', 'test'], - cwd: process.cwd(), - crashOnError: true, - humanReadableCommand: 'run', - isVerbose: DEFAULT_IS_VERBOSE, - timeout: Infinity, // <- TODO: [⏳] - }); - }); + it("can normalize single command and string args", () => { + expect($execCommandNormalizeOptions({ command: "npm run test" })).toEqual({ + command: expect.stringMatching(/npm(\.cmd)?/), + args: ["run", "test"], + cwd: process.cwd(), + crashOnError: true, + humanReadableCommand: "run", + isVerbose: DEFAULT_IS_VERBOSE, + timeout: Number.POSITIVE_INFINITY, // <- TODO: [⏳] + }); + }); - it('can normalize single command with args from array and string and also empty args ', () => { - expect( - $execCommandNormalizeOptions({ command: ' npm run ', args: ['test'], timeout: 1000, isVerbose: true }), - ).toEqual({ - command: expect.stringMatching(/npm(\.cmd)?/), - args: ['run', 'test'], - cwd: process.cwd(), - crashOnError: true, - humanReadableCommand: 'run', - isVerbose: true, - timeout: 1000, - }); - }); + it("can normalize single command with args from array and string and also empty args ", () => { + expect( + $execCommandNormalizeOptions({ + command: " npm run ", + args: ["test"], + timeout: 1000, + isVerbose: true, + }), + ).toEqual({ + command: expect.stringMatching(/npm(\.cmd)?/), + args: ["run", "test"], + cwd: process.cwd(), + crashOnError: true, + humanReadableCommand: "run", + isVerbose: true, + timeout: 1000, + }); + }); - it('can split arg flags', () => { - expect($execCommandNormalizeOptions(`git commit -m "Hello World"`)).toEqual({ - command: 'git', - args: ['commit', '-m', '"Hello World"'], - cwd: process.cwd(), - crashOnError: true, - humanReadableCommand: 'git', - isVerbose: DEFAULT_IS_VERBOSE, - timeout: Infinity, // <- TODO: [⏳] - }); - }); + it("can split arg flags", () => { + expect($execCommandNormalizeOptions(`git commit -m "Hello World"`)).toEqual( + { + command: "git", + args: ["commit", "-m", '"Hello World"'], + cwd: process.cwd(), + crashOnError: true, + humanReadableCommand: "git", + isVerbose: DEFAULT_IS_VERBOSE, + timeout: Number.POSITIVE_INFINITY, // <- TODO: [⏳] + }, + ); + }); - /* TODO: + /* TODO: it('can normalize multiple commands', async () => {}); */ }); diff --git a/src/utils/expectation-counters/countCharacters.test.ts b/src/utils/expectation-counters/countCharacters.test.ts index 939dae99db..c69b5c278b 100644 --- a/src/utils/expectation-counters/countCharacters.test.ts +++ b/src/utils/expectation-counters/countCharacters.test.ts @@ -1,62 +1,62 @@ -import { describe, expect, it } from '@jest/globals'; -import { countCharacters } from './countCharacters'; +import { describe, expect, it } from "@jest/globals"; +import { countCharacters } from "./countCharacters"; -describe('countCharacters', () => { - it('should return 0 for an empty string', () => { - expect(countCharacters('')).toBe(0); - expect(countCharacters('\0')).toBe(0); - expect(countCharacters('\0\0')).toBe(0); - }); +describe("countCharacters", () => { + it("should return 0 for an empty string", () => { + expect(countCharacters("")).toBe(0); + expect(countCharacters("\0")).toBe(0); + expect(countCharacters("\0\0")).toBe(0); + }); - it('should return the correct count for a string with one character', () => { - expect(countCharacters(' ')).toBe(1); - expect(countCharacters('a')).toBe(1); - expect(countCharacters('A')).toBe(1); - expect(countCharacters('1')).toBe(1); - expect(countCharacters('!')).toBe(1); - }); + it("should return the correct count for a string with one character", () => { + expect(countCharacters(" ")).toBe(1); + expect(countCharacters("a")).toBe(1); + expect(countCharacters("A")).toBe(1); + expect(countCharacters("1")).toBe(1); + expect(countCharacters("!")).toBe(1); + }); - it('should work with emojis', () => { - expect(countCharacters('♥')).toBe(1); - expect(countCharacters('❤')).toBe(1); - expect(countCharacters('💚')).toBe(1); - expect(countCharacters('💙')).toBe(1); - expect(countCharacters('💫')).toBe(1); - expect(countCharacters('🌟')).toBe(1); - expect(countCharacters('🌠')).toBe(1); - expect(countCharacters('🌈')).toBe(1); - expect(countCharacters('🎆')).toBe(1); - expect(countCharacters('🎇')).toBe(1); - expect(countCharacters('👩🏾')).toBe(1); - expect(countCharacters('👨')).toBe(1); - expect(countCharacters('👨🏻')).toBe(1); - expect(countCharacters('👨‍❤️‍👨')).toBe(1); - expect(countCharacters('❤♥')).toBe(2); - }); + it("should work with emojis", () => { + expect(countCharacters("♥")).toBe(1); + expect(countCharacters("❤")).toBe(1); + expect(countCharacters("💚")).toBe(1); + expect(countCharacters("💙")).toBe(1); + expect(countCharacters("💫")).toBe(1); + expect(countCharacters("🌟")).toBe(1); + expect(countCharacters("🌠")).toBe(1); + expect(countCharacters("🌈")).toBe(1); + expect(countCharacters("🎆")).toBe(1); + expect(countCharacters("🎇")).toBe(1); + expect(countCharacters("👩🏾")).toBe(1); + expect(countCharacters("👨")).toBe(1); + expect(countCharacters("👨🏻")).toBe(1); + expect(countCharacters("👨‍❤️‍👨")).toBe(1); + expect(countCharacters("❤♥")).toBe(2); + }); - it('should work with multiple emojis', () => { - expect(countCharacters('💚💙💫🌟🌠')).toBe(5); - expect(countCharacters('👩🏾👨')).toBe(2); - expect(countCharacters('👨')).toBe(1); - expect(countCharacters('👨‍❤️‍👨👨‍❤️‍👨👨‍❤️‍👨')).toBe(3); - }); + it("should work with multiple emojis", () => { + expect(countCharacters("💚💙💫🌟🌠")).toBe(5); + expect(countCharacters("👩🏾👨")).toBe(2); + expect(countCharacters("👨")).toBe(1); + expect(countCharacters("👨‍❤️‍👨👨‍❤️‍👨👨‍❤️‍👨")).toBe(3); + }); - it('should return the correct count for a string with multiple characters', () => { - expect(countCharacters('Hello')).toBe(5); - expect(countCharacters('I ♥ Programming')).toBe(15); - expect(countCharacters('I ❤ Programming')).toBe(15); - expect(countCharacters('I 💙 Programming')).toBe(15); - expect(countCharacters('I 💫 Programming')).toBe(15); - expect(countCharacters('I 👩🏾 Programming')).toBe(15); - expect(countCharacters('I 👨‍❤️‍👨 Programming')).toBe(15); - }); + it("should return the correct count for a string with multiple characters", () => { + expect(countCharacters("Hello")).toBe(5); + expect(countCharacters("I ♥ Programming")).toBe(15); + expect(countCharacters("I ❤ Programming")).toBe(15); + expect(countCharacters("I 💙 Programming")).toBe(15); + expect(countCharacters("I 💫 Programming")).toBe(15); + expect(countCharacters("I 👩🏾 Programming")).toBe(15); + expect(countCharacters("I 👨‍❤️‍👨 Programming")).toBe(15); + }); - it('should return the correct count for a string with special characters', () => { - expect(countCharacters('Hello!')).toBe(6); - }); + it("should return the correct count for a string with special characters", () => { + expect(countCharacters("Hello!")).toBe(6); + }); - it('should return the correct count for a string with whitespace characters', () => { - expect(countCharacters('Hello world')).toBe(11); - expect(countCharacters('Hello\nworld')).toBe(11); - }); + it("should return the correct count for a string with whitespace characters", () => { + expect(countCharacters("Hello world")).toBe(11); + expect(countCharacters("Hello\nworld")).toBe(11); + }); }); diff --git a/src/utils/expectation-counters/countCharacters.ts b/src/utils/expectation-counters/countCharacters.ts index a315c1040c..7133f23588 100644 --- a/src/utils/expectation-counters/countCharacters.ts +++ b/src/utils/expectation-counters/countCharacters.ts @@ -1,4 +1,4 @@ -import type { ExpectationAmount } from '../../pipeline/PipelineJson/Expectations'; +import type { ExpectationAmount } from "../../pipeline/PipelineJson/Expectations"; /** * Counts number of characters in the text @@ -6,13 +6,16 @@ import type { ExpectationAmount } from '../../pipeline/PipelineJson/Expectations * @public exported from `@promptbook/utils` */ export function countCharacters(text: string): ExpectationAmount { - // Remove null characters - text = text.replace(/\0/g, ''); + // Remove null characters + text = text.replace(/\0/g, ""); - // Replace emojis (and also ZWJ sequence) with hyphens - text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1'); - text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1'); - text = text.replace(/\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, '-'); + // Replace emojis (and also ZWJ sequence) with hyphens + text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, "$1"); + text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, "$1"); + text = text.replace( + /\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, + "-", + ); - return text.length; + return text.length; } diff --git a/src/utils/expectation-counters/countLines.test.ts b/src/utils/expectation-counters/countLines.test.ts index 2a471ae251..a0a74a71c5 100644 --- a/src/utils/expectation-counters/countLines.test.ts +++ b/src/utils/expectation-counters/countLines.test.ts @@ -1,19 +1,19 @@ -import { describe, expect, it } from '@jest/globals'; -import { countLines } from './countLines'; +import { describe, expect, it } from "@jest/globals"; +import { countLines } from "./countLines"; -describe('countLines', () => { - it('should return 0 for an empty string', () => { - expect(countLines('')).toBe(0); - }); +describe("countLines", () => { + it("should return 0 for an empty string", () => { + expect(countLines("")).toBe(0); + }); - it('should return the correct count for a string with one line', () => { - expect(countLines(' ')).toBe(1); - expect(countLines(' \t')).toBe(1); - expect(countLines('Hello')).toBe(1); - }); + it("should return the correct count for a string with one line", () => { + expect(countLines(" ")).toBe(1); + expect(countLines(" \t")).toBe(1); + expect(countLines("Hello")).toBe(1); + }); - it('should return the correct count for a string with multiple lines', () => { - expect(countLines('Hello\nworld')).toBe(2); - expect(countLines('Hello\nworld\n!')).toBe(3); - }); + it("should return the correct count for a string with multiple lines", () => { + expect(countLines("Hello\nworld")).toBe(2); + expect(countLines("Hello\nworld\n!")).toBe(3); + }); }); diff --git a/src/utils/expectation-counters/countLines.ts b/src/utils/expectation-counters/countLines.ts index 4b9c21009c..a7ad26f865 100644 --- a/src/utils/expectation-counters/countLines.ts +++ b/src/utils/expectation-counters/countLines.ts @@ -1,5 +1,5 @@ -import type { ExpectationAmount } from '../../pipeline/PipelineJson/Expectations'; -import { CHARACTERS_PER_STANDARD_LINE } from './constants'; +import type { ExpectationAmount } from "../../pipeline/PipelineJson/Expectations"; +import { CHARACTERS_PER_STANDARD_LINE } from "./constants"; /** * Counts number of lines in the text @@ -9,10 +9,14 @@ import { CHARACTERS_PER_STANDARD_LINE } from './constants'; * @public exported from `@promptbook/utils` */ export function countLines(text: string): ExpectationAmount { - text = text.replace('\r\n', '\n'); - text = text.replace('\r', '\n'); + text = text.replace("\r\n", "\n"); + text = text.replace("\r", "\n"); - const lines = text.split('\n'); + const lines = text.split("\n"); - return lines.reduce((count, line) => count + Math.ceil(line.length / CHARACTERS_PER_STANDARD_LINE), 0); + return lines.reduce( + (count, line) => + count + Math.ceil(line.length / CHARACTERS_PER_STANDARD_LINE), + 0, + ); } diff --git a/src/utils/expectation-counters/countPages.test.ts b/src/utils/expectation-counters/countPages.test.ts index f3077f587e..ceefb11072 100644 --- a/src/utils/expectation-counters/countPages.test.ts +++ b/src/utils/expectation-counters/countPages.test.ts @@ -1,11 +1,11 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { countPages } from './countPages'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { countPages } from "./countPages"; -describe('countPages', () => { - const ONE_PAGE_OF_CHARACTERS = - 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'; - const ONE_PAGE_OF_LINES = spaceTrim(` +describe("countPages", () => { + const ONE_PAGE_OF_CHARACTERS = + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + const ONE_PAGE_OF_LINES = spaceTrim(` Line Line Line @@ -52,40 +52,43 @@ describe('countPages', () => { Line `); - it('should return zero pages', () => { - expect(countPages('')).toBe(0); - }); + it("should return zero pages", () => { + expect(countPages("")).toBe(0); + }); - it('should return single page', () => { - expect(countPages('-')).toBe(1); - expect(countPages('Page 1')).toBe(1); - expect( - countPages( - spaceTrim(` + it("should return single page", () => { + expect(countPages("-")).toBe(1); + expect(countPages("Page 1")).toBe(1); + expect( + countPages( + spaceTrim(` Page Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris. Maecenas congue ligula ac quam viverra nec consectetur ante hendrerit. Donec et mollis dolor. Praesent et diam eget libero egestas mattis sit amet vitae augue. Nam tincidunt congue enim, ut porta lorem lacinia consectetur. `), - ), - ).toBe(1); - expect(countPages(ONE_PAGE_OF_LINES)).toBe(1); - expect(countPages(ONE_PAGE_OF_CHARACTERS)).toBe(1); - expect(countPages(ONE_PAGE_OF_LINES + '-')).toBe(1); - expect(countPages(ONE_PAGE_OF_LINES + 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA')).toBe( - 2, - ); - expect(countPages(ONE_PAGE_OF_LINES + 'a'.repeat(150))).toBe(2); - }); + ), + ).toBe(1); + expect(countPages(ONE_PAGE_OF_LINES)).toBe(1); + expect(countPages(ONE_PAGE_OF_CHARACTERS)).toBe(1); + expect(countPages(ONE_PAGE_OF_LINES + "-")).toBe(1); + expect( + countPages( + ONE_PAGE_OF_LINES + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ), + ).toBe(2); + expect(countPages(ONE_PAGE_OF_LINES + "a".repeat(150))).toBe(2); + }); - it('should return two pages', () => { - expect(countPages(ONE_PAGE_OF_LINES + '\n' + '-')).toBe(2); - expect(countPages(ONE_PAGE_OF_CHARACTERS + '-')).toBe(2); - expect( - countPages( - spaceTrim( - (block) => ` + it("should return two pages", () => { + expect(countPages(ONE_PAGE_OF_LINES + "\n" + "-")).toBe(2); + expect(countPages(ONE_PAGE_OF_CHARACTERS + "-")).toBe(2); + expect( + countPages( + spaceTrim( + (block) => ` Page Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris. Maecenas congue ligula ac quam viverra nec consectetur ante hendrerit. Donec et mollis dolor. @@ -95,30 +98,36 @@ describe('countPages', () => { ${block(ONE_PAGE_OF_CHARACTERS)} `, - ), - ), - ).toBe(2); - expect(countPages(ONE_PAGE_OF_LINES + ONE_PAGE_OF_CHARACTERS)).toBe(2); - expect(countPages(ONE_PAGE_OF_CHARACTERS + ONE_PAGE_OF_CHARACTERS)).toBe(2); - }); + ), + ), + ).toBe(2); + expect(countPages(ONE_PAGE_OF_LINES + ONE_PAGE_OF_CHARACTERS)).toBe(2); + expect(countPages(ONE_PAGE_OF_CHARACTERS + ONE_PAGE_OF_CHARACTERS)).toBe(2); + }); - it('should return multiple pages', () => { - expect(countPages(ONE_PAGE_OF_LINES + ONE_PAGE_OF_CHARACTERS + ONE_PAGE_OF_CHARACTERS)).toBe(3); - expect(countPages(ONE_PAGE_OF_CHARACTERS + ONE_PAGE_OF_CHARACTERS + '-')).toBe(3); + it("should return multiple pages", () => { + expect( + countPages( + ONE_PAGE_OF_LINES + ONE_PAGE_OF_CHARACTERS + ONE_PAGE_OF_CHARACTERS, + ), + ).toBe(3); + expect( + countPages(ONE_PAGE_OF_CHARACTERS + ONE_PAGE_OF_CHARACTERS + "-"), + ).toBe(3); - expect( - countPages( - ONE_PAGE_OF_CHARACTERS + - ONE_PAGE_OF_LINES + - ONE_PAGE_OF_CHARACTERS + - ONE_PAGE_OF_LINES + - ONE_PAGE_OF_CHARACTERS + - ONE_PAGE_OF_LINES + - ONE_PAGE_OF_CHARACTERS + - ONE_PAGE_OF_LINES + - ONE_PAGE_OF_CHARACTERS + - ONE_PAGE_OF_LINES, - ), - ).toBe(10); - }); + expect( + countPages( + ONE_PAGE_OF_CHARACTERS + + ONE_PAGE_OF_LINES + + ONE_PAGE_OF_CHARACTERS + + ONE_PAGE_OF_LINES + + ONE_PAGE_OF_CHARACTERS + + ONE_PAGE_OF_LINES + + ONE_PAGE_OF_CHARACTERS + + ONE_PAGE_OF_LINES + + ONE_PAGE_OF_CHARACTERS + + ONE_PAGE_OF_LINES, + ), + ).toBe(10); + }); }); diff --git a/src/utils/expectation-counters/countPages.ts b/src/utils/expectation-counters/countPages.ts index 7219bd3130..306162d642 100644 --- a/src/utils/expectation-counters/countPages.ts +++ b/src/utils/expectation-counters/countPages.ts @@ -1,6 +1,6 @@ -import type { ExpectationAmount } from '../../pipeline/PipelineJson/Expectations'; -import { LINES_PER_STANDARD_PAGE } from './constants'; -import { countLines } from './countLines'; +import type { ExpectationAmount } from "../../pipeline/PipelineJson/Expectations"; +import { LINES_PER_STANDARD_PAGE } from "./constants"; +import { countLines } from "./countLines"; /** * Counts number of pages in the text @@ -10,5 +10,5 @@ import { countLines } from './countLines'; * @public exported from `@promptbook/utils` */ export function countPages(text: string): ExpectationAmount { - return Math.ceil(countLines(text) / LINES_PER_STANDARD_PAGE); + return Math.ceil(countLines(text) / LINES_PER_STANDARD_PAGE); } diff --git a/src/utils/expectation-counters/countParagraphs.test.ts b/src/utils/expectation-counters/countParagraphs.test.ts index c064a2d102..8ec6634a4c 100644 --- a/src/utils/expectation-counters/countParagraphs.test.ts +++ b/src/utils/expectation-counters/countParagraphs.test.ts @@ -1,20 +1,22 @@ -import { describe, expect, it } from '@jest/globals'; -import { countParagraphs } from './countParagraphs'; +import { describe, expect, it } from "@jest/globals"; +import { countParagraphs } from "./countParagraphs"; -describe('countParagraphs', () => { - it('should return 0 for an empty string', () => { - expect(countParagraphs('')).toBe(0); - }); +describe("countParagraphs", () => { + it("should return 0 for an empty string", () => { + expect(countParagraphs("")).toBe(0); + }); - it('should return the correct count for a string with one paragraph', () => { - expect(countParagraphs('This is a paragraph.')).toBe(1); - }); + it("should return the correct count for a string with one paragraph", () => { + expect(countParagraphs("This is a paragraph.")).toBe(1); + }); - it('should return the correct count for a string with multiple paragraphs', () => { - expect(countParagraphs('This is paragraph 1.\n\nThis is paragraph 2.')).toBe(2); - }); + it("should return the correct count for a string with multiple paragraphs", () => { + expect( + countParagraphs("This is paragraph 1.\n\nThis is paragraph 2."), + ).toBe(2); + }); - it('should return the correct count for a string with leading/trailing whitespace', () => { - expect(countParagraphs(' This is a paragraph. ')).toBe(1); - }); + it("should return the correct count for a string with leading/trailing whitespace", () => { + expect(countParagraphs(" This is a paragraph. ")).toBe(1); + }); }); diff --git a/src/utils/expectation-counters/countParagraphs.ts b/src/utils/expectation-counters/countParagraphs.ts index 4168c77144..e09e8e4ac3 100644 --- a/src/utils/expectation-counters/countParagraphs.ts +++ b/src/utils/expectation-counters/countParagraphs.ts @@ -1,4 +1,4 @@ -import type { ExpectationAmount } from '../../pipeline/PipelineJson/Expectations'; +import type { ExpectationAmount } from "../../pipeline/PipelineJson/Expectations"; /** * Counts number of paragraphs in the text @@ -6,5 +6,6 @@ import type { ExpectationAmount } from '../../pipeline/PipelineJson/Expectations * @public exported from `@promptbook/utils` */ export function countParagraphs(text: string): ExpectationAmount { - return text.split(/\n\s*\n/).filter((paragraph) => paragraph.trim() !== '').length; + return text.split(/\n\s*\n/).filter((paragraph) => paragraph.trim() !== "") + .length; } diff --git a/src/utils/expectation-counters/countSentences.test.ts b/src/utils/expectation-counters/countSentences.test.ts index b176256154..8d64a1efa1 100644 --- a/src/utils/expectation-counters/countSentences.test.ts +++ b/src/utils/expectation-counters/countSentences.test.ts @@ -1,31 +1,37 @@ -import { describe, expect, it } from '@jest/globals'; -import { countSentences } from './countSentences'; +import { describe, expect, it } from "@jest/globals"; +import { countSentences } from "./countSentences"; -describe('countSentences', () => { - it('should return 0 for an empty string', () => { - expect(countSentences('')).toBe(0); - expect(countSentences(' ')).toBe(0); - expect(countSentences('\n\n')).toBe(0); - expect(countSentences(' \n \n')).toBe(0); - }); +describe("countSentences", () => { + it("should return 0 for an empty string", () => { + expect(countSentences("")).toBe(0); + expect(countSentences(" ")).toBe(0); + expect(countSentences("\n\n")).toBe(0); + expect(countSentences(" \n \n")).toBe(0); + }); - it('should return the correct count for a string with one sentence', () => { - expect(countSentences('Hello.')).toBe(1); - expect(countSentences('This is a sentence.')).toBe(1); - }); + it("should return the correct count for a string with one sentence", () => { + expect(countSentences("Hello.")).toBe(1); + expect(countSentences("This is a sentence.")).toBe(1); + }); - it('should return the correct count for a string with multiple sentences', () => { - expect(countSentences('Hello. How are you?')).toBe(2); - expect(countSentences('This is a sentence. This is another sentence.')).toBe(2); - }); + it("should return the correct count for a string with multiple sentences", () => { + expect(countSentences("Hello. How are you?")).toBe(2); + expect( + countSentences("This is a sentence. This is another sentence."), + ).toBe(2); + }); - it('should return the correct count for a string with special characters', () => { - expect(countSentences('Hello! How are you?')).toBe(2); - expect(countSentences('This is a sentence. This is another sentence!')).toBe(2); - }); + it("should return the correct count for a string with special characters", () => { + expect(countSentences("Hello! How are you?")).toBe(2); + expect( + countSentences("This is a sentence. This is another sentence!"), + ).toBe(2); + }); - it('should return the correct count for a string with whitespace characters', () => { - expect(countSentences('Hello world. How are you?')).toBe(2); - expect(countSentences('This is a sentence.\nThis is another sentence.')).toBe(2); - }); + it("should return the correct count for a string with whitespace characters", () => { + expect(countSentences("Hello world. How are you?")).toBe(2); + expect( + countSentences("This is a sentence.\nThis is another sentence."), + ).toBe(2); + }); }); diff --git a/src/utils/expectation-counters/countSentences.ts b/src/utils/expectation-counters/countSentences.ts index 9914bbce72..d4fc2aede6 100644 --- a/src/utils/expectation-counters/countSentences.ts +++ b/src/utils/expectation-counters/countSentences.ts @@ -1,4 +1,4 @@ -import type { ExpectationAmount } from '../../pipeline/PipelineJson/Expectations'; +import type { ExpectationAmount } from "../../pipeline/PipelineJson/Expectations"; /** * Split text into sentences @@ -6,7 +6,7 @@ import type { ExpectationAmount } from '../../pipeline/PipelineJson/Expectations * @public exported from `@promptbook/utils` */ export function splitIntoSentences(text: string): ReadonlyArray { - return text.split(/[.!?]+/).filter((sentence) => sentence.trim() !== ''); + return text.split(/[.!?]+/).filter((sentence) => sentence.trim() !== ""); } /** @@ -15,5 +15,5 @@ export function splitIntoSentences(text: string): ReadonlyArray { * @public exported from `@promptbook/utils` */ export function countSentences(text: string): ExpectationAmount { - return splitIntoSentences(text).length; + return splitIntoSentences(text).length; } diff --git a/src/utils/expectation-counters/countWords.test.ts b/src/utils/expectation-counters/countWords.test.ts index 8ac061fb09..b8101e3763 100644 --- a/src/utils/expectation-counters/countWords.test.ts +++ b/src/utils/expectation-counters/countWords.test.ts @@ -1,68 +1,68 @@ -import { describe, expect, it } from '@jest/globals'; -import { countWords } from './countWords'; +import { describe, expect, it } from "@jest/globals"; +import { countWords } from "./countWords"; -describe('countWords', () => { - it('should return 0 for an empty string', () => { - expect(countWords('')).toBe(0); - expect(countWords(' ')).toBe(0); - expect(countWords(' ')).toBe(0); - expect(countWords(' \n \n ')).toBe(0); - expect(countWords(' \r\n \r\n ')).toBe(0); - expect(countWords(' \n \n \t\t')).toBe(0); - expect(countWords('\0')).toBe(0); - }); +describe("countWords", () => { + it("should return 0 for an empty string", () => { + expect(countWords("")).toBe(0); + expect(countWords(" ")).toBe(0); + expect(countWords(" ")).toBe(0); + expect(countWords(" \n \n ")).toBe(0); + expect(countWords(" \r\n \r\n ")).toBe(0); + expect(countWords(" \n \n \t\t")).toBe(0); + expect(countWords("\0")).toBe(0); + }); - it('should return the correct count for a string with one word', () => { - expect(countWords('Hello')).toBe(1); - expect(countWords('World')).toBe(1); - expect(countWords('Hello.')).toBe(1); - expect(countWords('Hello!')).toBe(1); - expect(countWords('Hello!' + '!' + '!' + '!' + '!' + '!')).toBe(1); - expect(countWords('Hello!?')).toBe(1); - }); + it("should return the correct count for a string with one word", () => { + expect(countWords("Hello")).toBe(1); + expect(countWords("World")).toBe(1); + expect(countWords("Hello.")).toBe(1); + expect(countWords("Hello!")).toBe(1); + expect(countWords("Hello!" + "!" + "!" + "!" + "!" + "!")).toBe(1); + expect(countWords("Hello!?")).toBe(1); + }); - it('should return the correct count for a string with multiple words', () => { - expect(countWords('Hello World')).toBe(2); - expect(countWords('Hello World')).toBe(2); - expect(countWords('Count the words')).toBe(3); - }); + it("should return the correct count for a string with multiple words", () => { + expect(countWords("Hello World")).toBe(2); + expect(countWords("Hello World")).toBe(2); + expect(countWords("Count the words")).toBe(3); + }); - it('should work with numbers as words', () => { - expect(countWords('I have 6 functions')).toBe(4); - expect(countWords('My password is dsfgfwsedgt3asadsd58asd')).toBe(4); - }); + it("should work with numbers as words", () => { + expect(countWords("I have 6 functions")).toBe(4); + expect(countWords("My password is dsfgfwsedgt3asadsd58asd")).toBe(4); + }); - it('should work with diacritics', () => { - expect(countWords('My name is Pavol Hejný and I am a developer')).toBe(10); - expect(countWords('My name is Ján Pišta and I am a developer')).toBe(10); - }); + it("should work with diacritics", () => { + expect(countWords("My name is Pavol Hejný and I am a developer")).toBe(10); + expect(countWords("My name is Ján Pišta and I am a developer")).toBe(10); + }); - it('should work with cyrilic', () => { - expect(countWords('Меня зовут Павел Хейный и я разработчик')).toBe(7); - }); + it("should work with cyrilic", () => { + expect(countWords("Меня зовут Павел Хейный и я разработчик")).toBe(7); + }); - it('should work with emojis', () => { - expect(countWords('I ❤️ emojis')).toBe(3); - expect(countWords('1 l⭕ve emojis')).toBe(3); - }); + it("should work with emojis", () => { + expect(countWords("I ❤️ emojis")).toBe(3); + expect(countWords("1 l⭕ve emojis")).toBe(3); + }); - it('should ignore leading and trailing whitespace', () => { - expect(countWords(' Hello ')).toBe(1); - expect(countWords(' Count the words ')).toBe(3); - }); + it("should ignore leading and trailing whitespace", () => { + expect(countWords(" Hello ")).toBe(1); + expect(countWords(" Count the words ")).toBe(3); + }); - it('should handle special characters and punctuation', () => { - expect(countWords('Hello, World!')).toBe(2); - expect(countWords('Count the words...')).toBe(3); - expect(countWords('Case-sensitive')).toBe(2); - expect(countWords('Case+sensitive')).toBe(2); - expect(countWords('Case–⁠⁠⁠⁠⁠sensitive')).toBe(2); - expect(countWords('Case⁠⁠⁠⁠.sensitive')).toBe(2); - expect(countWords('Case⁠⁠⁠⁠~sensitive')).toBe(2); - }); + it("should handle special characters and punctuation", () => { + expect(countWords("Hello, World!")).toBe(2); + expect(countWords("Count the words...")).toBe(3); + expect(countWords("Case-sensitive")).toBe(2); + expect(countWords("Case+sensitive")).toBe(2); + expect(countWords("Case–⁠⁠⁠⁠⁠sensitive")).toBe(2); + expect(countWords("Case⁠⁠⁠⁠.sensitive")).toBe(2); + expect(countWords("Case⁠⁠⁠⁠~sensitive")).toBe(2); + }); - it('should handle newlines and tabs', () => { - expect(countWords('Hello\nWorld')).toBe(2); - expect(countWords('Count\tthe\twords')).toBe(3); - }); + it("should handle newlines and tabs", () => { + expect(countWords("Hello\nWorld")).toBe(2); + expect(countWords("Count\tthe\twords")).toBe(3); + }); }); diff --git a/src/utils/expectation-counters/countWords.ts b/src/utils/expectation-counters/countWords.ts index 08aff2d9b4..b373a3428c 100644 --- a/src/utils/expectation-counters/countWords.ts +++ b/src/utils/expectation-counters/countWords.ts @@ -1,5 +1,5 @@ -import type { ExpectationAmount } from '../../pipeline/PipelineJson/Expectations'; -import { removeDiacritics } from '../normalization/removeDiacritics'; +import type { ExpectationAmount } from "../../pipeline/PipelineJson/Expectations"; +import { removeDiacritics } from "../normalization/removeDiacritics"; /** * Counts number of words in the text @@ -7,8 +7,8 @@ import { removeDiacritics } from '../normalization/removeDiacritics'; * @public exported from `@promptbook/utils` */ export function countWords(text: string): ExpectationAmount { - text = text.replace(/[\p{Extended_Pictographic}]/gu, 'a'); - text = removeDiacritics(text); + text = text.replace(/[\p{Extended_Pictographic}]/gu, "a"); + text = removeDiacritics(text); - return text.split(/[^a-zа-я0-9]+/i).filter((word) => word.length > 0).length; + return text.split(/[^a-zа-я0-9]+/i).filter((word) => word.length > 0).length; } diff --git a/src/utils/expectation-counters/index.ts b/src/utils/expectation-counters/index.ts index 5e3761c4e6..078835df81 100644 --- a/src/utils/expectation-counters/index.ts +++ b/src/utils/expectation-counters/index.ts @@ -1,24 +1,27 @@ -import type { ExpectationAmount } from '../../pipeline/PipelineJson/Expectations'; -import type { ExpectationUnit } from '../../pipeline/PipelineJson/Expectations'; -import { countCharacters } from './countCharacters'; -import { countLines } from './countLines'; -import { countPages } from './countPages'; -import { countParagraphs } from './countParagraphs'; -import { countSentences } from './countSentences'; -import { countWords } from './countWords'; +import type { ExpectationAmount } from "../../pipeline/PipelineJson/Expectations"; +import type { ExpectationUnit } from "../../pipeline/PipelineJson/Expectations"; +import { countCharacters } from "./countCharacters"; +import { countLines } from "./countLines"; +import { countPages } from "./countPages"; +import { countParagraphs } from "./countParagraphs"; +import { countSentences } from "./countSentences"; +import { countWords } from "./countWords"; /** * Index of all counter functions * * @public exported from `@promptbook/utils` */ -export const CountUtils: Record ExpectationAmount> = { - CHARACTERS: countCharacters, - WORDS: countWords, - SENTENCES: countSentences, - PARAGRAPHS: countParagraphs, - LINES: countLines, - PAGES: countPages, +export const CountUtils: Record< + ExpectationUnit, + (text: string) => ExpectationAmount +> = { + CHARACTERS: countCharacters, + WORDS: countWords, + SENTENCES: countSentences, + PARAGRAPHS: countParagraphs, + LINES: countLines, + PAGES: countPages, }; /** diff --git a/src/utils/files/extensionToMimeType.test.ts b/src/utils/files/extensionToMimeType.test.ts index e0233c8800..71bf6bce83 100644 --- a/src/utils/files/extensionToMimeType.test.ts +++ b/src/utils/files/extensionToMimeType.test.ts @@ -1,31 +1,31 @@ -import { describe, expect, it } from '@jest/globals'; -import { extensionToMimeType } from './extensionToMimeType'; +import { describe, expect, it } from "@jest/globals"; +import { extensionToMimeType } from "./extensionToMimeType"; -describe('how `extensionToMimeType` works', () => { - it('should work with various exemples', () => { - expect(extensionToMimeType('exe')).toBe('application/x-msdos-program'); - expect(extensionToMimeType('pdf')).toBe('application/pdf'); - expect(extensionToMimeType('docx')).toBe( - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - ); - expect(extensionToMimeType('doc')).toBe('application/msword'); - expect(extensionToMimeType('jpeg')).toBe('image/jpeg'); - expect(extensionToMimeType('jpg')).toBe('image/jpeg'); - expect(extensionToMimeType('png')).toBe('image/png'); - expect(extensionToMimeType('gif')).toBe('image/gif'); - expect(extensionToMimeType('bmp')).toBe('image/bmp'); - expect(extensionToMimeType('svg')).toBe('image/svg+xml'); - expect(extensionToMimeType('html')).toBe('text/html'); - expect(extensionToMimeType('css')).toBe('text/css'); - expect(extensionToMimeType('js')).toBe('application/javascript'); - expect(extensionToMimeType('json')).toBe('application/json'); - expect(extensionToMimeType('xml')).toBe('application/xml'); - expect(extensionToMimeType('csv')).toBe('text/csv'); - }); +describe("how `extensionToMimeType` works", () => { + it("should work with various exemples", () => { + expect(extensionToMimeType("exe")).toBe("application/x-msdos-program"); + expect(extensionToMimeType("pdf")).toBe("application/pdf"); + expect(extensionToMimeType("docx")).toBe( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ); + expect(extensionToMimeType("doc")).toBe("application/msword"); + expect(extensionToMimeType("jpeg")).toBe("image/jpeg"); + expect(extensionToMimeType("jpg")).toBe("image/jpeg"); + expect(extensionToMimeType("png")).toBe("image/png"); + expect(extensionToMimeType("gif")).toBe("image/gif"); + expect(extensionToMimeType("bmp")).toBe("image/bmp"); + expect(extensionToMimeType("svg")).toBe("image/svg+xml"); + expect(extensionToMimeType("html")).toBe("text/html"); + expect(extensionToMimeType("css")).toBe("text/css"); + expect(extensionToMimeType("js")).toBe("application/javascript"); + expect(extensionToMimeType("json")).toBe("application/json"); + expect(extensionToMimeType("xml")).toBe("application/xml"); + expect(extensionToMimeType("csv")).toBe("text/csv"); + }); - it('should work with weird cases', () => { - expect(extensionToMimeType('')).toBe('application/octet-stream'); - expect(extensionToMimeType('document')).toBe('application/octet-stream'); - expect(extensionToMimeType('documentpdf')).toBe('application/octet-stream'); - }); + it("should work with weird cases", () => { + expect(extensionToMimeType("")).toBe("application/octet-stream"); + expect(extensionToMimeType("document")).toBe("application/octet-stream"); + expect(extensionToMimeType("documentpdf")).toBe("application/octet-stream"); + }); }); diff --git a/src/utils/files/extensionToMimeType.ts b/src/utils/files/extensionToMimeType.ts index f7130aef4f..ce3f981bf0 100644 --- a/src/utils/files/extensionToMimeType.ts +++ b/src/utils/files/extensionToMimeType.ts @@ -1,12 +1,14 @@ -import { lookup } from 'mime-types'; -import type { string_file_extension } from '../../types/typeAliases'; -import type { string_mime_type } from '../../types/typeAliases'; +import { lookup } from "mime-types"; +import type { string_file_extension } from "../../types/typeAliases"; +import type { string_mime_type } from "../../types/typeAliases"; /** * Convert file extension to mime type * * @private within the repository */ -export function extensionToMimeType(value: string_file_extension): string_mime_type { - return lookup(value) || 'application/octet-stream'; +export function extensionToMimeType( + value: string_file_extension, +): string_mime_type { + return lookup(value) || "application/octet-stream"; } diff --git a/src/utils/files/getFileExtension.test.ts b/src/utils/files/getFileExtension.test.ts index 27c836b9b7..718267bb7b 100644 --- a/src/utils/files/getFileExtension.test.ts +++ b/src/utils/files/getFileExtension.test.ts @@ -1,22 +1,22 @@ -import { describe, expect, it } from '@jest/globals'; -import { getFileExtension } from './getFileExtension'; +import { describe, expect, it } from "@jest/globals"; +import { getFileExtension } from "./getFileExtension"; -describe('how `getFileExtension` works', () => { - it('should work with various exemples', () => { - expect(getFileExtension('file.png')).toBe('png'); - expect(getFileExtension('file.jpeg')).toBe('jpeg'); - expect(getFileExtension('foo/bar/file.jpg')).toBe('jpg'); - expect(getFileExtension('C://foo/bar/document.docx')).toBe('docx'); - expect(getFileExtension('C://foo/bar/document.docx.pdf')).toBe('pdf'); - expect(getFileExtension('C://foo/bar/document.jpeg.exe')).toBe('exe'); - expect(getFileExtension('/foo/bar/document.pdf')).toBe('pdf'); - expect(getFileExtension('X://foo\\bar\\virus.exe')).toBe('exe'); - }); +describe("how `getFileExtension` works", () => { + it("should work with various exemples", () => { + expect(getFileExtension("file.png")).toBe("png"); + expect(getFileExtension("file.jpeg")).toBe("jpeg"); + expect(getFileExtension("foo/bar/file.jpg")).toBe("jpg"); + expect(getFileExtension("C://foo/bar/document.docx")).toBe("docx"); + expect(getFileExtension("C://foo/bar/document.docx.pdf")).toBe("pdf"); + expect(getFileExtension("C://foo/bar/document.jpeg.exe")).toBe("exe"); + expect(getFileExtension("/foo/bar/document.pdf")).toBe("pdf"); + expect(getFileExtension("X://foo\\bar\\virus.exe")).toBe("exe"); + }); - it('should work with weird cases', () => { - expect(getFileExtension('')).toBe(null); - expect(getFileExtension('document')).toBe(null); - expect(getFileExtension('/foo/bar/documentpdf')).toBe(null); - expect(getFileExtension('C://foo/bar/document')).toBe(null); - }); + it("should work with weird cases", () => { + expect(getFileExtension("")).toBe(null); + expect(getFileExtension("document")).toBe(null); + expect(getFileExtension("/foo/bar/documentpdf")).toBe(null); + expect(getFileExtension("C://foo/bar/document")).toBe(null); + }); }); diff --git a/src/utils/files/getFileExtension.ts b/src/utils/files/getFileExtension.ts index ab94a85170..1b6b404f5e 100644 --- a/src/utils/files/getFileExtension.ts +++ b/src/utils/files/getFileExtension.ts @@ -1,12 +1,14 @@ -import type { string_filename } from '../../types/typeAliases'; -import type { string_mime_type } from '../../types/typeAliases'; +import type { string_filename } from "../../types/typeAliases"; +import type { string_mime_type } from "../../types/typeAliases"; /** * Get the file extension from a file name * * @private within the repository */ -export function getFileExtension(value: string_filename): string_mime_type | null { - const match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i); - return match ? match[1]!.toLowerCase() : null; +export function getFileExtension( + value: string_filename, +): string_mime_type | null { + const match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i); + return match ? match[1]!.toLowerCase() : null; } diff --git a/src/utils/files/isDirectoryExisting.test.ts b/src/utils/files/isDirectoryExisting.test.ts index 83a44aa140..ccb4d3ba54 100644 --- a/src/utils/files/isDirectoryExisting.test.ts +++ b/src/utils/files/isDirectoryExisting.test.ts @@ -1,20 +1,29 @@ -import { describe, expect, it } from '@jest/globals'; -import { $provideFilesystemForNode } from '../../scrapers/_common/register/$provideFilesystemForNode'; -import { isDirectoryExisting } from './isDirectoryExisting'; +import { describe, expect, it } from "@jest/globals"; +import { $provideFilesystemForNode } from "../../scrapers/_common/register/$provideFilesystemForNode"; +import { isDirectoryExisting } from "./isDirectoryExisting"; -describe('how isDirectoryExisting works', () => { - it('works with existing directories', () => - expect(isDirectoryExisting('src/utils/files/existing-directory', $provideFilesystemForNode())).resolves.toBe( - true, - )); +describe("how isDirectoryExisting works", () => { + it("works with existing directories", () => + expect( + isDirectoryExisting( + "src/utils/files/existing-directory", + $provideFilesystemForNode(), + ), + ).resolves.toBe(true)); - it('works with non-existing directories', () => - expect(isDirectoryExisting('src/utils/files/non-existing-director', $provideFilesystemForNode())).resolves.toBe( - false, - )); + it("works with non-existing directories", () => + expect( + isDirectoryExisting( + "src/utils/files/non-existing-director", + $provideFilesystemForNode(), + ), + ).resolves.toBe(false)); - it('is not confused by file (which is not a directory)', () => - expect( - isDirectoryExisting('src/utils/files/existing-directory/existing-file.txt', $provideFilesystemForNode()), - ).resolves.toBe(false)); + it("is not confused by file (which is not a directory)", () => + expect( + isDirectoryExisting( + "src/utils/files/existing-directory/existing-file.txt", + $provideFilesystemForNode(), + ), + ).resolves.toBe(false)); }); diff --git a/src/utils/files/isDirectoryExisting.ts b/src/utils/files/isDirectoryExisting.ts index 7cd9cf8998..c5307b7a00 100644 --- a/src/utils/files/isDirectoryExisting.ts +++ b/src/utils/files/isDirectoryExisting.ts @@ -1,27 +1,30 @@ -import type { FilesystemTools } from '../../execution/FilesystemTools'; -import type { string_dirname } from '../../types/typeAliases'; +import type { FilesystemTools } from "../../execution/FilesystemTools"; +import type { string_dirname } from "../../types/typeAliases"; /** * Checks if the directory exists * * @private within the repository */ -export async function isDirectoryExisting(directoryPath: string_dirname, fs: FilesystemTools): Promise { - const isReadAccessAllowed = await fs - .access(directoryPath, fs.constants.R_OK) - .then(() => true) - .catch(() => false); +export async function isDirectoryExisting( + directoryPath: string_dirname, + fs: FilesystemTools, +): Promise { + const isReadAccessAllowed = await fs + .access(directoryPath, fs.constants.R_OK) + .then(() => true) + .catch(() => false); - if (!isReadAccessAllowed) { - return false; - } + if (!isReadAccessAllowed) { + return false; + } - const isDirectory = await fs - .stat(directoryPath) - .then((fileStat) => fileStat.isDirectory()) - .catch(() => false); + const isDirectory = await fs + .stat(directoryPath) + .then((fileStat) => fileStat.isDirectory()) + .catch(() => false); - return isDirectory; + return isDirectory; } /** diff --git a/src/utils/files/isExecutable.ts b/src/utils/files/isExecutable.ts index 77126a83ed..d513cd1da9 100644 --- a/src/utils/files/isExecutable.ts +++ b/src/utils/files/isExecutable.ts @@ -1,17 +1,20 @@ -import type { FilesystemTools } from '../../execution/FilesystemTools'; +import type { FilesystemTools } from "../../execution/FilesystemTools"; /** * Checks if the file is executable * * @private within the repository */ -export async function isExecutable(path: string, fs: FilesystemTools): Promise { - try { - await fs.access(path, fs.constants.X_OK); - return true; - } catch (error) { - return false; - } +export async function isExecutable( + path: string, + fs: FilesystemTools, +): Promise { + try { + await fs.access(path, fs.constants.X_OK); + return true; + } catch (error) { + return false; + } } /** diff --git a/src/utils/files/isFileExisting.test.ts b/src/utils/files/isFileExisting.test.ts index 6145e020bb..5288fdb793 100644 --- a/src/utils/files/isFileExisting.test.ts +++ b/src/utils/files/isFileExisting.test.ts @@ -1,18 +1,29 @@ -import { describe, expect, it } from '@jest/globals'; -import { $provideFilesystemForNode } from '../../scrapers/_common/register/$provideFilesystemForNode'; -import { isFileExisting } from './isFileExisting'; +import { describe, expect, it } from "@jest/globals"; +import { $provideFilesystemForNode } from "../../scrapers/_common/register/$provideFilesystemForNode"; +import { isFileExisting } from "./isFileExisting"; -describe('how isFileExisting works', () => { - it('works with existing files', () => - expect( - isFileExisting('src/utils/files/existing-directory/existing-file.txt', $provideFilesystemForNode()), - ).resolves.toBe(true)); +describe("how isFileExisting works", () => { + it("works with existing files", () => + expect( + isFileExisting( + "src/utils/files/existing-directory/existing-file.txt", + $provideFilesystemForNode(), + ), + ).resolves.toBe(true)); - it('works with non-existing files', () => - expect( - isFileExisting('src/utils/files/non-existing-directory/non-existing-file.txt', $provideFilesystemForNode()), - ).resolves.toBe(false)); + it("works with non-existing files", () => + expect( + isFileExisting( + "src/utils/files/non-existing-directory/non-existing-file.txt", + $provideFilesystemForNode(), + ), + ).resolves.toBe(false)); - it('is not confused by directory (which is not a file)', () => - expect(isFileExisting('src/utils/files/existing-directory', $provideFilesystemForNode())).resolves.toBe(false)); + it("is not confused by directory (which is not a file)", () => + expect( + isFileExisting( + "src/utils/files/existing-directory", + $provideFilesystemForNode(), + ), + ).resolves.toBe(false)); }); diff --git a/src/utils/files/isFileExisting.ts b/src/utils/files/isFileExisting.ts index 024a1baf36..3649a0c0d0 100644 --- a/src/utils/files/isFileExisting.ts +++ b/src/utils/files/isFileExisting.ts @@ -1,27 +1,30 @@ -import type { FilesystemTools } from '../../execution/FilesystemTools'; -import type { string_filename } from '../../types/typeAliases'; +import type { FilesystemTools } from "../../execution/FilesystemTools"; +import type { string_filename } from "../../types/typeAliases"; /** * Checks if the file exists * * @private within the repository */ -export async function isFileExisting(filename: string_filename, fs: FilesystemTools): Promise { - const isReadAccessAllowed = await fs - .access(filename, fs.constants.R_OK) - .then(() => true) - .catch(() => false); +export async function isFileExisting( + filename: string_filename, + fs: FilesystemTools, +): Promise { + const isReadAccessAllowed = await fs + .access(filename, fs.constants.R_OK) + .then(() => true) + .catch(() => false); - if (!isReadAccessAllowed) { - return false; - } + if (!isReadAccessAllowed) { + return false; + } - const isFile = await fs - .stat(filename) - .then((fileStat) => fileStat.isFile()) - .catch(() => false); + const isFile = await fs + .stat(filename) + .then((fileStat) => fileStat.isFile()) + .catch(() => false); - return isFile; + return isFile; } /** diff --git a/src/utils/files/listAllFiles.test.ts b/src/utils/files/listAllFiles.test.ts index 17a6efdba7..9640782970 100644 --- a/src/utils/files/listAllFiles.test.ts +++ b/src/utils/files/listAllFiles.test.ts @@ -1,20 +1,38 @@ -import { describe, expect, it } from '@jest/globals'; -import { $provideFilesystemForNode } from '../../scrapers/_common/register/$provideFilesystemForNode'; -import { listAllFiles } from './listAllFiles'; +import { describe, expect, it } from "@jest/globals"; +import { $provideFilesystemForNode } from "../../scrapers/_common/register/$provideFilesystemForNode"; +import { listAllFiles } from "./listAllFiles"; -describe('how listAllFiles works', () => { - it('lists all files', () => - expect( - listAllFiles('src/utils/files/existing-directory', false, $provideFilesystemForNode()), - ).resolves.toContain('src/utils/files/existing-directory/existing-file.txt')); +describe("how listAllFiles works", () => { + it("lists all files", () => + expect( + listAllFiles( + "src/utils/files/existing-directory", + false, + $provideFilesystemForNode(), + ), + ).resolves.toContain( + "src/utils/files/existing-directory/existing-file.txt", + )); - it('lists nested files in recursive listing', () => - expect( - listAllFiles('src/utils/files/existing-directory', true, $provideFilesystemForNode()), - ).resolves.toContain('src/utils/files/existing-directory/existing-subdirectory/existing-subfile.txt')); + it("lists nested files in recursive listing", () => + expect( + listAllFiles( + "src/utils/files/existing-directory", + true, + $provideFilesystemForNode(), + ), + ).resolves.toContain( + "src/utils/files/existing-directory/existing-subdirectory/existing-subfile.txt", + )); - it('not contain nested file in non-recursive listing', () => - expect( - listAllFiles('src/utils/files/existing-directory', false, $provideFilesystemForNode()), - ).resolves.not.toContain('src/utils/files/existing-directory/existing-subdirectory/existing-subfile.txt')); + it("not contain nested file in non-recursive listing", () => + expect( + listAllFiles( + "src/utils/files/existing-directory", + false, + $provideFilesystemForNode(), + ), + ).resolves.not.toContain( + "src/utils/files/existing-directory/existing-subdirectory/existing-subfile.txt", + )); }); diff --git a/src/utils/files/listAllFiles.ts b/src/utils/files/listAllFiles.ts index 3e74ab696b..8185990b7a 100644 --- a/src/utils/files/listAllFiles.ts +++ b/src/utils/files/listAllFiles.ts @@ -1,8 +1,8 @@ -import { join } from 'path'; -import type { FilesystemTools } from '../../execution/FilesystemTools'; -import type { string_dirname } from '../../types/typeAliases'; -import type { string_filename } from '../../types/typeAliases'; -import { isDirectoryExisting } from './isDirectoryExisting'; +import { join } from "path"; +import type { FilesystemTools } from "../../execution/FilesystemTools"; +import type { string_dirname } from "../../types/typeAliases"; +import type { string_filename } from "../../types/typeAliases"; +import { isDirectoryExisting } from "./isDirectoryExisting"; /** * Reads all files in the directory @@ -13,31 +13,35 @@ import { isDirectoryExisting } from './isDirectoryExisting'; * @private internal function of `createCollectionFromDirectory` */ export async function listAllFiles( - path: string_dirname, - isRecursive: boolean, - fs: FilesystemTools, + path: string_dirname, + isRecursive: boolean, + fs: FilesystemTools, ): Promise> { - if (!(await isDirectoryExisting(path, fs))) { - throw new Error(`Directory "${path}" does not exist or is not readable`); - // <- TODO: Use some custom error class - } + if (!(await isDirectoryExisting(path, fs))) { + throw new Error(`Directory "${path}" does not exist or is not readable`); + // <- TODO: Use some custom error class + } - const dirents = await fs.readdir(path, { - withFileTypes: true /* Note: This is not working: recursive: isRecursive */, - }); + const dirents = await fs.readdir(path, { + withFileTypes: true /* Note: This is not working: recursive: isRecursive */, + }); - const fileNames = dirents - .filter((dirent) => dirent.isFile()) - .map(({ name }) => join(path, name).split('\\').join('/')); + const fileNames = dirents + .filter((dirent) => dirent.isFile()) + .map(({ name }) => join(path, name).split("\\").join("/")); - if (isRecursive) { - for (const dirent of dirents.filter((dirent) => dirent.isDirectory())) { - const subPath = join(path, dirent.name); - fileNames.push(...(await listAllFiles(subPath, isRecursive, fs)).map((filename) => filename)); - } - } + if (isRecursive) { + for (const dirent of dirents.filter((dirent) => dirent.isDirectory())) { + const subPath = join(path, dirent.name); + fileNames.push( + ...(await listAllFiles(subPath, isRecursive, fs)).map( + (filename) => filename, + ), + ); + } + } - return fileNames; + return fileNames; } /** diff --git a/src/utils/files/mimeTypeToExtension.test.ts b/src/utils/files/mimeTypeToExtension.test.ts index 8d105c2947..1fd4399e39 100644 --- a/src/utils/files/mimeTypeToExtension.test.ts +++ b/src/utils/files/mimeTypeToExtension.test.ts @@ -1,30 +1,32 @@ -import { describe, expect, it } from '@jest/globals'; -import { mimeTypeToExtension } from './mimeTypeToExtension'; +import { describe, expect, it } from "@jest/globals"; +import { mimeTypeToExtension } from "./mimeTypeToExtension"; -describe('how `mimeTypeToExtension` works', () => { - it('should work with various examples', () => { - expect(mimeTypeToExtension('application/x-msdos-program')).toBe('exe'); - expect(mimeTypeToExtension('application/pdf')).toBe('pdf'); - expect(mimeTypeToExtension('application/vnd.openxmlformats-officedocument.wordprocessingml.document')).toBe( - 'docx', - ); - expect(mimeTypeToExtension('application/msword')).toBe('doc'); - expect(mimeTypeToExtension('image/jpeg')).toBe('jpeg'); - expect(mimeTypeToExtension('image/png')).toBe('png'); - expect(mimeTypeToExtension('image/gif')).toBe('gif'); - expect(mimeTypeToExtension('image/bmp')).toBe('bmp'); - expect(mimeTypeToExtension('image/svg+xml')).toBe('svg'); - expect(mimeTypeToExtension('text/html')).toBe('html'); - expect(mimeTypeToExtension('text/css')).toBe('css'); - expect(mimeTypeToExtension('application/javascript')).toBe('js'); - expect(mimeTypeToExtension('application/json')).toBe('json'); - expect(mimeTypeToExtension('application/xml')).toBe('xml'); - expect(mimeTypeToExtension('text/csv')).toBe('csv'); - }); +describe("how `mimeTypeToExtension` works", () => { + it("should work with various examples", () => { + expect(mimeTypeToExtension("application/x-msdos-program")).toBe("exe"); + expect(mimeTypeToExtension("application/pdf")).toBe("pdf"); + expect( + mimeTypeToExtension( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ), + ).toBe("docx"); + expect(mimeTypeToExtension("application/msword")).toBe("doc"); + expect(mimeTypeToExtension("image/jpeg")).toBe("jpeg"); + expect(mimeTypeToExtension("image/png")).toBe("png"); + expect(mimeTypeToExtension("image/gif")).toBe("gif"); + expect(mimeTypeToExtension("image/bmp")).toBe("bmp"); + expect(mimeTypeToExtension("image/svg+xml")).toBe("svg"); + expect(mimeTypeToExtension("text/html")).toBe("html"); + expect(mimeTypeToExtension("text/css")).toBe("css"); + expect(mimeTypeToExtension("application/javascript")).toBe("js"); + expect(mimeTypeToExtension("application/json")).toBe("json"); + expect(mimeTypeToExtension("application/xml")).toBe("xml"); + expect(mimeTypeToExtension("text/csv")).toBe("csv"); + }); - it('should work with weird cases', () => { - expect(mimeTypeToExtension('')).toBe(null); - expect(mimeTypeToExtension('invalid/mime-type')).toBe(null); - expect(mimeTypeToExtension('application/octet-stream')).toBe('bin'); - }); + it("should work with weird cases", () => { + expect(mimeTypeToExtension("")).toBe(null); + expect(mimeTypeToExtension("invalid/mime-type")).toBe(null); + expect(mimeTypeToExtension("application/octet-stream")).toBe("bin"); + }); }); diff --git a/src/utils/files/mimeTypeToExtension.ts b/src/utils/files/mimeTypeToExtension.ts index 3383c2734f..ce4f9ee9c8 100644 --- a/src/utils/files/mimeTypeToExtension.ts +++ b/src/utils/files/mimeTypeToExtension.ts @@ -1,6 +1,6 @@ -import { extension } from 'mime-types'; -import type { string_file_extension } from '../../types/typeAliases'; -import type { string_mime_type } from '../../types/typeAliases'; +import { extension } from "mime-types"; +import type { string_file_extension } from "../../types/typeAliases"; +import type { string_mime_type } from "../../types/typeAliases"; /** * Convert mime type to file extension @@ -9,6 +9,8 @@ import type { string_mime_type } from '../../types/typeAliases'; * * @private within the repository */ -export function mimeTypeToExtension(value: string_mime_type): string_file_extension | null { - return extension(value) || null; +export function mimeTypeToExtension( + value: string_mime_type, +): string_file_extension | null { + return extension(value) || null; } diff --git a/src/utils/markdown/addAutoGeneratedSection.test.ts b/src/utils/markdown/addAutoGeneratedSection.test.ts index 1c2a2e72f7..782c446b40 100644 --- a/src/utils/markdown/addAutoGeneratedSection.test.ts +++ b/src/utils/markdown/addAutoGeneratedSection.test.ts @@ -1,13 +1,13 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { just } from '../organization/just'; -import { addAutoGeneratedSection } from './addAutoGeneratedSection'; - -describe('how addAutoGeneratedSection works', () => { - it('should add the section', () => { - expect( - addAutoGeneratedSection( - spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { just } from "../organization/just"; +import { addAutoGeneratedSection } from "./addAutoGeneratedSection"; + +describe("how addAutoGeneratedSection works", () => { + it("should add the section", () => { + expect( + addAutoGeneratedSection( + spaceTrim(` # Foo Blah blah blah @@ -20,14 +20,14 @@ describe('how addAutoGeneratedSection works', () => { Blah blah blah `), - { - sectionName: 'Graph', - sectionContent: 'Bar\nBaz\nFoo', - }, - ), - ).toBe( - just( - spaceTrim(` + { + sectionName: "Graph", + sectionContent: "Bar\nBaz\nFoo", + }, + ), + ).toBe( + just( + spaceTrim(` # Foo Blah blah blah @@ -46,14 +46,14 @@ describe('how addAutoGeneratedSection works', () => { Blah blah blah `), - ), - ); - }); - - it('should modify the section', () => { - expect( - addAutoGeneratedSection( - spaceTrim(` + ), + ); + }); + + it("should modify the section", () => { + expect( + addAutoGeneratedSection( + spaceTrim(` # Foo Blah blah blah @@ -73,14 +73,14 @@ describe('how addAutoGeneratedSection works', () => { Blah blah blah `), - { - sectionName: 'Graph', - sectionContent: 'Brrrr', - }, - ), - ).toBe( - just( - spaceTrim(` + { + sectionName: "Graph", + sectionContent: "Brrrr", + }, + ), + ).toBe( + just( + spaceTrim(` # Foo Blah blah blah @@ -97,14 +97,14 @@ describe('how addAutoGeneratedSection works', () => { Blah blah blah `), - ), - ); - }); - - it('should not be confused by other comments', () => { - expect( - addAutoGeneratedSection( - spaceTrim(` + ), + ); + }); + + it("should not be confused by other comments", () => { + expect( + addAutoGeneratedSection( + spaceTrim(` # Foo Blah blah blah @@ -123,14 +123,14 @@ describe('how addAutoGeneratedSection works', () => { Blah blah blah `), - { - sectionName: 'Graph', - sectionContent: 'Bar\nBaz\nFoo', - }, - ), - ).toBe( - just( - spaceTrim(` + { + sectionName: "Graph", + sectionContent: "Bar\nBaz\nFoo", + }, + ), + ).toBe( + just( + spaceTrim(` # Foo Blah blah blah @@ -155,7 +155,7 @@ describe('how addAutoGeneratedSection works', () => { Blah blah blah `), - ), - ); - }); + ), + ); + }); }); diff --git a/src/utils/markdown/addAutoGeneratedSection.ts b/src/utils/markdown/addAutoGeneratedSection.ts index 60cfd16244..df8f81c248 100644 --- a/src/utils/markdown/addAutoGeneratedSection.ts +++ b/src/utils/markdown/addAutoGeneratedSection.ts @@ -1,8 +1,8 @@ -import { spaceTrim } from 'spacetrim'; -import { GENERATOR_WARNING } from '../../config'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_name } from '../../types/typeAliases'; -import { removeMarkdownComments } from './removeMarkdownComments'; +import { spaceTrim } from "spacetrim"; +import { GENERATOR_WARNING } from "../../config"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_name } from "../../types/typeAliases"; +import { removeMarkdownComments } from "./removeMarkdownComments"; /** * Add or modify an auto-generated section in a markdown file @@ -10,61 +10,66 @@ import { removeMarkdownComments } from './removeMarkdownComments'; * @public exported from `@promptbook/markdown-utils` */ export function addAutoGeneratedSection( - content: string_markdown, - options: { - readonly sectionName: string_name; - readonly sectionContent: string_markdown; - }, + content: string_markdown, + options: { + readonly sectionName: string_name; + readonly sectionContent: string_markdown; + }, ): string_markdown { - const { sectionName, sectionContent } = options; + const { sectionName, sectionContent } = options; - const warningLine: string_markdown = ``; - const sectionRegex = new RegExp(`([\\s\\S]*?)`, 'g'); + const warningLine: string_markdown = ``; + const sectionRegex = new RegExp( + `([\\s\\S]*?)`, + "g", + ); - const sectionMatch = content.match(sectionRegex); + const sectionMatch = content.match(sectionRegex); - const contentToInsert = spaceTrim( - (block) => ` + const contentToInsert = spaceTrim( + (block) => ` ${block(warningLine)} ${block(sectionContent)} `, - // <- Note: Spaces before and after comment blocks are added automatically by prettier later in the process - ); + // <- Note: Spaces before and after comment blocks are added automatically by prettier later in the process + ); - if (sectionMatch) { - return content.replace(sectionRegex, contentToInsert); - } + if (sectionMatch) { + return content.replace(sectionRegex, contentToInsert); + } - // Note: Following is the case when the section is not found in the file so we add it there + // Note: Following is the case when the section is not found in the file so we add it there - const placeForSection = removeMarkdownComments(content).match(/^##.*$/im); + const placeForSection = removeMarkdownComments(content).match(/^##.*$/im); - if (placeForSection !== null) { - const [heading] = placeForSection; - return content.replace( - heading, - spaceTrim( - (block) => ` + if (placeForSection !== null) { + const [heading] = placeForSection; + return content.replace( + heading, + spaceTrim( + (block) => ` ${block(contentToInsert)} ${block(heading)} `, - ), - ); - } + ), + ); + } - console.warn(`No place where to put the section , using the end of the file`); - // <- TODO: [🚎][💩] Some better way how to get warnings from pipeline parsing / logic + console.warn( + `No place where to put the section , using the end of the file`, + ); + // <- TODO: [🚎][💩] Some better way how to get warnings from pipeline parsing / logic - return spaceTrim( - (block) => ` + return spaceTrim( + (block) => ` ${block(content)} ${block(contentToInsert)} `, - ); + ); } /** diff --git a/src/utils/markdown/createMarkdownChart.test.ts b/src/utils/markdown/createMarkdownChart.test.ts index 2ce5f5cb12..07bbc9280f 100644 --- a/src/utils/markdown/createMarkdownChart.test.ts +++ b/src/utils/markdown/createMarkdownChart.test.ts @@ -1,25 +1,25 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { createMarkdownChart } from './createMarkdownChart'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { createMarkdownChart } from "./createMarkdownChart"; -describe('how createMarkdownChart works', () => { - it('should work advanced chart', () => { - expect( - createMarkdownChart({ - nameHeader: 'Task', - valueHeader: 'Timeline', - items: [ - { title: 'Task 1', from: 0, to: 10 }, - { title: 'Task 2', from: 4, to: 6 }, - { title: 'Task 3', from: 3, to: 9 }, - { title: 'Task 4', from: 6, to: 11 }, - { title: 'Task 5', from: 11, to: 12 }, - ], - width: 12, - unitName: 'seconds', - }), - ).toBe( - spaceTrim(` +describe("how createMarkdownChart works", () => { + it("should work advanced chart", () => { + expect( + createMarkdownChart({ + nameHeader: "Task", + valueHeader: "Timeline", + items: [ + { title: "Task 1", from: 0, to: 10 }, + { title: "Task 2", from: 4, to: 6 }, + { title: "Task 3", from: 3, to: 9 }, + { title: "Task 4", from: 6, to: 11 }, + { title: "Task 5", from: 11, to: 12 }, + ], + width: 12, + unitName: "seconds", + }), + ).toBe( + spaceTrim(` | Task | Timeline | |--------|--------------| | Task 1 | ██████████░░ | @@ -30,24 +30,24 @@ describe('how createMarkdownChart works', () => { _Note: Each █ represents 1 seconds, width of timeline is 12 seconds = 12 squares_ `), - ); - }); + ); + }); - it('should round boxes to nearest whole number', () => { - expect( - createMarkdownChart({ - nameHeader: 'Task', - valueHeader: 'Timeline', - items: [ - { title: 'Task 1', from: -1.2, to: 9 }, - { title: 'Task 2', from: 4.5, to: 5.2 }, - { title: 'Task 3', from: 3.3, to: 8.7 }, - ], - width: 4, - unitName: 'seconds', - }), - ).toBe( - spaceTrim(` + it("should round boxes to nearest whole number", () => { + expect( + createMarkdownChart({ + nameHeader: "Task", + valueHeader: "Timeline", + items: [ + { title: "Task 1", from: -1.2, to: 9 }, + { title: "Task 2", from: 4.5, to: 5.2 }, + { title: "Task 3", from: 3.3, to: 8.7 }, + ], + width: 4, + unitName: "seconds", + }), + ).toBe( + spaceTrim(` | Task | Timeline | |--------|----------| | Task 1 | ████ | @@ -57,6 +57,6 @@ describe('how createMarkdownChart works', () => { _Note: Each █ represents 2.55 seconds, width of timeline is 10.2 seconds = 4 squares_ `), - ); - }); + ); + }); }); diff --git a/src/utils/markdown/createMarkdownChart.ts b/src/utils/markdown/createMarkdownChart.ts index ea62f8d71a..9471116853 100644 --- a/src/utils/markdown/createMarkdownChart.ts +++ b/src/utils/markdown/createMarkdownChart.ts @@ -1,38 +1,38 @@ -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; -import type { FromtoItems } from '../FromtoItems'; -import { numberToString } from '../parameters/numberToString'; -import { removeEmojis } from '../removeEmojis'; -import { createMarkdownTable } from './createMarkdownTable'; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; +import type { FromtoItems } from "../FromtoItems"; +import { numberToString } from "../parameters/numberToString"; +import { removeEmojis } from "../removeEmojis"; +import { createMarkdownTable } from "./createMarkdownTable"; /** * Options for `CreateMarkdownChartOptions` */ type CreateMarkdownChartOptions = { - /** - * The header for the first column - the name of the item - */ - readonly nameHeader: string; + /** + * The header for the first column - the name of the item + */ + readonly nameHeader: string; - /** - * The header for the second column - the value of the item - */ - readonly valueHeader: string; + /** + * The header for the second column - the value of the item + */ + readonly valueHeader: string; - /** - * The items to be charted - */ - readonly items: FromtoItems; + /** + * The items to be charted + */ + readonly items: FromtoItems; - /** - * The width of the chart in squares - */ - readonly width: number; + /** + * The width of the chart in squares + */ + readonly width: number; - /** - * The name of the unit shown in the chart - */ - readonly unitName: string; + /** + * The name of the unit shown in the chart + */ + readonly unitName: string; }; /** @@ -40,30 +40,35 @@ type CreateMarkdownChartOptions = { * * @public exported from `@promptbook/markdown-utils` */ -export function createMarkdownChart(options: CreateMarkdownChartOptions): string_markdown { - const { nameHeader, valueHeader, items, width, unitName } = options; - const from = Math.min(...items.map((item) => item.from)); - const to = Math.max(...items.map((item) => item.to)); +export function createMarkdownChart( + options: CreateMarkdownChartOptions, +): string_markdown { + const { nameHeader, valueHeader, items, width, unitName } = options; + const from = Math.min(...items.map((item) => item.from)); + const to = Math.max(...items.map((item) => item.to)); - const scale = width / (to - from); + const scale = width / (to - from); - const table: Array> = [[nameHeader, valueHeader]]; + const table: Array> = [[nameHeader, valueHeader]]; - for (const item of items) { - const before = Math.round((item.from - from) * scale); - const during = Math.round((item.to - item.from) * scale); - const after = width - before - during; + for (const item of items) { + const before = Math.round((item.from - from) * scale); + const during = Math.round((item.to - item.from) * scale); + const after = width - before - during; - table.push([removeEmojis(item.title).trim(), '░'.repeat(before) + '█'.repeat(during) + '░'.repeat(after)]); - } + table.push([ + removeEmojis(item.title).trim(), + "░".repeat(before) + "█".repeat(during) + "░".repeat(after), + ]); + } - const legend = `_Note: Each █ represents ${numberToString( - 1 / scale, - )} ${unitName}, width of ${valueHeader.toLowerCase()} is ${numberToString( - to - from, - )} ${unitName} = ${width} squares_`; + const legend = `_Note: Each █ represents ${numberToString( + 1 / scale, + )} ${unitName}, width of ${valueHeader.toLowerCase()} is ${numberToString( + to - from, + )} ${unitName} = ${width} squares_`; - return createMarkdownTable(table) + '\n\n' + legend; + return createMarkdownTable(table) + "\n\n" + legend; } /** diff --git a/src/utils/markdown/createMarkdownTable.test.ts b/src/utils/markdown/createMarkdownTable.test.ts index a3a8569819..ecdd44be5a 100644 --- a/src/utils/markdown/createMarkdownTable.test.ts +++ b/src/utils/markdown/createMarkdownTable.test.ts @@ -1,92 +1,92 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { createMarkdownTable } from './createMarkdownTable'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { createMarkdownTable } from "./createMarkdownTable"; -describe('how createMarkdownTable works', () => { - it('should work with 1x1 table', () => { - expect(createMarkdownTable([[`Name`], [`Foo`]])).toBe( - spaceTrim(` +describe("how createMarkdownTable works", () => { + it("should work with 1x1 table", () => { + expect(createMarkdownTable([[`Name`], [`Foo`]])).toBe( + spaceTrim(` | Name | |------| | Foo | `), - ); - }); + ); + }); - it('should work with 1x2 table', () => { - expect( - createMarkdownTable([ - [`Name`, `Job`], - [`Foo`, `Bar`], - ]), - ).toBe( - spaceTrim(` + it("should work with 1x2 table", () => { + expect( + createMarkdownTable([ + [`Name`, `Job`], + [`Foo`, `Bar`], + ]), + ).toBe( + spaceTrim(` | Name | Job | |------|-----| | Foo | Bar | `), - ); - }); + ); + }); - it('should work with 2x1 table', () => { - expect(createMarkdownTable([[`Name`], [`Foo`], [`Bar`]])).toBe( - spaceTrim(` + it("should work with 2x1 table", () => { + expect(createMarkdownTable([[`Name`], [`Foo`], [`Bar`]])).toBe( + spaceTrim(` | Name | |------| | Foo | | Bar | `), - ); - }); + ); + }); - it('should work with 2x2 table', () => { - expect( - createMarkdownTable([ - [`Name`, `Job`], - [`Foo`, `Bar`], - [`Baz`, `Qux`], - ]), - ).toBe( - spaceTrim(` + it("should work with 2x2 table", () => { + expect( + createMarkdownTable([ + [`Name`, `Job`], + [`Foo`, `Bar`], + [`Baz`, `Qux`], + ]), + ).toBe( + spaceTrim(` | Name | Job | |------|-----| | Foo | Bar | | Baz | Qux | `), - ); - }); + ); + }); - it('should work with 3x3 table', () => { - expect( - createMarkdownTable([ - [`Name`, `Job`, `Hobby`], - [`Foo`, `Bar`, `Baz`], - [`Qux`, `Quux`, `Corge`], - [`Grault`, `Garply`, `Waldo`], - ]), - ).toBe( - spaceTrim(` + it("should work with 3x3 table", () => { + expect( + createMarkdownTable([ + [`Name`, `Job`, `Hobby`], + [`Foo`, `Bar`, `Baz`], + [`Qux`, `Quux`, `Corge`], + [`Grault`, `Garply`, `Waldo`], + ]), + ).toBe( + spaceTrim(` | Name | Job | Hobby | |--------|--------|-------| | Foo | Bar | Baz | | Qux | Quux | Corge | | Grault | Garply | Waldo | `), - ); - }); + ); + }); - it('should work with chart table', () => { - expect( - createMarkdownTable([ - ['Template', 'Time chart'], - ['Template 1', '██████████░░'], - ['Template 2', '░░░░██░░░░░░'], - ['Template 3', '░░░██████░░░'], - ['Template 4', '░░░░░░█████░'], - ['Template 5', '░░░░░░░░░░░█'], - ]), - ).toBe( - spaceTrim(` + it("should work with chart table", () => { + expect( + createMarkdownTable([ + ["Template", "Time chart"], + ["Template 1", "██████████░░"], + ["Template 2", "░░░░██░░░░░░"], + ["Template 3", "░░░██████░░░"], + ["Template 4", "░░░░░░█████░"], + ["Template 5", "░░░░░░░░░░░█"], + ]), + ).toBe( + spaceTrim(` | Template | Time chart | |------------|--------------| | Template 1 | ██████████░░ | @@ -95,6 +95,6 @@ describe('how createMarkdownTable works', () => { | Template 4 | ░░░░░░█████░ | | Template 5 | ░░░░░░░░░░░█ | `), - ); - }); + ); + }); }); diff --git a/src/utils/markdown/createMarkdownTable.ts b/src/utils/markdown/createMarkdownTable.ts index 7a522f30d5..5e3f3222d0 100644 --- a/src/utils/markdown/createMarkdownTable.ts +++ b/src/utils/markdown/createMarkdownTable.ts @@ -1,36 +1,46 @@ -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; /** * Create a markdown table from a 2D array of strings * * @public exported from `@promptbook/markdown-utils` */ -export function createMarkdownTable(table: Array>): string_markdown { - const columnWidths: Array = table.reduce((widths: Array, row: Array) => { - row.forEach((subformat: string_markdown_text, columnIndex: number) => { - const cellLength: number = subformat.length; - if (!widths[columnIndex] || cellLength > widths[columnIndex]!) { - widths[columnIndex] = cellLength; - } - }); - return widths; - }, []); +export function createMarkdownTable( + table: Array>, +): string_markdown { + const columnWidths: Array = table.reduce( + (widths: Array, row: Array) => { + row.forEach((subformat: string_markdown_text, columnIndex: number) => { + const cellLength: number = subformat.length; + if (!widths[columnIndex] || cellLength > widths[columnIndex]!) { + widths[columnIndex] = cellLength; + } + }); + return widths; + }, + [], + ); - const header: string = `| ${table[0]! - .map((subformat: string_markdown_text, columnIndex: number) => subformat.padEnd(columnWidths[columnIndex]!)) - .join(' | ')} |`; + const header: string = `| ${table[0]! + .map((subformat: string_markdown_text, columnIndex: number) => + subformat.padEnd(columnWidths[columnIndex]!), + ) + .join(" | ")} |`; - const separator: string = `|${columnWidths.map((width: number) => '-'.repeat(width + 2)).join('|')}|`; + const separator: string = `|${columnWidths.map((width: number) => "-".repeat(width + 2)).join("|")}|`; - const rows: Array = table.slice(1).map((row: Array) => { - const paddedRow: Array = row.map((subformat: string_markdown_text, columnIndex: number) => - subformat.padEnd(columnWidths[columnIndex]!), - ); - return `| ${paddedRow.join(' | ')} |`; - }); + const rows: Array = table + .slice(1) + .map((row: Array) => { + const paddedRow: Array = row.map( + (subformat: string_markdown_text, columnIndex: number) => + subformat.padEnd(columnWidths[columnIndex]!), + ); + return `| ${paddedRow.join(" | ")} |`; + }); - return [header, separator, ...rows].join('\n'); + return [header, separator, ...rows].join("\n"); } /** diff --git a/src/utils/markdown/escapeMarkdownBlock.test.ts b/src/utils/markdown/escapeMarkdownBlock.test.ts index 925ec5652f..ed8bd100e5 100644 --- a/src/utils/markdown/escapeMarkdownBlock.test.ts +++ b/src/utils/markdown/escapeMarkdownBlock.test.ts @@ -1,38 +1,38 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { just } from '../organization/just'; -import { escapeMarkdownBlock } from './escapeMarkdownBlock'; - -describe('how escapeMarkdownBlock works', () => { - it('should work with string that does not need any escaping', () => { - expect(escapeMarkdownBlock('Foo')).toBe('Foo'); - expect( - escapeMarkdownBlock( - spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { just } from "../organization/just"; +import { escapeMarkdownBlock } from "./escapeMarkdownBlock"; + +describe("how escapeMarkdownBlock works", () => { + it("should work with string that does not need any escaping", () => { + expect(escapeMarkdownBlock("Foo")).toBe("Foo"); + expect( + escapeMarkdownBlock( + spaceTrim(` Foo Bar Baz `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` Foo Bar Baz `), - ), - ); - }); - - it('should work with block in block', () => { - expect( - escapeMarkdownBlock( - spaceTrim(` + ), + ); + }); + + it("should work with block in block", () => { + expect( + escapeMarkdownBlock( + spaceTrim(` Foo \`\`\`javascript @@ -41,10 +41,10 @@ describe('how escapeMarkdownBlock works', () => { Baz `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` Foo \\\`\\\`\\\`javascript @@ -53,7 +53,7 @@ describe('how escapeMarkdownBlock works', () => { Baz `), - ), - ); - }); + ), + ); + }); }); diff --git a/src/utils/markdown/escapeMarkdownBlock.ts b/src/utils/markdown/escapeMarkdownBlock.ts index bd2a2bbc91..8558c7c810 100644 --- a/src/utils/markdown/escapeMarkdownBlock.ts +++ b/src/utils/markdown/escapeMarkdownBlock.ts @@ -1,4 +1,4 @@ -import type { string_markdown_text } from '../../types/typeAliases'; +import type { string_markdown_text } from "../../types/typeAliases"; /** * Function escapeMarkdownBlock will escape markdown block if needed @@ -6,8 +6,10 @@ import type { string_markdown_text } from '../../types/typeAliases'; * * @public exported from `@promptbook/markdown-utils` */ -export function escapeMarkdownBlock(value: string_markdown_text): string_markdown_text { - return value.replace(/```/g, '\\`\\`\\`'); +export function escapeMarkdownBlock( + value: string_markdown_text, +): string_markdown_text { + return value.replace(/```/g, "\\`\\`\\`"); } /** diff --git a/src/utils/markdown/extractAllBlocksFromMarkdown-real.test.ts b/src/utils/markdown/extractAllBlocksFromMarkdown-real.test.ts index ca32421abe..602c0b4c9f 100644 --- a/src/utils/markdown/extractAllBlocksFromMarkdown-real.test.ts +++ b/src/utils/markdown/extractAllBlocksFromMarkdown-real.test.ts @@ -1,18 +1,18 @@ -import { describe, expect, it } from '@jest/globals'; -import { readFileSync } from 'fs'; -import spaceTrim from 'spacetrim'; -import { extractAllBlocksFromMarkdown } from './extractAllBlocksFromMarkdown'; +import { readFileSync } from "fs"; +import { describe, expect, it } from "@jest/globals"; +import spaceTrim from "spacetrim"; +import { extractAllBlocksFromMarkdown } from "./extractAllBlocksFromMarkdown"; -describe('how extractAllBlocksFromMarkdown works in real example', () => { - it('should work with example with no code blocks', () => { - expect( - extractAllBlocksFromMarkdown( - readFileSync('examples/pipelines/30-escaping.book', 'utf-8'), - // <- Note: Its OK to use sync in tests - ), - ).toContainEqual({ - blockNotation: '>', - content: spaceTrim(` +describe("how extractAllBlocksFromMarkdown works in real example", () => { + it("should work with example with no code blocks", () => { + expect( + extractAllBlocksFromMarkdown( + readFileSync("examples/pipelines/30-escaping.book", "utf-8"), + // <- Note: Its OK to use sync in tests + ), + ).toContainEqual({ + blockNotation: ">", + content: spaceTrim(` Rewrite the function below: @@ -24,7 +24,7 @@ describe('how extractAllBlocksFromMarkdown works in real example', () => { To return "Goodbye" from the function instead of "Hello". `), - language: null, - }); - }); + language: null, + }); + }); }); diff --git a/src/utils/markdown/extractAllBlocksFromMarkdown.test.ts b/src/utils/markdown/extractAllBlocksFromMarkdown.test.ts index 9b724fd4af..8dc6e4a6a9 100644 --- a/src/utils/markdown/extractAllBlocksFromMarkdown.test.ts +++ b/src/utils/markdown/extractAllBlocksFromMarkdown.test.ts @@ -1,39 +1,39 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { extractAllBlocksFromMarkdown } from './extractAllBlocksFromMarkdown'; - -describe('how extractAllBlocksFromMarkdown works', () => { - it('should work with example with no code blocks', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { extractAllBlocksFromMarkdown } from "./extractAllBlocksFromMarkdown"; + +describe("how extractAllBlocksFromMarkdown works", () => { + it("should work with example with no code blocks", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` Hello World `), - ), - ).toEqual([]); + ), + ).toEqual([]); - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` Hello World Hello World `), - ), - ).toEqual([]); + ), + ).toEqual([]); - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` # Hello World Content with **bold** and *italic* text `), - ), - ).toEqual([]); + ), + ).toEqual([]); - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` # Prague Prague is the capital of Czech Republic. It is a beautiful city. In my opinion, it is the most beautiful city in the world. @@ -52,86 +52,86 @@ describe('how extractAllBlocksFromMarkdown works', () => { \`Lennon Wall\` is a wall in Prague. It is located in the center of Prague. On this wall, you can see many graffiti like %#2/*\`\`\`7#^ `), - ), - ).toEqual([]); - }); - - it('should work with example with one code block of one line', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + ), + ).toEqual([]); + }); + + it("should work with example with one code block of one line", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` \`\`\`python print('Hello World') \`\`\` `), - ), - ).toEqual([ - { - blockNotation: '```', - language: 'python', - content: "print('Hello World')", - }, - ]); - - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: "python", + content: "print('Hello World')", + }, + ]); + + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` # Hello World \`\`\`python print('Hello World') \`\`\` `), - ), - ).toEqual([ - { - blockNotation: '```', - language: 'python', - content: "print('Hello World')", - }, - ]); - }); - - it('should work with code block without language', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: "python", + content: "print('Hello World')", + }, + ]); + }); + + it("should work with code block without language", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` \`\`\` print('Hello World') \`\`\` `), - ), - ).toEqual([ - { - blockNotation: '```', - language: null, - content: "print('Hello World')", - }, - ]); - - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: null, + content: "print('Hello World')", + }, + ]); + + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` # Hello World \`\`\`python print('Hello World') \`\`\` `), - ), - ).toEqual([ - { - blockNotation: '```', - language: 'python', - content: "print('Hello World')", - }, - ]); - }); - - it('should work with example with one multiline code block made by gt char', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: "python", + content: "print('Hello World')", + }, + ]); + }); + + it("should work with example with one multiline code block made by gt char", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` # Hello World > print('Hello World 1') @@ -139,24 +139,24 @@ describe('how extractAllBlocksFromMarkdown works', () => { > print('Hello World 3') `), - ), - ).toEqual([ - { - blockNotation: '>', - language: null, - content: spaceTrim(` + ), + ).toEqual([ + { + blockNotation: ">", + language: null, + content: spaceTrim(` print('Hello World 1') print('Hello World 2') print('Hello World 3') `), - }, - ]); - }); + }, + ]); + }); - it('should work with example with one code block of multiple lines', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + it("should work with example with one code block of multiple lines", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` # Example @@ -168,26 +168,26 @@ describe('how extractAllBlocksFromMarkdown works', () => { print('Hello World') \`\`\` `), - ), - ).toEqual([ - { - blockNotation: '```', - language: 'python', - content: spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: "python", + content: spaceTrim(` print('Hello World') print('Hello World') print('Hello World') `), - }, - ]); - }); + }, + ]); + }); - it('should work with python+javascript+unknown code blocks', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + it("should work with python+javascript+unknown code blocks", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` # Python @@ -210,39 +210,39 @@ describe('how extractAllBlocksFromMarkdown works', () => { $5/-/++'=>Hello Mars \`\`\` `), - ), - ).toEqual([ - { - blockNotation: '```', - language: 'python', - content: spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: "python", + content: spaceTrim(` print('Hello World') print('Hello Mars') `), - }, - { - blockNotation: '```', - language: 'javascript', - content: spaceTrim(` + }, + { + blockNotation: "```", + language: "javascript", + content: spaceTrim(` console.info('Hello World') console.info('Hello Mars') `), - }, - { - blockNotation: '```', - language: null, - content: spaceTrim(` + }, + { + blockNotation: "```", + language: null, + content: spaceTrim(` $5/-/++'=>Hello World $5/-/++'=>Hello Mars `), - }, - ]); - }); - - it('should work with example with multiple code blocks of one line', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + }, + ]); + }); + + it("should work with example with multiple code blocks of one line", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` # Hello World Hello World in multiple languages: @@ -263,25 +263,25 @@ describe('how extractAllBlocksFromMarkdown works', () => { console.info('Hello World') \`\`\` `), - ), - ).toEqual([ - { - blockNotation: '```', - language: 'python', - content: "print('Hello World')", - }, - { - blockNotation: '```', - language: 'javascript', - content: "console.info('Hello World')", - }, - ]); - }); - - it('should work with example with multiple mixed style code blocks', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: "python", + content: "print('Hello World')", + }, + { + blockNotation: "```", + language: "javascript", + content: "console.info('Hello World')", + }, + ]); + }); + + it("should work with example with multiple mixed style code blocks", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` # Hello World Hello World in multiple languages: @@ -296,25 +296,25 @@ describe('how extractAllBlocksFromMarkdown works', () => { > Block notated by gt char `), - ), - ).toEqual([ - { - blockNotation: '```', - language: 'text', - content: 'Block notated by backticks', - }, - { - blockNotation: '>', - language: null, - content: 'Block notated by gt char', - }, - ]); - }); - - it('should not be confused by 3 backtics in codeblock which are not ending the codeblock', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: "text", + content: "Block notated by backticks", + }, + { + blockNotation: ">", + language: null, + content: "Block notated by gt char", + }, + ]); + }); + + it("should not be confused by 3 backtics in codeblock which are not ending the codeblock", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` ## Javascript @@ -324,20 +324,20 @@ describe('how extractAllBlocksFromMarkdown works', () => { `), - ), - ).toEqual([ - { - blockNotation: '```', - language: 'javascript', - content: "console.info('Hello 3 backtics ```')", - }, - ]); - }); - - it('should work with codeblock with escaped embeded codeblock as content', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: "javascript", + content: "console.info('Hello 3 backtics ```')", + }, + ]); + }); + + it("should work with codeblock with escaped embeded codeblock as content", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` This is a simple markdown with code block with escaped embeded code block as content: @@ -364,13 +364,13 @@ describe('how extractAllBlocksFromMarkdown works', () => { print('Hello World') \`\`\` `), - ), - ).toEqual([ - { - blockNotation: '```', - language: 'markdown', - content: - spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: "markdown", + content: + spaceTrim(` Markdown has simple formatting like **bold** and *italic* text. @@ -385,20 +385,20 @@ describe('how extractAllBlocksFromMarkdown works', () => { And loooot of other features. - `) + '\n', - }, - { - blockNotation: '```', - language: 'python', - content: "print('Hello World')", - }, - ]); - }); - - it('should throw error when code block is not propperly closed', () => { - expect(() => - extractAllBlocksFromMarkdown( - spaceTrim(` + `) + "\n", + }, + { + blockNotation: "```", + language: "python", + content: "print('Hello World')", + }, + ]); + }); + + it("should throw error when code block is not propperly closed", () => { + expect(() => + extractAllBlocksFromMarkdown( + spaceTrim(` This is a simple markdown code: @@ -411,12 +411,14 @@ describe('how extractAllBlocksFromMarkdown works', () => { \`\`\`python print('Hello World') `), - ), - ).toThrowError(/Python code block was not closed at the end of the markdown/i); + ), + ).toThrowError( + /Python code block was not closed at the end of the markdown/i, + ); - expect(() => - extractAllBlocksFromMarkdown( - spaceTrim(` + expect(() => + extractAllBlocksFromMarkdown( + spaceTrim(` This is a simple markdown code: @@ -429,12 +431,14 @@ describe('how extractAllBlocksFromMarkdown works', () => { print('Hello World') \`\`\` `), - ), - ).toThrowError(/Markdown code block was not closed and already opening new python code block/i); + ), + ).toThrowError( + /Markdown code block was not closed and already opening new python code block/i, + ); - expect(() => - extractAllBlocksFromMarkdown( - spaceTrim(` + expect(() => + extractAllBlocksFromMarkdown( + spaceTrim(` This is a simple markdown code: @@ -448,14 +452,16 @@ describe('how extractAllBlocksFromMarkdown works', () => { print('Hello World') \`\`\`python `), - ), - ).toThrowError(/Markdown code block was not closed and already opening new markdown code block/i); - }); + ), + ).toThrowError( + /Markdown code block was not closed and already opening new markdown code block/i, + ); + }); - it('should not be confused by nested blocks in blocks', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + it("should not be confused by nested blocks in blocks", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` This is a simple markdown with code block with escaped embeded code block as content: @@ -466,23 +472,23 @@ describe('how extractAllBlocksFromMarkdown works', () => { \\\`\\\`\\\` \`\`\` `), - ), - ).toEqual([ - { - blockNotation: '```', - language: 'markdown', - content: spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: "markdown", + content: spaceTrim(` Block \`\`\`markdown Block in block \`\`\` `), - }, - ]); - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + }, + ]); + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` This is a simple markdown with code block with escaped embeded code block as content: @@ -493,23 +499,23 @@ describe('how extractAllBlocksFromMarkdown works', () => { > \`\`\` `), - ), - ).toEqual([ - { - blockNotation: '```', - language: 'markdown', - content: spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: "markdown", + content: spaceTrim(` Block > markdown > Block in block > `), - }, - ]); - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + }, + ]); + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` This is a simple markdown with code block with escaped embeded code block as content: @@ -518,56 +524,56 @@ describe('how extractAllBlocksFromMarkdown works', () => { > Block in block > \`\`\` `), - ), - ).toEqual([ - { - blockNotation: '>', - language: null, - content: spaceTrim(` + ), + ).toEqual([ + { + blockNotation: ">", + language: null, + content: spaceTrim(` Block \`\`\`markdown Block in block \`\`\` `), - }, - ]); - }); - - it('should not crash when there is just a block, nothing else', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + }, + ]); + }); + + it("should not crash when there is just a block, nothing else", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` \`\`\` Block \`\`\` `), - ), - ).toEqual([ - { - blockNotation: '```', - language: null, - content: 'Block', - }, - ]); - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + language: null, + content: "Block", + }, + ]); + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` > Block `), - ), - ).toEqual([ - { - blockNotation: '>', - language: null, - content: 'Block', - }, - ]); - }); - - it('should not be confused zig-zag blocks with different notations', () => { - expect( - extractAllBlocksFromMarkdown( - spaceTrim(` + ), + ).toEqual([ + { + blockNotation: ">", + language: null, + content: "Block", + }, + ]); + }); + + it("should not be confused zig-zag blocks with different notations", () => { + expect( + extractAllBlocksFromMarkdown( + spaceTrim(` This is a simple markdown with code block with escaped embeded code block as content: @@ -594,26 +600,26 @@ describe('how extractAllBlocksFromMarkdown works', () => { M `), - ), - ).toEqual([ - { - blockNotation: '```', - content: spaceTrim(` + ), + ).toEqual([ + { + blockNotation: "```", + content: spaceTrim(` A > B C > D `), - language: 'markdown', - }, - { - blockNotation: '>', - content: 'F', - language: null, - }, - { - blockNotation: '>', - content: spaceTrim(` + language: "markdown", + }, + { + blockNotation: ">", + content: "F", + language: null, + }, + { + blockNotation: ">", + content: spaceTrim(` H \`\`\`i J @@ -621,8 +627,8 @@ describe('how extractAllBlocksFromMarkdown works', () => { \`\`\` L `), - language: null, - }, - ]); - }); + language: null, + }, + ]); + }); }); diff --git a/src/utils/markdown/extractAllBlocksFromMarkdown.ts b/src/utils/markdown/extractAllBlocksFromMarkdown.ts index d75ca72d85..6cca50b1cb 100644 --- a/src/utils/markdown/extractAllBlocksFromMarkdown.ts +++ b/src/utils/markdown/extractAllBlocksFromMarkdown.ts @@ -1,26 +1,26 @@ -import type { Writable } from 'type-fest'; -import { ParseError } from '../../errors/ParseError'; -import type { string_markdown } from '../../types/typeAliases'; -import { capitalize } from '../normalization/capitalize'; +import type { Writable } from "type-fest"; +import { ParseError } from "../../errors/ParseError"; +import type { string_markdown } from "../../types/typeAliases"; +import { capitalize } from "../normalization/capitalize"; /** * Single code block inside markdown. */ export type CodeBlock = { - /** - * Which notation was used to open the code block - */ - readonly blockNotation: '```' | '>'; - - /** - * Language of the code block OR null if the language is not specified in opening ``` - */ - readonly language: string | null; - - /** - * Content of the code block (unescaped) - */ - readonly content: string; + /** + * Which notation was used to open the code block + */ + readonly blockNotation: "```" | ">"; + + /** + * Language of the code block OR null if the language is not specified in opening ``` + */ + readonly language: string | null; + + /** + * Content of the code block (unescaped) + */ + readonly content: string; }; /** @@ -37,69 +37,79 @@ export type CodeBlock = { * @throws {ParseError} if block is not closed properly * @public exported from `@promptbook/markdown-utils` */ -export function extractAllBlocksFromMarkdown(markdown: string_markdown): ReadonlyArray { - const codeBlocks: Array = []; - const lines = markdown.split('\n'); - - // Note: [0] Ensure that the last block notated by gt > will be closed - lines.push(''); - - let currentCodeBlock: Writable | null = null; - - for (const line of lines) { - if (line.startsWith('> ') || line === '>') { - if (currentCodeBlock === null) { - currentCodeBlock = { blockNotation: '>', language: null, content: '' }; - } /* not else */ - - if (currentCodeBlock.blockNotation === '>') { - if (currentCodeBlock.content !== '') { - currentCodeBlock.content += '\n'; - } - - currentCodeBlock.content += line.slice(2); - } - } else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '>' /* <- Note: [0] */) { - codeBlocks.push(currentCodeBlock); - currentCodeBlock = null; - } - - /* not else */ - - if (line.startsWith('```')) { - const language = line.slice(3).trim() || null; - - if (currentCodeBlock === null) { - currentCodeBlock = { blockNotation: '```', language, content: '' }; - } else { - if (language !== null) { - throw new ParseError( - `${capitalize( - currentCodeBlock.language || 'the', - )} code block was not closed and already opening new ${language} code block`, - // <- [🚞] - ); - } - codeBlocks.push(currentCodeBlock); - currentCodeBlock = null; - } - } else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '```') { - if (currentCodeBlock.content !== '') { - currentCodeBlock.content += '\n'; - } - - currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make propper unescape */; - } - } - - if (currentCodeBlock !== null) { - throw new ParseError( - `${capitalize(currentCodeBlock.language || 'the')} code block was not closed at the end of the markdown`, - // <- [🚞] - ); - } - - return codeBlocks; +export function extractAllBlocksFromMarkdown( + markdown: string_markdown, +): ReadonlyArray { + const codeBlocks: Array = []; + const lines = markdown.split("\n"); + + // Note: [0] Ensure that the last block notated by gt > will be closed + lines.push(""); + + let currentCodeBlock: Writable | null = null; + + for (const line of lines) { + if (line.startsWith("> ") || line === ">") { + if (currentCodeBlock === null) { + currentCodeBlock = { blockNotation: ">", language: null, content: "" }; + } /* not else */ + + if (currentCodeBlock.blockNotation === ">") { + if (currentCodeBlock.content !== "") { + currentCodeBlock.content += "\n"; + } + + currentCodeBlock.content += line.slice(2); + } + } else if ( + currentCodeBlock !== null && + currentCodeBlock.blockNotation === ">" /* <- Note: [0] */ + ) { + codeBlocks.push(currentCodeBlock); + currentCodeBlock = null; + } + + /* not else */ + + if (line.startsWith("```")) { + const language = line.slice(3).trim() || null; + + if (currentCodeBlock === null) { + currentCodeBlock = { blockNotation: "```", language, content: "" }; + } else { + if (language !== null) { + throw new ParseError( + `${capitalize( + currentCodeBlock.language || "the", + )} code block was not closed and already opening new ${language} code block`, + // <- [🚞] + ); + } + codeBlocks.push(currentCodeBlock); + currentCodeBlock = null; + } + } else if ( + currentCodeBlock !== null && + currentCodeBlock.blockNotation === "```" + ) { + if (currentCodeBlock.content !== "") { + currentCodeBlock.content += "\n"; + } + + currentCodeBlock.content += line + .split("\\`\\`\\`") + .join("```") /* <- TODO: Maybe make propper unescape */; + } + } + + if (currentCodeBlock !== null) { + throw new ParseError( + `${capitalize(currentCodeBlock.language || "the")} code block was not closed at the end of the markdown`, + // <- [🚞] + ); + } + + return codeBlocks; } /** diff --git a/src/utils/markdown/extractAllListItemsFromMarkdown.test.ts b/src/utils/markdown/extractAllListItemsFromMarkdown.test.ts index 935712c392..ef4f929ce4 100644 --- a/src/utils/markdown/extractAllListItemsFromMarkdown.test.ts +++ b/src/utils/markdown/extractAllListItemsFromMarkdown.test.ts @@ -1,61 +1,61 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { extractAllListItemsFromMarkdown } from './extractAllListItemsFromMarkdown'; - -describe('how extractAllListItemsFromMarkdown works', () => { - it('should work with example with no items', () => { - expect( - extractAllListItemsFromMarkdown( - spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { extractAllListItemsFromMarkdown } from "./extractAllListItemsFromMarkdown"; + +describe("how extractAllListItemsFromMarkdown works", () => { + it("should work with example with no items", () => { + expect( + extractAllListItemsFromMarkdown( + spaceTrim(` Hello World `), - ), - ).toEqual([]); + ), + ).toEqual([]); - expect( - extractAllListItemsFromMarkdown( - spaceTrim(` + expect( + extractAllListItemsFromMarkdown( + spaceTrim(` Hello World Hello World `), - ), - ).toEqual([]); + ), + ).toEqual([]); - expect( - extractAllListItemsFromMarkdown( - spaceTrim(` + expect( + extractAllListItemsFromMarkdown( + spaceTrim(` # Hello World Content with **bold** and *italic* text `), - ), - ).toEqual([]); - }); - - it('should work with example with one item', () => { - expect( - extractAllListItemsFromMarkdown( - spaceTrim(` + ), + ).toEqual([]); + }); + + it("should work with example with one item", () => { + expect( + extractAllListItemsFromMarkdown( + spaceTrim(` - Hello World `), - ), - ).toEqual(['Hello World']); + ), + ).toEqual(["Hello World"]); - expect( - extractAllListItemsFromMarkdown( - spaceTrim(` + expect( + extractAllListItemsFromMarkdown( + spaceTrim(` # Hello World - Item with **bold** and *italic* text `), - ), - ).toEqual(['Item with **bold** and *italic* text']); - }); - - it('should work with example with multiple items', () => { - expect( - extractAllListItemsFromMarkdown( - spaceTrim(` + ), + ).toEqual(["Item with **bold** and *italic* text"]); + }); + + it("should work with example with multiple items", () => { + expect( + extractAllListItemsFromMarkdown( + spaceTrim(` # Hello World Some text that is not a list item @@ -67,14 +67,14 @@ describe('how extractAllListItemsFromMarkdown works', () => { Also not a list item - and not a list item too just a hyphen - `), - ), - ).toEqual(['First item', 'Second item', 'Third item', 'Fourth item']); - }); - - it('should work with example with multiple items and nested lists', () => { - expect( - extractAllListItemsFromMarkdown( - spaceTrim(` + ), + ).toEqual(["First item", "Second item", "Third item", "Fourth item"]); + }); + + it("should work with example with multiple items and nested lists", () => { + expect( + extractAllListItemsFromMarkdown( + spaceTrim(` # Hello World - First item @@ -85,14 +85,21 @@ describe('how extractAllListItemsFromMarkdown works', () => { - Fourth item `), - ), - ).toEqual(['First item', 'Second item', 'Nested item 1', 'Nested item 2', 'Third item', 'Fourth item']); - }); - - it('should work with example with multiple items and nested lists and markdown', () => { - expect( - extractAllListItemsFromMarkdown( - spaceTrim(` + ), + ).toEqual([ + "First item", + "Second item", + "Nested item 1", + "Nested item 2", + "Third item", + "Fourth item", + ]); + }); + + it("should work with example with multiple items and nested lists and markdown", () => { + expect( + extractAllListItemsFromMarkdown( + spaceTrim(` # Hello World - First item @@ -112,27 +119,27 @@ describe('how extractAllListItemsFromMarkdown works', () => { - Eighth item `), - ), - ).toEqual([ - 'First item', - 'Second item', - 'Nested item 1', - 'Nested item 2', - 'Third item', - 'Fourth item', - 'Fifth item', - 'Sixth item', - 'Nested item 3', - 'Nested item 4', - 'Seventh item', - 'Eighth item', - ]); - }); - - it('should work with both with ol and ul', () => { - expect( - extractAllListItemsFromMarkdown( - spaceTrim(` + ), + ).toEqual([ + "First item", + "Second item", + "Nested item 1", + "Nested item 2", + "Third item", + "Fourth item", + "Fifth item", + "Sixth item", + "Nested item 3", + "Nested item 4", + "Seventh item", + "Eighth item", + ]); + }); + + it("should work with both with ol and ul", () => { + expect( + extractAllListItemsFromMarkdown( + spaceTrim(` # Hello World - First item @@ -152,27 +159,27 @@ describe('how extractAllListItemsFromMarkdown works', () => { 4. Eighth item `), - ), - ).toEqual([ - 'First item', - 'Second item', - 'Nested item 1', - 'Nested item 2', - 'Third item', - 'Fourth item', - 'Fifth item', - 'Sixth item', - 'Nested item 3', - 'Nested item 4', - 'Seventh item', - 'Eighth item', - ]); - }); - - it('should omit items in nested code block', () => { - expect( - extractAllListItemsFromMarkdown( - spaceTrim(` + ), + ).toEqual([ + "First item", + "Second item", + "Nested item 1", + "Nested item 2", + "Third item", + "Fourth item", + "Fifth item", + "Sixth item", + "Nested item 3", + "Nested item 4", + "Seventh item", + "Eighth item", + ]); + }); + + it("should omit items in nested code block", () => { + expect( + extractAllListItemsFromMarkdown( + spaceTrim(` # Hello World - First item @@ -184,7 +191,7 @@ describe('how extractAllListItemsFromMarkdown works', () => { \`\`\` `), - ), - ).toEqual(['First item', 'Second item']); - }); + ), + ).toEqual(["First item", "Second item"]); + }); }); diff --git a/src/utils/markdown/extractAllListItemsFromMarkdown.ts b/src/utils/markdown/extractAllListItemsFromMarkdown.ts index 58463360ce..0122da7388 100644 --- a/src/utils/markdown/extractAllListItemsFromMarkdown.ts +++ b/src/utils/markdown/extractAllListItemsFromMarkdown.ts @@ -1,5 +1,5 @@ -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; /** * Utility function to extract all list items from markdown @@ -13,24 +13,29 @@ import type { string_markdown_text } from '../../types/typeAliases'; * @returns @@@ * @public exported from `@promptbook/markdown-utils` */ -export function extractAllListItemsFromMarkdown(markdown: string_markdown): string_markdown_text[] { - const lines = markdown.split('\n'); - const listItems: string_markdown_text[] = []; +export function extractAllListItemsFromMarkdown( + markdown: string_markdown, +): string_markdown_text[] { + const lines = markdown.split("\n"); + const listItems: string_markdown_text[] = []; - let isInCodeBlock = false; + let isInCodeBlock = false; - for (const line of lines) { - const trimmedLine = line.trim(); + for (const line of lines) { + const trimmedLine = line.trim(); - if (trimmedLine.startsWith('```')) { - isInCodeBlock = !isInCodeBlock; - } + if (trimmedLine.startsWith("```")) { + isInCodeBlock = !isInCodeBlock; + } - if (!isInCodeBlock && (trimmedLine.startsWith('-') || trimmedLine.match(/^\d+\./))) { - const listItem = trimmedLine.replace(/^-|\d+\./, '').trim(); - listItems.push(listItem); - } - } + if ( + !isInCodeBlock && + (trimmedLine.startsWith("-") || trimmedLine.match(/^\d+\./)) + ) { + const listItem = trimmedLine.replace(/^-|\d+\./, "").trim(); + listItems.push(listItem); + } + } - return listItems; + return listItems; } diff --git a/src/utils/markdown/extractOneBlockFromMarkdown.test.ts b/src/utils/markdown/extractOneBlockFromMarkdown.test.ts index d437976e43..7853975fbe 100644 --- a/src/utils/markdown/extractOneBlockFromMarkdown.test.ts +++ b/src/utils/markdown/extractOneBlockFromMarkdown.test.ts @@ -1,47 +1,47 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { extractOneBlockFromMarkdown } from './extractOneBlockFromMarkdown'; - -describe('how extractOneBlockFromMarkdown works', () => { - it('should work with example with one unknown code block of one line', () => { - expect( - extractOneBlockFromMarkdown( - spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { extractOneBlockFromMarkdown } from "./extractOneBlockFromMarkdown"; + +describe("how extractOneBlockFromMarkdown works", () => { + it("should work with example with one unknown code block of one line", () => { + expect( + extractOneBlockFromMarkdown( + spaceTrim(` # Hello World \`\`\` print('Hello World') \`\`\` `), - ), - ).toEqual({ - blockNotation: '```', - language: null, - content: "print('Hello World')", - }); - }); - - it('should work with example with one code block made by gt char', () => { - expect( - extractOneBlockFromMarkdown( - spaceTrim(` + ), + ).toEqual({ + blockNotation: "```", + language: null, + content: "print('Hello World')", + }); + }); + + it("should work with example with one code block made by gt char", () => { + expect( + extractOneBlockFromMarkdown( + spaceTrim(` # Hello World > print('Hello World') `), - ), - ).toEqual({ - blockNotation: '>', - language: null, - content: "print('Hello World')", - }); - }); - - it('should work with example with one multiline code block made by gt char', () => { - expect( - extractOneBlockFromMarkdown( - spaceTrim(` + ), + ).toEqual({ + blockNotation: ">", + language: null, + content: "print('Hello World')", + }); + }); + + it("should work with example with one multiline code block made by gt char", () => { + expect( + extractOneBlockFromMarkdown( + spaceTrim(` # Hello World > print('Hello World 1') @@ -49,22 +49,22 @@ describe('how extractOneBlockFromMarkdown works', () => { > print('Hello World 3') `), - ), - ).toEqual({ - blockNotation: '>', - language: null, - content: spaceTrim(` + ), + ).toEqual({ + blockNotation: ">", + language: null, + content: spaceTrim(` print('Hello World 1') print('Hello World 2') print('Hello World 3') `), - }); - }); + }); + }); - it('should work with example with block nested in block with mixed notations', () => { - expect( - extractOneBlockFromMarkdown( - spaceTrim(` + it("should work with example with block nested in block with mixed notations", () => { + expect( + extractOneBlockFromMarkdown( + spaceTrim(` # A B @@ -79,11 +79,11 @@ describe('how extractOneBlockFromMarkdown works', () => { H `), - ), - ).toEqual({ - blockNotation: '>', - language: null, - content: spaceTrim(` + ), + ).toEqual({ + blockNotation: ">", + language: null, + content: spaceTrim(` C D \`\`\`E @@ -91,10 +91,10 @@ describe('how extractOneBlockFromMarkdown works', () => { \`\`\` G `), - }); - expect( - extractOneBlockFromMarkdown( - spaceTrim(` + }); + expect( + extractOneBlockFromMarkdown( + spaceTrim(` # A B @@ -109,55 +109,55 @@ describe('how extractOneBlockFromMarkdown works', () => { H `), - ), - ).toEqual({ - blockNotation: '```', - language: 'C', - content: spaceTrim(` + ), + ).toEqual({ + blockNotation: "```", + language: "C", + content: spaceTrim(` D > E > F G `), - }); - }); + }); + }); - it('should work with example with one python code block of one line', () => { - expect( - extractOneBlockFromMarkdown( - spaceTrim(` + it("should work with example with one python code block of one line", () => { + expect( + extractOneBlockFromMarkdown( + spaceTrim(` \`\`\`python print('Hello World') \`\`\` `), - ), - ).toEqual({ - blockNotation: '```', - language: 'python', - content: "print('Hello World')", - }); - - expect( - extractOneBlockFromMarkdown( - spaceTrim(` + ), + ).toEqual({ + blockNotation: "```", + language: "python", + content: "print('Hello World')", + }); + + expect( + extractOneBlockFromMarkdown( + spaceTrim(` # Hello World \`\`\`python print('Hello World') \`\`\` `), - ), - ).toEqual({ - blockNotation: '```', - language: 'python', - content: "print('Hello World')", - }); - }); - - it('should work with codeblock with escaped embeded codeblock as content', () => { - expect( - extractOneBlockFromMarkdown( - spaceTrim(` + ), + ).toEqual({ + blockNotation: "```", + language: "python", + content: "print('Hello World')", + }); + }); + + it("should work with codeblock with escaped embeded codeblock as content", () => { + expect( + extractOneBlockFromMarkdown( + spaceTrim(` This is a simple markdown with code block with escaped embeded code block as content: @@ -174,12 +174,12 @@ describe('how extractOneBlockFromMarkdown works', () => { \`\`\` `), - ), - ).toEqual({ - blockNotation: '```', - language: 'markdown', - content: - spaceTrim(` + ), + ).toEqual({ + blockNotation: "```", + language: "markdown", + content: + spaceTrim(` Markdown has simple formatting like **bold** and *italic* text. @@ -190,41 +190,47 @@ describe('how extractOneBlockFromMarkdown works', () => { And loooot of other features. - `) + '\n', - }); - }); + `) + "\n", + }); + }); - it('should fail with example with no code blocks', () => { - expect(() => - extractOneBlockFromMarkdown( - spaceTrim(` + it("should fail with example with no code blocks", () => { + expect(() => + extractOneBlockFromMarkdown( + spaceTrim(` Hello World `), - ), - ).toThrowError(/There should be exactly 1 code block in task section, found 0 code blocks/i); - - expect(() => - extractOneBlockFromMarkdown( - spaceTrim(` + ), + ).toThrowError( + /There should be exactly 1 code block in task section, found 0 code blocks/i, + ); + + expect(() => + extractOneBlockFromMarkdown( + spaceTrim(` Hello World Hello World `), - ), - ).toThrowError(/There should be exactly 1 code block in task section, found 0 code blocks/i); - - expect(() => - extractOneBlockFromMarkdown( - spaceTrim(` + ), + ).toThrowError( + /There should be exactly 1 code block in task section, found 0 code blocks/i, + ); + + expect(() => + extractOneBlockFromMarkdown( + spaceTrim(` # Hello World Content with **bold** and *italic* text `), - ), - ).toThrowError(/There should be exactly 1 code block in task section, found 0 code blocks/i); - - expect(() => - extractOneBlockFromMarkdown( - spaceTrim(` + ), + ).toThrowError( + /There should be exactly 1 code block in task section, found 0 code blocks/i, + ); + + expect(() => + extractOneBlockFromMarkdown( + spaceTrim(` # Prague Prague is the capital of Czech Republic. It is a beautiful city. In my opinion, it is the most beautiful city in the world. @@ -243,14 +249,16 @@ describe('how extractOneBlockFromMarkdown works', () => { \`Lennon Wall\` is a wall in Prague. It is located in the center of Prague. On this wall, you can see many graffiti like %#2/*\`\`\`7#^ `), - ), - ).toThrowError(/There should be exactly 1 code block in task section, found 0 code blocks/i); - }); - - it('should fail with example with multiple code blocks of one line', () => { - expect(() => - extractOneBlockFromMarkdown( - spaceTrim(` + ), + ).toThrowError( + /There should be exactly 1 code block in task section, found 0 code blocks/i, + ); + }); + + it("should fail with example with multiple code blocks of one line", () => { + expect(() => + extractOneBlockFromMarkdown( + spaceTrim(` # Hello World Hello World in multiple languages: @@ -271,7 +279,9 @@ describe('how extractOneBlockFromMarkdown works', () => { console.info('Hello World') \`\`\` `), - ), - ).toThrowError(/There should be exactly 1 code block in task section, found 2 code blocks/i); - }); + ), + ).toThrowError( + /There should be exactly 1 code block in task section, found 2 code blocks/i, + ); + }); }); diff --git a/src/utils/markdown/extractOneBlockFromMarkdown.ts b/src/utils/markdown/extractOneBlockFromMarkdown.ts index 4cdbfba629..5276285439 100644 --- a/src/utils/markdown/extractOneBlockFromMarkdown.ts +++ b/src/utils/markdown/extractOneBlockFromMarkdown.ts @@ -1,8 +1,8 @@ -import spaceTrim from 'spacetrim'; -import { ParseError } from '../../errors/ParseError'; -import type { string_markdown } from '../../types/typeAliases'; -import type { CodeBlock } from './extractAllBlocksFromMarkdown'; -import { extractAllBlocksFromMarkdown } from './extractAllBlocksFromMarkdown'; +import spaceTrim from "spacetrim"; +import { ParseError } from "../../errors/ParseError"; +import type { string_markdown } from "../../types/typeAliases"; +import type { CodeBlock } from "./extractAllBlocksFromMarkdown"; +import { extractAllBlocksFromMarkdown } from "./extractAllBlocksFromMarkdown"; /** * Extracts exactly ONE code block from markdown. @@ -20,23 +20,25 @@ import { extractAllBlocksFromMarkdown } from './extractAllBlocksFromMarkdown'; * @public exported from `@promptbook/markdown-utils` * @throws {ParseError} if there is not exactly one code block in the markdown */ -export function extractOneBlockFromMarkdown(markdown: string_markdown): CodeBlock { - const codeBlocks = extractAllBlocksFromMarkdown(markdown); +export function extractOneBlockFromMarkdown( + markdown: string_markdown, +): CodeBlock { + const codeBlocks = extractAllBlocksFromMarkdown(markdown); - if (codeBlocks.length !== 1) { - throw new ParseError( - spaceTrim( - (block) => ` + if (codeBlocks.length !== 1) { + throw new ParseError( + spaceTrim( + (block) => ` There should be exactly 1 code block in task section, found ${codeBlocks.length} code blocks - ${block(codeBlocks.map((block, i) => `Block ${i + 1}:\n${block.content}`).join('\n\n\n'))} + ${block(codeBlocks.map((block, i) => `Block ${i + 1}:\n${block.content}`).join("\n\n\n"))} `, - ), - // <- [🚞] - ); - } + ), + // <- [🚞] + ); + } - return codeBlocks[0]!; + return codeBlocks[0]!; } /*** diff --git a/src/utils/markdown/flattenMarkdown.test.ts b/src/utils/markdown/flattenMarkdown.test.ts index c76d7b675b..f8d8b433f0 100644 --- a/src/utils/markdown/flattenMarkdown.test.ts +++ b/src/utils/markdown/flattenMarkdown.test.ts @@ -1,42 +1,42 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { DEFAULT_BOOK_TITLE } from '../../config'; -import { just } from '../organization/just'; -import { flattenMarkdown } from './flattenMarkdown'; - -describe('flattenMarkdown', () => { - it('keep simple case', () => { - expect(flattenMarkdown('# Title')).toBe('# Title'); - }); - - it('adds missing h1', () => { - expect(flattenMarkdown('')).toBe(`# ${DEFAULT_BOOK_TITLE}`); - }); - - it('keep simple case without h2', () => { - expect( - flattenMarkdown( - spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { DEFAULT_BOOK_TITLE } from "../../config"; +import { just } from "../organization/just"; +import { flattenMarkdown } from "./flattenMarkdown"; + +describe("flattenMarkdown", () => { + it("keep simple case", () => { + expect(flattenMarkdown("# Title")).toBe("# Title"); + }); + + it("adds missing h1", () => { + expect(flattenMarkdown("")).toBe(`# ${DEFAULT_BOOK_TITLE}`); + }); + + it("keep simple case without h2", () => { + expect( + flattenMarkdown( + spaceTrim(` # Title Text below title `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # Title Text below title `), - ), - ); - }); - - it('keep simple case with multi-line text', () => { - expect( - flattenMarkdown( - spaceTrim(` + ), + ); + }); + + it("keep simple case with multi-line text", () => { + expect( + flattenMarkdown( + spaceTrim(` # Title Text below title @@ -44,10 +44,10 @@ describe('flattenMarkdown', () => { Text below title Text below title `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # Title Text below title @@ -55,34 +55,34 @@ describe('flattenMarkdown', () => { Text below title Text below title `), - ), - ); - }); - - it('keep simple case with bold/italic text', () => { - expect( - flattenMarkdown( - spaceTrim(` + ), + ); + }); + + it("keep simple case with bold/italic text", () => { + expect( + flattenMarkdown( + spaceTrim(` # Title Text below title **bold** *italic* `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # Title Text below title **bold** *italic* `), - ), - ); - }); - - it('keep simple case with ul/ol text', () => { - expect( - flattenMarkdown( - spaceTrim(` + ), + ); + }); + + it("keep simple case with ul/ol text", () => { + expect( + flattenMarkdown( + spaceTrim(` # Title Text below title @@ -93,10 +93,10 @@ describe('flattenMarkdown', () => { 2. ol 2 3. ol 3 `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # Title Text below title @@ -107,14 +107,14 @@ describe('flattenMarkdown', () => { 2. ol 2 3. ol 3 `), - ), - ); - }); - - it('flatten simple case with text and section', () => { - expect( - flattenMarkdown( - spaceTrim(` + ), + ); + }); + + it("flatten simple case with text and section", () => { + expect( + flattenMarkdown( + spaceTrim(` # Title Text below title @@ -123,10 +123,10 @@ describe('flattenMarkdown', () => { Text below section 1 `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # Title Text below title @@ -135,14 +135,14 @@ describe('flattenMarkdown', () => { Text below section 1 `), - ), - ); - }); - - it('ignores structure in code blocks', () => { - expect( - flattenMarkdown( - spaceTrim(` + ), + ); + }); + + it("ignores structure in code blocks", () => { + expect( + flattenMarkdown( + spaceTrim(` # Title Text below title @@ -159,10 +159,10 @@ describe('flattenMarkdown', () => { \`\`\` `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # Title Text below title @@ -179,24 +179,24 @@ describe('flattenMarkdown', () => { \`\`\` `), - ), - ); - }); + ), + ); + }); - it('should work when the first heading is not h1', () => { - expect(flattenMarkdown(`## Section 1`)).toBe( - spaceTrim(` + it("should work when the first heading is not h1", () => { + expect(flattenMarkdown(`## Section 1`)).toBe( + spaceTrim(` # ${DEFAULT_BOOK_TITLE} ## Section 1 `), - ); - }); + ); + }); - it('should work when there is heading level mismatch', () => { - expect( - flattenMarkdown( - spaceTrim(` + it("should work when there is heading level mismatch", () => { + expect( + flattenMarkdown( + spaceTrim(` # Title Text below title @@ -205,10 +205,10 @@ describe('flattenMarkdown', () => { Text below subsection 1.1 `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # Title Text below title @@ -217,14 +217,14 @@ describe('flattenMarkdown', () => { Text below subsection 1.1 `), - ), - ); - }); - - it('should work when there are multiple h1', () => { - expect( - flattenMarkdown( - spaceTrim(` + ), + ); + }); + + it("should work when there are multiple h1", () => { + expect( + flattenMarkdown( + spaceTrim(` # Title Text below title @@ -233,10 +233,10 @@ describe('flattenMarkdown', () => { Text below title 2 `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # Title Text below title @@ -245,14 +245,14 @@ describe('flattenMarkdown', () => { Text below title 2 `), - ), - ); - }); - - it('should work when there is h2 at begining', () => { - expect( - flattenMarkdown( - spaceTrim(` + ), + ); + }); + + it("should work when there is h2 at begining", () => { + expect( + flattenMarkdown( + spaceTrim(` ## Subtitle Text below subtitle @@ -261,10 +261,10 @@ describe('flattenMarkdown', () => { Text below title `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # ${DEFAULT_BOOK_TITLE} ## Subtitle @@ -275,24 +275,24 @@ describe('flattenMarkdown', () => { Text below title `), - ), - ); - }); - - it('should work when there is no h1 is not at begining', () => { - expect( - flattenMarkdown( - spaceTrim(` + ), + ); + }); + + it("should work when there is no h1 is not at begining", () => { + expect( + flattenMarkdown( + spaceTrim(` Text before title # Title Text below title `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # ${DEFAULT_BOOK_TITLE} Text before title @@ -301,14 +301,14 @@ describe('flattenMarkdown', () => { Text below title `), - ), - ); - }); - - it('flatten advanced heading case', () => { - expect( - flattenMarkdown( - spaceTrim(` + ), + ); + }); + + it("flatten advanced heading case", () => { + expect( + flattenMarkdown( + spaceTrim(` # Title Text below title @@ -329,10 +329,10 @@ describe('flattenMarkdown', () => { Text below subsection 2.2 `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # Title Text below title @@ -353,14 +353,14 @@ describe('flattenMarkdown', () => { Text below subsection 2.2 `), - ), - ); - }); - - it('should not be cunfused by heading in comment (which should not be taken as heading)', () => { - expect( - flattenMarkdown( - spaceTrim(` + ), + ); + }); + + it("should not be cunfused by heading in comment (which should not be taken as heading)", () => { + expect( + flattenMarkdown( + spaceTrim(` # Title Text below title @@ -379,10 +379,10 @@ describe('flattenMarkdown', () => { Text below subsection 2.2 `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # Title Text below title @@ -401,14 +401,14 @@ describe('flattenMarkdown', () => { Text below subsection 2.2 `), - ), - ); - }); - - it('should not be cunfused by heading in code block (which should not be taken as heading)', () => { - expect( - flattenMarkdown( - spaceTrim(` + ), + ); + }); + + it("should not be cunfused by heading in code block (which should not be taken as heading)", () => { + expect( + flattenMarkdown( + spaceTrim(` # Title Text below title @@ -427,10 +427,10 @@ describe('flattenMarkdown', () => { Text below subsection 2.2 `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # Title Text below title @@ -449,7 +449,7 @@ describe('flattenMarkdown', () => { Text below subsection 2.2 `), - ), - ); - }); + ), + ); + }); }); diff --git a/src/utils/markdown/flattenMarkdown.ts b/src/utils/markdown/flattenMarkdown.ts index f60d5e94d1..83e5ec7725 100644 --- a/src/utils/markdown/flattenMarkdown.ts +++ b/src/utils/markdown/flattenMarkdown.ts @@ -1,8 +1,8 @@ -import spaceTrim from 'spacetrim'; -import { DEFAULT_BOOK_TITLE } from '../../config'; -import type { string_markdown } from '../../types/typeAliases'; -import { parseMarkdownSection } from './parseMarkdownSection'; -import { splitMarkdownIntoSections } from './splitMarkdownIntoSections'; +import spaceTrim from "spacetrim"; +import { DEFAULT_BOOK_TITLE } from "../../config"; +import type { string_markdown } from "../../types/typeAliases"; +import { parseMarkdownSection } from "./parseMarkdownSection"; +import { splitMarkdownIntoSections } from "./splitMarkdownIntoSections"; /** * Normalizes the markdown by flattening the structure @@ -12,32 +12,34 @@ import { splitMarkdownIntoSections } from './splitMarkdownIntoSections'; * * @public exported from `@promptbook/markdown-utils` */ -export function flattenMarkdown(markdown: TContent): TContent { - const sections = splitMarkdownIntoSections(markdown); - - if (sections.length === 0) { - return `# ${DEFAULT_BOOK_TITLE}` as TContent; - } - - let flattenedMarkdown: string_markdown = ''; - - const parsedSections = sections.map(parseMarkdownSection); - const firstSection = parsedSections.shift()!; - - if (firstSection.level === 1) { - flattenedMarkdown += `# ${firstSection.title}` + `\n\n`; - flattenedMarkdown += firstSection.content + `\n\n`; // <- [🧠] Maybe 3 new lines? - } else { - parsedSections.unshift(firstSection); - flattenedMarkdown += `# ${DEFAULT_BOOK_TITLE}` + `\n\n`; // <- [🧠] Maybe 3 new lines? - } - - for (const { title, content } of parsedSections) { - flattenedMarkdown += `## ${title}` + `\n\n`; - flattenedMarkdown += content + `\n\n`; // <- [🧠] Maybe 3 new lines? - } - - return spaceTrim(flattenedMarkdown) as TContent; +export function flattenMarkdown( + markdown: TContent, +): TContent { + const sections = splitMarkdownIntoSections(markdown); + + if (sections.length === 0) { + return `# ${DEFAULT_BOOK_TITLE}` as TContent; + } + + let flattenedMarkdown: string_markdown = ""; + + const parsedSections = sections.map(parseMarkdownSection); + const firstSection = parsedSections.shift()!; + + if (firstSection.level === 1) { + flattenedMarkdown += `# ${firstSection.title}` + `\n\n`; + flattenedMarkdown += firstSection.content + `\n\n`; // <- [🧠] Maybe 3 new lines? + } else { + parsedSections.unshift(firstSection); + flattenedMarkdown += `# ${DEFAULT_BOOK_TITLE}` + `\n\n`; // <- [🧠] Maybe 3 new lines? + } + + for (const { title, content } of parsedSections) { + flattenedMarkdown += `## ${title}` + `\n\n`; + flattenedMarkdown += content + `\n\n`; // <- [🧠] Maybe 3 new lines? + } + + return spaceTrim(flattenedMarkdown) as TContent; } /** diff --git a/src/utils/markdown/parseMarkdownSection.test.ts b/src/utils/markdown/parseMarkdownSection.test.ts index 84c1a13535..fe52d0562e 100644 --- a/src/utils/markdown/parseMarkdownSection.test.ts +++ b/src/utils/markdown/parseMarkdownSection.test.ts @@ -1,44 +1,46 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { parseMarkdownSection } from './parseMarkdownSection'; - -describe('how parseMarkdownSection works', () => { - it('should work with h1 section', () => - expect( - parseMarkdownSection( - spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { parseMarkdownSection } from "./parseMarkdownSection"; + +describe("how parseMarkdownSection works", () => { + it("should work with h1 section", () => + expect( + parseMarkdownSection( + spaceTrim(` # Title Content `), - ), - ).toEqual({ - title: 'Title', - level: 1, - content: 'Content', - })); - - it('should work with h2 section', () => - expect( - parseMarkdownSection( - spaceTrim(` + ), + ).toEqual({ + title: "Title", + level: 1, + content: "Content", + })); + + it("should work with h2 section", () => + expect( + parseMarkdownSection( + spaceTrim(` ## Subtitle Content `), - ), - ).toEqual({ - title: 'Subtitle', - level: 2, - content: 'Content', - })); - - it('should NOT work with non-markdown-section string', () => - expect(() => parseMarkdownSection(``)).toThrowError(/Markdown section must start with heading/)); - - expect(() => - parseMarkdownSection( - spaceTrim(` + ), + ).toEqual({ + title: "Subtitle", + level: 2, + content: "Content", + })); + + it("should NOT work with non-markdown-section string", () => + expect(() => parseMarkdownSection(``)).toThrowError( + /Markdown section must start with heading/, + )); + + expect(() => + parseMarkdownSection( + spaceTrim(` **This is not a title** Content without title @@ -46,6 +48,6 @@ describe('how parseMarkdownSection works', () => { - Foo - Bar `), - ), - ).toThrowError(/Markdown section must start with heading/); + ), + ).toThrowError(/Markdown section must start with heading/); }); diff --git a/src/utils/markdown/parseMarkdownSection.ts b/src/utils/markdown/parseMarkdownSection.ts index a454da7c6e..0b9d96564f 100644 --- a/src/utils/markdown/parseMarkdownSection.ts +++ b/src/utils/markdown/parseMarkdownSection.ts @@ -1,8 +1,8 @@ -import spaceTrim from 'spacetrim'; -import { ParseError } from '../../errors/ParseError'; -import type { string_markdown_section } from '../../types/typeAliases'; -import type { string_markdown_section_content } from '../../types/typeAliases'; -import type { string_markdown_text } from '../../types/typeAliases'; +import spaceTrim from "spacetrim"; +import { ParseError } from "../../errors/ParseError"; +import type { string_markdown_section } from "../../types/typeAliases"; +import type { string_markdown_section_content } from "../../types/typeAliases"; +import type { string_markdown_text } from "../../types/typeAliases"; /** * Parsed markdown section @@ -10,20 +10,20 @@ import type { string_markdown_text } from '../../types/typeAliases'; * @public exported from `@promptbook/markdown-utils` */ export type MarkdownSection = { - /** - * Title of the section - */ - readonly title: string_markdown_text; - - /** - * Level of the section like h1, h2, h3, h4, h5, h6 - */ - readonly level: 1 | 2 | 3 | 4 | 5 | 6; - - /** - * Content of the section with markdown formatting, blocks, lists, etc. - */ - readonly content: string_markdown_section_content; + /** + * Title of the section + */ + readonly title: string_markdown_text; + + /** + * Level of the section like h1, h2, h3, h4, h5, h6 + */ + readonly level: 1 | 2 | 3 | 4 | 5 | 6; + + /** + * Content of the section with markdown formatting, blocks, lists, etc. + */ + readonly content: string_markdown_section_content; }; /** @@ -31,28 +31,30 @@ export type MarkdownSection = { * * @public exported from `@promptbook/markdown-utils` */ -export function parseMarkdownSection(value: string_markdown_section): MarkdownSection { - const lines = value.split('\n'); - - if (!lines[0]!.startsWith('#')) { - throw new ParseError( - 'Markdown section must start with heading', - // <- [🚞] - ); - } - - const title = lines[0]!.replace(/^#+\s*/, ''); - const level = lines[0]!.match(/^#+/)?.[0].length ?? 0; - const content = spaceTrim(lines.slice(1).join('\n')); - - if (level < 1 || level > 6) { - throw new ParseError( - 'Markdown section must have heading level between 1 and 6', - // <- [🚞] - ); - } - - return { title, level: level as 1 | 2 | 3 | 4 | 5 | 6, content }; +export function parseMarkdownSection( + value: string_markdown_section, +): MarkdownSection { + const lines = value.split("\n"); + + if (!lines[0]!.startsWith("#")) { + throw new ParseError( + "Markdown section must start with heading", + // <- [🚞] + ); + } + + const title = lines[0]!.replace(/^#+\s*/, ""); + const level = lines[0]!.match(/^#+/)?.[0].length ?? 0; + const content = spaceTrim(lines.slice(1).join("\n")); + + if (level < 1 || level > 6) { + throw new ParseError( + "Markdown section must have heading level between 1 and 6", + // <- [🚞] + ); + } + + return { title, level: level as 1 | 2 | 3 | 4 | 5 | 6, content }; } /** diff --git a/src/utils/markdown/prettifyMarkdown.test.ts b/src/utils/markdown/prettifyMarkdown.test.ts index 087163caeb..b15feec194 100644 --- a/src/utils/markdown/prettifyMarkdown.test.ts +++ b/src/utils/markdown/prettifyMarkdown.test.ts @@ -1,13 +1,13 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { just } from '../organization/just'; -import { prettifyMarkdown } from './prettifyMarkdown'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { just } from "../organization/just"; +import { prettifyMarkdown } from "./prettifyMarkdown"; describe(`prettifyMarkdown`, () => { - it(`should prettify the markdown`, () => { - expect( - prettifyMarkdown( - spaceTrim(` + it(`should prettify the markdown`, () => { + expect( + prettifyMarkdown( + spaceTrim(` # Title - *Item* 1 @@ -22,10 +22,10 @@ describe(`prettifyMarkdown`, () => { - Item 1 `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` # Title - _Item_ 1 @@ -35,8 +35,8 @@ describe(`prettifyMarkdown`, () => { - Item 1 - `) + '\n', - ), - ); - }); + `) + "\n", + ), + ); + }); }); diff --git a/src/utils/markdown/prettifyMarkdown.ts b/src/utils/markdown/prettifyMarkdown.ts index db76615a28..ea384b75f4 100644 --- a/src/utils/markdown/prettifyMarkdown.ts +++ b/src/utils/markdown/prettifyMarkdown.ts @@ -1,6 +1,6 @@ -import { format } from 'prettier'; -import parserHtml from 'prettier/parser-html'; -import type { string_html } from '../../types/typeAliases'; +import { format } from "prettier"; +import parserHtml from "prettier/parser-html"; +import type { string_html } from "../../types/typeAliases"; /** * Prettify the html code @@ -9,29 +9,34 @@ import type { string_html } from '../../types/typeAliases'; * @returns formatted html code * @private withing the package because of HUGE size of prettier dependency */ -export function prettifyMarkdown(content: TContent): TContent { - try { - return format(content, { - parser: 'markdown', - plugins: [parserHtml], +export function prettifyMarkdown( + content: TContent, +): TContent { + try { + return format(content, { + parser: "markdown", + plugins: [parserHtml], - // TODO: DRY - make some import or auto-copy of .prettierrc - endOfLine: 'lf', - tabWidth: 4, - singleQuote: true, - trailingComma: 'all', - arrowParens: 'always', - printWidth: 120, - htmlWhitespaceSensitivity: 'ignore', - jsxBracketSameLine: false, - bracketSpacing: true, - }) as TContent; - } catch (error) { - // TODO: [🟥] Detect browser / node and make it colorfull - console.error('There was an error with prettifying the markdown, using the original as the fallback', { - error, - html: content, - }); - return content; - } + // TODO: DRY - make some import or auto-copy of .prettierrc + endOfLine: "lf", + tabWidth: 4, + singleQuote: true, + trailingComma: "all", + arrowParens: "always", + printWidth: 120, + htmlWhitespaceSensitivity: "ignore", + jsxBracketSameLine: false, + bracketSpacing: true, + }) as TContent; + } catch (error) { + // TODO: [🟥] Detect browser / node and make it colorfull + console.error( + "There was an error with prettifying the markdown, using the original as the fallback", + { + error, + html: content, + }, + ); + return content; + } } diff --git a/src/utils/markdown/removeMarkdownComments.test.ts b/src/utils/markdown/removeMarkdownComments.test.ts index 980eac400c..b3a8d8ea36 100644 --- a/src/utils/markdown/removeMarkdownComments.test.ts +++ b/src/utils/markdown/removeMarkdownComments.test.ts @@ -1,126 +1,126 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { removeMarkdownComments } from './removeMarkdownComments'; - -describe('removeMarkdownComments', () => { - it('should remove comments from simple text', () => { - expect( - removeMarkdownComments( - spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { removeMarkdownComments } from "./removeMarkdownComments"; + +describe("removeMarkdownComments", () => { + it("should remove comments from simple text", () => { + expect( + removeMarkdownComments( + spaceTrim(` Hello World `), - ), - ).toBe( - spaceTrim(` + ), + ).toBe( + spaceTrim(` Hello World `), - ); - }); + ); + }); - it('should remove comments from html', () => { - expect( - removeMarkdownComments( - spaceTrim(` + it("should remove comments from html", () => { + expect( + removeMarkdownComments( + spaceTrim(`

Hello World

Some content

`), - ), - ).toBe( - spaceTrim(` + ), + ).toBe( + spaceTrim(`

Hello World

Some content

`), - ); - expect( - removeMarkdownComments( - spaceTrim(` + ); + expect( + removeMarkdownComments( + spaceTrim(`

Hello World

Some content

`), - ), - ).toBe( - spaceTrim(` + ), + ).toBe( + spaceTrim(`

Hello World

Some content

`), - ); - }); + ); + }); - it('should remove comments from markdown', () => { - expect( - removeMarkdownComments( - spaceTrim(` + it("should remove comments from markdown", () => { + expect( + removeMarkdownComments( + spaceTrim(` # Hello World Some content `), - ), - ).toBe( - spaceTrim(` + ), + ).toBe( + spaceTrim(` # Hello World Some content `), - ); - expect( - removeMarkdownComments( - spaceTrim(` + ); + expect( + removeMarkdownComments( + spaceTrim(` # Hello World Some content `), - ), - ).toBe( - spaceTrim(` + ), + ).toBe( + spaceTrim(` # Hello World Some content `), - ); - }); + ); + }); - it('should not be confisused with comment content', () => { - expect( - removeMarkdownComments( - spaceTrim(` + it("should not be confisused with comment content", () => { + expect( + removeMarkdownComments( + spaceTrim(` Hello World `), - ), - ).toBe( - spaceTrim(` + ), + ).toBe( + spaceTrim(` Hello World `), - ); - }); + ); + }); - it('should remove multiple comments from simple text', () => { - expect( - removeMarkdownComments( - spaceTrim(` + it("should remove multiple comments from simple text", () => { + expect( + removeMarkdownComments( + spaceTrim(` Hello World `), - ), - ).toBe( - spaceTrim(` + ), + ).toBe( + spaceTrim(` Hello World `), - ); - }); + ); + }); - it('should remove multi-line comments from simple text', () => { - expect( - removeMarkdownComments( - spaceTrim(` + it("should remove multi-line comments from simple text", () => { + expect( + removeMarkdownComments( + spaceTrim(` World `), - ), - ).toBe( - spaceTrim(` + ), + ).toBe( + spaceTrim(` Hello World `), - ); - }); + ); + }); - /* + /* TODO: it('should not remove confusing non-comments', () => { }); diff --git a/src/utils/markdown/removeMarkdownComments.ts b/src/utils/markdown/removeMarkdownComments.ts index c827f08af2..bda2cb2b8c 100644 --- a/src/utils/markdown/removeMarkdownComments.ts +++ b/src/utils/markdown/removeMarkdownComments.ts @@ -1,6 +1,6 @@ -import { spaceTrim } from 'spacetrim'; -import type { string_html } from '../../types/typeAliases'; -import type { string_markdown } from '../../types/typeAliases'; +import { spaceTrim } from "spacetrim"; +import type { string_html } from "../../types/typeAliases"; +import type { string_markdown } from "../../types/typeAliases"; /** * Removes Markdown (or HTML) comments * @@ -8,6 +8,8 @@ import type { string_markdown } from '../../types/typeAliases'; * @returns {string} The input string with all comments removed. * @public exported from `@promptbook/markdown-utils` */ -export function removeMarkdownComments(content: TContent): TContent { - return spaceTrim(content.replace(//gs, '')) as TContent; +export function removeMarkdownComments< + TContent extends string_html | string_markdown, +>(content: TContent): TContent { + return spaceTrim(content.replace(//gs, "")) as TContent; } diff --git a/src/utils/markdown/removeMarkdownFormatting.test.ts b/src/utils/markdown/removeMarkdownFormatting.test.ts index dcb333debe..3cb2eec682 100644 --- a/src/utils/markdown/removeMarkdownFormatting.test.ts +++ b/src/utils/markdown/removeMarkdownFormatting.test.ts @@ -1,57 +1,58 @@ -import { describe, expect, it } from '@jest/globals'; -import { removeMarkdownFormatting } from './removeMarkdownFormatting'; - -describe('removeMarkdownFormatting', () => { - it('should remove bold formatting from a string', () => { - const str = 'This is **bold** text.'; - const expected = 'This is bold text.'; - expect(removeMarkdownFormatting(str)).toEqual(expected); - }); - - it('should remove multiple instances of bold formatting from a string', () => { - const str = 'This is **bold** and **also bold** text.'; - const expected = 'This is bold and also bold text.'; - expect(removeMarkdownFormatting(str)).toEqual(expected); - }); - - it('should remove italic formatting from a string', () => { - const str = 'This is *italic* text.'; - const expected = 'This is italic text.'; - expect(removeMarkdownFormatting(str)).toEqual(expected); - }); - - it('should remove code formatting from a string', () => { - const str = 'This is name of the function `spaceTrim`.'; - const expected = 'This is name of the function spaceTrim.'; - expect(removeMarkdownFormatting(str)).toEqual(expected); - }); - - it('should remove multiple instances of italic formatting from a string', () => { - const str = 'This is *italic* and *also italic* text.'; - const expected = 'This is italic and also italic text.'; - expect(removeMarkdownFormatting(str)).toEqual(expected); - }); - - it('should return the original string if it contains no Markdown tags', () => { - const str = 'This is a plain string.'; - expect(removeMarkdownFormatting(str)).toEqual(str); - }); - - it('should preserve just one mark text', () => { - const str1 = 'I am * You are'; - expect(removeMarkdownFormatting(str1)).toEqual(str1); - - const str2 = 'I am * You are ! He is ~ She is ? It is | We are / They are \\'; - expect(removeMarkdownFormatting(str2)).toEqual(str2); - - const str3 = 'I`m here'; - expect(removeMarkdownFormatting(str3)).toEqual(str3); - - const str4 = "I'm here You`re here"; - expect(removeMarkdownFormatting(str4)).toEqual(str4); - }); - - /* +import { describe, expect, it } from "@jest/globals"; +import { removeMarkdownFormatting } from "./removeMarkdownFormatting"; + +describe("removeMarkdownFormatting", () => { + it("should remove bold formatting from a string", () => { + const str = "This is **bold** text."; + const expected = "This is bold text."; + expect(removeMarkdownFormatting(str)).toEqual(expected); + }); + + it("should remove multiple instances of bold formatting from a string", () => { + const str = "This is **bold** and **also bold** text."; + const expected = "This is bold and also bold text."; + expect(removeMarkdownFormatting(str)).toEqual(expected); + }); + + it("should remove italic formatting from a string", () => { + const str = "This is *italic* text."; + const expected = "This is italic text."; + expect(removeMarkdownFormatting(str)).toEqual(expected); + }); + + it("should remove code formatting from a string", () => { + const str = "This is name of the function `spaceTrim`."; + const expected = "This is name of the function spaceTrim."; + expect(removeMarkdownFormatting(str)).toEqual(expected); + }); + + it("should remove multiple instances of italic formatting from a string", () => { + const str = "This is *italic* and *also italic* text."; + const expected = "This is italic and also italic text."; + expect(removeMarkdownFormatting(str)).toEqual(expected); + }); + + it("should return the original string if it contains no Markdown tags", () => { + const str = "This is a plain string."; + expect(removeMarkdownFormatting(str)).toEqual(str); + }); + + it("should preserve just one mark text", () => { + const str1 = "I am * You are"; + expect(removeMarkdownFormatting(str1)).toEqual(str1); + + const str2 = + "I am * You are ! He is ~ She is ? It is | We are / They are \\"; + expect(removeMarkdownFormatting(str2)).toEqual(str2); + + const str3 = "I`m here"; + expect(removeMarkdownFormatting(str3)).toEqual(str3); + + const str4 = "I'm here You`re here"; + expect(removeMarkdownFormatting(str4)).toEqual(str4); + }); + + /* TODO: it('should convert headings to texts', () => { const str = spaceTrim(` @@ -67,7 +68,7 @@ describe('removeMarkdownFormatting', () => { }); */ - /* + /* TODO: it('should convert blockquotes to texts', () => { const str = spaceTrim(` @@ -83,7 +84,7 @@ describe('removeMarkdownFormatting', () => { }); */ - /* + /* TODO: it('should remove horizontal line', () => { const str = spaceTrim(` diff --git a/src/utils/markdown/removeMarkdownFormatting.ts b/src/utils/markdown/removeMarkdownFormatting.ts index fed62dd1ad..084743fd44 100644 --- a/src/utils/markdown/removeMarkdownFormatting.ts +++ b/src/utils/markdown/removeMarkdownFormatting.ts @@ -1,4 +1,4 @@ -import type { string_markdown_text } from '../../types/typeAliases'; +import type { string_markdown_text } from "../../types/typeAliases"; /** * Removes Markdown formatting tags from a string. @@ -8,14 +8,14 @@ import type { string_markdown_text } from '../../types/typeAliases'; * @public exported from `@promptbook/markdown-utils` */ export function removeMarkdownFormatting(str: string_markdown_text): string { - // Remove bold formatting - str = str.replace(/\*\*(.*?)\*\*/g, '$1'); + // Remove bold formatting + str = str.replace(/\*\*(.*?)\*\*/g, "$1"); - // Remove italic formatting - str = str.replace(/\*(.*?)\*/g, '$1'); + // Remove italic formatting + str = str.replace(/\*(.*?)\*/g, "$1"); - // Remove code formatting - str = str.replace(/`(.*?)`/g, '$1'); + // Remove code formatting + str = str.replace(/`(.*?)`/g, "$1"); - return str; + return str; } diff --git a/src/utils/markdown/splitMarkdownIntoSections.test.ts b/src/utils/markdown/splitMarkdownIntoSections.test.ts index ae0c501cc6..ca954e53b7 100644 --- a/src/utils/markdown/splitMarkdownIntoSections.test.ts +++ b/src/utils/markdown/splitMarkdownIntoSections.test.ts @@ -1,34 +1,34 @@ -import { describe } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { DEFAULT_BOOK_TITLE } from '../../config'; -import { just } from '../organization/just'; -import { splitMarkdownIntoSections } from './splitMarkdownIntoSections'; - -describe('how splitMarkdownIntoSections works', () => { - it('flatten simple cases', () => { - expect(splitMarkdownIntoSections(``)).toEqual([]); - expect( - splitMarkdownIntoSections( - spaceTrim(` +import { describe } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { DEFAULT_BOOK_TITLE } from "../../config"; +import { just } from "../organization/just"; +import { splitMarkdownIntoSections } from "./splitMarkdownIntoSections"; + +describe("how splitMarkdownIntoSections works", () => { + it("flatten simple cases", () => { + expect(splitMarkdownIntoSections(``)).toEqual([]); + expect( + splitMarkdownIntoSections( + spaceTrim(` # Title Text below title `), - ), - ).toEqual( - just([ - spaceTrim(` + ), + ).toEqual( + just([ + spaceTrim(` # Title Text below title `), - ]), - ); + ]), + ); - expect( - splitMarkdownIntoSections( - spaceTrim(` + expect( + splitMarkdownIntoSections( + spaceTrim(` # Title Text below title @@ -37,38 +37,38 @@ describe('how splitMarkdownIntoSections works', () => { Text below section 1 `), - ), - ).toEqual( - just([ - spaceTrim(` + ), + ).toEqual( + just([ + spaceTrim(` # Title Text below title `), - spaceTrim(` + spaceTrim(` ## Section 1 Text below section 1 `), - ]), - ); - }); + ]), + ); + }); - it('adds "Untitled" if markdown does not starts with heading', () => { - expect(splitMarkdownIntoSections(`text`)).toEqual([ - spaceTrim(` + it('adds "Untitled" if markdown does not starts with heading', () => { + expect(splitMarkdownIntoSections(`text`)).toEqual([ + spaceTrim(` # ${DEFAULT_BOOK_TITLE} text `), - ]); - }); + ]); + }); - it('works withs mismatched headings', () => { - expect( - splitMarkdownIntoSections( - spaceTrim(` + it("works withs mismatched headings", () => { + expect( + splitMarkdownIntoSections( + spaceTrim(` ## Heading 2 a @@ -89,42 +89,42 @@ describe('how splitMarkdownIntoSections works', () => { e `), - ), - ).toEqual( - just([ - spaceTrim(` + ), + ).toEqual( + just([ + spaceTrim(` ## Heading 2 a `), - spaceTrim(` + spaceTrim(` # Heading 1 b `), - spaceTrim(` + spaceTrim(` # Heading 1 c `), - spaceTrim(` + spaceTrim(` #### Heading 4 d `), - spaceTrim(` + spaceTrim(` # Heading 1 e `), - ]), - ); - }); + ]), + ); + }); - it('flatten advanced case', () => { - expect( - splitMarkdownIntoSections( - spaceTrim(` + it("flatten advanced case", () => { + expect( + splitMarkdownIntoSections( + spaceTrim(` # Title Text below title @@ -165,10 +165,10 @@ describe('how splitMarkdownIntoSections works', () => { Text below subsection 2.4 \`\`\` `), - ), - ).toEqual( - just([ - spaceTrim(` + ), + ).toEqual( + just([ + spaceTrim(` # Title Text below title @@ -182,25 +182,25 @@ describe('how splitMarkdownIntoSections works', () => { 3) Cherry `), - spaceTrim(` + spaceTrim(` ## Section 1 Text below section 1 `), - spaceTrim(` + spaceTrim(` ## Section 2 Text below section 2 `), - spaceTrim(` + spaceTrim(` ### Subsection 2.1 Text below subsection 2.1 `), - spaceTrim(` + spaceTrim(` ### Subsection 2.2 Text below subsection 2.2 @@ -218,9 +218,9 @@ describe('how splitMarkdownIntoSections works', () => { \`\`\` `), - ]), - ); - }); + ]), + ); + }); - // Note: More things are tested in flattenMarkdown.test.ts which uses splitMarkdownIntoSections + // Note: More things are tested in flattenMarkdown.test.ts which uses splitMarkdownIntoSections }); diff --git a/src/utils/markdown/splitMarkdownIntoSections.ts b/src/utils/markdown/splitMarkdownIntoSections.ts index 1d80685aaa..0aa89d330f 100644 --- a/src/utils/markdown/splitMarkdownIntoSections.ts +++ b/src/utils/markdown/splitMarkdownIntoSections.ts @@ -1,68 +1,70 @@ -import spaceTrim from 'spacetrim'; -import { DEFAULT_BOOK_TITLE } from '../../config'; -import type { string_markdown } from '../../types/typeAliases'; -import type { string_markdown_section } from '../../types/typeAliases'; +import spaceTrim from "spacetrim"; +import { DEFAULT_BOOK_TITLE } from "../../config"; +import type { string_markdown } from "../../types/typeAliases"; +import type { string_markdown_section } from "../../types/typeAliases"; /** * Splits the markdown into sections by headings * * @public exported from `@promptbook/markdown-utils` */ -export function splitMarkdownIntoSections(markdown: string_markdown): ReadonlyArray { - const lines = markdown.split('\n'); - const sections: Array = []; +export function splitMarkdownIntoSections( + markdown: string_markdown, +): ReadonlyArray { + const lines = markdown.split("\n"); + const sections: Array = []; - // TODO: [🧽] DRY - let currentType: 'MARKDOWN' | 'CODE_BLOCK' | 'COMMENT' = 'MARKDOWN'; - let buffer: Array = []; + // TODO: [🧽] DRY + let currentType: "MARKDOWN" | "CODE_BLOCK" | "COMMENT" = "MARKDOWN"; + let buffer: Array = []; - const finishSection = () => { - if (buffer.length === 0) { - return; - } + const finishSection = () => { + if (buffer.length === 0) { + return; + } - let section = spaceTrim(buffer.join('\n')); + let section = spaceTrim(buffer.join("\n")); - if (section === '') { - return; - } + if (section === "") { + return; + } - if (!section.startsWith('#')) { - section = `# ${DEFAULT_BOOK_TITLE}\n\n${section}`; - } + if (!section.startsWith("#")) { + section = `# ${DEFAULT_BOOK_TITLE}\n\n${section}`; + } - sections.push(section); - buffer = []; - }; + sections.push(section); + buffer = []; + }; - for (const line of lines) { - if (currentType === 'MARKDOWN') { - if (line.startsWith('#')) { - finishSection(); - } + for (const line of lines) { + if (currentType === "MARKDOWN") { + if (line.startsWith("#")) { + finishSection(); + } - buffer.push(line); + buffer.push(line); - if (line.startsWith('```')) { - currentType = 'CODE_BLOCK'; - } else if (line.includes('')) { - currentType = 'MARKDOWN'; - } - } - } + if (line.startsWith("```")) { + currentType = "CODE_BLOCK"; + } else if (line.includes("")) { + currentType = "MARKDOWN"; + } + } + } - finishSection(); - return sections; + finishSection(); + return sections; } /** diff --git a/src/utils/normalization/DIACRITIC_VARIANTS_LETTERS.ts b/src/utils/normalization/DIACRITIC_VARIANTS_LETTERS.ts index ad2558b37b..1d1cdc8e1e 100644 --- a/src/utils/normalization/DIACRITIC_VARIANTS_LETTERS.ts +++ b/src/utils/normalization/DIACRITIC_VARIANTS_LETTERS.ts @@ -1,227 +1,245 @@ const defaultDiacriticsRemovalMap = [ - { - base: 'A', - letters: - '\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F', - }, - { base: 'AA', letters: '\uA732' }, - { base: 'AE', letters: '\u00C6\u01FC\u01E2' }, - { base: 'AO', letters: '\uA734' }, - { base: 'AU', letters: '\uA736' }, - { base: 'AV', letters: '\uA738\uA73A' }, - { base: 'AY', letters: '\uA73C' }, - { - base: 'B', - letters: '\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181', - }, - { - base: 'C', - letters: '\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E', - }, - { - base: 'D', - letters: '\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779\u00D0', - }, - { base: 'DZ', letters: '\u01F1\u01C4' }, - { base: 'Dz', letters: '\u01F2\u01C5' }, - { - base: 'E', - letters: - '\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E', - }, - { base: 'F', letters: '\u0046\u24BB\uFF26\u1E1E\u0191\uA77B' }, - { - base: 'G', - letters: '\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E', - }, - { - base: 'H', - letters: '\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D', - }, - { - base: 'I', - letters: - '\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197', - }, - { base: 'J', letters: '\u004A\u24BF\uFF2A\u0134\u0248' }, - { - base: 'K', - letters: '\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2', - }, - { - base: 'L', - letters: - '\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780', - }, - { base: 'LJ', letters: '\u01C7' }, - { base: 'Lj', letters: '\u01C8' }, - { base: 'M', letters: '\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C' }, - { - base: 'N', - letters: '\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4', - }, - { base: 'NJ', letters: '\u01CA' }, - { base: 'Nj', letters: '\u01CB' }, - { - base: 'O', - letters: - '\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C', - }, - { base: 'OI', letters: '\u01A2' }, - { base: 'OO', letters: '\uA74E' }, - { base: 'OU', letters: '\u0222' }, - { base: 'OE', letters: '\u008C\u0152' }, - { base: 'oe', letters: '\u009C\u0153' }, - { - base: 'P', - letters: '\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754', - }, - { base: 'Q', letters: '\u0051\u24C6\uFF31\uA756\uA758\u024A' }, - { - base: 'R', - letters: - '\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782', - }, - { - base: 'S', - letters: - '\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784', - }, - { - base: 'T', - letters: '\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786', - }, - { base: 'TZ', letters: '\uA728' }, - { - base: 'U', - letters: - '\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244', - }, - { base: 'V', letters: '\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245' }, - { base: 'VY', letters: '\uA760' }, - { - base: 'W', - letters: '\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72', - }, - { base: 'X', letters: '\u0058\u24CD\uFF38\u1E8A\u1E8C' }, - { - base: 'Y', - letters: '\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE', - }, - { - base: 'Z', - letters: '\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762', - }, - { - base: 'a', - letters: - '\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250', - }, - { base: 'aa', letters: '\uA733' }, - { base: 'ae', letters: '\u00E6\u01FD\u01E3' }, - { base: 'ao', letters: '\uA735' }, - { base: 'au', letters: '\uA737' }, - { base: 'av', letters: '\uA739\uA73B' }, - { base: 'ay', letters: '\uA73D' }, - { - base: 'b', - letters: '\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253', - }, - { - base: 'c', - letters: '\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184', - }, - { - base: 'd', - letters: '\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A', - }, - { base: 'dz', letters: '\u01F3\u01C6' }, - { - base: 'e', - letters: - '\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD', - }, - { base: 'f', letters: '\u0066\u24D5\uFF46\u1E1F\u0192\uA77C' }, - { - base: 'g', - letters: '\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F', - }, - { - base: 'h', - letters: '\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265', - }, - { base: 'hv', letters: '\u0195' }, - { - base: 'i', - letters: - '\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131', - }, - { base: 'j', letters: '\u006A\u24D9\uFF4A\u0135\u01F0\u0249' }, - { - base: 'k', - letters: '\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3', - }, - { - base: 'l', - letters: - '\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747', - }, - { base: 'lj', letters: '\u01C9' }, - { base: 'm', letters: '\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F' }, - { - base: 'n', - letters: - '\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5', - }, - { base: 'nj', letters: '\u01CC' }, - { - base: 'o', - letters: - '\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275', - }, - { base: 'oi', letters: '\u01A3' }, - { base: 'ou', letters: '\u0223' }, - { base: 'oo', letters: '\uA74F' }, - { - base: 'p', - letters: '\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755', - }, - { base: 'q', letters: '\u0071\u24E0\uFF51\u024B\uA757\uA759' }, - { - base: 'r', - letters: - '\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783', - }, - { - base: 's', - letters: - '\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B', - }, - { - base: 't', - letters: '\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787', - }, - { base: 'tz', letters: '\uA729' }, - { - base: 'u', - letters: - '\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289', - }, - { base: 'v', letters: '\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C' }, - { base: 'vy', letters: '\uA761' }, - { - base: 'w', - letters: '\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73', - }, - { base: 'x', letters: '\u0078\u24E7\uFF58\u1E8B\u1E8D' }, - { - base: 'y', - letters: '\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF', - }, - { - base: 'z', - letters: '\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763', - }, + { + base: "A", + letters: + "\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F", + }, + { base: "AA", letters: "\uA732" }, + { base: "AE", letters: "\u00C6\u01FC\u01E2" }, + { base: "AO", letters: "\uA734" }, + { base: "AU", letters: "\uA736" }, + { base: "AV", letters: "\uA738\uA73A" }, + { base: "AY", letters: "\uA73C" }, + { + base: "B", + letters: "\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181", + }, + { + base: "C", + letters: + "\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E", + }, + { + base: "D", + letters: + "\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779\u00D0", + }, + { base: "DZ", letters: "\u01F1\u01C4" }, + { base: "Dz", letters: "\u01F2\u01C5" }, + { + base: "E", + letters: + "\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E", + }, + { base: "F", letters: "\u0046\u24BB\uFF26\u1E1E\u0191\uA77B" }, + { + base: "G", + letters: + "\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E", + }, + { + base: "H", + letters: + "\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D", + }, + { + base: "I", + letters: + "\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197", + }, + { base: "J", letters: "\u004A\u24BF\uFF2A\u0134\u0248" }, + { + base: "K", + letters: + "\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2", + }, + { + base: "L", + letters: + "\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780", + }, + { base: "LJ", letters: "\u01C7" }, + { base: "Lj", letters: "\u01C8" }, + { base: "M", letters: "\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C" }, + { + base: "N", + letters: + "\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4", + }, + { base: "NJ", letters: "\u01CA" }, + { base: "Nj", letters: "\u01CB" }, + { + base: "O", + letters: + "\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C", + }, + { base: "OI", letters: "\u01A2" }, + { base: "OO", letters: "\uA74E" }, + { base: "OU", letters: "\u0222" }, + { base: "OE", letters: "\u008C\u0152" }, + { base: "oe", letters: "\u009C\u0153" }, + { + base: "P", + letters: "\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754", + }, + { base: "Q", letters: "\u0051\u24C6\uFF31\uA756\uA758\u024A" }, + { + base: "R", + letters: + "\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782", + }, + { + base: "S", + letters: + "\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784", + }, + { + base: "T", + letters: + "\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786", + }, + { base: "TZ", letters: "\uA728" }, + { + base: "U", + letters: + "\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244", + }, + { base: "V", letters: "\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245" }, + { base: "VY", letters: "\uA760" }, + { + base: "W", + letters: "\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72", + }, + { base: "X", letters: "\u0058\u24CD\uFF38\u1E8A\u1E8C" }, + { + base: "Y", + letters: + "\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE", + }, + { + base: "Z", + letters: + "\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762", + }, + { + base: "a", + letters: + "\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250", + }, + { base: "aa", letters: "\uA733" }, + { base: "ae", letters: "\u00E6\u01FD\u01E3" }, + { base: "ao", letters: "\uA735" }, + { base: "au", letters: "\uA737" }, + { base: "av", letters: "\uA739\uA73B" }, + { base: "ay", letters: "\uA73D" }, + { + base: "b", + letters: "\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253", + }, + { + base: "c", + letters: + "\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184", + }, + { + base: "d", + letters: + "\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A", + }, + { base: "dz", letters: "\u01F3\u01C6" }, + { + base: "e", + letters: + "\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD", + }, + { base: "f", letters: "\u0066\u24D5\uFF46\u1E1F\u0192\uA77C" }, + { + base: "g", + letters: + "\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F", + }, + { + base: "h", + letters: + "\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265", + }, + { base: "hv", letters: "\u0195" }, + { + base: "i", + letters: + "\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131", + }, + { base: "j", letters: "\u006A\u24D9\uFF4A\u0135\u01F0\u0249" }, + { + base: "k", + letters: + "\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3", + }, + { + base: "l", + letters: + "\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747", + }, + { base: "lj", letters: "\u01C9" }, + { base: "m", letters: "\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F" }, + { + base: "n", + letters: + "\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5", + }, + { base: "nj", letters: "\u01CC" }, + { + base: "o", + letters: + "\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275", + }, + { base: "oi", letters: "\u01A3" }, + { base: "ou", letters: "\u0223" }, + { base: "oo", letters: "\uA74F" }, + { + base: "p", + letters: "\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755", + }, + { base: "q", letters: "\u0071\u24E0\uFF51\u024B\uA757\uA759" }, + { + base: "r", + letters: + "\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783", + }, + { + base: "s", + letters: + "\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B", + }, + { + base: "t", + letters: + "\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787", + }, + { base: "tz", letters: "\uA729" }, + { + base: "u", + letters: + "\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289", + }, + { base: "v", letters: "\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C" }, + { base: "vy", letters: "\uA761" }, + { + base: "w", + letters: + "\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73", + }, + { base: "x", letters: "\u0078\u24E7\uFF58\u1E8B\u1E8D" }, + { + base: "y", + letters: + "\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF", + }, + { + base: "z", + letters: + "\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763", + }, ]; /** @@ -232,17 +250,18 @@ const defaultDiacriticsRemovalMap = [ * > "ě" => "e" * > "Ă" => "A" * > ... - * + * * @public exported from `@promptbook/utils` */ export const DIACRITIC_VARIANTS_LETTERS: Record = {}; // tslint:disable-next-line: prefer-for-of for (let i = 0; i < defaultDiacriticsRemovalMap.length; i++) { - const letters = defaultDiacriticsRemovalMap[i]!.letters; - // tslint:disable-next-line: prefer-for-of - for (let j = 0; j < letters.length; j++) { - DIACRITIC_VARIANTS_LETTERS[letters[j]!] = defaultDiacriticsRemovalMap[i]!.base; - } + const letters = defaultDiacriticsRemovalMap[i]!.letters; + // tslint:disable-next-line: prefer-for-of + for (let j = 0; j < letters.length; j++) { + DIACRITIC_VARIANTS_LETTERS[letters[j]!] = + defaultDiacriticsRemovalMap[i]!.base; + } } // <- TODO: [🍓] Put to maker function to save execution time if not needed diff --git a/src/utils/normalization/capitalize.test.ts b/src/utils/normalization/capitalize.test.ts index ac96c118d2..7d538e2017 100644 --- a/src/utils/normalization/capitalize.test.ts +++ b/src/utils/normalization/capitalize.test.ts @@ -1,19 +1,19 @@ -import { describe, expect, it } from '@jest/globals'; -import { capitalize } from './capitalize'; +import { describe, expect, it } from "@jest/globals"; +import { capitalize } from "./capitalize"; -describe('how capitalize works', () => { - it('will handle words beginning with lowercase', () => { - expect(capitalize(`foo`)).toEqual(`Foo`); - expect(capitalize(`bar`)).toEqual(`Bar`); - }); +describe("how capitalize works", () => { + it("will handle words beginning with lowercase", () => { + expect(capitalize(`foo`)).toEqual(`Foo`); + expect(capitalize(`bar`)).toEqual(`Bar`); + }); - it('will handle words beginning with uppercase', () => { - expect(capitalize(`Foo`)).toEqual(`Foo`); - expect(capitalize(`Bar`)).toEqual(`Bar`); - }); + it("will handle words beginning with uppercase", () => { + expect(capitalize(`Foo`)).toEqual(`Foo`); + expect(capitalize(`Bar`)).toEqual(`Bar`); + }); - it('will handle words beginning with number', () => { - expect(capitalize(`1foo`)).toEqual(`1foo`); - expect(capitalize(`1bar`)).toEqual(`1bar`); - }); + it("will handle words beginning with number", () => { + expect(capitalize(`1foo`)).toEqual(`1foo`); + expect(capitalize(`1bar`)).toEqual(`1bar`); + }); }); diff --git a/src/utils/normalization/capitalize.ts b/src/utils/normalization/capitalize.ts index bc1b9aa9ed..f04174995b 100644 --- a/src/utils/normalization/capitalize.ts +++ b/src/utils/normalization/capitalize.ts @@ -4,5 +4,5 @@ * @public exported from `@promptbook/utils` */ export function capitalize(word: string): string { - return word.substring(0, 1).toUpperCase() + word.substring(1); + return word.substring(0, 1).toUpperCase() + word.substring(1); } diff --git a/src/utils/normalization/decapitalize.test.ts b/src/utils/normalization/decapitalize.test.ts index efd4a980e4..4781a78a39 100644 --- a/src/utils/normalization/decapitalize.test.ts +++ b/src/utils/normalization/decapitalize.test.ts @@ -1,19 +1,19 @@ -import { describe, expect, it } from '@jest/globals'; -import { decapitalize } from './decapitalize'; +import { describe, expect, it } from "@jest/globals"; +import { decapitalize } from "./decapitalize"; -describe('how capitalize works', () => { - it('will handle words beginning with lowercase', () => { - expect(decapitalize(`foo`)).toEqual(`foo`); - expect(decapitalize(`bar`)).toEqual(`bar`); - }); +describe("how capitalize works", () => { + it("will handle words beginning with lowercase", () => { + expect(decapitalize(`foo`)).toEqual(`foo`); + expect(decapitalize(`bar`)).toEqual(`bar`); + }); - it('will handle words beginning with uppercase', () => { - expect(decapitalize(`Foo`)).toEqual(`foo`); - expect(decapitalize(`Bar`)).toEqual(`bar`); - }); + it("will handle words beginning with uppercase", () => { + expect(decapitalize(`Foo`)).toEqual(`foo`); + expect(decapitalize(`Bar`)).toEqual(`bar`); + }); - it('will handle words beginning with number', () => { - expect(decapitalize(`1foo`)).toEqual(`1foo`); - expect(decapitalize(`1bar`)).toEqual(`1bar`); - }); + it("will handle words beginning with number", () => { + expect(decapitalize(`1foo`)).toEqual(`1foo`); + expect(decapitalize(`1bar`)).toEqual(`1bar`); + }); }); diff --git a/src/utils/normalization/decapitalize.ts b/src/utils/normalization/decapitalize.ts index 7909ab986c..921b01e3fc 100644 --- a/src/utils/normalization/decapitalize.ts +++ b/src/utils/normalization/decapitalize.ts @@ -4,5 +4,5 @@ * @public exported from `@promptbook/utils` */ export function decapitalize(word: string): string { - return word.substring(0, 1).toLowerCase() + word.substring(1); + return word.substring(0, 1).toLowerCase() + word.substring(1); } diff --git a/src/utils/normalization/isValidKeyword.test.ts b/src/utils/normalization/isValidKeyword.test.ts index 66db7109dc..52e1c8c3df 100644 --- a/src/utils/normalization/isValidKeyword.test.ts +++ b/src/utils/normalization/isValidKeyword.test.ts @@ -1,17 +1,17 @@ -import { describe, expect, it } from '@jest/globals'; -import { isValidKeyword } from './isValidKeyword'; +import { describe, expect, it } from "@jest/globals"; +import { isValidKeyword } from "./isValidKeyword"; -describe('how validation of keyword works', () => { - it('is valid keyword', () => { - expect(isValidKeyword(`foo`)).toBe(true); - expect(isValidKeyword(`bar`)).toBe(true); - expect(isValidKeyword(`foobar`)).toBe(true); - }); +describe("how validation of keyword works", () => { + it("is valid keyword", () => { + expect(isValidKeyword(`foo`)).toBe(true); + expect(isValidKeyword(`bar`)).toBe(true); + expect(isValidKeyword(`foobar`)).toBe(true); + }); - it('is NOTvalid keyword', () => { - expect(isValidKeyword(``)).toBe(false); - expect(isValidKeyword(` `)).toBe(false); - expect(isValidKeyword(`foo bar`)).toBe(false); - expect(isValidKeyword(`fůů`)).toBe(false); - }); + it("is NOTvalid keyword", () => { + expect(isValidKeyword(``)).toBe(false); + expect(isValidKeyword(` `)).toBe(false); + expect(isValidKeyword(`foo bar`)).toBe(false); + expect(isValidKeyword(`fůů`)).toBe(false); + }); }); diff --git a/src/utils/normalization/isValidKeyword.ts b/src/utils/normalization/isValidKeyword.ts index 217cf624f0..f96c8d005a 100644 --- a/src/utils/normalization/isValidKeyword.ts +++ b/src/utils/normalization/isValidKeyword.ts @@ -1,23 +1,23 @@ -import type { string_keyword } from './IKeywords'; -import { parseKeywordsFromString } from './parseKeywordsFromString'; +import type { string_keyword } from "./IKeywords"; +import { parseKeywordsFromString } from "./parseKeywordsFromString"; /** * Tests if keyword is valid * * @param keyword to test * @returns if keyword is valid or not - * + * * @public exported from `@promptbook/utils` */ export function isValidKeyword(keyword: string): keyword is string_keyword { - const keywordParsed = parseKeywordsFromString(keyword); + const keywordParsed = parseKeywordsFromString(keyword); - if (keywordParsed.size !== 1) { - return false; - } + if (keywordParsed.size !== 1) { + return false; + } - const keywordParsedArray = Array.from(keywordParsed); - const keywordParsedFirst = keywordParsedArray[0]; + const keywordParsedArray = Array.from(keywordParsed); + const keywordParsedFirst = keywordParsedArray[0]; - return keywordParsedFirst === keyword; + return keywordParsedFirst === keyword; } diff --git a/src/utils/normalization/nameToUriPart.test.ts b/src/utils/normalization/nameToUriPart.test.ts index 98557ab49f..5a31e9eaac 100644 --- a/src/utils/normalization/nameToUriPart.test.ts +++ b/src/utils/normalization/nameToUriPart.test.ts @@ -1,16 +1,16 @@ -import { describe, expect, it } from '@jest/globals'; -import { nameToUriPart } from './nameToUriPart'; +import { describe, expect, it } from "@jest/globals"; +import { nameToUriPart } from "./nameToUriPart"; -describe('how converting name to part of URI works', () => { - it('can convert simple name to part of URI', () => { - expect(nameToUriPart(`foo`)).toBe(`foo`); - expect(nameToUriPart(`Foo`)).toBe(`foo`); - expect(nameToUriPart(`bar`)).toBe(`bar`); - expect(nameToUriPart(` foo Bar `)).toBe(`foo-bar`); - }); +describe("how converting name to part of URI works", () => { + it("can convert simple name to part of URI", () => { + expect(nameToUriPart(`foo`)).toBe(`foo`); + expect(nameToUriPart(`Foo`)).toBe(`foo`); + expect(nameToUriPart(`bar`)).toBe(`bar`); + expect(nameToUriPart(` foo Bar `)).toBe(`foo-bar`); + }); - it('can convert name with diacritics to part of URI', () => { - expect(nameToUriPart(`ěščřŽýáíéúů`)).toBe(`escrzyaieuu`); - expect(nameToUriPart(`ěščř--++++////---ŽÝÁÍÉÚŮ`)).toBe(`escr-zyaieuu`); - }); + it("can convert name with diacritics to part of URI", () => { + expect(nameToUriPart(`ěščřŽýáíéúů`)).toBe(`escrzyaieuu`); + expect(nameToUriPart(`ěščř--++++////---ŽÝÁÍÉÚŮ`)).toBe(`escr-zyaieuu`); + }); }); diff --git a/src/utils/normalization/nameToUriPart.ts b/src/utils/normalization/nameToUriPart.ts index d6df78903e..a264fb5688 100644 --- a/src/utils/normalization/nameToUriPart.ts +++ b/src/utils/normalization/nameToUriPart.ts @@ -1,20 +1,20 @@ -import { removeDiacritics } from './removeDiacritics'; +import { removeDiacritics } from "./removeDiacritics"; /** * @@@ - * + * * @param name @@@ * @returns @@@ * @example @@@ * @public exported from `@promptbook/utils` */ export function nameToUriPart(name: string): string { - let uriPart = name; + let uriPart = name; - uriPart = uriPart.toLowerCase(); - uriPart = removeDiacritics(uriPart); - uriPart = uriPart.replace(/[^a-zA-Z0-9]+/g, '-'); - uriPart = uriPart.replace(/^-+/, ''); - uriPart = uriPart.replace(/-+$/, ''); - return uriPart; + uriPart = uriPart.toLowerCase(); + uriPart = removeDiacritics(uriPart); + uriPart = uriPart.replace(/[^a-zA-Z0-9]+/g, "-"); + uriPart = uriPart.replace(/^-+/, ""); + uriPart = uriPart.replace(/-+$/, ""); + return uriPart; } diff --git a/src/utils/normalization/nameToUriParts.test.ts b/src/utils/normalization/nameToUriParts.test.ts index 3dab31442d..c0061005fb 100644 --- a/src/utils/normalization/nameToUriParts.test.ts +++ b/src/utils/normalization/nameToUriParts.test.ts @@ -1,23 +1,26 @@ -import { describe, expect, it } from '@jest/globals'; -import { nameToUriParts } from './nameToUriParts'; +import { describe, expect, it } from "@jest/globals"; +import { nameToUriParts } from "./nameToUriParts"; -describe('how converting name to parts of URI works', () => { - it('should convert simple name to parts of URI', () => { - expect(nameToUriParts(`foo`)).toEqual([`foo`]); - expect(nameToUriParts(`Foo`)).toEqual([`foo`]); - expect(nameToUriParts(`bar`)).toEqual([`bar`]); - expect(nameToUriParts(` foo Bar `)).toEqual([`foo`, `bar`]); - }); +describe("how converting name to parts of URI works", () => { + it("should convert simple name to parts of URI", () => { + expect(nameToUriParts(`foo`)).toEqual([`foo`]); + expect(nameToUriParts(`Foo`)).toEqual([`foo`]); + expect(nameToUriParts(`bar`)).toEqual([`bar`]); + expect(nameToUriParts(` foo Bar `)).toEqual([`foo`, `bar`]); + }); - it('should convert name with diacritics to parts of URI', () => { - expect(nameToUriParts(`ěščřŽýáíéúů`)).toEqual([`escrzyaieuu`]); - expect(nameToUriParts(`ěščř--++++////---ŽÝÁÍÉÚŮ`)).toEqual([`escr`, `zyaieuu`]); - }); + it("should convert name with diacritics to parts of URI", () => { + expect(nameToUriParts(`ěščřŽýáíéúů`)).toEqual([`escrzyaieuu`]); + expect(nameToUriParts(`ěščř--++++////---ŽÝÁÍÉÚŮ`)).toEqual([ + `escr`, + `zyaieuu`, + ]); + }); - it('should empty array on empty name', () => { - expect(nameToUriParts(``)).toEqual([]); - expect(nameToUriParts(` `)).toEqual([]); - expect(nameToUriParts(`___---:::`)).toEqual([]); - expect(nameToUriParts(`--++++////-`)).toEqual([]); - }); + it("should empty array on empty name", () => { + expect(nameToUriParts(``)).toEqual([]); + expect(nameToUriParts(` `)).toEqual([]); + expect(nameToUriParts(`___---:::`)).toEqual([]); + expect(nameToUriParts(`--++++////-`)).toEqual([]); + }); }); diff --git a/src/utils/normalization/nameToUriParts.ts b/src/utils/normalization/nameToUriParts.ts index e1e9130493..5eb2285eb8 100644 --- a/src/utils/normalization/nameToUriParts.ts +++ b/src/utils/normalization/nameToUriParts.ts @@ -1,4 +1,4 @@ -import { nameToUriPart } from './nameToUriPart'; +import { nameToUriPart } from "./nameToUriPart"; /** * @@@ @@ -9,7 +9,7 @@ import { nameToUriPart } from './nameToUriPart'; * @public exported from `@promptbook/utils` */ export function nameToUriParts(name: string): string[] { - return nameToUriPart(name) - .split('-') - .filter((value) => value !== ''); + return nameToUriPart(name) + .split("-") + .filter((value) => value !== ""); } diff --git a/src/utils/normalization/normalize-to-kebab-case.test.ts b/src/utils/normalization/normalize-to-kebab-case.test.ts index 54e0aa1ec5..743d267e80 100644 --- a/src/utils/normalization/normalize-to-kebab-case.test.ts +++ b/src/utils/normalization/normalize-to-kebab-case.test.ts @@ -1,29 +1,29 @@ -import { describe, expect, it } from '@jest/globals'; -import { normalizeToKebabCase } from './normalize-to-kebab-case'; +import { describe, expect, it } from "@jest/globals"; +import { normalizeToKebabCase } from "./normalize-to-kebab-case"; -describe('how normalizing to kebab-case works', () => { - it('will normalize one word', () => { - expect(normalizeToKebabCase('hello')).toEqual('hello'); - expect(normalizeToKebabCase(' hello ')).toEqual('hello'); - expect(normalizeToKebabCase('HELLO')).toEqual('hello'); - }); +describe("how normalizing to kebab-case works", () => { + it("will normalize one word", () => { + expect(normalizeToKebabCase("hello")).toEqual("hello"); + expect(normalizeToKebabCase(" hello ")).toEqual("hello"); + expect(normalizeToKebabCase("HELLO")).toEqual("hello"); + }); - it('will normalize sentence', () => { - expect(normalizeToKebabCase('hello world')).toEqual('hello-world'); - expect(normalizeToKebabCase(' hello world')).toEqual('hello-world'); - expect(normalizeToKebabCase('helloWorld')).toEqual('hello-world'); - expect(normalizeToKebabCase('hello___world')).toEqual('hello-world'); - expect(normalizeToKebabCase('hello_/_world')).toEqual('hello-world'); - expect(normalizeToKebabCase('hello_\\_world')).toEqual('hello-world'); - expect(normalizeToKebabCase('hello.world')).toEqual('hello-world'); - expect(normalizeToKebabCase('hello\nworld')).toEqual('hello-world'); - }); + it("will normalize sentence", () => { + expect(normalizeToKebabCase("hello world")).toEqual("hello-world"); + expect(normalizeToKebabCase(" hello world")).toEqual("hello-world"); + expect(normalizeToKebabCase("helloWorld")).toEqual("hello-world"); + expect(normalizeToKebabCase("hello___world")).toEqual("hello-world"); + expect(normalizeToKebabCase("hello_/_world")).toEqual("hello-world"); + expect(normalizeToKebabCase("hello_\\_world")).toEqual("hello-world"); + expect(normalizeToKebabCase("hello.world")).toEqual("hello-world"); + expect(normalizeToKebabCase("hello\nworld")).toEqual("hello-world"); + }); - it('will normalize chars with diacritics', () => { - expect(normalizeToKebabCase('hělló wórld')).toEqual('hello-world'); - }); + it("will normalize chars with diacritics", () => { + expect(normalizeToKebabCase("hělló wórld")).toEqual("hello-world"); + }); - it('can normalize word with numbers', () => { - expect(normalizeToKebabCase('1FooBar2')).toEqual('1foo-bar2'); - }); + it("can normalize word with numbers", () => { + expect(normalizeToKebabCase("1FooBar2")).toEqual("1foo-bar2"); + }); }); diff --git a/src/utils/normalization/normalize-to-kebab-case.ts b/src/utils/normalization/normalize-to-kebab-case.ts index 6fc6afa9a6..615c4e170f 100644 --- a/src/utils/normalization/normalize-to-kebab-case.ts +++ b/src/utils/normalization/normalize-to-kebab-case.ts @@ -1,4 +1,4 @@ -import { removeDiacritics } from './removeDiacritics'; +import { removeDiacritics } from "./removeDiacritics"; /** * Semantic helper for kebab-case strings @@ -12,7 +12,7 @@ export type string_kebab_case = string; /** * @private type of `normalizeToKebabCase` */ -type char_type = 'LOWERCASE' | 'UPPERCASE' | 'NUMBER' | 'SLASH' | 'OTHER'; +type char_type = "LOWERCASE" | "UPPERCASE" | "NUMBER" | "SLASH" | "OTHER"; /** * @@@ @@ -24,50 +24,50 @@ type char_type = 'LOWERCASE' | 'UPPERCASE' | 'NUMBER' | 'SLASH' | 'OTHER'; * @public exported from `@promptbook/utils` */ export function normalizeToKebabCase(text: string): string_kebab_case { - text = removeDiacritics(text); + text = removeDiacritics(text); - let charType: char_type; - let lastCharType: char_type = 'OTHER'; + let charType: char_type; + let lastCharType: char_type = "OTHER"; - let normalizedName = ''; + let normalizedName = ""; - for (const char of text) { - let normalizedChar: string; + for (const char of text) { + let normalizedChar: string; - if (/^[a-z]$/.test(char)) { - charType = 'LOWERCASE'; - normalizedChar = char; - } else if (/^[A-Z]$/.test(char)) { - charType = 'UPPERCASE'; - normalizedChar = char.toLowerCase(); - } else if (/^[0-9]$/.test(char)) { - charType = 'NUMBER'; - normalizedChar = char; - } else { - charType = 'OTHER'; - normalizedChar = '-'; - } + if (/^[a-z]$/.test(char)) { + charType = "LOWERCASE"; + normalizedChar = char; + } else if (/^[A-Z]$/.test(char)) { + charType = "UPPERCASE"; + normalizedChar = char.toLowerCase(); + } else if (/^[0-9]$/.test(char)) { + charType = "NUMBER"; + normalizedChar = char; + } else { + charType = "OTHER"; + normalizedChar = "-"; + } - if ( - charType !== lastCharType && - !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') && - !(lastCharType === 'NUMBER') && - !(charType === 'NUMBER') - ) { - normalizedName += '-'; - } + if ( + charType !== lastCharType && + !(lastCharType === "UPPERCASE" && charType === "LOWERCASE") && + !(lastCharType === "NUMBER") && + !(charType === "NUMBER") + ) { + normalizedName += "-"; + } - normalizedName += normalizedChar; + normalizedName += normalizedChar; - lastCharType = charType; - } + lastCharType = charType; + } - normalizedName = normalizedName.split(/-+/g).join('-'); - normalizedName = normalizedName.split(/-?\/-?/g).join('/'); - normalizedName = normalizedName.replace(/^-/, ''); - normalizedName = normalizedName.replace(/-$/, ''); + normalizedName = normalizedName.split(/-+/g).join("-"); + normalizedName = normalizedName.split(/-?\/-?/g).join("/"); + normalizedName = normalizedName.replace(/^-/, ""); + normalizedName = normalizedName.replace(/-$/, ""); - return normalizedName; + return normalizedName; } /** diff --git a/src/utils/normalization/normalizeTo_PascalCase.test.ts b/src/utils/normalization/normalizeTo_PascalCase.test.ts index 0443c2b60b..e29b3fc828 100644 --- a/src/utils/normalization/normalizeTo_PascalCase.test.ts +++ b/src/utils/normalization/normalizeTo_PascalCase.test.ts @@ -1,25 +1,25 @@ -import { describe, expect, it } from '@jest/globals'; -import { normalizeTo_PascalCase } from './normalizeTo_PascalCase'; +import { describe, expect, it } from "@jest/globals"; +import { normalizeTo_PascalCase } from "./normalizeTo_PascalCase"; -describe('how normalizing to camelCase works', () => { - it('can normalize one word', () => { - expect(normalizeTo_PascalCase('hello')).toEqual('Hello'); - expect(normalizeTo_PascalCase('HELLO')).toEqual('Hello'); - }); +describe("how normalizing to camelCase works", () => { + it("can normalize one word", () => { + expect(normalizeTo_PascalCase("hello")).toEqual("Hello"); + expect(normalizeTo_PascalCase("HELLO")).toEqual("Hello"); + }); - it('can normalize sentence', () => { - expect(normalizeTo_PascalCase('hello world')).toEqual('HelloWorld'); - expect(normalizeTo_PascalCase('helloWorld')).toEqual('HelloWorld'); - expect(normalizeTo_PascalCase('hello___world')).toEqual('HelloWorld'); - expect(normalizeTo_PascalCase('hello_/_world')).toEqual('HelloWorld'); - expect(normalizeTo_PascalCase('hello_\\_world')).toEqual('HelloWorld'); - expect(normalizeTo_PascalCase('hello.world')).toEqual('HelloWorld'); - expect(normalizeTo_PascalCase('hello\nworld')).toEqual('HelloWorld'); - }); + it("can normalize sentence", () => { + expect(normalizeTo_PascalCase("hello world")).toEqual("HelloWorld"); + expect(normalizeTo_PascalCase("helloWorld")).toEqual("HelloWorld"); + expect(normalizeTo_PascalCase("hello___world")).toEqual("HelloWorld"); + expect(normalizeTo_PascalCase("hello_/_world")).toEqual("HelloWorld"); + expect(normalizeTo_PascalCase("hello_\\_world")).toEqual("HelloWorld"); + expect(normalizeTo_PascalCase("hello.world")).toEqual("HelloWorld"); + expect(normalizeTo_PascalCase("hello\nworld")).toEqual("HelloWorld"); + }); - it('can normalize word with numbers', () => { - expect(normalizeTo_PascalCase('4STORY')).toEqual('4story'); - expect(normalizeTo_PascalCase('4Story')).toEqual('4story'); - expect(normalizeTo_PascalCase('Karel21')).toEqual('Karel21'); - }); + it("can normalize word with numbers", () => { + expect(normalizeTo_PascalCase("4STORY")).toEqual("4story"); + expect(normalizeTo_PascalCase("4Story")).toEqual("4story"); + expect(normalizeTo_PascalCase("Karel21")).toEqual("Karel21"); + }); }); diff --git a/src/utils/normalization/normalizeTo_PascalCase.ts b/src/utils/normalization/normalizeTo_PascalCase.ts index f7ae9abdf5..60cdcfd29c 100644 --- a/src/utils/normalization/normalizeTo_PascalCase.ts +++ b/src/utils/normalization/normalizeTo_PascalCase.ts @@ -1,4 +1,4 @@ -import { normalizeTo_camelCase } from './normalizeTo_camelCase'; +import { normalizeTo_camelCase } from "./normalizeTo_camelCase"; /** * Semantic helper for PascalCase strings @@ -18,5 +18,5 @@ export type string_PascalCase = string; * @public exported from `@promptbook/utils` */ export function normalizeTo_PascalCase(text: string): string_PascalCase { - return normalizeTo_camelCase(text, true); + return normalizeTo_camelCase(text, true); } diff --git a/src/utils/normalization/normalizeTo_SCREAMING_CASE.test.ts b/src/utils/normalization/normalizeTo_SCREAMING_CASE.test.ts index b2ce2b89ed..627166a757 100644 --- a/src/utils/normalization/normalizeTo_SCREAMING_CASE.test.ts +++ b/src/utils/normalization/normalizeTo_SCREAMING_CASE.test.ts @@ -1,25 +1,25 @@ -import { describe, expect, it } from '@jest/globals'; -import { normalizeTo_SCREAMING_CASE } from './normalizeTo_SCREAMING_CASE'; +import { describe, expect, it } from "@jest/globals"; +import { normalizeTo_SCREAMING_CASE } from "./normalizeTo_SCREAMING_CASE"; -describe('how normalizing to SCREAMING_CASE works', () => { - it('can normalize one word', () => { - expect(normalizeTo_SCREAMING_CASE('hello')).toEqual('HELLO'); - expect(normalizeTo_SCREAMING_CASE(' hello ')).toEqual('HELLO'); - expect(normalizeTo_SCREAMING_CASE('HELLO')).toEqual('HELLO'); - }); +describe("how normalizing to SCREAMING_CASE works", () => { + it("can normalize one word", () => { + expect(normalizeTo_SCREAMING_CASE("hello")).toEqual("HELLO"); + expect(normalizeTo_SCREAMING_CASE(" hello ")).toEqual("HELLO"); + expect(normalizeTo_SCREAMING_CASE("HELLO")).toEqual("HELLO"); + }); - it('can normalize sentence', () => { - expect(normalizeTo_SCREAMING_CASE('hello world')).toEqual('HELLO_WORLD'); - expect(normalizeTo_SCREAMING_CASE('helloWorld')).toEqual('HELLO_WORLD'); - expect(normalizeTo_SCREAMING_CASE('hello___world')).toEqual('HELLO_WORLD'); - expect(normalizeTo_SCREAMING_CASE('hello_/_world')).toEqual('HELLO_WORLD'); - expect(normalizeTo_SCREAMING_CASE('hello_\\_world')).toEqual('HELLO_WORLD'); - expect(normalizeTo_SCREAMING_CASE('hello.world')).toEqual('HELLO_WORLD'); - expect(normalizeTo_SCREAMING_CASE('hello\nworld')).toEqual('HELLO_WORLD'); - }); + it("can normalize sentence", () => { + expect(normalizeTo_SCREAMING_CASE("hello world")).toEqual("HELLO_WORLD"); + expect(normalizeTo_SCREAMING_CASE("helloWorld")).toEqual("HELLO_WORLD"); + expect(normalizeTo_SCREAMING_CASE("hello___world")).toEqual("HELLO_WORLD"); + expect(normalizeTo_SCREAMING_CASE("hello_/_world")).toEqual("HELLO_WORLD"); + expect(normalizeTo_SCREAMING_CASE("hello_\\_world")).toEqual("HELLO_WORLD"); + expect(normalizeTo_SCREAMING_CASE("hello.world")).toEqual("HELLO_WORLD"); + expect(normalizeTo_SCREAMING_CASE("hello\nworld")).toEqual("HELLO_WORLD"); + }); - it('can normalize word with numbers', () => { - expect(normalizeTo_SCREAMING_CASE('4story')).toEqual('4STORY'); - expect(normalizeTo_SCREAMING_CASE('karel21')).toEqual('KAREL21'); - }); + it("can normalize word with numbers", () => { + expect(normalizeTo_SCREAMING_CASE("4story")).toEqual("4STORY"); + expect(normalizeTo_SCREAMING_CASE("karel21")).toEqual("KAREL21"); + }); }); diff --git a/src/utils/normalization/normalizeTo_SCREAMING_CASE.ts b/src/utils/normalization/normalizeTo_SCREAMING_CASE.ts index 61b0f71c71..409421da24 100644 --- a/src/utils/normalization/normalizeTo_SCREAMING_CASE.ts +++ b/src/utils/normalization/normalizeTo_SCREAMING_CASE.ts @@ -10,7 +10,7 @@ export type string_SCREAMING_CASE = string; /** * @private type of `normalizeTo_SCREAMING_CASE` */ -type char_type = 'LOWERCASE' | 'UPPERCASE' | 'NUMBER' | 'SLASH' | 'OTHER'; +type char_type = "LOWERCASE" | "UPPERCASE" | "NUMBER" | "SLASH" | "OTHER"; /** * @@@ @@ -21,49 +21,51 @@ type char_type = 'LOWERCASE' | 'UPPERCASE' | 'NUMBER' | 'SLASH' | 'OTHER'; * @example 'I_LOVE_PROMPTBOOK' * @public exported from `@promptbook/utils` */ -export function normalizeTo_SCREAMING_CASE(text: string): string_SCREAMING_CASE { - let charType: char_type; - let lastCharType: char_type = 'OTHER'; +export function normalizeTo_SCREAMING_CASE( + text: string, +): string_SCREAMING_CASE { + let charType: char_type; + let lastCharType: char_type = "OTHER"; - let normalizedName = ''; + let normalizedName = ""; - for (const char of text) { - let normalizedChar: string; + for (const char of text) { + let normalizedChar: string; - if (/^[a-z]$/.test(char)) { - charType = 'LOWERCASE'; - normalizedChar = char.toUpperCase(); - } else if (/^[A-Z]$/.test(char)) { - charType = 'UPPERCASE'; - normalizedChar = char; - } else if (/^[0-9]$/.test(char)) { - charType = 'NUMBER'; - normalizedChar = char; - } else { - charType = 'OTHER'; - normalizedChar = '_'; - } + if (/^[a-z]$/.test(char)) { + charType = "LOWERCASE"; + normalizedChar = char.toUpperCase(); + } else if (/^[A-Z]$/.test(char)) { + charType = "UPPERCASE"; + normalizedChar = char; + } else if (/^[0-9]$/.test(char)) { + charType = "NUMBER"; + normalizedChar = char; + } else { + charType = "OTHER"; + normalizedChar = "_"; + } - if ( - charType !== lastCharType && - !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') && - !(lastCharType === 'NUMBER') && - !(charType === 'NUMBER') - ) { - normalizedName += '_'; - } + if ( + charType !== lastCharType && + !(lastCharType === "UPPERCASE" && charType === "LOWERCASE") && + !(lastCharType === "NUMBER") && + !(charType === "NUMBER") + ) { + normalizedName += "_"; + } - normalizedName += normalizedChar; + normalizedName += normalizedChar; - lastCharType = charType; - } + lastCharType = charType; + } - normalizedName = normalizedName.replace(/_+/g, '_'); - normalizedName = normalizedName.replace(/_?\/_?/g, '/'); - normalizedName = normalizedName.replace(/^_/, ''); - normalizedName = normalizedName.replace(/_$/, ''); + normalizedName = normalizedName.replace(/_+/g, "_"); + normalizedName = normalizedName.replace(/_?\/_?/g, "/"); + normalizedName = normalizedName.replace(/^_/, ""); + normalizedName = normalizedName.replace(/_$/, ""); - return normalizedName; + return normalizedName; } /** diff --git a/src/utils/normalization/normalizeTo_camelCase.test.ts b/src/utils/normalization/normalizeTo_camelCase.test.ts index dfe41e63c1..6ebbb92892 100644 --- a/src/utils/normalization/normalizeTo_camelCase.test.ts +++ b/src/utils/normalization/normalizeTo_camelCase.test.ts @@ -1,25 +1,25 @@ -import { describe, expect, it } from '@jest/globals'; -import { normalizeTo_camelCase } from './normalizeTo_camelCase'; +import { describe, expect, it } from "@jest/globals"; +import { normalizeTo_camelCase } from "./normalizeTo_camelCase"; -describe('how normalizing to camelCase works', () => { - it('can normalize one word', () => { - expect(normalizeTo_camelCase('hello')).toEqual('hello'); - expect(normalizeTo_camelCase('HELLO')).toEqual('hello'); - }); +describe("how normalizing to camelCase works", () => { + it("can normalize one word", () => { + expect(normalizeTo_camelCase("hello")).toEqual("hello"); + expect(normalizeTo_camelCase("HELLO")).toEqual("hello"); + }); - it('can normalize sentence', () => { - expect(normalizeTo_camelCase('hello world')).toEqual('helloWorld'); - expect(normalizeTo_camelCase('helloWorld')).toEqual('helloWorld'); - expect(normalizeTo_camelCase('hello___world')).toEqual('helloWorld'); - expect(normalizeTo_camelCase('hello_/_world')).toEqual('helloWorld'); - expect(normalizeTo_camelCase('hello_\\_world')).toEqual('helloWorld'); - expect(normalizeTo_camelCase('hello.world')).toEqual('helloWorld'); - expect(normalizeTo_camelCase('hello\nworld')).toEqual('helloWorld'); - }); + it("can normalize sentence", () => { + expect(normalizeTo_camelCase("hello world")).toEqual("helloWorld"); + expect(normalizeTo_camelCase("helloWorld")).toEqual("helloWorld"); + expect(normalizeTo_camelCase("hello___world")).toEqual("helloWorld"); + expect(normalizeTo_camelCase("hello_/_world")).toEqual("helloWorld"); + expect(normalizeTo_camelCase("hello_\\_world")).toEqual("helloWorld"); + expect(normalizeTo_camelCase("hello.world")).toEqual("helloWorld"); + expect(normalizeTo_camelCase("hello\nworld")).toEqual("helloWorld"); + }); - it('can normalize word with numbers', () => { - expect(normalizeTo_camelCase('4STORY')).toEqual('4story'); - expect(normalizeTo_camelCase('4Story')).toEqual('4story'); - expect(normalizeTo_camelCase('Karel21')).toEqual('karel21'); - }); + it("can normalize word with numbers", () => { + expect(normalizeTo_camelCase("4STORY")).toEqual("4story"); + expect(normalizeTo_camelCase("4Story")).toEqual("4story"); + expect(normalizeTo_camelCase("Karel21")).toEqual("karel21"); + }); }); diff --git a/src/utils/normalization/normalizeTo_camelCase.ts b/src/utils/normalization/normalizeTo_camelCase.ts index b4d0a361b6..d3619741cc 100644 --- a/src/utils/normalization/normalizeTo_camelCase.ts +++ b/src/utils/normalization/normalizeTo_camelCase.ts @@ -10,12 +10,11 @@ export type string_camelCase = string; /** * @private type of `normalizeTo_camelCase` */ -type char_type = 'LOWERCASE' | 'UPPERCASE' | 'NUMBER' | 'OTHER'; - +type char_type = "LOWERCASE" | "UPPERCASE" | "NUMBER" | "OTHER"; /** * @@@ - * + * * @param text @@@ * @param _isFirstLetterCapital @@@ * @returns @@@ @@ -23,48 +22,51 @@ type char_type = 'LOWERCASE' | 'UPPERCASE' | 'NUMBER' | 'OTHER'; * @example 'iLovePromptbook' * @public exported from `@promptbook/utils` */ -export function normalizeTo_camelCase(text: string, _isFirstLetterCapital = false): string_camelCase { - let charType: char_type; - let lastCharType: char_type | null = null; +export function normalizeTo_camelCase( + text: string, + _isFirstLetterCapital = false, +): string_camelCase { + let charType: char_type; + let lastCharType: char_type | null = null; - let normalizedName = ''; + let normalizedName = ""; - for (const char of text) { - let normalizedChar: string; + for (const char of text) { + let normalizedChar: string; - if (/^[a-z]$/.test(char)) { - charType = 'LOWERCASE'; - normalizedChar = char; - } else if (/^[A-Z]$/.test(char)) { - charType = 'UPPERCASE'; - normalizedChar = char.toLowerCase(); - } else if (/^[0-9]$/.test(char)) { - charType = 'NUMBER'; - normalizedChar = char; - } else { - charType = 'OTHER'; - normalizedChar = ''; - } + if (/^[a-z]$/.test(char)) { + charType = "LOWERCASE"; + normalizedChar = char; + } else if (/^[A-Z]$/.test(char)) { + charType = "UPPERCASE"; + normalizedChar = char.toLowerCase(); + } else if (/^[0-9]$/.test(char)) { + charType = "NUMBER"; + normalizedChar = char; + } else { + charType = "OTHER"; + normalizedChar = ""; + } - if (!lastCharType) { - if (_isFirstLetterCapital) { - normalizedChar = normalizedChar.toUpperCase(); //TODO: DRY - } - } else if ( - charType !== lastCharType && - !(charType === 'LOWERCASE' && lastCharType === 'UPPERCASE') && - !(lastCharType === 'NUMBER') && - !(charType === 'NUMBER') - ) { - normalizedChar = normalizedChar.toUpperCase(); //TODO: [🌺] DRY - } + if (!lastCharType) { + if (_isFirstLetterCapital) { + normalizedChar = normalizedChar.toUpperCase(); //TODO: DRY + } + } else if ( + charType !== lastCharType && + !(charType === "LOWERCASE" && lastCharType === "UPPERCASE") && + !(lastCharType === "NUMBER") && + !(charType === "NUMBER") + ) { + normalizedChar = normalizedChar.toUpperCase(); //TODO: [🌺] DRY + } - normalizedName += normalizedChar; + normalizedName += normalizedChar; - lastCharType = charType; - } + lastCharType = charType; + } - return normalizedName; + return normalizedName; } /** diff --git a/src/utils/normalization/normalizeTo_snake_case.test.ts b/src/utils/normalization/normalizeTo_snake_case.test.ts index 5875b88da0..48138635e7 100644 --- a/src/utils/normalization/normalizeTo_snake_case.test.ts +++ b/src/utils/normalization/normalizeTo_snake_case.test.ts @@ -1,25 +1,25 @@ -import { describe, expect, it } from '@jest/globals'; -import { normalizeTo_snake_case } from './normalizeTo_snake_case'; +import { describe, expect, it } from "@jest/globals"; +import { normalizeTo_snake_case } from "./normalizeTo_snake_case"; -describe('how normalizing to snake_case works', () => { - it('can normalize one word', () => { - expect(normalizeTo_snake_case('hello')).toEqual('hello'); - expect(normalizeTo_snake_case('HELLO')).toEqual('hello'); - }); +describe("how normalizing to snake_case works", () => { + it("can normalize one word", () => { + expect(normalizeTo_snake_case("hello")).toEqual("hello"); + expect(normalizeTo_snake_case("HELLO")).toEqual("hello"); + }); - it('can normalize sentence', () => { - expect(normalizeTo_snake_case('hello world')).toEqual('hello_world'); - expect(normalizeTo_snake_case('helloWorld')).toEqual('hello_world'); - expect(normalizeTo_snake_case('hello___world')).toEqual('hello_world'); - expect(normalizeTo_snake_case('hello_/_world')).toEqual('hello_world'); - expect(normalizeTo_snake_case('hello_\\_world')).toEqual('hello_world'); - expect(normalizeTo_snake_case('hello.world')).toEqual('hello_world'); - expect(normalizeTo_snake_case('hello\nworld')).toEqual('hello_world'); - }); + it("can normalize sentence", () => { + expect(normalizeTo_snake_case("hello world")).toEqual("hello_world"); + expect(normalizeTo_snake_case("helloWorld")).toEqual("hello_world"); + expect(normalizeTo_snake_case("hello___world")).toEqual("hello_world"); + expect(normalizeTo_snake_case("hello_/_world")).toEqual("hello_world"); + expect(normalizeTo_snake_case("hello_\\_world")).toEqual("hello_world"); + expect(normalizeTo_snake_case("hello.world")).toEqual("hello_world"); + expect(normalizeTo_snake_case("hello\nworld")).toEqual("hello_world"); + }); - it('can normalize word with numbers', () => { - expect(normalizeTo_snake_case('4STORY')).toEqual('4story'); - expect(normalizeTo_snake_case('4Story')).toEqual('4story'); - expect(normalizeTo_snake_case('Karel21')).toEqual('karel21'); - }); + it("can normalize word with numbers", () => { + expect(normalizeTo_snake_case("4STORY")).toEqual("4story"); + expect(normalizeTo_snake_case("4Story")).toEqual("4story"); + expect(normalizeTo_snake_case("Karel21")).toEqual("karel21"); + }); }); diff --git a/src/utils/normalization/normalizeTo_snake_case.ts b/src/utils/normalization/normalizeTo_snake_case.ts index a13bde630f..1c33fcbf8f 100644 --- a/src/utils/normalization/normalizeTo_snake_case.ts +++ b/src/utils/normalization/normalizeTo_snake_case.ts @@ -1,4 +1,4 @@ -import { normalizeTo_SCREAMING_CASE } from './normalizeTo_SCREAMING_CASE'; +import { normalizeTo_SCREAMING_CASE } from "./normalizeTo_SCREAMING_CASE"; /** * Semantic helper for snake_case strings @@ -18,5 +18,5 @@ export type string_snake_case = string; * @public exported from `@promptbook/utils` */ export function normalizeTo_snake_case(text: string): string_snake_case { - return normalizeTo_SCREAMING_CASE(text).toLowerCase(); + return normalizeTo_SCREAMING_CASE(text).toLowerCase(); } diff --git a/src/utils/normalization/normalizeWhitespaces.test.ts b/src/utils/normalization/normalizeWhitespaces.test.ts index 3ae58a8783..ac37f6cba4 100644 --- a/src/utils/normalization/normalizeWhitespaces.test.ts +++ b/src/utils/normalization/normalizeWhitespaces.test.ts @@ -1,24 +1,28 @@ // test normalizeWhitespaces -import { describe, expect, it } from '@jest/globals'; -import { normalizeWhitespaces } from './normalizeWhitespaces'; +import { describe, expect, it } from "@jest/globals"; +import { normalizeWhitespaces } from "./normalizeWhitespaces"; -describe('normalizeWhitespaces', () => { - it('will normalize multiple whitespaces', () => { - expect(normalizeWhitespaces(`foo bar baz`)).toEqual(`foo bar baz`); - }); - it('will normalize new lines', () => { - expect( - normalizeWhitespaces( - ` +describe("normalizeWhitespaces", () => { + it("will normalize multiple whitespaces", () => { + expect(normalizeWhitespaces(`foo bar baz`)).toEqual( + `foo bar baz`, + ); + }); + it("will normalize new lines", () => { + expect( + normalizeWhitespaces( + ` foo bar baz `, - ), - ).toEqual(`foo bar baz`); - }); + ), + ).toEqual(`foo bar baz`); + }); - it('will normalize tabs', () => { - expect(normalizeWhitespaces(`foo\t\tbar baz`)).toEqual(`foo bar baz`); - }); + it("will normalize tabs", () => { + expect(normalizeWhitespaces(`foo\t\tbar baz`)).toEqual( + `foo bar baz`, + ); + }); }); diff --git a/src/utils/normalization/normalizeWhitespaces.ts b/src/utils/normalization/normalizeWhitespaces.ts index 275e316cfc..8ded24b8de 100644 --- a/src/utils/normalization/normalizeWhitespaces.ts +++ b/src/utils/normalization/normalizeWhitespaces.ts @@ -4,5 +4,5 @@ * @public exported from `@promptbook/utils` */ export function normalizeWhitespaces(sentence: string): string { - return sentence.replace(/\s+/gs, ' ').trim(); + return sentence.replace(/\s+/gs, " ").trim(); } diff --git a/src/utils/normalization/orderJson.test.ts b/src/utils/normalization/orderJson.test.ts index 3c8372510b..ce19083ef6 100644 --- a/src/utils/normalization/orderJson.test.ts +++ b/src/utils/normalization/orderJson.test.ts @@ -1,36 +1,36 @@ -import { describe, expect, it } from '@jest/globals'; -import { orderJson } from './orderJson'; +import { describe, expect, it } from "@jest/globals"; +import { orderJson } from "./orderJson"; -describe('how `orderJson` works', () => { - it('should keep object when no order given', () => - expect( - JSON.stringify( - orderJson({ - value: { - a: 1, - b: 2, - c: 3, - d: 4, - }, - order: ['d'], - }), - ), - ).toBe(`{"d":4,"a":1,"b":2,"c":3}`)); +describe("how `orderJson` works", () => { + it("should keep object when no order given", () => + expect( + JSON.stringify( + orderJson({ + value: { + a: 1, + b: 2, + c: 3, + d: 4, + }, + order: ["d"], + }), + ), + ).toBe(`{"d":4,"a":1,"b":2,"c":3}`)); - it('should order simple object', () => - expect( - JSON.stringify( - orderJson({ - value: { - a: 1, - b: 2, - c: 3, - d: 4, - }, - order: ['d'], - }), - ), - ).toBe(`{"d":4,"a":1,"b":2,"c":3}`)); + it("should order simple object", () => + expect( + JSON.stringify( + orderJson({ + value: { + a: 1, + b: 2, + c: 3, + d: 4, + }, + order: ["d"], + }), + ), + ).toBe(`{"d":4,"a":1,"b":2,"c":3}`)); }); /** diff --git a/src/utils/normalization/orderJson.ts b/src/utils/normalization/orderJson.ts index 9a2d95fd78..911a55a430 100644 --- a/src/utils/normalization/orderJson.ts +++ b/src/utils/normalization/orderJson.ts @@ -1,19 +1,19 @@ -import type { JsonArray, JsonObject } from 'type-fest'; +import type { JsonArray, JsonObject } from "type-fest"; /** * Options for the `orderJson` function */ export type OrderJsonOptions = { - /** - * Value to checked, ordered and deeply frozen - */ - value: TObject; + /** + * Value to checked, ordered and deeply frozen + */ + value: TObject; - /** - * Order of the object properties - */ - order: Array; - // <- TODO: !!6 Deep keyof + /** + * Order of the object properties + */ + order: Array; + // <- TODO: !!6 Deep keyof }; /** @@ -22,13 +22,17 @@ export type OrderJsonOptions = { * @returns The same type of object as the input re-ordered * @public exported from `@promptbook/utils` */ -export function orderJson(options: OrderJsonOptions): TObject { - const { value, order } = options; +export function orderJson( + options: OrderJsonOptions, +): TObject { + const { value, order } = options; - const orderedValue = { - ...(order === undefined ? {} : Object.fromEntries(order.map((key) => [key, undefined]))), - ...value, - }; + const orderedValue = { + ...(order === undefined + ? {} + : Object.fromEntries(order.map((key) => [key, undefined]))), + ...value, + }; - return orderedValue; + return orderedValue; } diff --git a/src/utils/normalization/parseKeywords.test.ts b/src/utils/normalization/parseKeywords.test.ts index feba0c23b0..a9b5b6f8be 100644 --- a/src/utils/normalization/parseKeywords.test.ts +++ b/src/utils/normalization/parseKeywords.test.ts @@ -1,63 +1,73 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseKeywords } from './parseKeywords'; - -describe('how parsing of keywords from objects works', () => { - it('can parse keywords from arrays', () => { - expect(parseKeywords(['foo'])).toEqual(new Set(['foo'])); - expect(parseKeywords(['foo bar', 'bzz'])).toEqual(new Set(['foo', 'bar', 'bzz'])); - }); - - it('can parse keywords from objects', () => { - expect(parseKeywords({ a: 'foo' })).toEqual(new Set(['foo'])); - expect(parseKeywords({ b: ['foo bar', 'bzz'] })).toEqual(new Set(['foo', 'bar', 'bzz'])); - }); - - it('can parse numecic keywords', () => { - expect(parseKeywords(['foo 123'])).toEqual(new Set(['foo', '123'])); - expect(parseKeywords(['foo 0 bar 1', 'bzz', '4story', 'hevesh5'])).toEqual( - new Set(['foo', '0', 'bar', '1', 'bzz', '4story', 'hevesh5']), - ); - // TODO: What about decimal and negative numbers - }); - - it('will skip numbers values', () => { - expect(parseKeywords(['a', 1])).toEqual(new Set(['a'])); - expect(parseKeywords({ foo: 'a', bar: 1 })).toEqual(new Set(['a'])); - }); - - it('will skip boolean values', () => { - expect(parseKeywords(['a', true, false])).toEqual(new Set(['a'])); - expect(parseKeywords({ foo: 'a', bar: true })).toEqual(new Set(['a'])); - }); - - it('will skip Dates', () => { - expect(parseKeywords(['a', new Date('2020-04-13T00:00:00.000+08:00')])).toEqual(new Set(['a'])); - expect( - parseKeywords({ - foo: 'a', - bar: new Date('2020-04-13T00:00:00.000+08:00'), - }), - ).toEqual(new Set(['a'])); - }); - - it('will not crash when there is null value', () => { - expect(parseKeywords(['a', null])).toEqual(new Set(['a'])); - expect(parseKeywords({ foo: 'a', bar: null })).toEqual(new Set(['a'])); - }); - - it('will not crash when there is undefined value', () => { - expect(parseKeywords(['a', undefined])).toEqual(new Set(['a'])); - expect(parseKeywords({ foo: 'a', bar: undefined })).toEqual(new Set(['a'])); - }); - - it('can parse from multiple inputs at once', () => { - // TODO: This test is bit redundant - expect(parseKeywords(['a', 'b', 'c'])).toEqual(new Set(['a', 'b', 'c'])); - expect(parseKeywords(['a', ['b', 'c']])).toEqual(new Set(['a', 'b', 'c'])); - expect(parseKeywords(['a', ['b', { foo: 'c' }]])).toEqual(new Set(['a', 'b', 'c'])); - }); - - it('will return only unique keywords', () => { - expect(parseKeywords(['a', ['a', { foo: 'a' }, { bar: { 1: 'a' } }]])).toEqual(new Set(['a'])); - }); +import { describe, expect, it } from "@jest/globals"; +import { parseKeywords } from "./parseKeywords"; + +describe("how parsing of keywords from objects works", () => { + it("can parse keywords from arrays", () => { + expect(parseKeywords(["foo"])).toEqual(new Set(["foo"])); + expect(parseKeywords(["foo bar", "bzz"])).toEqual( + new Set(["foo", "bar", "bzz"]), + ); + }); + + it("can parse keywords from objects", () => { + expect(parseKeywords({ a: "foo" })).toEqual(new Set(["foo"])); + expect(parseKeywords({ b: ["foo bar", "bzz"] })).toEqual( + new Set(["foo", "bar", "bzz"]), + ); + }); + + it("can parse numecic keywords", () => { + expect(parseKeywords(["foo 123"])).toEqual(new Set(["foo", "123"])); + expect(parseKeywords(["foo 0 bar 1", "bzz", "4story", "hevesh5"])).toEqual( + new Set(["foo", "0", "bar", "1", "bzz", "4story", "hevesh5"]), + ); + // TODO: What about decimal and negative numbers + }); + + it("will skip numbers values", () => { + expect(parseKeywords(["a", 1])).toEqual(new Set(["a"])); + expect(parseKeywords({ foo: "a", bar: 1 })).toEqual(new Set(["a"])); + }); + + it("will skip boolean values", () => { + expect(parseKeywords(["a", true, false])).toEqual(new Set(["a"])); + expect(parseKeywords({ foo: "a", bar: true })).toEqual(new Set(["a"])); + }); + + it("will skip Dates", () => { + expect( + parseKeywords(["a", new Date("2020-04-13T00:00:00.000+08:00")]), + ).toEqual(new Set(["a"])); + expect( + parseKeywords({ + foo: "a", + bar: new Date("2020-04-13T00:00:00.000+08:00"), + }), + ).toEqual(new Set(["a"])); + }); + + it("will not crash when there is null value", () => { + expect(parseKeywords(["a", null])).toEqual(new Set(["a"])); + expect(parseKeywords({ foo: "a", bar: null })).toEqual(new Set(["a"])); + }); + + it("will not crash when there is undefined value", () => { + expect(parseKeywords(["a", undefined])).toEqual(new Set(["a"])); + expect(parseKeywords({ foo: "a", bar: undefined })).toEqual(new Set(["a"])); + }); + + it("can parse from multiple inputs at once", () => { + // TODO: This test is bit redundant + expect(parseKeywords(["a", "b", "c"])).toEqual(new Set(["a", "b", "c"])); + expect(parseKeywords(["a", ["b", "c"]])).toEqual(new Set(["a", "b", "c"])); + expect(parseKeywords(["a", ["b", { foo: "c" }]])).toEqual( + new Set(["a", "b", "c"]), + ); + }); + + it("will return only unique keywords", () => { + expect( + parseKeywords(["a", ["a", { foo: "a" }, { bar: { 1: "a" } }]]), + ).toEqual(new Set(["a"])); + }); }); diff --git a/src/utils/normalization/parseKeywords.ts b/src/utils/normalization/parseKeywords.ts index 8b63cdf622..1f55680a29 100644 --- a/src/utils/normalization/parseKeywords.ts +++ b/src/utils/normalization/parseKeywords.ts @@ -1,6 +1,6 @@ -import type { really_unknown } from '../organization/really_unknown'; -import type { Keywords } from './IKeywords'; -import { parseKeywordsFromString } from './parseKeywordsFromString'; +import type { really_unknown } from "../organization/really_unknown"; +import type { Keywords } from "./IKeywords"; +import { parseKeywordsFromString } from "./parseKeywordsFromString"; /** * Parses keywords from any object and recursively walks through @@ -12,19 +12,21 @@ import { parseKeywordsFromString } from './parseKeywordsFromString'; * @public exported from `@promptbook/utils` */ export function parseKeywords(input: really_unknown): Keywords { - if (typeof input === 'string') { - return parseKeywordsFromString(input); - } else if (typeof input === 'object') { - if (Array.isArray(input)) { - return input.map(parseKeywords).reduce((a, b) => new Set([...a, ...b]), new Set()); - } else if (input === null) { - return new Set(); - } else { - return parseKeywords(Object.values(input)); - } - } else { - return new Set(); - } + if (typeof input === "string") { + return parseKeywordsFromString(input); + } else if (typeof input === "object") { + if (Array.isArray(input)) { + return input + .map(parseKeywords) + .reduce((a, b) => new Set([...a, ...b]), new Set()); + } else if (input === null) { + return new Set(); + } else { + return parseKeywords(Object.values(input)); + } + } else { + return new Set(); + } } /** diff --git a/src/utils/normalization/parseKeywordsFromString.test.ts b/src/utils/normalization/parseKeywordsFromString.test.ts index 4d247e8f2c..62f26fa325 100644 --- a/src/utils/normalization/parseKeywordsFromString.test.ts +++ b/src/utils/normalization/parseKeywordsFromString.test.ts @@ -1,79 +1,97 @@ -import { describe, expect, it } from '@jest/globals'; -import { parseKeywordsFromString } from './parseKeywordsFromString'; +import { describe, expect, it } from "@jest/globals"; +import { parseKeywordsFromString } from "./parseKeywordsFromString"; -describe('how parsing of keywords from string works', () => { - it('can parse keywords from strings', () => { - expect(parseKeywordsFromString('foo')).toEqual(new Set(['foo'])); - expect(parseKeywordsFromString('foo bar')).toEqual(new Set(['foo', 'bar'])); - }); +describe("how parsing of keywords from string works", () => { + it("can parse keywords from strings", () => { + expect(parseKeywordsFromString("foo")).toEqual(new Set(["foo"])); + expect(parseKeywordsFromString("foo bar")).toEqual(new Set(["foo", "bar"])); + }); - it('will make keywords lowercase', () => { - expect(parseKeywordsFromString('ĚSČŘŽYÁÍÉUU')).toEqual(new Set(['escrzyaieuu'])); - expect(parseKeywordsFromString('ěsčřžyáíéuu')).toEqual(new Set(['escrzyaieuu'])); - }); - it('can parse keywords with complex whitespacing', () => { - expect(parseKeywordsFromString(' foo bar')).toEqual(new Set(['foo', 'bar'])); - expect( - parseKeywordsFromString(` foo + it("will make keywords lowercase", () => { + expect(parseKeywordsFromString("ĚSČŘŽYÁÍÉUU")).toEqual( + new Set(["escrzyaieuu"]), + ); + expect(parseKeywordsFromString("ěsčřžyáíéuu")).toEqual( + new Set(["escrzyaieuu"]), + ); + }); + it("can parse keywords with complex whitespacing", () => { + expect(parseKeywordsFromString(" foo bar")).toEqual( + new Set(["foo", "bar"]), + ); + expect( + parseKeywordsFromString(` foo bar `), - ).toEqual(new Set(['foo', 'bar'])); - }); + ).toEqual(new Set(["foo", "bar"])); + }); - it('will strip diacritics from the keywords', () => { - expect(parseKeywordsFromString('ěščřžýáíéúů')).toEqual(new Set(['escrzyaieuu'])); - }); + it("will strip diacritics from the keywords", () => { + expect(parseKeywordsFromString("ěščřžýáíéúů")).toEqual( + new Set(["escrzyaieuu"]), + ); + }); - it('will strip interpunction from the keywords', () => { - expect(parseKeywordsFromString('---aaá+++')).toEqual(new Set(['aaa'])); - }); - it('will take interpunction as a separator', () => { - expect(parseKeywordsFromString('---aaá+++úůu:h')).toEqual(new Set(['aaa', 'uuu', 'h'])); - }); + it("will strip interpunction from the keywords", () => { + expect(parseKeywordsFromString("---aaá+++")).toEqual(new Set(["aaa"])); + }); + it("will take interpunction as a separator", () => { + expect(parseKeywordsFromString("---aaá+++úůu:h")).toEqual( + new Set(["aaa", "uuu", "h"]), + ); + }); - it('will split camelČaseWords with diacritics', () => { - expect(parseKeywordsFromString('PavolHejný')).toEqual( - new Set([ - 'pavol', - 'hejny', - // TODO: [0] Maybe also> 'pavolhejny', - ]), - ); - }); + it("will split camelČaseWords with diacritics", () => { + expect(parseKeywordsFromString("PavolHejný")).toEqual( + new Set([ + "pavol", + "hejny", + // TODO: [0] Maybe also> 'pavolhejny', + ]), + ); + }); - it('will split camelCaseWords', () => { - expect(parseKeywordsFromString('FOO_BAR_HELLO')).toEqual( - new Set([ - 'foo', - 'bar', - 'hello', - // [0] - ]), - ); - }); + it("will split camelCaseWords", () => { + expect(parseKeywordsFromString("FOO_BAR_HELLO")).toEqual( + new Set([ + "foo", + "bar", + "hello", + // [0] + ]), + ); + }); - it('will split SCREAMING_CASE_WORDS', () => { - expect(parseKeywordsFromString('FOO_BAR_HELLO')).toEqual( - new Set([ - 'foo', - 'bar', - 'hello', - // [0] - ]), - ); - }); + it("will split SCREAMING_CASE_WORDS", () => { + expect(parseKeywordsFromString("FOO_BAR_HELLO")).toEqual( + new Set([ + "foo", + "bar", + "hello", + // [0] + ]), + ); + }); - it('can parse keywords from strings with special chars', () => { - expect(parseKeywordsFromString('foo:')).toEqual(new Set(['foo'])); - expect(parseKeywordsFromString('foo->bar')).toEqual(new Set(['foo', 'bar'])); - expect(parseKeywordsFromString('foo: bar**hello--?WORLD')).toEqual(new Set(['foo', 'bar', 'hello', 'world'])); - }); + it("can parse keywords from strings with special chars", () => { + expect(parseKeywordsFromString("foo:")).toEqual(new Set(["foo"])); + expect(parseKeywordsFromString("foo->bar")).toEqual( + new Set(["foo", "bar"]), + ); + expect(parseKeywordsFromString("foo: bar**hello--?WORLD")).toEqual( + new Set(["foo", "bar", "hello", "world"]), + ); + }); - it('will return only unique keywords', () => { - expect(parseKeywordsFromString('a a a a a a a a a a a a a a a a a a a')).toEqual(new Set(['a'])); - expect(parseKeywordsFromString('a b a a a a a b a a a a a a a a a a a a a')).toEqual(new Set(['a', 'b'])); - }); + it("will return only unique keywords", () => { + expect( + parseKeywordsFromString("a a a a a a a a a a a a a a a a a a a"), + ).toEqual(new Set(["a"])); + expect( + parseKeywordsFromString("a b a a a a a b a a a a a a a a a a a a a"), + ).toEqual(new Set(["a", "b"])); + }); }); diff --git a/src/utils/normalization/parseKeywordsFromString.ts b/src/utils/normalization/parseKeywordsFromString.ts index 1596c6b412..c14205b8cd 100644 --- a/src/utils/normalization/parseKeywordsFromString.ts +++ b/src/utils/normalization/parseKeywordsFromString.ts @@ -1,6 +1,6 @@ -import type { Keywords } from './IKeywords'; -import { normalizeTo_SCREAMING_CASE } from './normalizeTo_SCREAMING_CASE'; -import { removeDiacritics } from './removeDiacritics'; +import type { Keywords } from "./IKeywords"; +import { normalizeTo_SCREAMING_CASE } from "./normalizeTo_SCREAMING_CASE"; +import { removeDiacritics } from "./removeDiacritics"; /** * Parses keywords from a string @@ -10,10 +10,10 @@ import { removeDiacritics } from './removeDiacritics'; * @public exported from `@promptbook/utils` */ export function parseKeywordsFromString(input: string): Keywords { - const keywords = normalizeTo_SCREAMING_CASE(removeDiacritics(input)) - .toLowerCase() - .split(/[^a-z0-9]+/gs) - .filter((value) => value); + const keywords = normalizeTo_SCREAMING_CASE(removeDiacritics(input)) + .toLowerCase() + .split(/[^a-z0-9]+/gs) + .filter((value) => value); - return new Set(keywords); + return new Set(keywords); } diff --git a/src/utils/normalization/removeDiacritics.test.ts b/src/utils/normalization/removeDiacritics.test.ts index 26cce14457..5100066fb0 100644 --- a/src/utils/normalization/removeDiacritics.test.ts +++ b/src/utils/normalization/removeDiacritics.test.ts @@ -1,23 +1,23 @@ -import { describe, expect, it } from '@jest/globals'; -import { removeDiacritics } from './removeDiacritics'; +import { describe, expect, it } from "@jest/globals"; +import { removeDiacritics } from "./removeDiacritics"; -describe('how stripping of diacritics works', () => { - it('can strip diacritics diacritics from lowercase', () => { - expect(removeDiacritics(`ěščřžýáíéúů`)).toEqual(`escrzyaieuu`); - }); +describe("how stripping of diacritics works", () => { + it("can strip diacritics diacritics from lowercase", () => { + expect(removeDiacritics(`ěščřžýáíéúů`)).toEqual(`escrzyaieuu`); + }); - it('can strip diacritics diacritics from uppercase', () => { - expect(removeDiacritics(`ĚŠČŘŽÝÁÍÉÚŮ`)).toEqual(`ESCRZYAIEUU`); - }); + it("can strip diacritics diacritics from uppercase", () => { + expect(removeDiacritics(`ĚŠČŘŽÝÁÍÉÚŮ`)).toEqual(`ESCRZYAIEUU`); + }); - it('can strip diacritics from misc edge cases', () => { - expect(removeDiacritics(``)).toEqual(``); - expect(removeDiacritics(` `)).toEqual(` `); - expect( - removeDiacritics(` . + it("can strip diacritics from misc edge cases", () => { + expect(removeDiacritics(``)).toEqual(``); + expect(removeDiacritics(` `)).toEqual(` `); + expect( + removeDiacritics(` . .`), - ).toEqual(` . + ).toEqual(` . .`); - expect(removeDiacritics(` ě ě`)).toEqual(` e e`); - }); + expect(removeDiacritics(` ě ě`)).toEqual(` e e`); + }); }); diff --git a/src/utils/normalization/removeDiacritics.ts b/src/utils/normalization/removeDiacritics.ts index 01ca082e70..53b2feded4 100644 --- a/src/utils/normalization/removeDiacritics.ts +++ b/src/utils/normalization/removeDiacritics.ts @@ -1,4 +1,4 @@ -import { DIACRITIC_VARIANTS_LETTERS } from './DIACRITIC_VARIANTS_LETTERS'; +import { DIACRITIC_VARIANTS_LETTERS } from "./DIACRITIC_VARIANTS_LETTERS"; /** * @@@ @@ -8,10 +8,10 @@ import { DIACRITIC_VARIANTS_LETTERS } from './DIACRITIC_VARIANTS_LETTERS'; * @public exported from `@promptbook/utils` */ export function removeDiacritics(input: string): string { - /*eslint no-control-regex: "off"*/ - return input.replace(/[^\u0000-\u007E]/g, (a) => { - return DIACRITIC_VARIANTS_LETTERS[a] || a; - }); + /*eslint no-control-regex: "off"*/ + return input.replace(/[^\u0000-\u007E]/g, (a) => { + return DIACRITIC_VARIANTS_LETTERS[a] || a; + }); } /** diff --git a/src/utils/normalization/searchKeywords.test.ts b/src/utils/normalization/searchKeywords.test.ts index 9be62437b6..cdb3b66733 100644 --- a/src/utils/normalization/searchKeywords.test.ts +++ b/src/utils/normalization/searchKeywords.test.ts @@ -1,24 +1,34 @@ -import { describe, expect, it } from '@jest/globals'; -import { searchKeywords } from './searchKeywords'; +import { describe, expect, it } from "@jest/globals"; +import { searchKeywords } from "./searchKeywords"; -describe('how searching through keywords works', () => { - it('can search one keyword', () => { - expect(searchKeywords(new Set(['a', 'b']), new Set(['a']))).toEqual(true); - expect(searchKeywords(new Set(['a', 'b']), new Set(['c']))).toEqual(false); - }); +describe("how searching through keywords works", () => { + it("can search one keyword", () => { + expect(searchKeywords(new Set(["a", "b"]), new Set(["a"]))).toEqual(true); + expect(searchKeywords(new Set(["a", "b"]), new Set(["c"]))).toEqual(false); + }); - it('can search multiple keyword', () => { - expect(searchKeywords(new Set(['a', 'b']), new Set(['a', 'b']))).toEqual(true); - expect(searchKeywords(new Set(['a', 'b']), new Set(['b', 'a']))).toEqual(true); - expect(searchKeywords(new Set(['a', 'b']), new Set(['a', 'c']))).toEqual(false); - }); + it("can search multiple keyword", () => { + expect(searchKeywords(new Set(["a", "b"]), new Set(["a", "b"]))).toEqual( + true, + ); + expect(searchKeywords(new Set(["a", "b"]), new Set(["b", "a"]))).toEqual( + true, + ); + expect(searchKeywords(new Set(["a", "b"]), new Set(["a", "c"]))).toEqual( + false, + ); + }); - it('can search incomplete keyword', () => { - expect(searchKeywords(new Set(['aaa', 'bbbb']), new Set(['a']))).toEqual(true); - expect(searchKeywords(new Set(['aaa', 'bbbb']), new Set(['ab']))).toEqual(false); - }); + it("can search incomplete keyword", () => { + expect(searchKeywords(new Set(["aaa", "bbbb"]), new Set(["a"]))).toEqual( + true, + ); + expect(searchKeywords(new Set(["aaa", "bbbb"]), new Set(["ab"]))).toEqual( + false, + ); + }); - /* TODO: [🌮] + /* TODO: [🌮] it('will take on mind amount of needles', () => { expect(searchKeywords(['aaa', 'abbb'], ['a','a'])).toEqual(true); expect(searchKeywords(['aaa', 'bbbb'], ['a','a'])).toEqual(false); diff --git a/src/utils/normalization/searchKeywords.ts b/src/utils/normalization/searchKeywords.ts index 8cb1a34c9c..f9c5cab69d 100644 --- a/src/utils/normalization/searchKeywords.ts +++ b/src/utils/normalization/searchKeywords.ts @@ -1,4 +1,4 @@ -import type { Keywords } from './IKeywords'; +import type { Keywords } from "./IKeywords"; /** * @@@ @@ -9,10 +9,15 @@ import type { Keywords } from './IKeywords'; * @public exported from `@promptbook/utils` */ export function searchKeywords(haystack: Keywords, needle: Keywords): boolean { - for (const needleWord of needle) { - if (![...haystack].some((haystackWord) => haystackWord.substring(0, needleWord.length) === needleWord)) { - return false; - } - } - return true; + for (const needleWord of needle) { + if ( + ![...haystack].some( + (haystackWord) => + haystackWord.substring(0, needleWord.length) === needleWord, + ) + ) { + return false; + } + } + return true; } diff --git a/src/utils/normalization/suffixUrl.test.ts b/src/utils/normalization/suffixUrl.test.ts index 6464321e5a..b7862e9f76 100644 --- a/src/utils/normalization/suffixUrl.test.ts +++ b/src/utils/normalization/suffixUrl.test.ts @@ -1,14 +1,24 @@ -import { describe, expect, it } from '@jest/globals'; -import { suffixUrl } from './suffixUrl'; +import { describe, expect, it } from "@jest/globals"; +import { suffixUrl } from "./suffixUrl"; -describe('how `suffixUrl` works', () => { - it('should add suffix to urls in multiple forms', () => { - expect(suffixUrl(new URL('http://example.com'), '/')).toBe('http://example.com/'); - expect(suffixUrl(new URL('http://example.com/'), '/')).toBe('http://example.com/'); - expect(suffixUrl(new URL('http://example.com'), '/books')).toBe('http://example.com/books'); - expect(suffixUrl(new URL('http://example.com/'), '/books')).toBe('http://example.com/books'); - // expect(suffixUrl(new URL('http://example.com//'), '/books')).toBe('http://example.com/books'); - // expect(suffixUrl(new URL('http://example.com//'), '//books')).toBe('http://example.com/books'); - expect(suffixUrl(new URL('http://example.com'), '///////books')).toBe('http://example.com/books'); - }); +describe("how `suffixUrl` works", () => { + it("should add suffix to urls in multiple forms", () => { + expect(suffixUrl(new URL("http://example.com"), "/")).toBe( + "http://example.com/", + ); + expect(suffixUrl(new URL("http://example.com/"), "/")).toBe( + "http://example.com/", + ); + expect(suffixUrl(new URL("http://example.com"), "/books")).toBe( + "http://example.com/books", + ); + expect(suffixUrl(new URL("http://example.com/"), "/books")).toBe( + "http://example.com/books", + ); + // expect(suffixUrl(new URL('http://example.com//'), '/books')).toBe('http://example.com/books'); + // expect(suffixUrl(new URL('http://example.com//'), '//books')).toBe('http://example.com/books'); + expect(suffixUrl(new URL("http://example.com"), "///////books")).toBe( + "http://example.com/books", + ); + }); }); diff --git a/src/utils/normalization/suffixUrl.ts b/src/utils/normalization/suffixUrl.ts index 6d3e080688..3d83953761 100644 --- a/src/utils/normalization/suffixUrl.ts +++ b/src/utils/normalization/suffixUrl.ts @@ -1,4 +1,4 @@ -import type { string_url } from '../../types/typeAliases'; +import type { string_url } from "../../types/typeAliases"; /** * Adds suffix to the URL @@ -6,7 +6,9 @@ import type { string_url } from '../../types/typeAliases'; * @public exported from `@promptbook/utils` */ export function suffixUrl(value: URL, suffix: `/${string}`): string_url { - const baseUrl = value.href.endsWith('/') ? value.href.slice(0, -1) : value.href; - const normalizedSuffix = suffix.replace(/\/+/g, '/'); - return (baseUrl + normalizedSuffix) as string_url; + const baseUrl = value.href.endsWith("/") + ? value.href.slice(0, -1) + : value.href; + const normalizedSuffix = suffix.replace(/\/+/g, "/"); + return (baseUrl + normalizedSuffix) as string_url; } diff --git a/src/utils/normalization/titleToName.test.ts b/src/utils/normalization/titleToName.test.ts index 499e79b0c5..c61e5551c3 100644 --- a/src/utils/normalization/titleToName.test.ts +++ b/src/utils/normalization/titleToName.test.ts @@ -1,41 +1,53 @@ -import { describe, expect, it } from '@jest/globals'; -import { titleToName } from './titleToName'; +import { describe, expect, it } from "@jest/globals"; +import { titleToName } from "./titleToName"; -describe('how titleToName works', () => { - it('should make name from title', () => { - expect(titleToName(`hello`)).toBe(`hello`); - expect(titleToName(`hello 🤘`)).toBe(`hello`); - expect(titleToName(`hello world`)).toBe(`hello-world`); - expect(titleToName(`Hello, how are ýóů?`)).toBe(`hello-how-are-you`); - expect( - titleToName(` Hello, how +describe("how titleToName works", () => { + it("should make name from title", () => { + expect(titleToName(`hello`)).toBe(`hello`); + expect(titleToName(`hello 🤘`)).toBe(`hello`); + expect(titleToName(`hello world`)).toBe(`hello-world`); + expect(titleToName(`Hello, how are ýóů?`)).toBe(`hello-how-are-you`); + expect( + titleToName(` Hello, how are ýóů? `), - ).toBe(`hello-how-are-you`); - }); + ).toBe(`hello-how-are-you`); + }); - it('should make name url', () => { - expect(titleToName(`https://promptbook.studio/webgpt/write-website-content.book`)).toBe( - `promptbook-studio-webgpt-write-website-content-book`, - ); + it("should make name url", () => { + expect( + titleToName( + `https://promptbook.studio/webgpt/write-website-content.book`, + ), + ).toBe(`promptbook-studio-webgpt-write-website-content-book`); - expect(titleToName(`https://promptbook.studio/webgpt/write-website-content.html`)).toBe( - `promptbook-studio-webgpt-write-website-content`, - ); - }); + expect( + titleToName( + `https://promptbook.studio/webgpt/write-website-content.html`, + ), + ).toBe(`promptbook-studio-webgpt-write-website-content`); + }); - it('should make relative file path', () => { - expect(titleToName(`../webgpt/write-website-content.book`)).toBe(`write-website-content-book`); - expect(titleToName(`../webgpt/write-website-content.book.html`)).toBe(`write-website-content-book-html`); - expect(titleToName(`./webgpt/write-website-content.book`)).toBe(`write-website-content-book`); - expect(titleToName(`./webgpt/write-website-content.book.html`)).toBe(`write-website-content-book-html`); - }); + it("should make relative file path", () => { + expect(titleToName(`../webgpt/write-website-content.book`)).toBe( + `write-website-content-book`, + ); + expect(titleToName(`../webgpt/write-website-content.book.html`)).toBe( + `write-website-content-book-html`, + ); + expect(titleToName(`./webgpt/write-website-content.book`)).toBe( + `write-website-content-book`, + ); + expect(titleToName(`./webgpt/write-website-content.book.html`)).toBe( + `write-website-content-book-html`, + ); + }); - it('should make absolute file path', () => { - expect(titleToName(`C://Users/pavol/projects/webgpt/write-website-content.book`)).toBe( - `write-website-content-book`, - ); - expect(titleToName(`/home/pavol/projects/webgpt/write-website-content.book`)).toBe( - `write-website-content-book`, - ); - }); + it("should make absolute file path", () => { + expect( + titleToName(`C://Users/pavol/projects/webgpt/write-website-content.book`), + ).toBe(`write-website-content-book`); + expect( + titleToName(`/home/pavol/projects/webgpt/write-website-content.book`), + ).toBe(`write-website-content-book`); + }); }); diff --git a/src/utils/normalization/titleToName.ts b/src/utils/normalization/titleToName.ts index 27ad141f59..1280ccc8b3 100644 --- a/src/utils/normalization/titleToName.ts +++ b/src/utils/normalization/titleToName.ts @@ -1,8 +1,8 @@ -import { basename } from 'path'; -import { removeEmojis } from '../removeEmojis'; -import { isValidFilePath } from '../validators/filePath/isValidFilePath'; -import { isValidUrl } from '../validators/url/isValidUrl'; -import { normalizeToKebabCase } from './normalize-to-kebab-case'; +import { basename } from "path"; +import { removeEmojis } from "../removeEmojis"; +import { isValidFilePath } from "../validators/filePath/isValidFilePath"; +import { isValidUrl } from "../validators/url/isValidUrl"; +import { normalizeToKebabCase } from "./normalize-to-kebab-case"; /** * @@@ @@ -13,19 +13,19 @@ import { normalizeToKebabCase } from './normalize-to-kebab-case'; * @public exported from `@promptbook/utils` */ export function titleToName(value: string): string { - if (isValidUrl(value)) { - value = value.replace(/^https?:\/\//, ''); - value = value.replace(/\.html$/, ''); - } else if (isValidFilePath(value)) { - value = basename(value); - // Note: Keeping extension in the name - } + if (isValidUrl(value)) { + value = value.replace(/^https?:\/\//, ""); + value = value.replace(/\.html$/, ""); + } else if (isValidFilePath(value)) { + value = basename(value); + // Note: Keeping extension in the name + } - value = value.split('/').join('-'); + value = value.split("/").join("-"); - value = removeEmojis(value); - value = normalizeToKebabCase(value); + value = removeEmojis(value); + value = normalizeToKebabCase(value); - // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers - return value; + // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers + return value; } diff --git a/src/utils/organization/$sideEffect.ts b/src/utils/organization/$sideEffect.ts index c788581316..7e23f06cf9 100644 --- a/src/utils/organization/$sideEffect.ts +++ b/src/utils/organization/$sideEffect.ts @@ -1,5 +1,5 @@ -import { keepUnused } from './keepUnused'; -import type { really_any } from './really_any'; +import { keepUnused } from "./keepUnused"; +import type { really_any } from "./really_any"; /** * Just says that the variable is not used directlys but should be kept because the existence of the variable is important @@ -9,7 +9,7 @@ import type { really_any } from './really_any'; * @private within the repository */ export function $sideEffect( - ...sideEffectSubjects: ReadonlyArray + ...sideEffectSubjects: ReadonlyArray ): void { - keepUnused(...sideEffectSubjects); + keepUnused(...sideEffectSubjects); } diff --git a/src/utils/organization/TODO_USE.ts b/src/utils/organization/TODO_USE.ts index 3c4c90de23..fdda01f18a 100644 --- a/src/utils/organization/TODO_USE.ts +++ b/src/utils/organization/TODO_USE.ts @@ -1,4 +1,4 @@ -import type { really_any } from './really_any'; +import type { really_any } from "./really_any"; /** * Just marks a place of place where should be something implemented @@ -11,6 +11,6 @@ import type { really_any } from './really_any'; * @private within the repository */ export function TODO_USE(...value: ReadonlyArray): void { - // Note: Do nothing - value; + // Note: Do nothing + value; } diff --git a/src/utils/organization/___and___.ts b/src/utils/organization/___and___.ts index 340527674d..d1b85d4f29 100644 --- a/src/utils/organization/___and___.ts +++ b/src/utils/organization/___and___.ts @@ -1,4 +1,4 @@ -import type { really_unknown } from './really_unknown'; +import type { really_unknown } from "./really_unknown"; /** * Formatting helper to put void to keep longer version of prettier diff --git a/src/utils/organization/just.ts b/src/utils/organization/just.ts index 9c56d669aa..53a5457771 100644 --- a/src/utils/organization/just.ts +++ b/src/utils/organization/just.ts @@ -1,4 +1,4 @@ -import type { really_any } from './really_any'; +import type { really_any } from "./really_any"; /** * Returns the same value that is passed as argument. @@ -14,8 +14,8 @@ import type { really_any } from './really_any'; * @private within the repository */ export function just(value?: T): T { - if (value === undefined) { - return undefined as really_any as T; - } - return value; + if (value === undefined) { + return undefined as really_any as T; + } + return value; } diff --git a/src/utils/organization/keepTypeImported.ts b/src/utils/organization/keepTypeImported.ts index 4a9e937d43..ec9c14e152 100644 --- a/src/utils/organization/keepTypeImported.ts +++ b/src/utils/organization/keepTypeImported.ts @@ -1,4 +1,4 @@ -import { keepUnused } from './keepUnused'; +import { keepUnused } from "./keepUnused"; /** * Just says that the type is used but `organize-imports-cli` does not recognize it. @@ -9,6 +9,6 @@ import { keepUnused } from './keepUnused'; * @private within the repository */ export function keepTypeImported(): void { - // Note: Do nothing - keepUnused(null as TTypeToKeep); + // Note: Do nothing + keepUnused(null as TTypeToKeep); } diff --git a/src/utils/organization/keepUnused.ts b/src/utils/organization/keepUnused.ts index a899a7116e..95c60d5f51 100644 --- a/src/utils/organization/keepUnused.ts +++ b/src/utils/organization/keepUnused.ts @@ -1,4 +1,4 @@ -import type { really_any } from './really_any'; +import type { really_any } from "./really_any"; /** * Just says that the variable is not used but should be kept @@ -14,11 +14,13 @@ import type { really_any } from './really_any'; * @returns void * @private within the repository */ -export function keepUnused( - ...valuesToKeep: ReadonlyArray -): void { - // Note: Do nothing - valuesToKeep as TTypeToKeep1; - valuesToKeep as TTypeToKeep2; - valuesToKeep as TTypeToKeep3; +export function keepUnused< + TTypeToKeep1 = really_any, + TTypeToKeep2 = really_any, + TTypeToKeep3 = really_any, +>(...valuesToKeep: ReadonlyArray): void { + // Note: Do nothing + valuesToKeep as TTypeToKeep1; + valuesToKeep as TTypeToKeep2; + valuesToKeep as TTypeToKeep3; } diff --git a/src/utils/organization/spaceTrim.ts b/src/utils/organization/spaceTrim.ts index d47c39f760..a1fe4f41a9 100644 --- a/src/utils/organization/spaceTrim.ts +++ b/src/utils/organization/spaceTrim.ts @@ -1,4 +1,4 @@ -import { spaceTrim as spaceTrimReexported } from 'spacetrim'; +import { spaceTrim as spaceTrimReexported } from "spacetrim"; /** * Trims string from all 4 sides diff --git a/src/utils/parameters/extractParameterNames.test.ts b/src/utils/parameters/extractParameterNames.test.ts index 80811751b1..03fcd4b75f 100644 --- a/src/utils/parameters/extractParameterNames.test.ts +++ b/src/utils/parameters/extractParameterNames.test.ts @@ -1,77 +1,92 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { extractParameterNames } from './extractParameterNames'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { extractParameterNames } from "./extractParameterNames"; -describe('extractParameterNames', () => { - it('should work in supersimple case without any parameters', () => { - expect([...extractParameterNames('')]).toEqual([]); - expect([...extractParameterNames('Hello')]).toEqual([]); - expect([...extractParameterNames('Hello World')]).toEqual([]); - }); +describe("extractParameterNames", () => { + it("should work in supersimple case without any parameters", () => { + expect([...extractParameterNames("")]).toEqual([]); + expect([...extractParameterNames("Hello")]).toEqual([]); + expect([...extractParameterNames("Hello World")]).toEqual([]); + }); - it('should parse one parameter', () => { - expect([...extractParameterNames('Hello {name}')]).toEqual(['name']); - expect([...extractParameterNames('Hello {name}, how are you?')]).toEqual(['name']); - expect([...extractParameterNames('{name}, how are you?')]).toEqual(['name']); - }); + it("should parse one parameter", () => { + expect([...extractParameterNames("Hello {name}")]).toEqual(["name"]); + expect([...extractParameterNames("Hello {name}, how are you?")]).toEqual([ + "name", + ]); + expect([...extractParameterNames("{name}, how are you?")]).toEqual([ + "name", + ]); + }); - it('should parse multiple parameters', () => { - expect([...extractParameterNames('{greeting} {name}, how are you?')]).toEqual(['greeting', 'name']); - }); + it("should parse multiple parameters", () => { + expect([ + ...extractParameterNames("{greeting} {name}, how are you?"), + ]).toEqual(["greeting", "name"]); + }); - it('should not be confused by JSON', () => { - expect([...extractParameterNames('{greeting} {name}, this is how JSON look like {"key": 1}.')]).toEqual([ - 'greeting', - 'name', - ]); - expect([...extractParameterNames('{greeting} {name}, this is how JSON look like {}.')]).toEqual([ - 'greeting', - 'name', - ]); - expect([ - ...extractParameterNames( - '{greeting} {name}, this is how JSON look like {"greeting": "{greeting}", "name": "{name}"}.', - ), - ]).toEqual(['greeting', 'name']); - expect([ - ...extractParameterNames( - '{greeting} {name}, this is how JSON look like {"params": {"greeting": "{greeting}", "name": "{name}"}}.', - ), - ]).toEqual(['greeting', 'name']); - expect([ - ...extractParameterNames( - '{greeting} {name}, this is how invalid JSON look like {"params": {"greeting": "{greeting}", "name": "{name}"}.', - ), - ]).toEqual(['greeting', 'name']); - }); + it("should not be confused by JSON", () => { + expect([ + ...extractParameterNames( + '{greeting} {name}, this is how JSON look like {"key": 1}.', + ), + ]).toEqual(["greeting", "name"]); + expect([ + ...extractParameterNames( + "{greeting} {name}, this is how JSON look like {}.", + ), + ]).toEqual(["greeting", "name"]); + expect([ + ...extractParameterNames( + '{greeting} {name}, this is how JSON look like {"greeting": "{greeting}", "name": "{name}"}.', + ), + ]).toEqual(["greeting", "name"]); + expect([ + ...extractParameterNames( + '{greeting} {name}, this is how JSON look like {"params": {"greeting": "{greeting}", "name": "{name}"}}.', + ), + ]).toEqual(["greeting", "name"]); + expect([ + ...extractParameterNames( + '{greeting} {name}, this is how invalid JSON look like {"params": {"greeting": "{greeting}", "name": "{name}"}.', + ), + ]).toEqual(["greeting", "name"]); + }); - it('should parse parameter included multiple times', () => { - expect([...extractParameterNames('{greeting} {name}, how are you? {greeting} {name}')]).toEqual([ - 'greeting', - 'name', - ]); - }); + it("should parse parameter included multiple times", () => { + expect([ + ...extractParameterNames( + "{greeting} {name}, how are you? {greeting} {name}", + ), + ]).toEqual(["greeting", "name"]); + }); - it('should parse multi-line tasks', () => { - expect([ - ...extractParameterNames( - spaceTrim(` + it("should parse multi-line tasks", () => { + expect([ + ...extractParameterNames( + spaceTrim(` Hello {name}, how are you? I am {greeting} `), - ), - ]).toEqual(['name', 'greeting']); - }); + ), + ]).toEqual(["name", "greeting"]); + }); - it('should not be confused by some NON-JSON structure containing chars {}', () => { - expect([...extractParameterNames('{greeting {name}}, how are you?')]).toEqual(['name']); - expect([...extractParameterNames('')]).toEqual(['name']); - expect([...extractParameterNames('<{greeting {name}}>')]).toEqual(['name']); - expect([...extractParameterNames('<{{{greeting {name}}}}>')]).toEqual(['name']); - expect([...extractParameterNames('{greeting} }{}{}{')]).toEqual(['greeting']); - }); + it("should not be confused by some NON-JSON structure containing chars {}", () => { + expect([ + ...extractParameterNames("{greeting {name}}, how are you?"), + ]).toEqual(["name"]); + expect([...extractParameterNames("")]).toEqual(["name"]); + expect([...extractParameterNames("<{greeting {name}}>")]).toEqual(["name"]); + expect([...extractParameterNames("<{{{greeting {name}}}}>")]).toEqual([ + "name", + ]); + expect([...extractParameterNames("{greeting} }{}{}{")]).toEqual([ + "greeting", + ]); + }); - /* + /* TODO: [🧠][💫] Should be this done in extractParameterNames OR only in replaceParams? it('should throw error when parameter is not closed', () => { expect(() => extractParameterNames('Hello {name')).toThrowError(/Parameter is not closed/i); diff --git a/src/utils/parameters/extractParameterNames.ts b/src/utils/parameters/extractParameterNames.ts index 0668d0951e..aa8b681383 100644 --- a/src/utils/parameters/extractParameterNames.ts +++ b/src/utils/parameters/extractParameterNames.ts @@ -1,6 +1,6 @@ -import type { string_name } from '../../types/typeAliases'; -import type { string_parameter_name } from '../../types/typeAliases'; -import type { string_template } from '../../types/typeAliases'; +import type { string_name } from "../../types/typeAliases"; +import type { string_parameter_name } from "../../types/typeAliases"; +import type { string_template } from "../../types/typeAliases"; /** * Parses the task and returns the list of all parameter names @@ -9,14 +9,16 @@ import type { string_template } from '../../types/typeAliases'; * @returns the list of parameter names * @public exported from `@promptbook/utils` */ -export function extractParameterNames(template: string_template): Set { - const matches = template.matchAll(/{\w+}/g); - const parameterNames = new Set(); - for (const match of matches) { - const parameterName = match[0].slice(1, -1); +export function extractParameterNames( + template: string_template, +): Set { + const matches = template.matchAll(/{\w+}/g); + const parameterNames = new Set(); + for (const match of matches) { + const parameterName = match[0].slice(1, -1); - parameterNames.add(parameterName); - } + parameterNames.add(parameterName); + } - return parameterNames; + return parameterNames; } diff --git a/src/utils/parameters/mapAvailableToExpectedParameters.test.ts b/src/utils/parameters/mapAvailableToExpectedParameters.test.ts index 04e9bbca0d..161723f892 100644 --- a/src/utils/parameters/mapAvailableToExpectedParameters.test.ts +++ b/src/utils/parameters/mapAvailableToExpectedParameters.test.ts @@ -1,102 +1,102 @@ -import { describe, expect, it } from '@jest/globals'; -import { mapAvailableToExpectedParameters } from './mapAvailableToExpectedParameters'; +import { describe, expect, it } from "@jest/globals"; +import { mapAvailableToExpectedParameters } from "./mapAvailableToExpectedParameters"; -describe('how `mapAvailableToExpectedParameters` works', () => { - it('should work with supersimple empty example', () => - expect( - mapAvailableToExpectedParameters({ - expectedParameters: {}, - availableParameters: {}, - }), - ).toEqual({})); +describe("how `mapAvailableToExpectedParameters` works", () => { + it("should work with supersimple empty example", () => + expect( + mapAvailableToExpectedParameters({ + expectedParameters: {}, + availableParameters: {}, + }), + ).toEqual({})); - it('should work with simple 1:1 match', () => - expect( - mapAvailableToExpectedParameters({ - expectedParameters: { foo: null }, - availableParameters: { foo: 'foo' }, - }), - ).toEqual({ foo: 'foo' })); + it("should work with simple 1:1 match", () => + expect( + mapAvailableToExpectedParameters({ + expectedParameters: { foo: null }, + availableParameters: { foo: "foo" }, + }), + ).toEqual({ foo: "foo" })); - it('should work with simple N:N match', () => { - expect( - mapAvailableToExpectedParameters({ - expectedParameters: { foo: null, bar: null }, - availableParameters: { foo: 'foo', bar: 'bar' }, - }), - ).toEqual({ foo: 'foo', bar: 'bar' }); - expect( - mapAvailableToExpectedParameters({ - expectedParameters: { foo: null, bar: null, baz: null }, - availableParameters: { foo: 'foo', bar: 'bar', baz: 'baz' }, - }), - ).toEqual({ foo: 'foo', bar: 'bar', baz: 'baz' }); - }); + it("should work with simple N:N match", () => { + expect( + mapAvailableToExpectedParameters({ + expectedParameters: { foo: null, bar: null }, + availableParameters: { foo: "foo", bar: "bar" }, + }), + ).toEqual({ foo: "foo", bar: "bar" }); + expect( + mapAvailableToExpectedParameters({ + expectedParameters: { foo: null, bar: null, baz: null }, + availableParameters: { foo: "foo", bar: "bar", baz: "baz" }, + }), + ).toEqual({ foo: "foo", bar: "bar", baz: "baz" }); + }); - it('should ignore extra available parameters', () => - // TODO: [🧠][🐱‍👤] There should be some option for throw error or log warning in this situation - expect( - mapAvailableToExpectedParameters({ - expectedParameters: { foo: null }, - availableParameters: { foo: 'foo', bar: 'bar' }, - }), - ).toEqual({ foo: 'foo' })); + it("should ignore extra available parameters", () => + // TODO: [🧠][🐱‍👤] There should be some option for throw error or log warning in this situation + expect( + mapAvailableToExpectedParameters({ + expectedParameters: { foo: null }, + availableParameters: { foo: "foo", bar: "bar" }, + }), + ).toEqual({ foo: "foo" })); - it('should throw error when parameter not available', () => - expect(() => - mapAvailableToExpectedParameters({ - expectedParameters: { foo: null, bar: null }, - availableParameters: { foo: 'foo' }, - }), - ).toThrowError(/Can not map available parameters to expected parameters/i)); + it("should throw error when parameter not available", () => + expect(() => + mapAvailableToExpectedParameters({ + expectedParameters: { foo: null, bar: null }, + availableParameters: { foo: "foo" }, + }), + ).toThrowError(/Can not map available parameters to expected parameters/i)); - it('should match 1:1 parameters which are not matching by name BUT matching by count', () => - // TODO: [🧠][👩🏾‍🤝‍👩🏻] There should be option for turning on/off to match non-matching parameters by its name - expect( - mapAvailableToExpectedParameters({ - expectedParameters: { foo: null }, - availableParameters: { bar: 'bar' }, - }), - ).toEqual({ foo: 'bar' })); + it("should match 1:1 parameters which are not matching by name BUT matching by count", () => + // TODO: [🧠][👩🏾‍🤝‍👩🏻] There should be option for turning on/off to match non-matching parameters by its name + expect( + mapAvailableToExpectedParameters({ + expectedParameters: { foo: null }, + availableParameters: { bar: "bar" }, + }), + ).toEqual({ foo: "bar" })); - it('should match N:N parameters which are not matching by name BUT matching by count', () => - // TODO: [🧠][👩🏾‍🤝‍👩🏻] There should be option for turning on/off to match non-matching parameters by its name - expect( - mapAvailableToExpectedParameters({ - expectedParameters: { foo: null, bar: null, baz: null }, - availableParameters: { foox: 'foo', barx: 'bar', bazx: 'baz' }, - }), - ).toEqual({ foo: 'foo', bar: 'bar', baz: 'baz' })); + it("should match N:N parameters which are not matching by name BUT matching by count", () => + // TODO: [🧠][👩🏾‍🤝‍👩🏻] There should be option for turning on/off to match non-matching parameters by its name + expect( + mapAvailableToExpectedParameters({ + expectedParameters: { foo: null, bar: null, baz: null }, + availableParameters: { foox: "foo", barx: "bar", bazx: "baz" }, + }), + ).toEqual({ foo: "foo", bar: "bar", baz: "baz" })); - it('should match mixed situation', () => { - // TODO: [🧠][👩🏾‍🤝‍👩🏻] There should be option for turning on/off to match non-matching parameters by its name - expect( - mapAvailableToExpectedParameters({ - expectedParameters: { foo: null, baz: null }, - availableParameters: { baz: 'baz', bar: 'bar' }, - }), - ).toEqual({ foo: 'bar', baz: 'baz' }); - expect( - mapAvailableToExpectedParameters({ - expectedParameters: { a: null, b: null, c: null }, - availableParameters: { aa: 'aa', b: 'b', cc: 'cc' }, - }), - ).toEqual({ a: 'aa', b: 'b', c: 'cc' }); - }); + it("should match mixed situation", () => { + // TODO: [🧠][👩🏾‍🤝‍👩🏻] There should be option for turning on/off to match non-matching parameters by its name + expect( + mapAvailableToExpectedParameters({ + expectedParameters: { foo: null, baz: null }, + availableParameters: { baz: "baz", bar: "bar" }, + }), + ).toEqual({ foo: "bar", baz: "baz" }); + expect( + mapAvailableToExpectedParameters({ + expectedParameters: { a: null, b: null, c: null }, + availableParameters: { aa: "aa", b: "b", cc: "cc" }, + }), + ).toEqual({ a: "aa", b: "b", c: "cc" }); + }); - it('should throw error when parameters are not matching and also count is not matching', () => { - expect(() => - mapAvailableToExpectedParameters({ - expectedParameters: { foo: null, baz: null }, - availableParameters: { foox: 'foo', barx: 'bar', bazx: 'baz' }, - }), - ).toThrowError(/Can not map available parameters to expected parameters/i); + it("should throw error when parameters are not matching and also count is not matching", () => { + expect(() => + mapAvailableToExpectedParameters({ + expectedParameters: { foo: null, baz: null }, + availableParameters: { foox: "foo", barx: "bar", bazx: "baz" }, + }), + ).toThrowError(/Can not map available parameters to expected parameters/i); - expect(() => - mapAvailableToExpectedParameters({ - expectedParameters: { foo: null, bar: null, baz: null }, - availableParameters: { foox: 'foo', bazx: 'baz' }, - }), - ).toThrowError(/Can not map available parameters to expected parameters/i); - }); + expect(() => + mapAvailableToExpectedParameters({ + expectedParameters: { foo: null, bar: null, baz: null }, + availableParameters: { foox: "foo", bazx: "baz" }, + }), + ).toThrowError(/Can not map available parameters to expected parameters/i); + }); }); diff --git a/src/utils/parameters/mapAvailableToExpectedParameters.ts b/src/utils/parameters/mapAvailableToExpectedParameters.ts index 4c7472ef79..62f176c6a0 100644 --- a/src/utils/parameters/mapAvailableToExpectedParameters.ts +++ b/src/utils/parameters/mapAvailableToExpectedParameters.ts @@ -1,22 +1,24 @@ -import spaceTrim from 'spacetrim'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import type { string_parameter_name } from '../../types/typeAliases'; -import type { string_parameter_value } from '../../types/typeAliases'; -import { union } from '../sets/union'; +import spaceTrim from "spacetrim"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import type { string_parameter_name } from "../../types/typeAliases"; +import type { string_parameter_value } from "../../types/typeAliases"; +import { union } from "../sets/union"; /** * @@@ */ type MakeapAvailableToExpectedParametersOptions = { - /** - * @@@ - */ - readonly expectedParameters: Readonly>; - - /** - * @@@ - */ - readonly availableParameters: Readonly>; + /** + * @@@ + */ + readonly expectedParameters: Readonly>; + + /** + * @@@ + */ + readonly availableParameters: Readonly< + Record + >; }; /** @@ -30,83 +32,98 @@ type MakeapAvailableToExpectedParametersOptions = { * @private within the repository used in `createPipelineExecutor` */ export function mapAvailableToExpectedParameters( - options: MakeapAvailableToExpectedParametersOptions, + options: MakeapAvailableToExpectedParametersOptions, ): Readonly> { - const { expectedParameters, availableParameters } = options; - - const availableParametersNames = new Set(Object.keys(availableParameters)); - const expectedParameterNames = new Set(Object.keys(expectedParameters)); - - const mappedParameters: Record = {}; - - // Phase 1️⃣: Matching mapping - for (const parameterName of Array.from(union(availableParametersNames, expectedParameterNames))) { - // Situation: Parameter is available and expected - if (availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) { - mappedParameters[parameterName] = availableParameters[parameterName]!; - // <- Note: [👩‍👩‍👧] Maybe detect parameter collision here? - availableParametersNames.delete(parameterName); - expectedParameterNames.delete(parameterName); - } - - // Situation: Parameter is available but NOT expected - else if (availableParametersNames.has(parameterName) && !expectedParameterNames.has(parameterName)) { - // [🐱‍👤] Do not pass this parameter to prompt - Maybe use it non-matching mapping - } - - // Situation: Parameter is NOT available BUT expected - else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) { - // Do nothing here - this will be maybe fixed in the non-matching mapping - } - } - - if (expectedParameterNames.size === 0) { - // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@ - Object.freeze(mappedParameters); - return mappedParameters; - } - - // Phase 2️⃣: Non-matching mapping - if (expectedParameterNames.size !== availableParametersNames.size) { - throw new PipelineExecutionError( - spaceTrim( - (block) => ` + const { expectedParameters, availableParameters } = options; + + const availableParametersNames = new Set(Object.keys(availableParameters)); + const expectedParameterNames = new Set(Object.keys(expectedParameters)); + + const mappedParameters: Record< + string_parameter_name, + string_parameter_value + > = {}; + + // Phase 1️⃣: Matching mapping + for (const parameterName of Array.from( + union(availableParametersNames, expectedParameterNames), + )) { + // Situation: Parameter is available and expected + if ( + availableParametersNames.has(parameterName) && + expectedParameterNames.has(parameterName) + ) { + mappedParameters[parameterName] = availableParameters[parameterName]!; + // <- Note: [👩‍👩‍👧] Maybe detect parameter collision here? + availableParametersNames.delete(parameterName); + expectedParameterNames.delete(parameterName); + } + + // Situation: Parameter is available but NOT expected + else if ( + availableParametersNames.has(parameterName) && + !expectedParameterNames.has(parameterName) + ) { + // [🐱‍👤] Do not pass this parameter to prompt - Maybe use it non-matching mapping + } + + // Situation: Parameter is NOT available BUT expected + else if ( + !availableParametersNames.has(parameterName) && + expectedParameterNames.has(parameterName) + ) { + // Do nothing here - this will be maybe fixed in the non-matching mapping + } + } + + if (expectedParameterNames.size === 0) { + // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@ + Object.freeze(mappedParameters); + return mappedParameters; + } + + // Phase 2️⃣: Non-matching mapping + if (expectedParameterNames.size !== availableParametersNames.size) { + throw new PipelineExecutionError( + spaceTrim( + (block) => ` Can not map available parameters to expected parameters Mapped parameters: ${block( - Object.keys(mappedParameters) - .map((parameterName) => `- {${parameterName}}`) - .join('\n'), - )} + Object.keys(mappedParameters) + .map((parameterName) => `- {${parameterName}}`) + .join("\n"), + )} Expected parameters which can not be mapped: ${block( - Array.from(expectedParameterNames) - .map((parameterName) => `- {${parameterName}}`) - .join('\n'), - )} + Array.from(expectedParameterNames) + .map((parameterName) => `- {${parameterName}}`) + .join("\n"), + )} Remaining available parameters: ${block( - Array.from(availableParametersNames) - .map((parameterName) => `- {${parameterName}}`) - .join('\n'), - )} + Array.from(availableParametersNames) + .map((parameterName) => `- {${parameterName}}`) + .join("\n"), + )} `, - ), - ); - } + ), + ); + } - const expectedParameterNamesArray = Array.from(expectedParameterNames); - const availableParametersNamesArray = Array.from(availableParametersNames); + const expectedParameterNamesArray = Array.from(expectedParameterNames); + const availableParametersNamesArray = Array.from(availableParametersNames); - for (let i = 0; i < expectedParameterNames.size; i++) { - mappedParameters[expectedParameterNamesArray[i]!] = availableParameters[availableParametersNamesArray[i]!]!; - } + for (let i = 0; i < expectedParameterNames.size; i++) { + mappedParameters[expectedParameterNamesArray[i]!] = + availableParameters[availableParametersNamesArray[i]!]!; + } - // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@ - Object.freeze(mappedParameters); - return mappedParameters; + // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@ + Object.freeze(mappedParameters); + return mappedParameters; } diff --git a/src/utils/parameters/numberToString.test.ts b/src/utils/parameters/numberToString.test.ts index d982b7da76..60f993e31f 100644 --- a/src/utils/parameters/numberToString.test.ts +++ b/src/utils/parameters/numberToString.test.ts @@ -1,27 +1,27 @@ -import { describe, expect, it } from '@jest/globals'; -import { numberToString } from './numberToString'; +import { describe, expect, it } from "@jest/globals"; +import { numberToString } from "./numberToString"; -describe('how numberToString works', () => { - it('should format midrange numbers', () => { - expect(numberToString(0)).toBe('0'); - expect(numberToString(1)).toBe('1'); - expect(numberToString(-1)).toBe('-1'); - expect(numberToString(5)).toBe('5'); - expect(numberToString(5.000000003)).toBe('5'); - expect(numberToString(1000)).toBe('1000'); - expect(numberToString(0.2160000006)).toBe('0.216'); - expect(numberToString(0.2948055532)).toBe('0.295'); - }); +describe("how numberToString works", () => { + it("should format midrange numbers", () => { + expect(numberToString(0)).toBe("0"); + expect(numberToString(1)).toBe("1"); + expect(numberToString(-1)).toBe("-1"); + expect(numberToString(5)).toBe("5"); + expect(numberToString(5.000000003)).toBe("5"); + expect(numberToString(1000)).toBe("1000"); + expect(numberToString(0.2160000006)).toBe("0.216"); + expect(numberToString(0.2948055532)).toBe("0.295"); + }); - it('should format big numbers', () => { - expect(numberToString(10000000000)).toBe('10000000000'); - expect(numberToString(10000000000.123)).toBe('10000000000'); - expect(numberToString(564348645)).toBe('564348645'); - }); + it("should format big numbers", () => { + expect(numberToString(10000000000)).toBe("10000000000"); + expect(numberToString(10000000000.123)).toBe("10000000000"); + expect(numberToString(564348645)).toBe("564348645"); + }); - it('should format small numbers', () => { - expect(numberToString(0.0001)).toBe('0.0001'); - expect(numberToString(0.00010000005)).toBe('0.0001'); - expect(numberToString(0.00000000001)).toBe('0.00000000001'); - }); + it("should format small numbers", () => { + expect(numberToString(0.0001)).toBe("0.0001"); + expect(numberToString(0.00010000005)).toBe("0.0001"); + expect(numberToString(0.00000000001)).toBe("0.00000000001"); + }); }); diff --git a/src/utils/parameters/numberToString.ts b/src/utils/parameters/numberToString.ts index b52ce5bf05..3100312c82 100644 --- a/src/utils/parameters/numberToString.ts +++ b/src/utils/parameters/numberToString.ts @@ -1,6 +1,6 @@ -import { SMALL_NUMBER } from '../../config'; -import { VALUE_STRINGS } from '../../config'; -import type { string_parameter_value } from '../../types/typeAliases'; +import { SMALL_NUMBER } from "../../config"; +import { VALUE_STRINGS } from "../../config"; +import type { string_parameter_value } from "../../types/typeAliases"; /** * Format either small or big number @@ -8,24 +8,24 @@ import type { string_parameter_value } from '../../types/typeAliases'; * @public exported from `@promptbook/utils` */ export function numberToString(value: number): string_parameter_value { - if (value === 0) { - return '0'; - } else if (Number.isNaN(value)) { - return VALUE_STRINGS.nan; - } else if (value === Infinity) { - return VALUE_STRINGS.infinity; - } else if (value === -Infinity) { - return VALUE_STRINGS.negativeInfinity; - } + if (value === 0) { + return "0"; + } else if (Number.isNaN(value)) { + return VALUE_STRINGS.nan; + } else if (value === Number.POSITIVE_INFINITY) { + return VALUE_STRINGS.infinity; + } else if (value === Number.NEGATIVE_INFINITY) { + return VALUE_STRINGS.negativeInfinity; + } - for (let exponent = 0; exponent < 15; exponent++) { - const factor = 10 ** exponent; - const valueRounded = Math.round(value * factor) / factor; + for (let exponent = 0; exponent < 15; exponent++) { + const factor = 10 ** exponent; + const valueRounded = Math.round(value * factor) / factor; - if (Math.abs(value - valueRounded) / value < SMALL_NUMBER) { - return valueRounded.toFixed(exponent); - } - } + if (Math.abs(value - valueRounded) / value < SMALL_NUMBER) { + return valueRounded.toFixed(exponent); + } + } - return value.toString(); + return value.toString(); } diff --git a/src/utils/parameters/templateParameters.test.ts b/src/utils/parameters/templateParameters.test.ts index 1e0ef0bd46..32c2793b77 100644 --- a/src/utils/parameters/templateParameters.test.ts +++ b/src/utils/parameters/templateParameters.test.ts @@ -1,159 +1,203 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { just } from '../organization/just'; -import { templateParameters } from './templateParameters'; - -describe('templateParameters', () => { - it('should work in supersimple case', () => { - expect(templateParameters('', {})).toBe(''); - }); - - it('should keep template without parameters as it is', () => { - expect(templateParameters('Hello', {})).toBe('Hello'); - expect(templateParameters('Hello World', {})).toBe('Hello World'); - }); - - it('should replace parameter at the end', () => { - expect(templateParameters('Hello {name}', { name: 'World' })).toBe('Hello World'); - }); - - it('should replace parameter in the middle', () => { - expect(templateParameters('Hello {name}, how are you?', { name: 'World' })).toBe('Hello World, how are you?'); - }); - - it('should replace parameter at the beginning', () => { - expect(templateParameters('{name}, how are you?', { name: 'World' })).toBe('World, how are you?'); - }); - - it('should replace multiple parameters', () => { - expect(templateParameters('{greeting} {name}, how are you?', { greeting: 'Hello', name: 'World' })).toBe( - 'Hello World, how are you?', - ); - }); - - it('should not be confused by JSON', () => { - expect( - templateParameters('{greeting} {name}, this is how JSON look like {"key": 1}.', { - greeting: 'Hi', - name: 'Pavol', - }), - ).toBe('Hi Pavol, this is how JSON look like {"key": 1}.'); - expect( - templateParameters('{greeting} {name}, this is how JSON look like {}.', { - greeting: 'Hi', - name: 'Pavol', - }), - ).toBe('Hi Pavol, this is how JSON look like {}.'); - expect( - templateParameters( - '{greeting} {name}, this is how JSON look like {"greeting": "{greeting}", "name": "{name}"}.', - { - greeting: 'Hi', - name: 'Pavol', - }, - ), - ).toBe('Hi Pavol, this is how JSON look like {"greeting": "Hi", "name": "Pavol"}.'); - expect( - templateParameters( - '{greeting} {name}, this is how JSON look like {"params": {"greeting": "{greeting}", "name": "{name}"}}.', - { - greeting: 'Hi', - name: 'Pavol', - }, - ), - ).toBe('Hi Pavol, this is how JSON look like {"params": {"greeting": "Hi", "name": "Pavol"}}.'); - expect( - templateParameters( - '{greeting} {name}, this is how invalid JSON look like {"params": {"greeting": "{greeting}", "name": "{name}"}.', - { - greeting: 'Hi', - name: 'Pavol', - }, - ), - ).toBe('Hi Pavol, this is how invalid JSON look like {"params": {"greeting": "Hi", "name": "Pavol"}.'); - }); - - it('should replace same parameter multiple times', () => { - expect( - templateParameters('{greeting} {name}, how are you? {greeting} {name}', { - greeting: 'Hello', - name: 'World', - }), - ).toBe('Hello World, how are you? Hello World'); - }); - - it('should replace multi-line tasks', () => { - expect( - templateParameters( - spaceTrim(` +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { just } from "../organization/just"; +import { templateParameters } from "./templateParameters"; + +describe("templateParameters", () => { + it("should work in supersimple case", () => { + expect(templateParameters("", {})).toBe(""); + }); + + it("should keep template without parameters as it is", () => { + expect(templateParameters("Hello", {})).toBe("Hello"); + expect(templateParameters("Hello World", {})).toBe("Hello World"); + }); + + it("should replace parameter at the end", () => { + expect(templateParameters("Hello {name}", { name: "World" })).toBe( + "Hello World", + ); + }); + + it("should replace parameter in the middle", () => { + expect( + templateParameters("Hello {name}, how are you?", { name: "World" }), + ).toBe("Hello World, how are you?"); + }); + + it("should replace parameter at the beginning", () => { + expect(templateParameters("{name}, how are you?", { name: "World" })).toBe( + "World, how are you?", + ); + }); + + it("should replace multiple parameters", () => { + expect( + templateParameters("{greeting} {name}, how are you?", { + greeting: "Hello", + name: "World", + }), + ).toBe("Hello World, how are you?"); + }); + + it("should not be confused by JSON", () => { + expect( + templateParameters( + '{greeting} {name}, this is how JSON look like {"key": 1}.', + { + greeting: "Hi", + name: "Pavol", + }, + ), + ).toBe('Hi Pavol, this is how JSON look like {"key": 1}.'); + expect( + templateParameters("{greeting} {name}, this is how JSON look like {}.", { + greeting: "Hi", + name: "Pavol", + }), + ).toBe("Hi Pavol, this is how JSON look like {}."); + expect( + templateParameters( + '{greeting} {name}, this is how JSON look like {"greeting": "{greeting}", "name": "{name}"}.', + { + greeting: "Hi", + name: "Pavol", + }, + ), + ).toBe( + 'Hi Pavol, this is how JSON look like {"greeting": "Hi", "name": "Pavol"}.', + ); + expect( + templateParameters( + '{greeting} {name}, this is how JSON look like {"params": {"greeting": "{greeting}", "name": "{name}"}}.', + { + greeting: "Hi", + name: "Pavol", + }, + ), + ).toBe( + 'Hi Pavol, this is how JSON look like {"params": {"greeting": "Hi", "name": "Pavol"}}.', + ); + expect( + templateParameters( + '{greeting} {name}, this is how invalid JSON look like {"params": {"greeting": "{greeting}", "name": "{name}"}.', + { + greeting: "Hi", + name: "Pavol", + }, + ), + ).toBe( + 'Hi Pavol, this is how invalid JSON look like {"params": {"greeting": "Hi", "name": "Pavol"}.', + ); + }); + + it("should replace same parameter multiple times", () => { + expect( + templateParameters("{greeting} {name}, how are you? {greeting} {name}", { + greeting: "Hello", + name: "World", + }), + ).toBe("Hello World, how are you? Hello World"); + }); + + it("should replace multi-line tasks", () => { + expect( + templateParameters( + spaceTrim(` Hello {name}, how are you? I am {greeting} `), - { greeting: 'fine', name: 'World' }, - ), - ).toBe( - just( - spaceTrim(` + { greeting: "fine", name: "World" }, + ), + ).toBe( + just( + spaceTrim(` Hello World, how are you? I am fine `), - ), - ); - }); - - it('should not be confused by some NON-JSON structure containing chars {}', () => { - expect(templateParameters('{greeting {name}}, how are you?', { greeting: 'Hello', name: 'world' })).toBe( - '{greeting world}, how are you?', - ); - expect(templateParameters('', { greeting: 'Hello', name: 'world' })).toBe(''); - - expect(templateParameters('<{greeting {name}}>', { greeting: 'Hello', name: 'world' })).toBe( - '<{greeting world}>', - ); - expect(templateParameters('<{{{greeting {name}}}}>', { greeting: 'Hello', name: 'world' })).toBe( - '<{{{greeting world}}}>', - ); - - expect(templateParameters('{greeting} }{}{}{', { greeting: 'Hello', name: 'world' })).toBe('Hello }{}{}{'); - }); - - it('should throw error when parameter is not defined', () => { - expect(() => templateParameters('{greeting} {name}, how are you?', { greeting: 'Hello' })).toThrowError( - /Parameter `{name}` is not defined/i, - ); - }); - - it('should throw error when parameter is not closed', () => { - expect(() => templateParameters('Hello {name', { name: 'world' })).toThrowError(/Parameter is not closed/i); - }); - - it('should throw error when parameter is not opened', () => { - expect(() => - templateParameters('greeting} {name}, how are you?', { greeting: 'Hello', name: 'World' }), - ).toThrowError(/Parameter is not opened/i); - }); - - it('should preserve indentation in multi-line tasks', () => { - expect( - templateParameters( - spaceTrim(` + ), + ); + }); + + it("should not be confused by some NON-JSON structure containing chars {}", () => { + expect( + templateParameters("{greeting {name}}, how are you?", { + greeting: "Hello", + name: "world", + }), + ).toBe("{greeting world}, how are you?"); + expect( + templateParameters("", { + greeting: "Hello", + name: "world", + }), + ).toBe(""); + + expect( + templateParameters("<{greeting {name}}>", { + greeting: "Hello", + name: "world", + }), + ).toBe("<{greeting world}>"); + expect( + templateParameters("<{{{greeting {name}}}}>", { + greeting: "Hello", + name: "world", + }), + ).toBe("<{{{greeting world}}}>"); + + expect( + templateParameters("{greeting} }{}{}{", { + greeting: "Hello", + name: "world", + }), + ).toBe("Hello }{}{}{"); + }); + + it("should throw error when parameter is not defined", () => { + expect(() => + templateParameters("{greeting} {name}, how are you?", { + greeting: "Hello", + }), + ).toThrowError(/Parameter `{name}` is not defined/i); + }); + + it("should throw error when parameter is not closed", () => { + expect(() => + templateParameters("Hello {name", { name: "world" }), + ).toThrowError(/Parameter is not closed/i); + }); + + it("should throw error when parameter is not opened", () => { + expect(() => + templateParameters("greeting} {name}, how are you?", { + greeting: "Hello", + name: "World", + }), + ).toThrowError(/Parameter is not opened/i); + }); + + it("should preserve indentation in multi-line tasks", () => { + expect( + templateParameters( + spaceTrim(` Hello {name}, how are you? {response} `), - { - name: 'Pavol', - response: spaceTrim(` + { + name: "Pavol", + response: spaceTrim(` I am fine. And you? But I need some bananas 🍌 `), - }, - ), - ).toBe( - just( - spaceTrim(` + }, + ), + ).toBe( + just( + spaceTrim(` Hello Pavol, how are you? I am fine. @@ -161,31 +205,31 @@ describe('templateParameters', () => { But I need some bananas 🍌 `), - ), - ); - }); - - it('should preserve multiple indentations in multi-line tasks', () => { - expect( - templateParameters( - spaceTrim(` + ), + ); + }); + + it("should preserve multiple indentations in multi-line tasks", () => { + expect( + templateParameters( + spaceTrim(` Hello {name}, how are you? {response} `), - { - name: 'Pavol', - response: spaceTrim(` + { + name: "Pavol", + response: spaceTrim(` I am fine. And you? But I need some bananas 🍌 `), - }, - ), - ).toBe( - just( - spaceTrim(` + }, + ), + ).toBe( + just( + spaceTrim(` Hello Pavol, how are you? I am fine. @@ -193,31 +237,31 @@ describe('templateParameters', () => { ${`` /* <- Preserve indentation */} But I need some bananas 🍌 `), - ), - ); - }); - - it('should preserve col-chars in multi-line tasks', () => { - expect( - templateParameters( - spaceTrim(` + ), + ); + }); + + it("should preserve col-chars in multi-line tasks", () => { + expect( + templateParameters( + spaceTrim(` Hello {name}, how are you? > {response} `), - { - name: 'Pavol', - response: spaceTrim(` + { + name: "Pavol", + response: spaceTrim(` I am fine. And you? But I need some bananas 🍌 `), - }, - ), - ).toBe( - just( - spaceTrim(` + }, + ), + ).toBe( + just( + spaceTrim(` Hello Pavol, how are you? > I am fine. @@ -225,28 +269,28 @@ describe('templateParameters', () => { > ${`` /* <- Preserve space */} > But I need some bananas 🍌 `), - ), - ); - expect( - templateParameters( - spaceTrim(` + ), + ); + expect( + templateParameters( + spaceTrim(` Hello {name}, how are you? $ {response} `), - { - name: 'Pavol', - response: spaceTrim(` + { + name: "Pavol", + response: spaceTrim(` I am fine. And you? But I need some bananas 🍌 `), - }, - ), - ).toBe( - just( - spaceTrim(` + }, + ), + ).toBe( + just( + spaceTrim(` Hello Pavol, how are you? $ I am fine. @@ -254,31 +298,31 @@ describe('templateParameters', () => { $ ${`` /* <- Preserve space */} $ But I need some bananas 🍌 `), - ), - ); - }); - - it('should not-preserve non-col-chars in multi-line tasks', () => { - expect( - templateParameters( - spaceTrim(` + ), + ); + }); + + it("should not-preserve non-col-chars in multi-line tasks", () => { + expect( + templateParameters( + spaceTrim(` Hello {name}, how are you? The response from {name} is: {response} `), - { - name: 'Pavol', - response: spaceTrim(` + { + name: "Pavol", + response: spaceTrim(` I am fine. And you? ${`` /* <- Preserve indentation */} But I need some bananas 🍌 `), - }, - ), - ).toBe( - just( - spaceTrim(` + }, + ), + ).toBe( + just( + spaceTrim(` Hello Pavol, how are you? The response from Pavol is: I am fine. @@ -286,14 +330,14 @@ describe('templateParameters', () => { ${`` /* <- Preserve indentation */} But I need some bananas 🍌 `), - ), - ); - }); - - it('should work with combination of col-chars and non-col-chars in multi-line tasks', () => { - expect( - templateParameters( - spaceTrim(` + ), + ); + }); + + it("should work with combination of col-chars and non-col-chars in multi-line tasks", () => { + expect( + templateParameters( + spaceTrim(` Hello {name}, how are you? > {response} @@ -302,19 +346,19 @@ describe('templateParameters', () => { Pavol said "{response}" and "{response}" `), - { - name: 'Pavol', - response: spaceTrim(` + { + name: "Pavol", + response: spaceTrim(` I am fine. And you? But I need some bananas 🍌 `), - }, - ), - ).toBe( - just( - spaceTrim(` + }, + ), + ).toBe( + just( + spaceTrim(` Hello Pavol, how are you? > I am fine. @@ -335,24 +379,24 @@ describe('templateParameters', () => { ${`` /* <- Preserve indentation */} But I need some bananas 🍌" `), - ), - ); - }); - - it('should work with escaping', () => { - expect( - templateParameters('\\{name\\}, how are you?', { - name: 'Paul', - }), - ).toBe('\\{name\\}, how are you?'); - }); - - it('should prevent prompt-injection', () => { - expect( - templateParameters('{name}, how are you?', { - name: '{instructions} + {context}', - instructions: 'Some secret information', - }), - ).toBe('\\{instructions\\} + \\{context\\}, how are you?'); - }); + ), + ); + }); + + it("should work with escaping", () => { + expect( + templateParameters("\\{name\\}, how are you?", { + name: "Paul", + }), + ).toBe("\\{name\\}, how are you?"); + }); + + it("should prevent prompt-injection", () => { + expect( + templateParameters("{name}, how are you?", { + name: "{instructions} + {context}", + instructions: "Some secret information", + }), + ).toBe("\\{instructions\\} + \\{context\\}, how are you?"); + }); }); diff --git a/src/utils/parameters/templateParameters.ts b/src/utils/parameters/templateParameters.ts index 402d7a6be8..5e2f6930ca 100644 --- a/src/utils/parameters/templateParameters.ts +++ b/src/utils/parameters/templateParameters.ts @@ -1,14 +1,14 @@ -import { LOOP_LIMIT } from '../../config'; -import { RESERVED_PARAMETER_MISSING_VALUE } from '../../constants'; -import { RESERVED_PARAMETER_RESTRICTED } from '../../constants'; -import { LimitReachedError } from '../../errors/LimitReachedError'; -import { PipelineExecutionError } from '../../errors/PipelineExecutionError'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { Parameters } from '../../types/typeAliases'; -import type { string_parameter_name } from '../../types/typeAliases'; -import type { string_template } from '../../types/typeAliases'; -import type { really_unknown } from '../organization/really_unknown'; -import { valueToString } from './valueToString'; +import { LOOP_LIMIT } from "../../config"; +import { RESERVED_PARAMETER_MISSING_VALUE } from "../../constants"; +import { RESERVED_PARAMETER_RESTRICTED } from "../../constants"; +import { LimitReachedError } from "../../errors/LimitReachedError"; +import { PipelineExecutionError } from "../../errors/PipelineExecutionError"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { Parameters } from "../../types/typeAliases"; +import type { string_parameter_name } from "../../types/typeAliases"; +import type { string_template } from "../../types/typeAliases"; +import type { really_unknown } from "../organization/really_unknown"; +import { valueToString } from "./valueToString"; /** * Replaces parameters in template with values from parameters object @@ -23,79 +23,96 @@ import { valueToString } from './valueToString'; * @public exported from `@promptbook/utils` */ export function templateParameters( - template: string_template, - parameters: Record, + template: string_template, + parameters: Record, ): string { - for (const [parameterName, parameterValue] of Object.entries(parameters)) { - if (parameterValue === RESERVED_PARAMETER_MISSING_VALUE) { - throw new UnexpectedError(`Parameter \`{${parameterName}}\` has missing value`); - } else if (parameterValue === RESERVED_PARAMETER_RESTRICTED) { - // TODO: [🍵] - throw new UnexpectedError(`Parameter \`{${parameterName}}\` is restricted to use`); - } - } - - let replacedTemplates = template; - let match: RegExpExecArray | null; - - let loopLimit = LOOP_LIMIT; - while ( - (match = /^(?.*){(?\w+)}(.*)/m /* <- Not global */ - .exec(replacedTemplates)) - ) { - if (loopLimit-- < 0) { - throw new LimitReachedError('Loop limit reached during parameters replacement in `templateParameters`'); - } - - const precol = match.groups!.precol!; - const parameterName = match.groups!.parameterName!; - - if (parameterName === '') { - // Note: Skip empty placeholders. It's used to avoid confusion with JSON-like strings - continue; - } - - if (parameterName.indexOf('{') !== -1 || parameterName.indexOf('}') !== -1) { - throw new PipelineExecutionError('Parameter is already opened or not closed'); - } - - if ((parameters as Parameters)[parameterName] === undefined) { - throw new PipelineExecutionError(`Parameter \`{${parameterName}}\` is not defined`); - } - - let parameterValue = (parameters as Parameters)[parameterName]; - - if (parameterValue === undefined) { - throw new PipelineExecutionError(`Parameter \`{${parameterName}}\` is not defined`); - } - - parameterValue = valueToString(parameterValue); - - // Escape curly braces in parameter values to prevent prompt-injection - parameterValue = parameterValue.replace(/[{}]/g, '\\$&'); - - if (parameterValue.includes('\n') && /^\s*\W{0,3}\s*$/.test(precol)) { - parameterValue = parameterValue - .split('\n') - .map((line, index) => (index === 0 ? line : `${precol}${line}`)) - .join('\n'); - } - - replacedTemplates = - replacedTemplates.substring(0, match.index + precol.length) + - parameterValue + - replacedTemplates.substring(match.index + precol.length + parameterName.length + 2); - } - - // [💫] Check if there are parameters that are not closed properly - if (/{\w+$/.test(replacedTemplates)) { - throw new PipelineExecutionError('Parameter is not closed'); - } - - // [💫] Check if there are parameters that are not opened properly - if (/^\w+}/.test(replacedTemplates)) { - throw new PipelineExecutionError('Parameter is not opened'); - } - - return replacedTemplates; + for (const [parameterName, parameterValue] of Object.entries(parameters)) { + if (parameterValue === RESERVED_PARAMETER_MISSING_VALUE) { + throw new UnexpectedError( + `Parameter \`{${parameterName}}\` has missing value`, + ); + } else if (parameterValue === RESERVED_PARAMETER_RESTRICTED) { + // TODO: [🍵] + throw new UnexpectedError( + `Parameter \`{${parameterName}}\` is restricted to use`, + ); + } + } + + let replacedTemplates = template; + let match: RegExpExecArray | null; + + let loopLimit = LOOP_LIMIT; + while ( + (match = /^(?.*){(?\w+)}(.*)/m /* <- Not global */ + .exec(replacedTemplates)) + ) { + if (loopLimit-- < 0) { + throw new LimitReachedError( + "Loop limit reached during parameters replacement in `templateParameters`", + ); + } + + const precol = match.groups!.precol!; + const parameterName = match.groups!.parameterName!; + + if (parameterName === "") { + // Note: Skip empty placeholders. It's used to avoid confusion with JSON-like strings + continue; + } + + if ( + parameterName.indexOf("{") !== -1 || + parameterName.indexOf("}") !== -1 + ) { + throw new PipelineExecutionError( + "Parameter is already opened or not closed", + ); + } + + if ((parameters as Parameters)[parameterName] === undefined) { + throw new PipelineExecutionError( + `Parameter \`{${parameterName}}\` is not defined`, + ); + } + + let parameterValue = (parameters as Parameters)[parameterName]; + + if (parameterValue === undefined) { + throw new PipelineExecutionError( + `Parameter \`{${parameterName}}\` is not defined`, + ); + } + + parameterValue = valueToString(parameterValue); + + // Escape curly braces in parameter values to prevent prompt-injection + parameterValue = parameterValue.replace(/[{}]/g, "\\$&"); + + if (parameterValue.includes("\n") && /^\s*\W{0,3}\s*$/.test(precol)) { + parameterValue = parameterValue + .split("\n") + .map((line, index) => (index === 0 ? line : `${precol}${line}`)) + .join("\n"); + } + + replacedTemplates = + replacedTemplates.substring(0, match.index + precol.length) + + parameterValue + + replacedTemplates.substring( + match.index + precol.length + parameterName.length + 2, + ); + } + + // [💫] Check if there are parameters that are not closed properly + if (/{\w+$/.test(replacedTemplates)) { + throw new PipelineExecutionError("Parameter is not closed"); + } + + // [💫] Check if there are parameters that are not opened properly + if (/^\w+}/.test(replacedTemplates)) { + throw new PipelineExecutionError("Parameter is not opened"); + } + + return replacedTemplates; } diff --git a/src/utils/parameters/valueToString.test.ts b/src/utils/parameters/valueToString.test.ts index 338b8d5cc7..9a93a98b02 100644 --- a/src/utils/parameters/valueToString.test.ts +++ b/src/utils/parameters/valueToString.test.ts @@ -1,25 +1,25 @@ -import { describe, expect, it } from '@jest/globals'; -import { VALUE_STRINGS } from '../../config'; -import { valueToString } from './valueToString'; - -describe('how `valueToString` works', () => { - it('should work with strings', () => { - expect(valueToString(``)).toBe(VALUE_STRINGS.empty); - expect(valueToString(`foo`)).toBe(`foo`); - expect(valueToString(`bar`)).toBe(`bar`); - }); - - it('should work with numbers', () => { - expect(valueToString(1)).toBe(`1`); - // Note: For advanced formatting, see `numberToString` tests - }); - - it('should work with `null` and `undefined`', () => { - expect(valueToString(null)).toBe(VALUE_STRINGS.null); - expect(valueToString(undefined)).toBe(VALUE_STRINGS.undefined); - }); - - /* +import { describe, expect, it } from "@jest/globals"; +import { VALUE_STRINGS } from "../../config"; +import { valueToString } from "./valueToString"; + +describe("how `valueToString` works", () => { + it("should work with strings", () => { + expect(valueToString(``)).toBe(VALUE_STRINGS.empty); + expect(valueToString(`foo`)).toBe(`foo`); + expect(valueToString(`bar`)).toBe(`bar`); + }); + + it("should work with numbers", () => { + expect(valueToString(1)).toBe(`1`); + // Note: For advanced formatting, see `numberToString` tests + }); + + it("should work with `null` and `undefined`", () => { + expect(valueToString(null)).toBe(VALUE_STRINGS.null); + expect(valueToString(undefined)).toBe(VALUE_STRINGS.undefined); + }); + + /* TODO: Some clever way how to convert objects to LLM-friendly strings it('should work with objects', () => expect( diff --git a/src/utils/parameters/valueToString.ts b/src/utils/parameters/valueToString.ts index 2f86f3acf9..96ca93aba0 100644 --- a/src/utils/parameters/valueToString.ts +++ b/src/utils/parameters/valueToString.ts @@ -1,7 +1,7 @@ -import { VALUE_STRINGS } from '../../config'; -import type { string_parameter_value } from '../../types/typeAliases'; -import type { really_unknown } from '../organization/really_unknown'; -import { numberToString } from './numberToString'; +import { VALUE_STRINGS } from "../../config"; +import type { string_parameter_value } from "../../types/typeAliases"; +import type { really_unknown } from "../organization/really_unknown"; +import { numberToString } from "./numberToString"; /** * Function `valueToString` will convert the given value to string @@ -18,28 +18,28 @@ import { numberToString } from './numberToString'; * @public exported from `@promptbook/utils` */ export function valueToString(value: really_unknown): string_parameter_value { - try { - if (value === '') { - return VALUE_STRINGS.empty; - } else if (value === null) { - return VALUE_STRINGS.null; - } else if (value === undefined) { - return VALUE_STRINGS.undefined; - } else if (typeof value === 'string') { - return value; - } else if (typeof value === 'number') { - return numberToString(value); - } else if (value instanceof Date) { - return value.toISOString(); - } else { - return JSON.stringify(value); - } - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } + try { + if (value === "") { + return VALUE_STRINGS.empty; + } else if (value === null) { + return VALUE_STRINGS.null; + } else if (value === undefined) { + return VALUE_STRINGS.undefined; + } else if (typeof value === "string") { + return value; + } else if (typeof value === "number") { + return numberToString(value); + } else if (value instanceof Date) { + return value.toISOString(); + } else { + return JSON.stringify(value); + } + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } - console.error(error); - return VALUE_STRINGS.unserializable; - } + console.error(error); + return VALUE_STRINGS.unserializable; + } } diff --git a/src/utils/parseNumber.test.ts b/src/utils/parseNumber.test.ts index c7bdffac65..6220b0ddca 100644 --- a/src/utils/parseNumber.test.ts +++ b/src/utils/parseNumber.test.ts @@ -1,96 +1,106 @@ -import { describe, expect, it } from '@jest/globals'; -import type { TODO_any } from './organization/TODO_any'; -import { parseNumber } from './parseNumber'; +import { describe, expect, it } from "@jest/globals"; +import type { TODO_any } from "./organization/TODO_any"; +import { parseNumber } from "./parseNumber"; -describe('how parseNumber works', () => { - it('should keeps number', () => { - expect(parseNumber(0)).toBe(0); - expect(parseNumber(-0)).toBe(0); - expect(parseNumber(1)).toBe(1); - expect(parseNumber(2)).toBe(2); - expect(parseNumber(100)).toBe(100); - expect(parseNumber(1000)).toBe(1000); - expect(parseNumber(-2)).toBe(-2); - expect(parseNumber(-100)).toBe(-100); - expect(parseNumber(10.5)).toBe(10.5); - expect(parseNumber(-11.11)).toBe(-11.11); - }); +describe("how parseNumber works", () => { + it("should keeps number", () => { + expect(parseNumber(0)).toBe(0); + expect(parseNumber(-0)).toBe(0); + expect(parseNumber(1)).toBe(1); + expect(parseNumber(2)).toBe(2); + expect(parseNumber(100)).toBe(100); + expect(parseNumber(1000)).toBe(1000); + expect(parseNumber(-2)).toBe(-2); + expect(parseNumber(-100)).toBe(-100); + expect(parseNumber(10.5)).toBe(10.5); + expect(parseNumber(-11.11)).toBe(-11.11); + }); - it('should parse number', () => { - expect(parseNumber('0')).toBe(0); - expect(parseNumber('1')).toBe(1); - expect(parseNumber('2')).toBe(2); - expect(parseNumber('100')).toBe(100); - expect(parseNumber('1000')).toBe(1000); - expect(parseNumber('-2')).toBe(-2); - expect(parseNumber('-100')).toBe(-100); - expect(parseNumber('10.5')).toBe(10.5); - expect(parseNumber('-11.11')).toBe(-11.11); - expect(parseNumber('+11.11')).toBe(11.11); - }); + it("should parse number", () => { + expect(parseNumber("0")).toBe(0); + expect(parseNumber("1")).toBe(1); + expect(parseNumber("2")).toBe(2); + expect(parseNumber("100")).toBe(100); + expect(parseNumber("1000")).toBe(1000); + expect(parseNumber("-2")).toBe(-2); + expect(parseNumber("-100")).toBe(-100); + expect(parseNumber("10.5")).toBe(10.5); + expect(parseNumber("-11.11")).toBe(-11.11); + expect(parseNumber("+11.11")).toBe(11.11); + }); - it('should parse Infinity', () => { - expect(parseNumber('Infinity')).toBe(Infinity); - expect(parseNumber('infinity')).toBe(Infinity); - expect(parseNumber('-Infinity')).toBe(-Infinity); - expect(parseNumber('♾')).toBe(Infinity); - expect(parseNumber('-♾')).toBe(-Infinity); - expect(parseNumber('+INFINITY')).toBe(Infinity); - expect(parseNumber('-INFINITY')).toBe(-Infinity); - expect(parseNumber('INF')).toBe(Infinity); - expect(parseNumber('-INF')).toBe(-Infinity); - expect(parseNumber(Infinity)).toBe(Infinity); - expect(parseNumber(-Infinity)).toBe(-Infinity); - }); + it("should parse Infinity", () => { + expect(parseNumber("Infinity")).toBe(Number.POSITIVE_INFINITY); + expect(parseNumber("infinity")).toBe(Number.POSITIVE_INFINITY); + expect(parseNumber("-Infinity")).toBe(Number.NEGATIVE_INFINITY); + expect(parseNumber("♾")).toBe(Number.POSITIVE_INFINITY); + expect(parseNumber("-♾")).toBe(Number.NEGATIVE_INFINITY); + expect(parseNumber("+INFINITY")).toBe(Number.POSITIVE_INFINITY); + expect(parseNumber("-INFINITY")).toBe(Number.NEGATIVE_INFINITY); + expect(parseNumber("INF")).toBe(Number.POSITIVE_INFINITY); + expect(parseNumber("-INF")).toBe(Number.NEGATIVE_INFINITY); + expect(parseNumber(Number.POSITIVE_INFINITY)).toBe( + Number.POSITIVE_INFINITY, + ); + expect(parseNumber(Number.NEGATIVE_INFINITY)).toBe( + Number.NEGATIVE_INFINITY, + ); + }); - it('should parse non-trimmed value', () => { - expect(parseNumber(' 0')).toBe(0); - expect(parseNumber(' 1')).toBe(1); - expect(parseNumber('2 ')).toBe(2); - expect(parseNumber(' 100 ')).toBe(100); - expect(parseNumber(' 1000 ')).toBe(1000); - expect(parseNumber('\n-2')).toBe(-2); - expect(parseNumber('-100\r\n')).toBe(-100); - expect(parseNumber('10.5\t')).toBe(10.5); - expect(parseNumber(' - 11.11 ')).toBe(-11.11); - }); + it("should parse non-trimmed value", () => { + expect(parseNumber(" 0")).toBe(0); + expect(parseNumber(" 1")).toBe(1); + expect(parseNumber("2 ")).toBe(2); + expect(parseNumber(" 100 ")).toBe(100); + expect(parseNumber(" 1000 ")).toBe(1000); + expect(parseNumber("\n-2")).toBe(-2); + expect(parseNumber("-100\r\n")).toBe(-100); + expect(parseNumber("10.5\t")).toBe(10.5); + expect(parseNumber(" - 11.11 ")).toBe(-11.11); + }); - it('should parse fractions', () => { - expect(parseNumber('1/2')).toBe(0.5); - expect(parseNumber('1/3')).toBe(0.3333333333333333); - expect(parseNumber('1/4')).toBe(0.25); - expect(parseNumber('1/5')).toBe(0.2); - expect(parseNumber('1/6')).toBe(0.16666666666666666); - expect(parseNumber('1/7')).toBe(0.14285714285714285); - expect(parseNumber('-1/8')).toBe(-0.125); - expect(parseNumber('2/5')).toBe(0.4); - }); + it("should parse fractions", () => { + expect(parseNumber("1/2")).toBe(0.5); + expect(parseNumber("1/3")).toBe(0.3333333333333333); + expect(parseNumber("1/4")).toBe(0.25); + expect(parseNumber("1/5")).toBe(0.2); + expect(parseNumber("1/6")).toBe(0.16666666666666666); + expect(parseNumber("1/7")).toBe(0.14285714285714285); + expect(parseNumber("-1/8")).toBe(-0.125); + expect(parseNumber("2/5")).toBe(0.4); + }); - it('should parse value with misc formatting', () => { - expect(parseNumber('-11,11')).toBe(-11.11); - expect(parseNumber('11e5')).toBe(1100000); - }); + it("should parse value with misc formatting", () => { + expect(parseNumber("-11,11")).toBe(-11.11); + expect(parseNumber("11e5")).toBe(1100000); + }); - it('should NOT result in NaN', () => { - expect(parseNumber('')).toBe(0); - expect(parseNumber('-')).toBe(0); - expect(parseNumber('NaN')).toBe(0); - expect(parseNumber('null')).toBe(0); - expect(parseNumber('undefined')).toBe(0); - expect(parseNumber(NaN)).toBe(0); - expect(parseNumber(null as TODO_any)).toBe(0); - expect(parseNumber(undefined as TODO_any)).toBe(0); - }); + it("should NOT result in NaN", () => { + expect(parseNumber("")).toBe(0); + expect(parseNumber("-")).toBe(0); + expect(parseNumber("NaN")).toBe(0); + expect(parseNumber("null")).toBe(0); + expect(parseNumber("undefined")).toBe(0); + expect(parseNumber(Number.NaN)).toBe(0); + expect(parseNumber(null as TODO_any)).toBe(0); + expect(parseNumber(undefined as TODO_any)).toBe(0); + }); - it('should throw error on invalid entries', () => { - expect(() => parseNumber('wtf')).toThrowError(/Unable to parse number/i); - expect(() => parseNumber('abc')).toThrowError(/Unable to parse number/i); - expect(() => parseNumber('123abc')).toThrowError(/Unable to parse number/i); - expect(() => parseNumber('86abc123')).toThrowError(/Unable to parse number/i); - expect(() => parseNumber('abc123')).toThrowError(/Unable to parse number/i); - expect(() => parseNumber('abc123xyz')).toThrowError(/Unable to parse number/i); - expect(() => parseNumber('1/0')).toThrowError(/Unable to parse number/i); - expect(() => parseNumber('1.2.3')).toThrowError(/Unable to parse number/i); - expect(() => parseNumber('1.2.3.4')).toThrowError(/Unable to parse number/i); - }); + it("should throw error on invalid entries", () => { + expect(() => parseNumber("wtf")).toThrowError(/Unable to parse number/i); + expect(() => parseNumber("abc")).toThrowError(/Unable to parse number/i); + expect(() => parseNumber("123abc")).toThrowError(/Unable to parse number/i); + expect(() => parseNumber("86abc123")).toThrowError( + /Unable to parse number/i, + ); + expect(() => parseNumber("abc123")).toThrowError(/Unable to parse number/i); + expect(() => parseNumber("abc123xyz")).toThrowError( + /Unable to parse number/i, + ); + expect(() => parseNumber("1/0")).toThrowError(/Unable to parse number/i); + expect(() => parseNumber("1.2.3")).toThrowError(/Unable to parse number/i); + expect(() => parseNumber("1.2.3.4")).toThrowError( + /Unable to parse number/i, + ); + }); }); diff --git a/src/utils/parseNumber.ts b/src/utils/parseNumber.ts index 15d102e8e4..efb53c7396 100644 --- a/src/utils/parseNumber.ts +++ b/src/utils/parseNumber.ts @@ -1,4 +1,4 @@ -import { ParseError } from '../errors/ParseError'; +import { ParseError } from "../errors/ParseError"; /** * Function parseNumber will parse number from string @@ -12,82 +12,82 @@ import { ParseError } from '../errors/ParseError'; * @public exported from `@promptbook/utils` */ export function parseNumber(value: string | number): number { - const originalValue = value; - - if (typeof value === 'number') { - value = value.toString(); // <- TODO: Maybe more efficient way to do this - } - - if (typeof value !== 'string') { - return 0; - } - - value = value.trim(); - - if (value.startsWith('+')) { - return parseNumber(value.substring(1)); - } - - if (value.startsWith('-')) { - const number = parseNumber(value.substring(1)); - if (number === 0) { - return 0; // <- Note: To prevent -0 - } - return -number; - } - - value = value.replace(/,/g, '.'); - value = value.toUpperCase(); - - if (value === '') { - return 0; - } - - if (value === '♾' || value.startsWith('INF')) { - return Infinity; - } - - if (value.includes('/')) { - const [numerator_, denominator_] = value.split('/'); - const numerator = parseNumber(numerator_!); - const denominator = parseNumber(denominator_!); - - if (denominator === 0) { - throw new ParseError( - `Unable to parse number from "${originalValue}" because denominator is zero`, - // <- TODO: [🚞] Pass from consumer(s) of `parseNumber` - ); - } - - return numerator / denominator; - } - - if (/^(NAN|NULL|NONE|UNDEFINED|ZERO|NO.*)$/.test(value)) { - return 0; - } - - if (value.includes('E')) { - const [significand, exponent] = value.split('E'); - return parseNumber(significand!) * 10 ** parseNumber(exponent!); - } - - if (!/^[0-9.]+$/.test(value) || value.split('.').length > 2) { - throw new ParseError( - `Unable to parse number from "${originalValue}"`, - // <- TODO: [🚞] Pass from consumer(s) of `parseNumber` - ); - } - - const num = parseFloat(value); - - if (isNaN(num)) { - throw new ParseError( - `Unexpected NaN when parsing number from "${originalValue}"`, - // <- TODO: [🚞] Pass from consumer(s) of `parseNumber` - ); - } - - return num; + const originalValue = value; + + if (typeof value === "number") { + value = value.toString(); // <- TODO: Maybe more efficient way to do this + } + + if (typeof value !== "string") { + return 0; + } + + value = value.trim(); + + if (value.startsWith("+")) { + return parseNumber(value.substring(1)); + } + + if (value.startsWith("-")) { + const number = parseNumber(value.substring(1)); + if (number === 0) { + return 0; // <- Note: To prevent -0 + } + return -number; + } + + value = value.replace(/,/g, "."); + value = value.toUpperCase(); + + if (value === "") { + return 0; + } + + if (value === "♾" || value.startsWith("INF")) { + return Number.POSITIVE_INFINITY; + } + + if (value.includes("/")) { + const [numerator_, denominator_] = value.split("/"); + const numerator = parseNumber(numerator_!); + const denominator = parseNumber(denominator_!); + + if (denominator === 0) { + throw new ParseError( + `Unable to parse number from "${originalValue}" because denominator is zero`, + // <- TODO: [🚞] Pass from consumer(s) of `parseNumber` + ); + } + + return numerator / denominator; + } + + if (/^(NAN|NULL|NONE|UNDEFINED|ZERO|NO.*)$/.test(value)) { + return 0; + } + + if (value.includes("E")) { + const [significand, exponent] = value.split("E"); + return parseNumber(significand!) * 10 ** parseNumber(exponent!); + } + + if (!/^[0-9.]+$/.test(value) || value.split(".").length > 2) { + throw new ParseError( + `Unable to parse number from "${originalValue}"`, + // <- TODO: [🚞] Pass from consumer(s) of `parseNumber` + ); + } + + const num = Number.parseFloat(value); + + if (isNaN(num)) { + throw new ParseError( + `Unexpected NaN when parsing number from "${originalValue}"`, + // <- TODO: [🚞] Pass from consumer(s) of `parseNumber` + ); + } + + return num; } /** diff --git a/src/utils/random/$randomSeed.ts b/src/utils/random/$randomSeed.ts index 2667fe8bd0..e5dcc29a5e 100644 --- a/src/utils/random/$randomSeed.ts +++ b/src/utils/random/$randomSeed.ts @@ -1,13 +1,13 @@ -import type { number_seed } from '../../types/typeAliases'; +import type { number_seed } from "../../types/typeAliases"; /** * Generates random seed * * Note: `$` is used to indicate that this function is not a pure function - it is not deterministic * Warning: This function is NOT cryptographically secure (it uses Math.random internally) - * + * * @private internal helper function */ export function $randomSeed(): number_seed { - return Math.random(); + return Math.random(); } diff --git a/src/utils/random/$randomToken.ts b/src/utils/random/$randomToken.ts index fd94eb7cd1..1fe6248476 100644 --- a/src/utils/random/$randomToken.ts +++ b/src/utils/random/$randomToken.ts @@ -1,5 +1,5 @@ -import { randomBytes } from 'crypto'; -import type { string_token } from '../../types/typeAliases'; +import { randomBytes } from "crypto"; +import type { string_token } from "../../types/typeAliases"; /** * Generates random token @@ -10,7 +10,7 @@ import type { string_token } from '../../types/typeAliases'; * @returns secure random token */ export function $randomToken(randomness: number): string_token { - return randomBytes(randomness).toString('hex'); + return randomBytes(randomness).toString("hex"); } /** diff --git a/src/utils/removeEmojis.test.ts b/src/utils/removeEmojis.test.ts index ed1d71c19f..4ff48c833d 100644 --- a/src/utils/removeEmojis.test.ts +++ b/src/utils/removeEmojis.test.ts @@ -1,46 +1,48 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { just } from './organization/just'; -import { removeEmojis } from './removeEmojis'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { just } from "./organization/just"; +import { removeEmojis } from "./removeEmojis"; -describe('removeEmojis', () => { - it('should preserve text without emojis', () => { - expect(removeEmojis('')).toBe(''); - expect(removeEmojis('Hello')).toBe('Hello'); - expect(removeEmojis('Hello World')).toBe('Hello World'); - expect(removeEmojis('Hello 1')).toBe('Hello 1'); - }); +describe("removeEmojis", () => { + it("should preserve text without emojis", () => { + expect(removeEmojis("")).toBe(""); + expect(removeEmojis("Hello")).toBe("Hello"); + expect(removeEmojis("Hello World")).toBe("Hello World"); + expect(removeEmojis("Hello 1")).toBe("Hello 1"); + }); - it('should remove one emoji', () => { - expect(removeEmojis('💗')).toBe(''); - expect(removeEmojis('🈚')).toBe(''); - expect(removeEmojis('Hello 🖖')).toBe('Hello '); - expect(removeEmojis('Hello 💅 World')).toBe('Hello World'); - }); + it("should remove one emoji", () => { + expect(removeEmojis("💗")).toBe(""); + expect(removeEmojis("🈚")).toBe(""); + expect(removeEmojis("Hello 🖖")).toBe("Hello "); + expect(removeEmojis("Hello 💅 World")).toBe("Hello World"); + }); - it('should remove multiple emojis', () => { - expect(removeEmojis('💗💙')).toBe(''); - expect(removeEmojis(`♥♦♠♣`)).toBe(``); - expect(removeEmojis('Hello 💗💚')).toBe('Hello '); - expect(removeEmojis('Hello 💗🔰🟩 World')).toBe('Hello World'); - expect(removeEmojis('Hello 💗 🟩 World 🏡')).toBe('Hello World '); - expect(removeEmojis(`👸👨‍🦱👨👴👨‍🦰 Hello 💗🥗💗 World`)).toBe(` Hello World`); - expect(removeEmojis('💚💙💫🌟🌠')).toBe(''); - expect(removeEmojis('👩🏾👨')).toBe(''); - expect(removeEmojis('👨')).toBe(''); - expect(removeEmojis('👨‍❤️‍👨👨‍❤️‍👨👨‍❤️‍👨')).toBe(''); - expect(removeEmojis('I ♥ Programming')).toBe('I Programming'); - expect(removeEmojis('I ❤ Programming')).toBe('I Programming'); - expect(removeEmojis('I 💙 Programming')).toBe('I Programming'); - expect(removeEmojis('I 💫 Programming')).toBe('I Programming'); - expect(removeEmojis('I 👩🏾 Programming')).toBe('I Programming'); - expect(removeEmojis('I 👨‍❤️‍👨 Programming')).toBe('I Programming'); - }); + it("should remove multiple emojis", () => { + expect(removeEmojis("💗💙")).toBe(""); + expect(removeEmojis(`♥♦♠♣`)).toBe(``); + expect(removeEmojis("Hello 💗💚")).toBe("Hello "); + expect(removeEmojis("Hello 💗🔰🟩 World")).toBe("Hello World"); + expect(removeEmojis("Hello 💗 🟩 World 🏡")).toBe("Hello World "); + expect(removeEmojis(`👸👨‍🦱👨👴👨‍🦰 Hello 💗🥗💗 World`)).toBe( + ` Hello World`, + ); + expect(removeEmojis("💚💙💫🌟🌠")).toBe(""); + expect(removeEmojis("👩🏾👨")).toBe(""); + expect(removeEmojis("👨")).toBe(""); + expect(removeEmojis("👨‍❤️‍👨👨‍❤️‍👨👨‍❤️‍👨")).toBe(""); + expect(removeEmojis("I ♥ Programming")).toBe("I Programming"); + expect(removeEmojis("I ❤ Programming")).toBe("I Programming"); + expect(removeEmojis("I 💙 Programming")).toBe("I Programming"); + expect(removeEmojis("I 💫 Programming")).toBe("I Programming"); + expect(removeEmojis("I 👩🏾 Programming")).toBe("I Programming"); + expect(removeEmojis("I 👨‍❤️‍👨 Programming")).toBe("I Programming"); + }); - it('should remove multiple emojis from multi-line text', () => { - expect( - spaceTrim( - removeEmojis(` + it("should remove multiple emojis from multi-line text", () => { + expect( + spaceTrim( + removeEmojis(` 🌲 Pine tree 🌳 Deciduous tree 🌴 Palm tree @@ -51,10 +53,10 @@ describe('removeEmojis', () => { 🎍 Pine decoration 🎋 Tanabata tree `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` Pine tree Deciduous tree Palm tree @@ -65,7 +67,7 @@ describe('removeEmojis', () => { Pine decoration Tanabata tree `), - ), - ); - }); + ), + ); + }); }); diff --git a/src/utils/removeEmojis.ts b/src/utils/removeEmojis.ts index 2dae456a13..3e1a8524c2 100644 --- a/src/utils/removeEmojis.ts +++ b/src/utils/removeEmojis.ts @@ -6,12 +6,15 @@ * @public exported from `@promptbook/utils` */ export function removeEmojis(text: string): string { - // Replace emojis (and also ZWJ sequence) with hyphens - text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1'); - text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1'); - text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1'); + // Replace emojis (and also ZWJ sequence) with hyphens + text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, "$1"); + text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, "$1"); + text = text.replace( + /(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, + "$1", + ); - text = text.replace(/\p{Extended_Pictographic}/gu, ''); + text = text.replace(/\p{Extended_Pictographic}/gu, ""); - return text; + return text; } diff --git a/src/utils/removeQuotes.test.ts b/src/utils/removeQuotes.test.ts index c30911dece..3e09825f9c 100644 --- a/src/utils/removeQuotes.test.ts +++ b/src/utils/removeQuotes.test.ts @@ -1,31 +1,31 @@ -import { describe, expect, it } from '@jest/globals'; -import { removeQuotes } from './removeQuotes'; +import { describe, expect, it } from "@jest/globals"; +import { removeQuotes } from "./removeQuotes"; -describe('removeQuotes', () => { - it('should keep unquoted string', () => { - expect(removeQuotes('Hello')).toBe('Hello'); - }); +describe("removeQuotes", () => { + it("should keep unquoted string", () => { + expect(removeQuotes("Hello")).toBe("Hello"); + }); - it('should remove single quotes', () => { - expect(removeQuotes('\'Hello\'')).toBe('Hello'); - }); + it("should remove single quotes", () => { + expect(removeQuotes("'Hello'")).toBe("Hello"); + }); - it('should remove double quotes', () => { - expect(removeQuotes('"Hello"')).toBe('Hello'); - }); + it("should remove double quotes", () => { + expect(removeQuotes('"Hello"')).toBe("Hello"); + }); - it('should NOT remove single quote from the beginning', () => { - expect(removeQuotes('\'Hello')).toBe('\'Hello'); - expect(removeQuotes('"Hello')).toBe('"Hello'); - }); + it("should NOT remove single quote from the beginning", () => { + expect(removeQuotes("'Hello")).toBe("'Hello"); + expect(removeQuotes('"Hello')).toBe('"Hello'); + }); - it('should NOT remove single quote from the end', () => { - expect(removeQuotes('Hello\'')).toBe('Hello\''); - expect(removeQuotes('Hello"')).toBe('Hello"'); - }); + it("should NOT remove single quote from the end", () => { + expect(removeQuotes("Hello'")).toBe("Hello'"); + expect(removeQuotes('Hello"')).toBe('Hello"'); + }); - it('should NOT remove quote from the middle', () => { - expect(removeQuotes('Hel\'lo')).toBe('Hel\'lo'); - expect(removeQuotes('Hel"lo')).toBe('Hel"lo'); - }); + it("should NOT remove quote from the middle", () => { + expect(removeQuotes("Hel'lo")).toBe("Hel'lo"); + expect(removeQuotes('Hel"lo')).toBe('Hel"lo'); + }); }); diff --git a/src/utils/removeQuotes.ts b/src/utils/removeQuotes.ts index 8055cc7d6c..ea9ef4176a 100644 --- a/src/utils/removeQuotes.ts +++ b/src/utils/removeQuotes.ts @@ -12,13 +12,13 @@ * @public exported from `@promptbook/utils` */ export function removeQuotes(text: string): string { - if (text.startsWith('"') && text.endsWith('"')) { - return text.slice(1, -1); - } + if (text.startsWith('"') && text.endsWith('"')) { + return text.slice(1, -1); + } - if (text.startsWith('\'') && text.endsWith('\'')) { - return text.slice(1, -1); - } + if (text.startsWith("'") && text.endsWith("'")) { + return text.slice(1, -1); + } - return text; + return text; } diff --git a/src/utils/serialization/$deepFreeze.ts b/src/utils/serialization/$deepFreeze.ts index b6ace56897..f5a3aa5047 100644 --- a/src/utils/serialization/$deepFreeze.ts +++ b/src/utils/serialization/$deepFreeze.ts @@ -1,6 +1,6 @@ -import type { ReadonlyDeep } from 'type-fest'; -import type { really_any } from '../organization/really_any'; -import type { TODO_any } from '../organization/TODO_any'; +import type { ReadonlyDeep } from "type-fest"; +import type { TODO_any } from "../organization/TODO_any"; +import type { really_any } from "../organization/really_any"; /** * Freezes the given object and all its nested objects recursively @@ -11,22 +11,26 @@ import type { TODO_any } from '../organization/TODO_any'; * @returns The same object as the input, but deeply frozen * @public exported from `@promptbook/utils` */ -export function $deepFreeze(objectValue: TObject): ReadonlyDeep { - if (Array.isArray(objectValue)) { - return Object.freeze(objectValue.map((item) => $deepFreeze(item))) as TODO_any; - } +export function $deepFreeze( + objectValue: TObject, +): ReadonlyDeep { + if (Array.isArray(objectValue)) { + return Object.freeze( + objectValue.map((item) => $deepFreeze(item)), + ) as TODO_any; + } - const propertyNames = Object.getOwnPropertyNames(objectValue); - for (const propertyName of propertyNames) { - const value = (objectValue as really_any)[propertyName]; - if (value && typeof value === 'object') { - $deepFreeze(value); - } - } + const propertyNames = Object.getOwnPropertyNames(objectValue); + for (const propertyName of propertyNames) { + const value = (objectValue as really_any)[propertyName]; + if (value && typeof value === "object") { + $deepFreeze(value); + } + } - Object.freeze(objectValue); + Object.freeze(objectValue); - return objectValue as ReadonlyDeep; + return objectValue as ReadonlyDeep; } /** diff --git a/src/utils/serialization/asSerializable.test.ts b/src/utils/serialization/asSerializable.test.ts index 99bb871c74..31deaa01dd 100644 --- a/src/utils/serialization/asSerializable.test.ts +++ b/src/utils/serialization/asSerializable.test.ts @@ -1,22 +1,22 @@ -import { describe, expect, it } from '@jest/globals'; -import { asSerializable } from './asSerializable'; +import { describe, expect, it } from "@jest/globals"; +import { asSerializable } from "./asSerializable"; -describe('how `asSerializable` works', () => { - it('should serialize the simple object', () => - expect( - asSerializable({ - value: true, - }), - ).toEqual({ - value: true, - })); +describe("how `asSerializable` works", () => { + it("should serialize the simple object", () => + expect( + asSerializable({ + value: true, + }), + ).toEqual({ + value: true, + })); - it('should serialize the date', () => - expect( - asSerializable({ - value: new Date(`2024-12-10T13:04:19.025Z`), - }), - ).toEqual({ - value: `2024-12-10T13:04:19.025Z`, - })); + it("should serialize the date", () => + expect( + asSerializable({ + value: new Date(`2024-12-10T13:04:19.025Z`), + }), + ).toEqual({ + value: `2024-12-10T13:04:19.025Z`, + })); }); diff --git a/src/utils/serialization/asSerializable.ts b/src/utils/serialization/asSerializable.ts index 37894827b9..0259070d59 100644 --- a/src/utils/serialization/asSerializable.ts +++ b/src/utils/serialization/asSerializable.ts @@ -1,5 +1,5 @@ -import type { really_any } from '../organization/really_any'; -import type { really_unknown } from '../organization/really_unknown'; +import type { really_any } from "../organization/really_any"; +import type { really_unknown } from "../organization/really_unknown"; /** * Function `asSerializable` will convert values which are not serializable to serializable values @@ -15,19 +15,19 @@ import type { really_unknown } from '../organization/really_unknown'; * @private Internal helper function */ export function asSerializable(value: really_any): really_any { - if (value instanceof Date) { - return value.toISOString(); - } else if (Array.isArray(value)) { - return value.map(asSerializable); - } else if (value !== null && typeof value === 'object') { - const result: really_unknown = {}; - for (const key in value) { - if (Object.prototype.hasOwnProperty.call(value, key)) { - (result as really_any)[key] = asSerializable(value[key]); - } - } - return result; - } else { - return value; - } + if (value instanceof Date) { + return value.toISOString(); + } else if (Array.isArray(value)) { + return value.map(asSerializable); + } else if (value !== null && typeof value === "object") { + const result: really_unknown = {}; + for (const key in value) { + if (Object.prototype.hasOwnProperty.call(value, key)) { + (result as really_any)[key] = asSerializable(value[key]); + } + } + return result; + } else { + return value; + } } diff --git a/src/utils/serialization/checkSerializableAsJson.ts b/src/utils/serialization/checkSerializableAsJson.ts index 0218db2035..e65974dfd8 100644 --- a/src/utils/serialization/checkSerializableAsJson.ts +++ b/src/utils/serialization/checkSerializableAsJson.ts @@ -1,26 +1,26 @@ -import spaceTrim from 'spacetrim'; -import { UnexpectedError } from '../../errors/UnexpectedError'; -import type { string_name } from '../../types/typeAliases'; -import type { really_unknown } from '../organization/really_unknown'; +import spaceTrim from "spacetrim"; +import { UnexpectedError } from "../../errors/UnexpectedError"; +import type { string_name } from "../../types/typeAliases"; +import type { really_unknown } from "../organization/really_unknown"; /** * Options for the `checkSerializableAsJson` function */ export type CheckSerializableAsJsonOptions = { - /** - * Value to be checked - */ - value: really_unknown; - - /** - * Semantic name of the value for debugging purposes - */ - name?: string_name; - - /** - * Message alongside the value for debugging purposes - */ - message?: string; + /** + * Value to be checked + */ + value: really_unknown; + + /** + * Semantic name of the value for debugging purposes + */ + name?: string_name; + + /** + * Message alongside the value for debugging purposes + */ + message?: string; }; /** @@ -43,92 +43,102 @@ export type CheckSerializableAsJsonOptions = { * @throws UnexpectedError if the value is not serializable as JSON * @public exported from `@promptbook/utils` */ -export function checkSerializableAsJson(options: CheckSerializableAsJsonOptions): void { - const { value, name, message } = options; - - if (value === undefined) { - throw new UnexpectedError(`${name} is undefined`); - } else if (value === null) { - return; - } else if (typeof value === 'boolean') { - return; - } else if (typeof value === 'number' && !isNaN(value)) { - return; - } else if (typeof value === 'string') { - return; - } else if (typeof value === 'symbol') { - throw new UnexpectedError(`${name} is symbol`); - } else if (typeof value === 'function') { - throw new UnexpectedError(`${name} is function`); - } else if (typeof value === 'object' && Array.isArray(value)) { - for (let i = 0; i < value.length; i++) { - checkSerializableAsJson({ name: `${name}[${i}]`, value: value[i], message }); - } - } else if (typeof value === 'object') { - if (value instanceof Date) { - throw new UnexpectedError( - spaceTrim( - (block) => ` +export function checkSerializableAsJson( + options: CheckSerializableAsJsonOptions, +): void { + const { value, name, message } = options; + + if (value === undefined) { + throw new UnexpectedError(`${name} is undefined`); + } else if (value === null) { + return; + } else if (typeof value === "boolean") { + return; + } else if (typeof value === "number" && !isNaN(value)) { + return; + } else if (typeof value === "string") { + return; + } else if (typeof value === "symbol") { + throw new UnexpectedError(`${name} is symbol`); + } else if (typeof value === "function") { + throw new UnexpectedError(`${name} is function`); + } else if (typeof value === "object" && Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + checkSerializableAsJson({ + name: `${name}[${i}]`, + value: value[i], + message, + }); + } + } else if (typeof value === "object") { + if (value instanceof Date) { + throw new UnexpectedError( + spaceTrim( + (block) => ` \`${name}\` is Date Use \`string_date_iso8601\` instead Additional message for \`${name}\`: - ${block(message || '(nothing)')} + ${block(message || "(nothing)")} `, - ), - ); - } else if (value instanceof Map) { - throw new UnexpectedError(`${name} is Map`); - } else if (value instanceof Set) { - throw new UnexpectedError(`${name} is Set`); - } else if (value instanceof RegExp) { - throw new UnexpectedError(`${name} is RegExp`); - } else if (value instanceof Error) { - throw new UnexpectedError( - spaceTrim( - (block) => ` + ), + ); + } else if (value instanceof Map) { + throw new UnexpectedError(`${name} is Map`); + } else if (value instanceof Set) { + throw new UnexpectedError(`${name} is Set`); + } else if (value instanceof RegExp) { + throw new UnexpectedError(`${name} is RegExp`); + } else if (value instanceof Error) { + throw new UnexpectedError( + spaceTrim( + (block) => ` \`${name}\` is unserialized Error Use function \`serializeError\` Additional message for \`${name}\`: - ${block(message || '(nothing)')} + ${block(message || "(nothing)")} `, - ), - ); - } else { - for (const [subName, subValue] of Object.entries(value)) { - if (subValue === undefined) { - // Note: undefined in object is serializable - it is just omited - continue; - } - checkSerializableAsJson({ name: `${name}.${subName}`, value: subValue, message }); - } - - try { - JSON.stringify(value); // <- TODO: [0] - } catch (error) { - if (!(error instanceof Error)) { - throw error; - } - - throw new UnexpectedError( - spaceTrim( - (block) => ` + ), + ); + } else { + for (const [subName, subValue] of Object.entries(value)) { + if (subValue === undefined) { + // Note: undefined in object is serializable - it is just omited + continue; + } + checkSerializableAsJson({ + name: `${name}.${subName}`, + value: subValue, + message, + }); + } + + try { + JSON.stringify(value); // <- TODO: [0] + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + throw new UnexpectedError( + spaceTrim( + (block) => ` \`${name}\` is not serializable ${block((error as Error).stack || (error as Error).message)} Additional message for \`${name}\`: - ${block(message || '(nothing)')} + ${block(message || "(nothing)")} `, - ), - ); - } + ), + ); + } - /* + /* TODO: [0] Is there some more elegant way to check circular references? const seen = new Set(); const stack = [{ value }]; @@ -148,20 +158,20 @@ export function checkSerializableAsJson(options: CheckSerializableAsJsonOptions) } */ - return; - } - } else { - throw new UnexpectedError( - spaceTrim( - (block) => ` + return; + } + } else { + throw new UnexpectedError( + spaceTrim( + (block) => ` \`${name}\` is unknown type Additional message for \`${name}\`: - ${block(message || '(nothing)')} + ${block(message || "(nothing)")} `, - ), - ); - } + ), + ); + } } /** diff --git a/src/utils/serialization/clonePipeline.ts b/src/utils/serialization/clonePipeline.ts index 0dd126e593..69ea3f7009 100644 --- a/src/utils/serialization/clonePipeline.ts +++ b/src/utils/serialization/clonePipeline.ts @@ -1,4 +1,4 @@ -import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; +import type { PipelineJson } from "../../pipeline/PipelineJson/PipelineJson"; /** * @@@ @@ -9,39 +9,39 @@ import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson'; * @public exported from `@promptbook/utils` */ export function clonePipeline(pipeline: PipelineJson): PipelineJson { - // Note: Not using spread operator (...) because @@@ + // Note: Not using spread operator (...) because @@@ - const { - pipelineUrl, - sourceFile, - title, - bookVersion, - description, - formfactorName, - parameters, - tasks, - knowledgeSources, - knowledgePieces, - personas, - preparations, - sources, - } = pipeline; + const { + pipelineUrl, + sourceFile, + title, + bookVersion, + description, + formfactorName, + parameters, + tasks, + knowledgeSources, + knowledgePieces, + personas, + preparations, + sources, + } = pipeline; - return { - pipelineUrl, - sourceFile, - title, - bookVersion, - description, - formfactorName, - parameters, - tasks, - knowledgeSources, - knowledgePieces, - personas, - preparations, - sources, - }; + return { + pipelineUrl, + sourceFile, + title, + bookVersion, + description, + formfactorName, + parameters, + tasks, + knowledgeSources, + knowledgePieces, + personas, + preparations, + sources, + }; } /** diff --git a/src/utils/serialization/deepClone.test.ts b/src/utils/serialization/deepClone.test.ts index 93631ceaaf..869860a44f 100644 --- a/src/utils/serialization/deepClone.test.ts +++ b/src/utils/serialization/deepClone.test.ts @@ -1,56 +1,56 @@ -import { describe, expect, it } from '@jest/globals'; -import { deepClone } from './deepClone'; +import { describe, expect, it } from "@jest/globals"; +import { deepClone } from "./deepClone"; -describe('how `deepClone` works', () => { - it('should clone simple primitives and objects', () => { - expect(deepClone(null)).toEqual(null); - expect(deepClone(true)).toEqual(true); - expect(deepClone(42)).toEqual(42); - expect(deepClone('Foo')).toEqual('Foo'); - expect(deepClone({ foo: 'bar' })).toEqual({ foo: 'bar' }); - expect(deepClone([1, 2, 3])).toEqual([1, 2, 3]); - }); +describe("how `deepClone` works", () => { + it("should clone simple primitives and objects", () => { + expect(deepClone(null)).toEqual(null); + expect(deepClone(true)).toEqual(true); + expect(deepClone(42)).toEqual(42); + expect(deepClone("Foo")).toEqual("Foo"); + expect(deepClone({ foo: "bar" })).toEqual({ foo: "bar" }); + expect(deepClone([1, 2, 3])).toEqual([1, 2, 3]); + }); - it('should clone deep objects', () => - expect( - deepClone({ - foo: { bar: 'baz' }, - }), - ).toEqual({ - foo: { bar: 'baz' }, - })); + it("should clone deep objects", () => + expect( + deepClone({ + foo: { bar: "baz" }, + }), + ).toEqual({ + foo: { bar: "baz" }, + })); - it('should crash on circular structures', () => { - const obj: Record = {}; - obj.obj = obj; - expect(() => deepClone(obj)).toThrowError(/circular structure/i); - }); + it("should crash on circular structures", () => { + const obj: Record = {}; + obj.obj = obj; + expect(() => deepClone(obj)).toThrowError(/circular structure/i); + }); - it('should clone really advanced objects', () => { - const advancedObject = { - level1: { - level2: { - level3: { - level4: { - level5: { - level6: { - level7: { - level8: { - level9: { - level10: ['deep value', 'foo'], - }, - }, - }, - }, - }, - }, - }, - }, - }, - }; + it("should clone really advanced objects", () => { + const advancedObject = { + level1: { + level2: { + level3: { + level4: { + level5: { + level6: { + level7: { + level8: { + level9: { + level10: ["deep value", "foo"], + }, + }, + }, + }, + }, + }, + }, + }, + }, + }; - const clonedObject = deepClone(advancedObject); - expect(clonedObject).toEqual(advancedObject); - expect(clonedObject).not.toBe(advancedObject); // <- Note: Ensure it's a deep clone - }); + const clonedObject = deepClone(advancedObject); + expect(clonedObject).toEqual(advancedObject); + expect(clonedObject).not.toBe(advancedObject); // <- Note: Ensure it's a deep clone + }); }); diff --git a/src/utils/serialization/deepClone.ts b/src/utils/serialization/deepClone.ts index f9083e1dfa..807e11da75 100644 --- a/src/utils/serialization/deepClone.ts +++ b/src/utils/serialization/deepClone.ts @@ -1,14 +1,16 @@ -import type { WritableDeep } from 'type-fest'; +import type { WritableDeep } from "type-fest"; /** * @@@ * * @public exported from `@promptbook/utils` */ -export function deepClone(objectValue: TObject): WritableDeep { - return JSON.parse(JSON.stringify(objectValue)) as WritableDeep; +export function deepClone( + objectValue: TObject, +): WritableDeep { + return JSON.parse(JSON.stringify(objectValue)) as WritableDeep; - /* + /* TODO: [🧠] Is there a better implementation? > const propertyNames = Object.getOwnPropertyNames(objectValue); > for (const propertyName of propertyNames) { diff --git a/src/utils/serialization/exportJson.ts b/src/utils/serialization/exportJson.ts index 43af3ffff8..ed260e543a 100644 --- a/src/utils/serialization/exportJson.ts +++ b/src/utils/serialization/exportJson.ts @@ -1,22 +1,24 @@ -import type { JsonArray, JsonObject } from 'type-fest'; -import type { OrderJsonOptions } from '../normalization/orderJson'; -import { orderJson } from '../normalization/orderJson'; -import type { TODO_any } from '../organization/TODO_any'; -import { $deepFreeze } from './$deepFreeze'; -import type { CheckSerializableAsJsonOptions } from './checkSerializableAsJson'; -import { checkSerializableAsJson } from './checkSerializableAsJson'; -import { deepClone } from './deepClone'; +import type { JsonArray, JsonObject } from "type-fest"; +import type { OrderJsonOptions } from "../normalization/orderJson"; +import { orderJson } from "../normalization/orderJson"; +import type { TODO_any } from "../organization/TODO_any"; +import { $deepFreeze } from "./$deepFreeze"; +import type { CheckSerializableAsJsonOptions } from "./checkSerializableAsJson"; +import { checkSerializableAsJson } from "./checkSerializableAsJson"; +import { deepClone } from "./deepClone"; /** * Options for the `$exportJson` function */ export type ExportJsonOptions = CheckSerializableAsJsonOptions & - Partial, 'order'>> & { - /** - * Value to be checked, ordered and deeply frozen - */ - value: TObject; - }; + Partial< + Pick, "order"> + > & { + /** + * Value to be checked, ordered and deeply frozen + */ + value: TObject; + }; /** * Utility to export a JSON object from a function @@ -31,27 +33,29 @@ export type ExportJsonOptions = CheckSerializableAsJsonOptions & * @returns The same type of object as the input but read-only and re-ordered * @public exported from `@promptbook/utils` */ -export function exportJson(options: ExportJsonOptions): TObject { - const { name, value, order, message } = options; +export function exportJson( + options: ExportJsonOptions, +): TObject { + const { name, value, order, message } = options; - checkSerializableAsJson({ name, value, message }); + checkSerializableAsJson({ name, value, message }); - const orderedValue = - // TODO: Fix error "Type instantiation is excessively deep and possibly infinite." - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - order === undefined - ? deepClone(value) - : orderJson({ - value: value as JsonObject | JsonArray, - // <- Note: checkSerializableAsJson asserts that the value is serializable as JSON + const orderedValue = + // TODO: Fix error "Type instantiation is excessively deep and possibly infinite." + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + order === undefined + ? deepClone(value) + : orderJson({ + value: value as JsonObject | JsonArray, + // <- Note: checkSerializableAsJson asserts that the value is serializable as JSON - order: order as TODO_any, - }); + order: order as TODO_any, + }); - $deepFreeze(orderedValue); + $deepFreeze(orderedValue); - return orderedValue as TODO_any; + return orderedValue as TODO_any; } /** diff --git a/src/utils/serialization/isSerializableAsJson.test.ts b/src/utils/serialization/isSerializableAsJson.test.ts index a6736895fe..6cf806fd3e 100644 --- a/src/utils/serialization/isSerializableAsJson.test.ts +++ b/src/utils/serialization/isSerializableAsJson.test.ts @@ -1,104 +1,112 @@ -import { describe, it } from '@jest/globals'; -import type { really_any } from '../organization/really_any'; -import { isSerializableAsJson } from './isSerializableAsJson'; - -describe('how `isSerializableAsJson` works', () => { - it('almost all primitives are serializable', () => { - expect(isSerializableAsJson('Foo')).toBe(true); - expect(isSerializableAsJson(42)).toBe(true); - expect(isSerializableAsJson(true)).toBe(true); - expect(isSerializableAsJson(null)).toBe(true); - }); - - it('undefined alone is not serializable', () => { - expect(isSerializableAsJson(undefined)).toBe(false); - }); - - it('undefined in array is not serializable - it brokes order', () => { - expect(isSerializableAsJson([undefined])).toBe(false); - expect(isSerializableAsJson([1, undefined])).toBe(false); - expect(isSerializableAsJson([1, undefined, 3])).toBe(false); - expect(isSerializableAsJson({ foo: 'Hi', bar: ['hello', undefined] })).toBe(false); - }); - - it('undefined in object is serializable - it is just omited', () => { - expect(isSerializableAsJson({ foo: undefined })).toBe(true); - expect(isSerializableAsJson({ foo: undefined, bar: 'hello' })).toBe(true); - }); - - it('NaN is not serializable', () => { - expect(isSerializableAsJson(NaN)).toBe(false); - }); - - it('objects and arrays are serializable if all their properties are serializable', () => { - expect(isSerializableAsJson({ foo: 'bar' })).toBe(true); - expect(isSerializableAsJson([1, 2, 3])).toBe(true); - }); - - it('deep objects and arrays are serializable if all their properties are serializable', () => { - expect(isSerializableAsJson({ foo: { bar: 'baz' } })).toBe(true); - expect(isSerializableAsJson([1, [2, 3], 4])).toBe(true); - expect(isSerializableAsJson([1, [2, { foo: { bar: 'baz', brr: [1, 2, 3, false] } }], 4])).toBe(true); - }); - - it('objects instantiated from classes are not serializable.', () => { - class Foo { - constructor(public bar: string) {} - } - expect(isSerializableAsJson(new Foo('baz'))); - }); - - it('functions are not serializable', () => { - expect(isSerializableAsJson(() => 'Foo')).toBe(false); - }); - - it('circular references are not serializable', () => { - const obj: Record = {}; - obj.obj = obj; - expect(isSerializableAsJson(obj)).toBe(false); - }); - - it('advanced circular references are not serializable', () => { - const obj1: Record = {}; - const obj2: Record = {}; - obj1.obj = [obj2]; - obj2.obj = { foo: [obj1, obj1, obj1], bar: { baz: obj1 } }; - expect(isSerializableAsJson(obj1)).toBe(false); - }); - - it('objects with same sibling references are serializable *(no circular references)*', () => { - const obj: really_any = {}; - obj.a = {}; - obj.b = obj.a; - expect(isSerializableAsJson(obj)).toBe(true); - }); - - it('objects with same family references are serializable *(no circular references)*', () => { - const obj: really_any = {}; - obj.a = {}; - obj.b = obj.a; - obj.b = { d: { e: [obj.a] } }; - expect(isSerializableAsJson(obj)).toBe(true); - }); - - it('Date objects are not serializable', () => { - expect(isSerializableAsJson(new Date())).toBe(false); - }); - - it('Map and Set objects are not serializable', () => { - expect(isSerializableAsJson(new Map())).toBe(false); - expect(isSerializableAsJson(new Set())).toBe(false); - }); - - it('RegExp objects are not serializable', () => { - expect(isSerializableAsJson(/foo/)).toBe(false); - }); - - it('Error objects are not serializable', () => { - expect(isSerializableAsJson(new Error())).toBe(false); - }); - - it('Symbol objects are not serializable', () => { - expect(isSerializableAsJson(Symbol('foo'))).toBe(false); - }); +import { describe, it } from "@jest/globals"; +import type { really_any } from "../organization/really_any"; +import { isSerializableAsJson } from "./isSerializableAsJson"; + +describe("how `isSerializableAsJson` works", () => { + it("almost all primitives are serializable", () => { + expect(isSerializableAsJson("Foo")).toBe(true); + expect(isSerializableAsJson(42)).toBe(true); + expect(isSerializableAsJson(true)).toBe(true); + expect(isSerializableAsJson(null)).toBe(true); + }); + + it("undefined alone is not serializable", () => { + expect(isSerializableAsJson(undefined)).toBe(false); + }); + + it("undefined in array is not serializable - it brokes order", () => { + expect(isSerializableAsJson([undefined])).toBe(false); + expect(isSerializableAsJson([1, undefined])).toBe(false); + expect(isSerializableAsJson([1, undefined, 3])).toBe(false); + expect(isSerializableAsJson({ foo: "Hi", bar: ["hello", undefined] })).toBe( + false, + ); + }); + + it("undefined in object is serializable - it is just omited", () => { + expect(isSerializableAsJson({ foo: undefined })).toBe(true); + expect(isSerializableAsJson({ foo: undefined, bar: "hello" })).toBe(true); + }); + + it("NaN is not serializable", () => { + expect(isSerializableAsJson(Number.NaN)).toBe(false); + }); + + it("objects and arrays are serializable if all their properties are serializable", () => { + expect(isSerializableAsJson({ foo: "bar" })).toBe(true); + expect(isSerializableAsJson([1, 2, 3])).toBe(true); + }); + + it("deep objects and arrays are serializable if all their properties are serializable", () => { + expect(isSerializableAsJson({ foo: { bar: "baz" } })).toBe(true); + expect(isSerializableAsJson([1, [2, 3], 4])).toBe(true); + expect( + isSerializableAsJson([ + 1, + [2, { foo: { bar: "baz", brr: [1, 2, 3, false] } }], + 4, + ]), + ).toBe(true); + }); + + it("objects instantiated from classes are not serializable.", () => { + class Foo { + constructor(public bar: string) {} + } + expect(isSerializableAsJson(new Foo("baz"))); + }); + + it("functions are not serializable", () => { + expect(isSerializableAsJson(() => "Foo")).toBe(false); + }); + + it("circular references are not serializable", () => { + const obj: Record = {}; + obj.obj = obj; + expect(isSerializableAsJson(obj)).toBe(false); + }); + + it("advanced circular references are not serializable", () => { + const obj1: Record = {}; + const obj2: Record = {}; + obj1.obj = [obj2]; + obj2.obj = { foo: [obj1, obj1, obj1], bar: { baz: obj1 } }; + expect(isSerializableAsJson(obj1)).toBe(false); + }); + + it("objects with same sibling references are serializable *(no circular references)*", () => { + const obj: really_any = {}; + obj.a = {}; + obj.b = obj.a; + expect(isSerializableAsJson(obj)).toBe(true); + }); + + it("objects with same family references are serializable *(no circular references)*", () => { + const obj: really_any = {}; + obj.a = {}; + obj.b = obj.a; + obj.b = { d: { e: [obj.a] } }; + expect(isSerializableAsJson(obj)).toBe(true); + }); + + it("Date objects are not serializable", () => { + expect(isSerializableAsJson(new Date())).toBe(false); + }); + + it("Map and Set objects are not serializable", () => { + expect(isSerializableAsJson(new Map())).toBe(false); + expect(isSerializableAsJson(new Set())).toBe(false); + }); + + it("RegExp objects are not serializable", () => { + expect(isSerializableAsJson(/foo/)).toBe(false); + }); + + it("Error objects are not serializable", () => { + expect(isSerializableAsJson(new Error())).toBe(false); + }); + + it("Symbol objects are not serializable", () => { + expect(isSerializableAsJson(Symbol("foo"))).toBe(false); + }); }); diff --git a/src/utils/serialization/isSerializableAsJson.ts b/src/utils/serialization/isSerializableAsJson.ts index 7da28215a3..587aa3d4f1 100644 --- a/src/utils/serialization/isSerializableAsJson.ts +++ b/src/utils/serialization/isSerializableAsJson.ts @@ -1,6 +1,6 @@ -import type { JsonValue } from 'type-fest'; +import type { JsonValue } from "type-fest"; // <- TODO: Auto convert to type `import { ... } from 'type-fest';` -import { checkSerializableAsJson } from './checkSerializableAsJson'; +import { checkSerializableAsJson } from "./checkSerializableAsJson"; /** * Tests if the value is [🚉] serializable as JSON @@ -22,12 +22,12 @@ import { checkSerializableAsJson } from './checkSerializableAsJson'; * @public exported from `@promptbook/utils` */ export function isSerializableAsJson(value: unknown): value is JsonValue { - try { - checkSerializableAsJson({ value }); - return true; - } catch (error) { - return false; - } + try { + checkSerializableAsJson({ value }); + return true; + } catch (error) { + return false; + } } /** diff --git a/src/utils/sets/difference.test.ts b/src/utils/sets/difference.test.ts index 00ba3fdca3..4cce2b9fec 100644 --- a/src/utils/sets/difference.test.ts +++ b/src/utils/sets/difference.test.ts @@ -1,52 +1,52 @@ -import { describe, expect, it } from '@jest/globals'; -import { difference } from './difference'; +import { describe, expect, it } from "@jest/globals"; +import { difference } from "./difference"; -describe('difference', () => { - it('should return a new set with elements that are in set a but not in set b', () => { - const setA = new Set([1, 2, 3, 4, 5]); - const setB = new Set([3, 4, 5, 6, 7]); +describe("difference", () => { + it("should return a new set with elements that are in set a but not in set b", () => { + const setA = new Set([1, 2, 3, 4, 5]); + const setB = new Set([3, 4, 5, 6, 7]); - const result = difference(setA, setB); + const result = difference(setA, setB); - expect(result).toEqual(new Set([1, 2])); - }); + expect(result).toEqual(new Set([1, 2])); + }); - it('should handle empty sets', () => { - const setA = new Set([1, 2, 3]); - const setB = new Set(); + it("should handle empty sets", () => { + const setA = new Set([1, 2, 3]); + const setB = new Set(); - const result = difference(setA, setB); + const result = difference(setA, setB); - expect(result).toEqual(new Set([1, 2, 3])); - }); + expect(result).toEqual(new Set([1, 2, 3])); + }); - it('should handle custom equality check', () => { - const setA = new Set([{ id: 1 }, { id: 2 }, { id: 3 }]); - const setB = new Set([{ id: 2 }, { id: 3 }, { id: 4 }]); + it("should handle custom equality check", () => { + const setA = new Set([{ id: 1 }, { id: 2 }, { id: 3 }]); + const setB = new Set([{ id: 2 }, { id: 3 }, { id: 4 }]); - const result = difference(setA, setB, (a, b) => a.id === b.id); + const result = difference(setA, setB, (a, b) => a.id === b.id); - expect(result).toEqual(new Set([{ id: 1 }])); - }); + expect(result).toEqual(new Set([{ id: 1 }])); + }); }); -describe('difference', () => { - it('should return the difference set of two sets', () => { - const setA = new Set([1, 2, 3, 4]); - const setB = new Set([3, 4, 5, 6]); - const expectedDifference = new Set([1, 2]); +describe("difference", () => { + it("should return the difference set of two sets", () => { + const setA = new Set([1, 2, 3, 4]); + const setB = new Set([3, 4, 5, 6]); + const expectedDifference = new Set([1, 2]); - const result = difference(setA, setB); + const result = difference(setA, setB); - expect(result).toEqual(expectedDifference); - }); + expect(result).toEqual(expectedDifference); + }); - it('should use the custom equality function if provided', () => { - const setA = new Set([{ id: 1 }, { id: 2 }, { id: 3 }]); - const setB = new Set([{ id: 2 }, { id: 3 }, { id: 4 }]); - const expectedDifference = new Set([{ id: 1 }]); + it("should use the custom equality function if provided", () => { + const setA = new Set([{ id: 1 }, { id: 2 }, { id: 3 }]); + const setB = new Set([{ id: 2 }, { id: 3 }, { id: 4 }]); + const expectedDifference = new Set([{ id: 1 }]); - const result = difference(setA, setB, (a, b) => a.id === b.id); + const result = difference(setA, setB, (a, b) => a.id === b.id); - expect(result).toEqual(expectedDifference); - }); + expect(result).toEqual(expectedDifference); + }); }); diff --git a/src/utils/sets/difference.ts b/src/utils/sets/difference.ts index 7bd7087e62..b96913b69b 100644 --- a/src/utils/sets/difference.ts +++ b/src/utils/sets/difference.ts @@ -5,19 +5,19 @@ * @public exported from `@promptbook/utils` */ export function difference( - a: Set, - b: Set, - isEqual: (a: TItem, b: TItem) => boolean = (a, b) => a === b, + a: Set, + b: Set, + isEqual: (a: TItem, b: TItem) => boolean = (a, b) => a === b, ): Set { - const diff = new Set(); + const diff = new Set(); - for (const itemA of Array.from(a)) { - if (!Array.from(b).some((itemB) => isEqual(itemA, itemB))) { - diff.add(itemA); - } - } + for (const itemA of Array.from(a)) { + if (!Array.from(b).some((itemB) => isEqual(itemA, itemB))) { + diff.add(itemA); + } + } - return diff; + return diff; } /** diff --git a/src/utils/sets/intersection.test.ts b/src/utils/sets/intersection.test.ts index 3e4a1bf8ce..9e6432257d 100644 --- a/src/utils/sets/intersection.test.ts +++ b/src/utils/sets/intersection.test.ts @@ -1,33 +1,33 @@ -import { describe, expect, it } from '@jest/globals'; -import { intersection } from './intersection'; +import { describe, expect, it } from "@jest/globals"; +import { intersection } from "./intersection"; -describe('intersection', () => { - it('should return a new set with elements that are present in all input sets', () => { - const set1 = new Set([1, 2, 3]); - const set2 = new Set([2, 3, 4]); - const set3 = new Set([3, 4, 5]); +describe("intersection", () => { + it("should return a new set with elements that are present in all input sets", () => { + const set1 = new Set([1, 2, 3]); + const set2 = new Set([2, 3, 4]); + const set3 = new Set([3, 4, 5]); - const result = intersection(set1, set2, set3); + const result = intersection(set1, set2, set3); - expect(result).toEqual(new Set([3])); - }); + expect(result).toEqual(new Set([3])); + }); - it('should handle empty sets', () => { - const set1 = new Set([1, 2, 3]); - const set2 = new Set(); + it("should handle empty sets", () => { + const set1 = new Set([1, 2, 3]); + const set2 = new Set(); - const result = intersection(set1, set2); + const result = intersection(set1, set2); - expect(result).toEqual(new Set()); - }); + expect(result).toEqual(new Set()); + }); - it('should handle duplicate elements', () => { - const set1 = new Set([1, 2, 3]); - const set2 = new Set([2, 3, 4]); - const set3 = new Set([3, 4, 5]); + it("should handle duplicate elements", () => { + const set1 = new Set([1, 2, 3]); + const set2 = new Set([2, 3, 4]); + const set3 = new Set([3, 4, 5]); - const result = intersection(set1, set2, set3, set1); + const result = intersection(set1, set2, set3, set1); - expect(result).toEqual(new Set([3])); - }); + expect(result).toEqual(new Set([3])); + }); }); diff --git a/src/utils/sets/intersection.ts b/src/utils/sets/intersection.ts index 3e7bfdb946..fdb2e6f5c1 100644 --- a/src/utils/sets/intersection.ts +++ b/src/utils/sets/intersection.ts @@ -4,23 +4,25 @@ * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set * @public exported from `@promptbook/utils` */ -export function intersection(...sets: ReadonlyArray>): Set { - const intersection = new Set(); +export function intersection( + ...sets: ReadonlyArray> +): Set { + const intersection = new Set(); - if (sets[0]) { - for (const item of Array.from(sets[0])) { - let isPresentInAllSets = true; - for (let i = 1; i < sets.length; i++) { - if (sets[i] !== undefined && !sets[i]!.has(item)) { - isPresentInAllSets = false; - break; - } - } - if (isPresentInAllSets) { - intersection.add(item); - } - } - } + if (sets[0]) { + for (const item of Array.from(sets[0])) { + let isPresentInAllSets = true; + for (let i = 1; i < sets.length; i++) { + if (sets[i] !== undefined && !sets[i]!.has(item)) { + isPresentInAllSets = false; + break; + } + } + if (isPresentInAllSets) { + intersection.add(item); + } + } + } - return intersection; + return intersection; } diff --git a/src/utils/sets/union.test.ts b/src/utils/sets/union.test.ts index 0dfbbf6f8a..471967b391 100644 --- a/src/utils/sets/union.test.ts +++ b/src/utils/sets/union.test.ts @@ -1,33 +1,33 @@ -import { describe, expect, it } from '@jest/globals'; -import { union } from './union'; +import { describe, expect, it } from "@jest/globals"; +import { union } from "./union"; -describe('union', () => { - it('should return a new set with elements from all input sets', () => { - const set1 = new Set([1, 2, 3]); - const set2 = new Set([3, 4, 5]); - const set3 = new Set([5, 6, 7]); +describe("union", () => { + it("should return a new set with elements from all input sets", () => { + const set1 = new Set([1, 2, 3]); + const set2 = new Set([3, 4, 5]); + const set3 = new Set([5, 6, 7]); - const result = union(set1, set2, set3); + const result = union(set1, set2, set3); - expect(result).toEqual(new Set([1, 2, 3, 4, 5, 6, 7])); - }); + expect(result).toEqual(new Set([1, 2, 3, 4, 5, 6, 7])); + }); - it('should handle empty sets', () => { - const set1 = new Set([1, 2, 3]); - const set2 = new Set(); + it("should handle empty sets", () => { + const set1 = new Set([1, 2, 3]); + const set2 = new Set(); - const result = union(set1, set2); + const result = union(set1, set2); - expect(result).toEqual(new Set([1, 2, 3])); - }); + expect(result).toEqual(new Set([1, 2, 3])); + }); - it('should handle duplicate elements', () => { - const set1 = new Set([1, 2, 3]); - const set2 = new Set([3, 4, 5]); - const set3 = new Set([5, 6, 7]); + it("should handle duplicate elements", () => { + const set1 = new Set([1, 2, 3]); + const set2 = new Set([3, 4, 5]); + const set3 = new Set([5, 6, 7]); - const result = union(set1, set2, set3, set1); + const result = union(set1, set2, set3, set1); - expect(result).toEqual(new Set([1, 2, 3, 4, 5, 6, 7])); - }); + expect(result).toEqual(new Set([1, 2, 3, 4, 5, 6, 7])); + }); }); diff --git a/src/utils/sets/union.ts b/src/utils/sets/union.ts index a5f6eda8af..c6eb864e34 100644 --- a/src/utils/sets/union.ts +++ b/src/utils/sets/union.ts @@ -5,13 +5,13 @@ * @public exported from `@promptbook/utils` */ export function union(...sets: ReadonlyArray>): Set { - const union = new Set(); + const union = new Set(); - for (const set of sets) { - for (const item of Array.from(set)) { - union.add(item); - } - } + for (const set of sets) { + for (const item of Array.from(set)) { + union.add(item); + } + } - return union; + return union; } diff --git a/src/utils/trimCodeBlock.test.ts b/src/utils/trimCodeBlock.test.ts index b857f27d0d..e2f12660f1 100644 --- a/src/utils/trimCodeBlock.test.ts +++ b/src/utils/trimCodeBlock.test.ts @@ -1,163 +1,163 @@ -import { describe, expect, it } from '@jest/globals'; -import { spaceTrim } from 'spacetrim'; -import { just } from './organization/just'; -import { trimCodeBlock } from './trimCodeBlock'; +import { describe, expect, it } from "@jest/globals"; +import { spaceTrim } from "spacetrim"; +import { just } from "./organization/just"; +import { trimCodeBlock } from "./trimCodeBlock"; -describe('how trimCodeBlock works', () => { - it('should preserve string without code block', () => { - expect( - trimCodeBlock( - spaceTrim(` +describe("how trimCodeBlock works", () => { + it("should preserve string without code block", () => { + expect( + trimCodeBlock( + spaceTrim(` Foo `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` Foo `), - ), - ); - expect( - trimCodeBlock( - spaceTrim(` + ), + ); + expect( + trimCodeBlock( + spaceTrim(` Hello: "Bar" `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` Hello: "Bar" `), - ), - ); - }); + ), + ); + }); - it('should preserve just block starting or ending', () => { - expect( - trimCodeBlock( - spaceTrim(` + it("should preserve just block starting or ending", () => { + expect( + trimCodeBlock( + spaceTrim(` \`\`\`markdown Foo `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` \`\`\`markdown Foo `), - ), - ); + ), + ); - expect( - trimCodeBlock( - spaceTrim(` + expect( + trimCodeBlock( + spaceTrim(` \`\`\` Foo `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` \`\`\` Foo `), - ), - ); + ), + ); - expect( - trimCodeBlock( - spaceTrim(` + expect( + trimCodeBlock( + spaceTrim(` Foo \`\`\` `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` Foo \`\`\` `), - ), - ); - }); + ), + ); + }); - it('should trim code block', () => { - expect( - trimCodeBlock( - spaceTrim(` + it("should trim code block", () => { + expect( + trimCodeBlock( + spaceTrim(` \`\`\` Foo \`\`\` `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` Foo `), - ), - ); - expect( - trimCodeBlock( - spaceTrim(` + ), + ); + expect( + trimCodeBlock( + spaceTrim(` \`\`\`text Foo \`\`\` `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` Foo `), - ), - ); + ), + ); - expect( - trimCodeBlock( - spaceTrim(` + expect( + trimCodeBlock( + spaceTrim(` \`\`\`markdown "Bar" \`\`\` `), - ), - ).toBe( - just( - spaceTrim(` + ), + ).toBe( + just( + spaceTrim(` "Bar" `), - ), - ); - }); + ), + ); + }); - it('should trim ending code block and some whitespace', () => { - expect( - trimCodeBlock( - spaceTrim(` + it("should trim ending code block and some whitespace", () => { + expect( + trimCodeBlock( + spaceTrim(` \`\`\` Foo \`\`\` - `) + '\n\n ', - ), - ).toBe( - just( - spaceTrim(` + `) + "\n\n ", + ), + ).toBe( + just( + spaceTrim(` Foo `), - ), - ); - }); + ), + ); + }); - it('will work on real-life example', () => { - expect( - trimCodeBlock( - spaceTrim(` + it("will work on real-life example", () => { + expect( + trimCodeBlock( + spaceTrim(` \`\`\`html