diff --git a/docs/components/data-sources/image.mdx b/docs/components/data-sources/image.mdx new file mode 100644 index 00000000..bdaa2e44 --- /dev/null +++ b/docs/components/data-sources/image.mdx @@ -0,0 +1,31 @@ +--- +title: '🖼️ Image' +--- + +You can load most images using an LLM to describe the contents of the image. This LLM by default is the one used with the application, +but a special LLM can also be specified for the content description. To load images, follow the steps below. + +## Install Image addon + +```bash +npm install @llm-tools/embedjs-loader-image +``` + +## Usage + +### Load from a local file + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { ImageLoader } from '@llm-tools/embedjs-loader-image'; +import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAiEmbeddings()) +.setVectorDatabase(new HNSWDb()) +.build(); + +app.addLoader(new ImageLoader({ filePathOrUrl: '/path/to/file.jpeg' })) +``` \ No newline at end of file diff --git a/docs/components/data-sources/overview.mdx b/docs/components/data-sources/overview.mdx index 8c826508..3c63ecd3 100644 --- a/docs/components/data-sources/overview.mdx +++ b/docs/components/data-sources/overview.mdx @@ -22,6 +22,7 @@ We handle the complexity of loading unstructured data from these data sources, a + diff --git a/docs/mint.json b/docs/mint.json index 56164fc9..07e00dba 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -84,6 +84,7 @@ "components/data-sources/markdown", "components/data-sources/xml", "components/data-sources/directory", + "components/data-sources/image", "components/data-sources/custom" ] } diff --git a/examples/image/assets/test.jpg b/examples/image/assets/test.jpg new file mode 100644 index 00000000..3bdea555 Binary files /dev/null and b/examples/image/assets/test.jpg differ diff --git a/examples/image/eslint.config.js b/examples/image/eslint.config.js new file mode 100644 index 00000000..e2a15f2a --- /dev/null +++ b/examples/image/eslint.config.js @@ -0,0 +1,3 @@ +import baseConfig from '../../eslint.config.js'; + +export default [...baseConfig]; diff --git a/examples/image/package.json b/examples/image/package.json new file mode 100644 index 00000000..2609813b --- /dev/null +++ b/examples/image/package.json @@ -0,0 +1,11 @@ +{ + "name": "@llm-tools/embedjs-examples-image", + "version": "0.1.1", + "type": "module", + "dependencies": { + "dotenv": "^16.4.7" + }, + "scripts": { + "start": "nx run examples-image:serve" + } +} diff --git a/examples/image/project.json b/examples/image/project.json new file mode 100644 index 00000000..11fc57ea --- /dev/null +++ b/examples/image/project.json @@ -0,0 +1,57 @@ +{ + "name": "examples-image", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "examples/image/src", + "projectType": "application", + "tags": [], + "targets": { + "build": { + "executor": "@nx/esbuild:esbuild", + "outputs": ["{options.outputPath}"], + "defaultConfiguration": "development", + "options": { + "platform": "node", + "outputPath": "dist/examples/image", + "format": ["esm"], + "bundle": true, + "main": "examples/image/src/main.ts", + "tsConfig": "examples/image/tsconfig.app.json", + "generatePackageJson": false, + "esbuildOptions": { + "sourcemap": true, + "outExtension": { + ".js": ".js" + } + } + }, + "configurations": { + "development": {}, + "production": { + "esbuildOptions": { + "sourcemap": false, + "outExtension": { + ".js": ".js" + } + } + } + } + }, + "serve": { + "executor": "@nx/js:node", + "defaultConfiguration": "development", + "dependsOn": ["build"], + "options": { + "buildTarget": "examples-image:build", + "runBuildTargetDependencies": true + }, + "configurations": { + "development": { + "buildTarget": "examples-image:build:development" + }, + "production": { + "buildTarget": "examples-image:build:production" + } + } + } + } +} diff --git a/examples/image/src/main.ts b/examples/image/src/main.ts new file mode 100644 index 00000000..5a546a5b --- /dev/null +++ b/examples/image/src/main.ts @@ -0,0 +1,17 @@ +import 'dotenv/config'; +import path from 'node:path'; +import { RAGApplicationBuilder, SIMPLE_MODELS } from '@llm-tools/embedjs'; +import { ImageLoader } from '@llm-tools/embedjs-loader-image'; +import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const ragApplication = await new RAGApplicationBuilder() + .setModel(SIMPLE_MODELS.OPENAI_GPT4_O) + .setEmbeddingModel(new OpenAiEmbeddings()) + .setVectorDatabase(new HNSWDb()) + .build(); + +const imagePath = path.resolve('./examples/image/assets/test.jpg'); +await ragApplication.addLoader(new ImageLoader({ filePathOrUrl: imagePath })); + +await ragApplication.query('How does deep learning relate to artifical intelligence'); diff --git a/examples/image/tsconfig.app.json b/examples/image/tsconfig.app.json new file mode 100644 index 00000000..5a8c35da --- /dev/null +++ b/examples/image/tsconfig.app.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "types": ["node"] + }, + "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"], + "include": ["src/**/*.ts"] +} diff --git a/examples/image/tsconfig.json b/examples/image/tsconfig.json new file mode 100644 index 00000000..c60cf5c2 --- /dev/null +++ b/examples/image/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.app.json" + } + ], + "compilerOptions": { + "esModuleInterop": true, + "target": "ES2022", + "lib": ["ES2022", "ES2022.Object"], + "module": "NodeNext", + "moduleResolution": "nodenext" + } +} diff --git a/loaders/embedjs-loader-image/package.json b/loaders/embedjs-loader-image/package.json index a2f66cab..faa42582 100644 --- a/loaders/embedjs-loader-image/package.json +++ b/loaders/embedjs-loader-image/package.json @@ -9,7 +9,8 @@ "debug": "^4.4.0", "md5": "^2.3.0", "mime": "^4.0.6", - "stream-mime-type": "^2.0.0" + "stream-mime-type": "^2.0.0", + "exifremove": "^1.0.1" }, "type": "module", "main": "./src/index.js", diff --git a/loaders/embedjs-loader-image/src/image-loader.ts b/loaders/embedjs-loader-image/src/image-loader.ts index 491a1b2f..7f5e669c 100644 --- a/loaders/embedjs-loader-image/src/image-loader.ts +++ b/loaders/embedjs-loader-image/src/image-loader.ts @@ -1,11 +1,12 @@ import { HumanMessage } from '@langchain/core/messages'; import { getMimeType } from 'stream-mime-type'; import createDebugMessages from 'debug'; +import exifremove from 'exifremove'; import fs from 'node:fs'; import md5 from 'md5'; import { BaseLoader, BaseModel } from '@llm-tools/embedjs-interfaces'; -import { cleanString, contentTypeToMimeType, getSafe, isValidURL, streamToString } from '@llm-tools/embedjs-utils'; +import { cleanString, contentTypeToMimeType, getSafe, isValidURL, streamToBuffer } from '@llm-tools/embedjs-utils'; export class ImageLoader extends BaseLoader<{ type: 'ImageLoader' }> { private readonly debug = createDebugMessages('embedjs:loader:ImageLoader'); @@ -60,9 +61,11 @@ export class ImageLoader extends BaseLoader<{ type: 'ImageLoader' }> { } this.debug(`Image stream detected type '${this.mime}'`); - const text = this.isUrl - ? (await getSafe(this.filePathOrUrl, { format: 'text' })).body - : await streamToString(fs.createReadStream(this.filePathOrUrl)); + const buffer = this.isUrl + ? (await getSafe(this.filePathOrUrl, { format: 'buffer' })).body + : await streamToBuffer(fs.createReadStream(this.filePathOrUrl)); + + const plainImageBuffer = exifremove.remove(buffer); const message = new HumanMessage({ content: [ @@ -73,7 +76,7 @@ export class ImageLoader extends BaseLoader<{ type: 'ImageLoader' }> { { type: 'image_url', image_url: { - url: `data:${this.mime};base64,${btoa(text)}`, + url: `data:${this.mime};base64,${plainImageBuffer.toString('base64')}`, }, }, ], @@ -81,7 +84,7 @@ export class ImageLoader extends BaseLoader<{ type: 'ImageLoader' }> { this.debug('Asking LLM to describe image'); const response = await this.captionModel.simpleQuery([message]); - this.debug('LLM describes image as', response.result); + this.debug('LLM describes image as: ', response.result); yield { pageContent: cleanString(response.result), diff --git a/package-lock.json b/package-lock.json index 0407ecff..dd19c991 100644 --- a/package-lock.json +++ b/package-lock.json @@ -346,6 +346,7 @@ "@llm-tools/embedjs-interfaces": "0.1.27", "@llm-tools/embedjs-utils": "0.1.27", "debug": "^4.4.0", + "exifremove": "^1.0.1", "md5": "^2.3.0", "mime": "^4.0.6", "stream-mime-type": "^2.0.0" @@ -12728,6 +12729,12 @@ "node": ">=0.8.x" } }, + "node_modules/exifremove": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/exifremove/-/exifremove-1.0.1.tgz", + "integrity": "sha512-a4VrNsSgpKBo9dHFsRi098XYm/X8dHP2NPA4N/3WAIxInQ8VHOfic/F8uAMIlWk4QRurTFwClFUb0QkMj4nqHg==", + "license": "MIT" + }, "node_modules/exit": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", diff --git a/scripts/publish-via-nx.js b/scripts/publish-via-nx.js index 273f604f..4751e4bd 100644 --- a/scripts/publish-via-nx.js +++ b/scripts/publish-via-nx.js @@ -12,6 +12,19 @@ function abs(relativePath) { return resolve(dirname(fileURLToPath(import.meta.url)), relativePath); } +/** + * @param {string} version - The version to update the root package to + * @param {boolean} dryRun - Whether to perform a dry run or not + */ +async function updateRootPackageVersion(version, dryRun) { + const absPath = abs('..'); + console.log(`Updating root package at path '${absPath}' to version '${version}' ${dryRun ? '[dry run]' : ''}`); + const pkgJson = await PackageJson.load(absPath); + pkgJson.update({ version }); + + if (!dryRun) await pkgJson.save(); +} + /** * @param {pkgName} pkgName - The name of the package to update * @param {string} version - The version to update the package to @@ -76,6 +89,7 @@ async function createRelease(dryRun, version, makeGitCommit) { } console.log('Updating projects actual version to match NX computed values'); + await updateRootPackageVersion(newVersion, dryRun); for await (const [pkgName, { newVersion }] of Object.entries(projectsVersionData)) { if (newVersion !== null) await updatePackageVersion(pkgName, newVersion, versionMap, dryRun); else console.log(`Skipping '${pkgName}' version update as it's already up to date`);