Added image loader

docs/components/data-sources/image.mdx

@@ -0,0 +1,31 @@
+---
+title: '🖼️ Image'
+---
+
+You can load most images using an LLM to describe the contents of the image. This LLM by default is the one used with the application, 
+but a special LLM can also be specified for the content description. To load images, follow the steps below.
+
+## Install Image addon
+
+```bash
+npm install @llm-tools/embedjs-loader-image
+```
+
+## Usage
+
+### Load from a local file
+
+```ts
+import { RAGApplicationBuilder } from '@llm-tools/embedjs';
+import { ImageLoader } from '@llm-tools/embedjs-loader-image';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
+import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
+
+const app = await new RAGApplicationBuilder()
+.setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
+.setEmbeddingModel(new OpenAiEmbeddings())
+.setVectorDatabase(new HNSWDb())
+.build();
+
+app.addLoader(new ImageLoader({ filePathOrUrl: '/path/to/file.jpeg' }))
+```

docs/components/data-sources/overview.mdx

@@ -22,6 +22,7 @@ We handle the complexity of loading unstructured data from these data sources, a
     <Card title="Excel file" href="/components/data-sources/excel" />
     <Card title="Sitemap" href="/components/data-sources/sitemap" />
     <Card title="Markdown / MDX" href="/components/data-sources/markdown" />
+    <Card title="Image" href="/components/data-sources/image" />
     <Card title="Custom" href="/components/data-sources/custom" />
 </CardGroup>
 

docs/mint.json

@@ -84,6 +84,7 @@
                                 "components/data-sources/markdown",
                                 "components/data-sources/xml",
                                 "components/data-sources/directory",
+                                "components/data-sources/image",
                                 "components/data-sources/custom"
                             ]
                         }

examples/image/eslint.config.js

@@ -0,0 +1,3 @@
+import baseConfig from '../../eslint.config.js';
+
+export default [...baseConfig];

examples/image/package.json

@@ -0,0 +1,11 @@
+{
+    "name": "@llm-tools/embedjs-examples-image",
+    "version": "0.1.1",
+    "type": "module",
+    "dependencies": {
+        "dotenv": "^16.4.7"
+    },
+    "scripts": {
+        "start": "nx run examples-image:serve"
+    }
+}

examples/image/project.json

@@ -0,0 +1,57 @@
+{
+    "name": "examples-image",
+    "$schema": "../../node_modules/nx/schemas/project-schema.json",
+    "sourceRoot": "examples/image/src",
+    "projectType": "application",
+    "tags": [],
+    "targets": {
+        "build": {
+            "executor": "@nx/esbuild:esbuild",
+            "outputs": ["{options.outputPath}"],
+            "defaultConfiguration": "development",
+            "options": {
+                "platform": "node",
+                "outputPath": "dist/examples/image",
+                "format": ["esm"],
+                "bundle": true,
+                "main": "examples/image/src/main.ts",
+                "tsConfig": "examples/image/tsconfig.app.json",
+                "generatePackageJson": false,
+                "esbuildOptions": {
+                    "sourcemap": true,
+                    "outExtension": {
+                        ".js": ".js"
+                    }
+                }
+            },
+            "configurations": {
+                "development": {},
+                "production": {
+                    "esbuildOptions": {
+                        "sourcemap": false,
+                        "outExtension": {
+                            ".js": ".js"
+                        }
+                    }
+                }
+            }
+        },
+        "serve": {
+            "executor": "@nx/js:node",
+            "defaultConfiguration": "development",
+            "dependsOn": ["build"],
+            "options": {
+                "buildTarget": "examples-image:build",
+                "runBuildTargetDependencies": true
+            },
+            "configurations": {
+                "development": {
+                    "buildTarget": "examples-image:build:development"
+                },
+                "production": {
+                    "buildTarget": "examples-image:build:production"
+                }
+            }
+        }
+    }
+}

examples/image/src/main.ts

@@ -0,0 +1,17 @@
+import 'dotenv/config';
+import path from 'node:path';
+import { RAGApplicationBuilder, SIMPLE_MODELS } from '@llm-tools/embedjs';
+import { ImageLoader } from '@llm-tools/embedjs-loader-image';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
+import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
+
+const ragApplication = await new RAGApplicationBuilder()
+    .setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
+    .setEmbeddingModel(new OpenAiEmbeddings())
+    .setVectorDatabase(new HNSWDb())
+    .build();
+
+const imagePath = path.resolve('./examples/image/assets/test.jpg');
+await ragApplication.addLoader(new ImageLoader({ filePathOrUrl: imagePath }));
+
+await ragApplication.query('How does deep learning relate to artifical intelligence');

examples/image/tsconfig.app.json

@@ -0,0 +1,9 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../dist/out-tsc",
+        "types": ["node"]
+    },
+    "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"],
+    "include": ["src/**/*.ts"]
+}

examples/image/tsconfig.json

@@ -0,0 +1,17 @@
+{
+    "extends": "../../tsconfig.base.json",
+    "files": [],
+    "include": [],
+    "references": [
+        {
+            "path": "./tsconfig.app.json"
+        }
+    ],
+    "compilerOptions": {
+        "esModuleInterop": true,
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
+        "module": "NodeNext",
+        "moduleResolution": "nodenext"
+    }
+}

loaders/embedjs-loader-image/package.json

@@ -9,7 +9,8 @@
         "debug": "^4.4.0",
         "md5": "^2.3.0",
         "mime": "^4.0.6",
-        "stream-mime-type": "^2.0.0"
+        "stream-mime-type": "^2.0.0",
+        "exifremove": "^1.0.1"
     },
     "type": "module",
     "main": "./src/index.js",

loaders/embedjs-loader-image/src/image-loader.ts

@@ -1,11 +1,12 @@
 import { HumanMessage } from '@langchain/core/messages';
 import { getMimeType } from 'stream-mime-type';
 import createDebugMessages from 'debug';
+import exifremove from 'exifremove';
 import fs from 'node:fs';
 import md5 from 'md5';
 
 import { BaseLoader, BaseModel } from '@llm-tools/embedjs-interfaces';
-import { cleanString, contentTypeToMimeType, getSafe, isValidURL, streamToString } from '@llm-tools/embedjs-utils';
+import { cleanString, contentTypeToMimeType, getSafe, isValidURL, streamToBuffer } from '@llm-tools/embedjs-utils';
 
 export class ImageLoader extends BaseLoader<{ type: 'ImageLoader' }> {
     private readonly debug = createDebugMessages('embedjs:loader:ImageLoader');
@@ -60,9 +61,11 @@ export class ImageLoader extends BaseLoader<{ type: 'ImageLoader' }> {
         }
 
         this.debug(`Image stream detected type '${this.mime}'`);
-        const text = this.isUrl
-            ? (await getSafe(this.filePathOrUrl, { format: 'text' })).body
-            : await streamToString(fs.createReadStream(this.filePathOrUrl));
+        const buffer = this.isUrl
+            ? (await getSafe(this.filePathOrUrl, { format: 'buffer' })).body
+            : await streamToBuffer(fs.createReadStream(this.filePathOrUrl));
+
+        const plainImageBuffer = exifremove.remove(buffer);
 
         const message = new HumanMessage({
             content: [
@@ -73,15 +76,15 @@ export class ImageLoader extends BaseLoader<{ type: 'ImageLoader' }> {
                 {
                     type: 'image_url',
                     image_url: {
-                        url: `data:${this.mime};base64,${btoa(text)}`,
+                        url: `data:${this.mime};base64,${plainImageBuffer.toString('base64')}`,
                     },
                 },
             ],
         });
 
         this.debug('Asking LLM to describe image');
         const response = await this.captionModel.simpleQuery([message]);
-        this.debug('LLM describes image as', response.result);
+        this.debug('LLM describes image as: ', response.result);
 
         yield {
             pageContent: cleanString(response.result),

scripts/publish-via-nx.js

@@ -12,6 +12,19 @@ function abs(relativePath) {
     return resolve(dirname(fileURLToPath(import.meta.url)), relativePath);
 }
 
+/**
+ * @param {string} version - The version to update the root package to
+ * @param {boolean} dryRun - Whether to perform a dry run or not
+ */
+async function updateRootPackageVersion(version, dryRun) {
+    const absPath = abs('..');
+    console.log(`Updating root package at path '${absPath}' to version '${version}' ${dryRun ? '[dry run]' : ''}`);
+    const pkgJson = await PackageJson.load(absPath);
+    pkgJson.update({ version });
+
+    if (!dryRun) await pkgJson.save();
+}
+
 /**
  * @param {pkgName} pkgName - The name of the package to update
  * @param {string} version - The version to update the package to
@@ -76,6 +89,7 @@ async function createRelease(dryRun, version, makeGitCommit) {
     }
 
     console.log('Updating projects actual version to match NX computed values');
+    await updateRootPackageVersion(newVersion, dryRun);
     for await (const [pkgName, { newVersion }] of Object.entries(projectsVersionData)) {
         if (newVersion !== null) await updatePackageVersion(pkgName, newVersion, versionMap, dryRun);
         else console.log(`Skipping '${pkgName}' version update as it's already up to date`);