Skip to content

Updates #193

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adhityan merged 7 commits into from
Dec 27, 2024
Merged

Updates #193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
517cc7b
fix: resolved wrong score for libsql
adhityan Dec 27, 2024
3962082
feat: added image loader using content description
adhityan Dec 27, 2024
654affb
chore: renamed internal methods to be consistent
adhityan Dec 27, 2024
bdb4cdd
chore(release): create release 0.1.27
adhityan Dec 27, 2024
49f2c8c
chore: bump package version
adhityan Dec 27, 2024
3d9009a
Merge branch 'confluence_updates' of github.com:llm-tools/embedJs int…
adhityan Dec 27, 2024
bcfbe83
Merge branch 'main' into confluence_updates
adhityan Dec 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/embedjs-interfaces/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@llm-tools/embedjs-interfaces",
"version": "0.1.26",
"version": "0.1.27",
"description": "Interfaces for extending the embedjs ecosystem",
"dependencies": {
"@langchain/core": "^0.3.26",
Expand Down
10 changes: 7 additions & 3 deletions core/embedjs-interfaces/src/interfaces/base-loader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { EventEmitter } from 'node:events';

import { BaseStore } from './base-store.js';
import { LoaderChunk, UnfilteredLoaderChunk } from '../types.js';
import { BaseModel } from './base-model.js';

export abstract class BaseLoader<
MetadataTemplate extends Record<string, string | number | boolean> = Record<string, string | number | boolean>,
Expand Down Expand Up @@ -39,13 +40,16 @@ export abstract class BaseLoader<
createDebugMessages('embedjs:loader:BaseLoader')(`New loader class initalized with key ${uniqueId}`);
}

// eslint-disable-next-line @typescript-eslint/no-empty-function
public async init(): Promise<void> {}

public getUniqueId(): string {
return this.uniqueId;
}

// eslint-disable-next-line @typescript-eslint/no-empty-function
public async init(): Promise<void> {}

// eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars
public injectModel(_model: BaseModel) {}

private async recordLoaderInCache(chunksProcessed: number) {
if (!BaseLoader.store) return;

Expand Down
12 changes: 12 additions & 0 deletions core/embedjs-interfaces/src/interfaces/base-model.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,5 +135,17 @@ export abstract class BaseModel {
};
}

public async simpleQuery(messages: (AIMessage | SystemMessage | HumanMessage)[]) {
const response = await this.runQuery(messages);

return {
result: response.result,
tokenUse: {
inputTokens: response.tokenUse?.inputTokens ?? 'UNKNOWN',
outputTokens: response.tokenUse?.outputTokens ?? 'UNKNOWN',
},
};
}

protected abstract runQuery(messages: (AIMessage | SystemMessage | HumanMessage)[]): Promise<ModelResponse>;
}
4 changes: 2 additions & 2 deletions core/embedjs-utils/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-utils",
"version": "0.1.26",
"version": "0.1.27",
"description": "Useful util functions when extending the embedjs ecosystem",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.26"
"@llm-tools/embedjs-interfaces": "0.1.27"
},
"type": "module",
"main": "./src/index.js",
Expand Down
12 changes: 11 additions & 1 deletion core/embedjs-utils/src/util/stream.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Stream } from 'stream';

export async function stream2buffer(stream: Stream): Promise<Buffer> {
export async function streamToBuffer(stream: Stream): Promise<Buffer> {
return new Promise<Buffer>((resolve, reject) => {
const _buf = Array<Uint8Array>();

Expand All @@ -10,6 +10,16 @@ export async function stream2buffer(stream: Stream): Promise<Buffer> {
});
}

export async function streamToString(stream: Stream): Promise<string> {
return new Promise((resolve, reject) => {
const chunks = [];

stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
stream.on('error', (err) => reject(`error converting stream - ${err}`));
});
}

export function contentTypeToMimeType(contentType: string) {
if (!contentType) return contentType;
if (contentType.includes(';')) return contentType.split(';')[0];
Expand Down
6 changes: 3 additions & 3 deletions core/embedjs/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
{
"type": "module",
"name": "@llm-tools/embedjs",
"version": "0.1.26",
"version": "0.1.27",
"description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
"dependencies": {
"@langchain/textsplitters": "^0.1.0",
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-utils": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"@llm-tools/embedjs-utils": "0.1.27",
"debug": "^4.4.0",
"langchain": "^0.3.8",
"md5": "^2.3.0",
Expand Down
1 change: 1 addition & 0 deletions core/embedjs/src/core/rag-application.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ export class RAGApplication {
private async _addLoader(loader: BaseLoader, forceReload: boolean): Promise<AddLoaderReturn> {
const uniqueId = loader.getUniqueId();
this.debug('Exploring loader', uniqueId);
if (this.model) loader.injectModel(this.model);

if (this.store && (await this.store.hasLoaderMetadata(uniqueId))) {
if (forceReload) {
Expand Down
9 changes: 9 additions & 0 deletions core/embedjs/src/util/mime.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,15 @@ export async function createLoaderFromMimeType(loaderData: string, mimeType: str
createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported MarkdownLoader');
return new MarkdownLoader({ filePathOrUrl: loaderData });
}
case 'image/png':
case 'image/jpeg': {
const { ImageLoader } = await import('@llm-tools/embedjs-loader-image').catch(() => {
throw new Error('Package `@llm-tools/embedjs-loader-image` needs to be installed to load images');
});
createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported ImageLoader');
return new ImageLoader({ filePathOrUrl: loaderData, mime: mimeType });
}

case undefined:
throw new Error(`MIME type could not be detected. Please file an issue if you think this is a bug.`);
default:
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-astra/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-astradb",
"version": "0.1.26",
"version": "0.1.27",
"description": "Add AstraDB support to embedjs",
"dependencies": {
"@datastax/astra-db-ts": "^1.5.0",
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"debug": "^4.4.0"
},
"type": "module",
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-cosmos/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-cosmos",
"version": "0.1.26",
"version": "0.1.27",
"description": "Add CosmosDB support to embedjs",
"dependencies": {
"@azure/cosmos": "^4.2.0",
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"debug": "^4.4.0"
},
"type": "module",
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-hnswlib/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-hnswlib",
"version": "0.1.26",
"version": "0.1.27",
"description": "Add HNSWLib support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"debug": "^4.4.0",
"hnswlib-node": "^3.0.0"
},
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-lancedb/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-lancedb",
"version": "0.1.26",
"version": "0.1.27",
"description": "Add LanceDb support to embedjs",
"dependencies": {
"@lancedb/lancedb": "^0.14.1",
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"compute-cosine-similarity": "^1.1.0",
"debug": "^4.4.0"
},
Expand Down
6 changes: 3 additions & 3 deletions databases/embedjs-libsql/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
{
"name": "@llm-tools/embedjs-libsql",
"version": "0.1.26",
"version": "0.1.27",
"description": "Add LibSQL support to embedjs",
"dependencies": {
"@libsql/client": "^0.14.0",
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-utils": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"@llm-tools/embedjs-utils": "0.1.27",
"debug": "^4.4.0"
},
"type": "module",
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-libsql/src/libsql-db.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ export class LibSqlDb implements BaseVectorDatabase {

async similaritySearch(query: number[], k: number): Promise<ExtractChunkData[]> {
const statement = `SELECT id, pageContent, uniqueLoaderId, source, metadata,
vector_distance_cos(vector, vector32('[${query.join(',')}]'))
vector_distance_cos(vector, vector32('[${query.join(',')}]')) as distance
FROM ${this.tableName}
ORDER BY vector_distance_cos(vector, vector32('[${query.join(',')}]')) ASC
LIMIT ${k};`;
Expand All @@ -63,7 +63,7 @@ export class LibSqlDb implements BaseVectorDatabase {
return {
metadata,
pageContent: result.pageContent.toString(),
score: 1,
score: 1 - <number>result.distance,
};
});
}
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-lmdb/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-lmdb",
"version": "0.1.26",
"version": "0.1.27",
"description": "Add LMDB support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"debug": "^4.4.0",
"lmdb": "^3.2.0"
},
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-mongodb/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-mongodb",
"version": "0.1.26",
"version": "0.1.27",
"description": "Add MongoDB support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"debug": "^4.4.0",
"mongodb": "^6.12.0"
},
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-pinecone/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-pinecone",
"version": "0.1.26",
"version": "0.1.27",
"description": "Add Pinecone support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"@pinecone-database/pinecone": "^4.0.0",
"debug": "^4.4.0"
},
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-qdrant/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-qdrant",
"version": "0.1.26",
"version": "0.1.27",
"description": "Add Qdrant support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"@qdrant/js-client-rest": "^1.12.0",
"debug": "^4.4.0",
"uuid": "^11.0.3"
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-redis/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-redis",
"version": "0.1.26",
"version": "0.1.27",
"description": "Add Redis support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"ioredis": "^5.4.2"
},
"type": "module",
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-weaviate/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-weaviate",
"version": "0.1.26",
"version": "0.1.27",
"description": "Add Weaviate support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"compute-cosine-similarity": "^1.1.0",
"debug": "^4.4.0",
"weaviate-ts-client": "^2.2.0"
Expand Down
6 changes: 3 additions & 3 deletions loaders/embedjs-loader-confluence/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-loader-confluence",
"version": "0.1.26",
"version": "0.1.27",
"description": "Confluence loader for embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-loader-web": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"@llm-tools/embedjs-loader-web": "0.1.27",
"confluence.js": "^1.7.4",
"debug": "^4.4.0",
"md5": "^2.3.0"
Expand Down
6 changes: 3 additions & 3 deletions loaders/embedjs-loader-confluence/src/confluence-loader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,23 +21,23 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
confluenceToken,
chunkSize,
chunkOverlap,
options,
filterOptions,
}: {
spaceName: string;
confluenceBaseUrl?: string;
confluenceUsername?: string;
confluenceToken?: string;
chunkSize?: number;
chunkOverlap?: number;
options?: {
filterOptions?: {
lastUpdatedFilter: Date;
};
}) {
super(`ConfluenceLoader_${md5(spaceName)}`, { spaceName }, chunkSize ?? 2000, chunkOverlap ?? 200);

this.spaceName = spaceName;
this.confluenceBaseUrl = confluenceBaseUrl ?? process.env.CONFLUENCE_BASE_URL;
this.lastUpdatedFilter = options?.lastUpdatedFilter ?? null;
this.lastUpdatedFilter = filterOptions?.lastUpdatedFilter ?? null;

this.confluence = new ConfluenceClient({
host: this.confluenceBaseUrl,
Expand Down
6 changes: 3 additions & 3 deletions loaders/embedjs-loader-csv/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-loader-csv",
"version": "0.1.26",
"version": "0.1.27",
"description": "CSV loader for embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.26",
"@llm-tools/embedjs-utils": "0.1.26",
"@llm-tools/embedjs-interfaces": "0.1.27",
"@llm-tools/embedjs-utils": "0.1.27",
"csv-parse": "^5.6.0",
"debug": "^4.4.0",
"md5": "^2.3.0"
Expand Down
4 changes: 2 additions & 2 deletions loaders/embedjs-loader-csv/src/csv-loader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import fs from 'node:fs';
import md5 from 'md5';

import { BaseLoader } from '@llm-tools/embedjs-interfaces';
import { cleanString, getSafe, isValidURL, stream2buffer } from '@llm-tools/embedjs-utils';
import { cleanString, getSafe, isValidURL, streamToBuffer } from '@llm-tools/embedjs-utils';

export class CsvLoader extends BaseLoader<{ type: 'CsvLoader' }> {
private readonly debug = createDebugMessages('embedjs:loader:CsvLoader');
Expand Down Expand Up @@ -33,7 +33,7 @@ export class CsvLoader extends BaseLoader<{ type: 'CsvLoader' }> {
override async *getUnfilteredChunks() {
const buffer = this.isUrl
? (await getSafe(this.filePathOrUrl, { format: 'buffer' })).body
: await stream2buffer(fs.createReadStream(this.filePathOrUrl));
: await streamToBuffer(fs.createReadStream(this.filePathOrUrl));

this.debug('CsvParser stream created');
const parser = parse(buffer, this.csvParseOptions);
Expand Down
8 changes: 8 additions & 0 deletions loaders/embedjs-loader-image/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# embedjs-loader-image

<p>
<a href="https://www.npmjs.com/package/@llm-tools/embedjs" target="_blank"><img alt="NPM Version" src="https://img.shields.io/npm/v/%40llm-tools/embedjs?style=for-the-badge"></a>
<a href="https://www.npmjs.com/package/@llm-tools/embedjs" target="_blank"><img alt="License" src="https://img.shields.io/npm/l/%40llm-tools%2Fembedjs?style=for-the-badge"></a>
</p>

This package extends and offers additional functionality to [embedJs](https://www.npmjs.com/package/@llm-tools/embedjs). Refer to the documentation there for more details.
20 changes: 20 additions & 0 deletions loaders/embedjs-loader-image/eslint.config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import baseConfig from '../../eslint.config.js';
import parser from '@nx/eslint-plugin';

export default [
...baseConfig,
{
files: ['**/*.json'],
rules: {
'@nx/dependency-checks': [
'error',
{
ignoredFiles: ['{projectRoot}/eslint.config.{js,cjs,mjs}'],
},
],
},
languageOptions: {
parser,
},
},
];
Loading
Loading