From 3216c29b83fead8218e936b25bc91dc4fd33c593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Fri, 31 Jan 2025 10:29:43 +0000 Subject: [PATCH 01/13] Add method for creating milvus vector db --- dockers/llm.vdb.service/common.py | 19 +++++++++++++++++++ dockers/llm.vdb.service/requirements.txt | 1 + 2 files changed, 20 insertions(+) diff --git a/dockers/llm.vdb.service/common.py b/dockers/llm.vdb.service/common.py index c0080a8..4879f77 100644 --- a/dockers/llm.vdb.service/common.py +++ b/dockers/llm.vdb.service/common.py @@ -4,6 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_community.vectorstores import FAISS from langchain_huggingface import HuggingFaceEmbeddings +from langchain_milvus import Milvus def load_jsonl_files_from_directory(directory): @@ -77,3 +78,21 @@ def create_vectordb_from_data( print("Convert to FAISS vectorstore") vectorstore = FAISS.from_texts(texts, embeddings, metadatas=metadatas) return vectorstore + + +def create_milvus_vectordb_from_data( + data, + embedding_model_name: str, + milvus_uri: str, + collection_name: str, +): + embeddings = HuggingFaceEmbeddings(model_name=embedding_model_name) + print("Convert to Milvus vectorstore") + + vectorstore = Milvus( + data, + embedding_function=embeddings, + collection_name=collection_name, + connection_args={"uri": milvus_uri}, + ) + return vectorstore diff --git a/dockers/llm.vdb.service/requirements.txt b/dockers/llm.vdb.service/requirements.txt index 46a7b98..d79c0d6 100644 --- a/dockers/llm.vdb.service/requirements.txt +++ b/dockers/llm.vdb.service/requirements.txt @@ -2,5 +2,6 @@ click faiss-cpu langchain_community langchain_huggingface +langchain_milvus pydantic_settings s3fs \ No newline at end of file From f3c8356b6e5330f5d007f0bbc7324fb40538318e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Fri, 31 Jan 2025 10:30:04 +0000 Subject: [PATCH 02/13] Add config for milvus --- dockers/llm.vdb.service/.env_local_template | 6 +++++- dockers/llm.vdb.service/.env_s3_template | 6 +++++- dockers/llm.vdb.service/config.py | 18 ++++++++++++++++++ dockers/llm.vdb.service/test_data/.env_local | 6 +++++- dockers/llm.vdb.service/test_data/.env_s3 | 6 +++++- 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/dockers/llm.vdb.service/.env_local_template b/dockers/llm.vdb.service/.env_local_template index 5f7fefd..66c5dd0 100644 --- a/dockers/llm.vdb.service/.env_local_template +++ b/dockers/llm.vdb.service/.env_local_template @@ -5,4 +5,8 @@ OUTPUT_FILENAME=/path/to/local/output_pickled.obj # Vector DB Optional Settings # EMBEDDING_CHUNK_SIZE=1000 # EMBEDDING_CHUNK_OVERLAP=100 -# EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 \ No newline at end of file +# EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 + +# Milvus Vector DB Optional Settings +# MILVUS_URI="./milvus_example.db" +# MILVUS_COLLECTION_NAME="test_milvus_collection" \ No newline at end of file diff --git a/dockers/llm.vdb.service/.env_s3_template b/dockers/llm.vdb.service/.env_s3_template index 28e0d4a..4dde0cf 100644 --- a/dockers/llm.vdb.service/.env_s3_template +++ b/dockers/llm.vdb.service/.env_s3_template @@ -9,4 +9,8 @@ AWS_SECRET_ACCESS_KEY=my-secret-key # Vector DB Optional Settings # EMBEDDING_CHUNK_SIZE=1000 # EMBEDDING_CHUNK_OVERLAP=100 -# EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 \ No newline at end of file +# EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 + +# Milvus Vector DB Optional Settings +# MILVUS_URI="./milvus_example.db" +# MILVUS_COLLECTION_NAME="test_milvus_collection" \ No newline at end of file diff --git a/dockers/llm.vdb.service/config.py b/dockers/llm.vdb.service/config.py index ee67095..2e48ed8 100644 --- a/dockers/llm.vdb.service/config.py +++ b/dockers/llm.vdb.service/config.py @@ -54,6 +54,15 @@ class S3Settings(BaseSettings): description="Name of the embedding model to use" ) + milvus_uri: str = Field( + default="", + description="Milvus connection URI" + ) + milvus_collection_name: str = Field( + default="", + description="Milvus collection name" + ) + class Config: env_file = ".env" @@ -80,6 +89,15 @@ class LocalSettings(BaseSettings): description="Name of the embedding model to use" ) + milvus_uri: str = Field( + default="", + description="Milvus connection URI" + ) + milvus_collection_name: str = Field( + default="", + description="Milvus collection name" + ) + class Config: env_file = ".env" diff --git a/dockers/llm.vdb.service/test_data/.env_local b/dockers/llm.vdb.service/test_data/.env_local index e625831..8c3fa71 100644 --- a/dockers/llm.vdb.service/test_data/.env_local +++ b/dockers/llm.vdb.service/test_data/.env_local @@ -5,4 +5,8 @@ OUTPUT_FILENAME=test_data/output/output_pickled.obj # Vector DB Optional Settings # EMBEDDING_CHUNK_SIZE=1000 # EMBEDDING_CHUNK_OVERLAP=100 -# EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 \ No newline at end of file +# EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 + +# Milvus Vector DB Optional Settings +# MILVUS_URI="./milvus_example.db" +# MILVUS_COLLECTION_NAME="test_milvus_collection" \ No newline at end of file diff --git a/dockers/llm.vdb.service/test_data/.env_s3 b/dockers/llm.vdb.service/test_data/.env_s3 index 33b004b..bb5eddb 100644 --- a/dockers/llm.vdb.service/test_data/.env_s3 +++ b/dockers/llm.vdb.service/test_data/.env_s3 @@ -10,4 +10,8 @@ AWS_REGION=us-east-2 # Vector DB Optional Settings # EMBEDDING_CHUNK_SIZE=1000 # EMBEDDING_CHUNK_OVERLAP=100 -# EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 \ No newline at end of file +# EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 + +# Milvus Vector DB Optional Settings +# MILVUS_URI="./milvus_example.db" +# MILVUS_COLLECTION_NAME="test_milvus_collection" \ No newline at end of file From 0cd05c94fc528b06a6bafa0193842bdc77bf2ea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Fri, 31 Jan 2025 11:33:24 +0000 Subject: [PATCH 03/13] Add service and fix creation --- dockers/llm.vdb.service/common.py | 18 +++++++++++++- dockers/llm.vdb.service/service.py | 24 +++++++++++++++++++ .../test_data/.env_local_milvus | 12 ++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 dockers/llm.vdb.service/test_data/.env_local_milvus diff --git a/dockers/llm.vdb.service/common.py b/dockers/llm.vdb.service/common.py index 4879f77..34a907b 100644 --- a/dockers/llm.vdb.service/common.py +++ b/dockers/llm.vdb.service/common.py @@ -3,6 +3,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_community.vectorstores import FAISS +from langchain_core.documents import Document from langchain_huggingface import HuggingFaceEmbeddings from langchain_milvus import Milvus @@ -61,6 +62,13 @@ def chunk_documents_with_metadata(data, chunk_size=1000, chunk_overlap=200): return all_chunks, all_metadatas +def json_to_document(json_dic): + return Document( + page_content=json_dic["text"], + metadata=json_dic["metadata"] + ) + + def create_vectordb_from_data( data, embedding_model_name: str, @@ -90,9 +98,17 @@ def create_milvus_vectordb_from_data( print("Convert to Milvus vectorstore") vectorstore = Milvus( - data, embedding_function=embeddings, collection_name=collection_name, connection_args={"uri": milvus_uri}, + # TODO: these index settings are required for local setup + # make sure it also makes sense for other setups + index_params={ + "metric_type": "L2", + "index_type": "IVF_FLAT", + "params": {"nlist": 1024} + }, + auto_id=True, ) + vectorstore.add_documents(documents=data) return vectorstore diff --git a/dockers/llm.vdb.service/service.py b/dockers/llm.vdb.service/service.py index ee23e3b..90f424d 100644 --- a/dockers/llm.vdb.service/service.py +++ b/dockers/llm.vdb.service/service.py @@ -4,7 +4,9 @@ from dataclasses import dataclass from common import ( + create_milvus_vectordb_from_data, create_vectordb_from_data, + json_to_document, load_jsonl_files_from_directory, ) from config import LocalSettings, S3Settings @@ -61,3 +63,25 @@ def create(self): with open(self.config.output_filename, "wb") as file: file.write(pickle_byte_obj) print(f"Pickle byte object saved to {self.config.output_filename}") + + +@dataclass +class LocalDirMilvusDbCreationService: + config: LocalSettings + + def create(self): + print("Load JSON files") + data = load_jsonl_files_from_directory(self.config.local_directory) + docs = [] + for d in data: + docs.append(json_to_document(d)) + + print("Convert to Milvus vectorstore") + create_milvus_vectordb_from_data( + docs, + self.config.embedding_model_name, + self.config.milvus_uri, + self.config.milvus_collection_name, + ) + + print(f"Milvus collection saved {self.config.milvus_collection_name}") diff --git a/dockers/llm.vdb.service/test_data/.env_local_milvus b/dockers/llm.vdb.service/test_data/.env_local_milvus new file mode 100644 index 0000000..8c3fa71 --- /dev/null +++ b/dockers/llm.vdb.service/test_data/.env_local_milvus @@ -0,0 +1,12 @@ +# Local Settings +LOCAL_DIRECTORY=test_data/input/ +OUTPUT_FILENAME=test_data/output/output_pickled.obj + +# Vector DB Optional Settings +# EMBEDDING_CHUNK_SIZE=1000 +# EMBEDDING_CHUNK_OVERLAP=100 +# EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 + +# Milvus Vector DB Optional Settings +# MILVUS_URI="./milvus_example.db" +# MILVUS_COLLECTION_NAME="test_milvus_collection" \ No newline at end of file From 1621262def530232d9e636768abc9f6cee67f9b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Fri, 31 Jan 2025 11:33:47 +0000 Subject: [PATCH 04/13] Enable creating milvus with local files and test --- dockers/llm.vdb.service/createvectordb.py | 14 ++++++++--- .../llm.vdb.service/createvectordb_test.py | 23 +++++++++++++++++++ .../test_data/.env_local_milvus | 4 ++-- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/dockers/llm.vdb.service/createvectordb.py b/dockers/llm.vdb.service/createvectordb.py index 4b18eb5..aa93e27 100644 --- a/dockers/llm.vdb.service/createvectordb.py +++ b/dockers/llm.vdb.service/createvectordb.py @@ -2,7 +2,11 @@ import sys from config import try_load_settings -from service import LocalDirDbCreationService, S3VectorDbCreationService +from service import ( + LocalDirDbCreationService, + LocalDirMilvusDbCreationService, + S3VectorDbCreationService, +) @click.command() @@ -15,8 +19,12 @@ def run(env_file: str): service.create() elif local_settings: - service = LocalDirDbCreationService(local_settings) - service.create() + if local_settings: + service = LocalDirMilvusDbCreationService(local_settings) + service.create() + else: + service = LocalDirDbCreationService(local_settings) + service.create() else: # TODO: not really needed, error will be thrown earlier diff --git a/dockers/llm.vdb.service/createvectordb_test.py b/dockers/llm.vdb.service/createvectordb_test.py index 7db0f1f..21726c1 100644 --- a/dockers/llm.vdb.service/createvectordb_test.py +++ b/dockers/llm.vdb.service/createvectordb_test.py @@ -4,6 +4,8 @@ import s3fs from botocore.session import Session +from langchain_huggingface import HuggingFaceEmbeddings +from langchain_milvus import Milvus from moto.moto_server.threaded_moto_server import ThreadedMotoServer from s3fs.core import S3FileSystem @@ -23,6 +25,27 @@ def test_create_faiss_vector_db_using_local_files(): os.remove("test_data/output/output_pickled.obj") +def test_create_milvus_vector_db_using_local_files(): + ctx = click.Context(run) + try: + ctx.forward(run, env_file="test_data/.env_local_milvus") + except SystemExit as e: + assert e.code == 0 + + URI = "./milvus_example.db" + embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2") + vector_store_loaded = Milvus( + embeddings, + connection_args={"uri": URI}, + collection_name="langchain_example", + ) + + vector_store_loaded.similarity_search_with_score("Enhance error tracking?", k=2) + + if os.path.exists("milvus_example.db"): + os.remove("milvus_example.db") + + @pytest.fixture(scope="module") def s3_base(): # writable local S3 system diff --git a/dockers/llm.vdb.service/test_data/.env_local_milvus b/dockers/llm.vdb.service/test_data/.env_local_milvus index 8c3fa71..ff1abb6 100644 --- a/dockers/llm.vdb.service/test_data/.env_local_milvus +++ b/dockers/llm.vdb.service/test_data/.env_local_milvus @@ -8,5 +8,5 @@ OUTPUT_FILENAME=test_data/output/output_pickled.obj # EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 # Milvus Vector DB Optional Settings -# MILVUS_URI="./milvus_example.db" -# MILVUS_COLLECTION_NAME="test_milvus_collection" \ No newline at end of file +MILVUS_URI="./milvus_example.db" +MILVUS_COLLECTION_NAME="test_milvus_collection" \ No newline at end of file From 9e3fa7def3da3289a8b6b6421d740b5b336e4cb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Fri, 31 Jan 2025 11:44:38 +0000 Subject: [PATCH 05/13] Use chunking as for faiss --- dockers/llm.vdb.service/common.py | 23 +++++++++++++++-------- dockers/llm.vdb.service/service.py | 8 +++----- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/dockers/llm.vdb.service/common.py b/dockers/llm.vdb.service/common.py index 34a907b..ed9880e 100644 --- a/dockers/llm.vdb.service/common.py +++ b/dockers/llm.vdb.service/common.py @@ -62,13 +62,6 @@ def chunk_documents_with_metadata(data, chunk_size=1000, chunk_overlap=200): return all_chunks, all_metadatas -def json_to_document(json_dic): - return Document( - page_content=json_dic["text"], - metadata=json_dic["metadata"] - ) - - def create_vectordb_from_data( data, embedding_model_name: str, @@ -93,7 +86,21 @@ def create_milvus_vectordb_from_data( embedding_model_name: str, milvus_uri: str, collection_name: str, + chunk_size, + chunk_overlap, ): + print("Start chunking documents") + texts, metadatas = chunk_documents_with_metadata(data, chunk_size, chunk_overlap) + + docs = [] + for text, metadata in zip(texts, metadatas): + docs.append( + Document( + page_content=text, + metadata=metadata, + ) + ) + embeddings = HuggingFaceEmbeddings(model_name=embedding_model_name) print("Convert to Milvus vectorstore") @@ -110,5 +117,5 @@ def create_milvus_vectordb_from_data( }, auto_id=True, ) - vectorstore.add_documents(documents=data) + vectorstore.add_documents(documents=docs) return vectorstore diff --git a/dockers/llm.vdb.service/service.py b/dockers/llm.vdb.service/service.py index 90f424d..dde50fe 100644 --- a/dockers/llm.vdb.service/service.py +++ b/dockers/llm.vdb.service/service.py @@ -6,7 +6,6 @@ from common import ( create_milvus_vectordb_from_data, create_vectordb_from_data, - json_to_document, load_jsonl_files_from_directory, ) from config import LocalSettings, S3Settings @@ -72,16 +71,15 @@ class LocalDirMilvusDbCreationService: def create(self): print("Load JSON files") data = load_jsonl_files_from_directory(self.config.local_directory) - docs = [] - for d in data: - docs.append(json_to_document(d)) print("Convert to Milvus vectorstore") create_milvus_vectordb_from_data( - docs, + data, self.config.embedding_model_name, self.config.milvus_uri, self.config.milvus_collection_name, + self.config.embedding_chunk_size, + self.config.embedding_chunk_overlap, ) print(f"Milvus collection saved {self.config.milvus_collection_name}") From 54caf33f6cf1797500f8b0c35d164b1c913d764b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Fri, 31 Jan 2025 19:29:14 +0000 Subject: [PATCH 06/13] Add local rag service using milvus --- .../serveragllm_milvus_local.py | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 dockers/llm.rag.service/serveragllm_milvus_local.py diff --git a/dockers/llm.rag.service/serveragllm_milvus_local.py b/dockers/llm.rag.service/serveragllm_milvus_local.py new file mode 100644 index 0000000..739d4fc --- /dev/null +++ b/dockers/llm.rag.service/serveragllm_milvus_local.py @@ -0,0 +1,120 @@ +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "faiss-cpu", +# "fastapi", +# "langchain-community", +# "langchain-huggingface", +# "openai", +# "uvicorn", +# "weaviate", +# "langchain_milvus", +# ] +# /// + +import os +import sys +import uvicorn + +from functools import partial +from typing import Union + +import click +from fastapi import FastAPI +from openai import OpenAI + +from common import get_answer_with_settings + + +SYSTEM_PROMPT="""You are a specialized support ticket assistant. Format your responses following these rules: + 1. Answer the provided question only using the provided context. + 2. Do not add the provided context to the generated answer. + 3. Include relevant technical details when present or provide a summary of the comments in the ticket. + 4. Include the submitter, assignee and collaborator for a ticket when this info is available. + 5. If the question cannot be answered with the given context, please say so and do not attempt to provide an answer. + 6. Do not create new questions related to the given question, instead answer only the provided question. + 7. Provide a clear, direct and factual answer.""" + + +def setup( + relevant_docs: int, + llm_server_url:str, + model_id: str, + max_tokens: int, + model_temperature: float, +): + app = FastAPI() + + # TODO: move to imports + from langchain_huggingface import HuggingFaceEmbeddings + from langchain_milvus import Milvus + + # TODO: pass through settings or params + URI = "http://localhost:19530" + embedding_model_name = "sentence-transformers/all-MiniLM-L6-v2" + embeddings = HuggingFaceEmbeddings(model_name=embedding_model_name) + vectorstore = Milvus( + embeddings, + connection_args={"uri": URI}, + collection_name="langchain_example", + ) + + retriever = vectorstore.as_retriever( + search_kwargs={ + "k": relevant_docs, + "metric_type": "L2", + } + ) + print("Created Vector DB retriever successfully. \n") + + print("Creating an OpenAI client to the hosted model at URL: ", llm_server_url) + try: + client = OpenAI(base_url=llm_server_url, api_key="na") + except Exception as e: + print("Error creating client:", e) + sys.exit(1) + + get_answer = partial( + get_answer_with_settings, + retriever=retriever, + client=client, + model_id=model_id, + max_tokens=max_tokens, + model_temperature=model_temperature, + system_prompt=SYSTEM_PROMPT, + ) + + @app.get("/answer/{question}") + def read_item(question: Union[str, None] = None): + print(f"Received question: {question}") + answer = get_answer(question) + return {"question": question, "answer": answer} + + return app + + +MICROSOFT_MODEL_ID = "microsoft/Phi-3-mini-4k-instruct" +MOSAICML_MODEL_ID = "mosaicml/mpt-7b-chat" +RELEVANT_DOCS_DEFAULT = 2 +MAX_TOKENS_DEFAULT = 64 +MODEL_TEMPERATURE_DEFAULT = 0.01 + +relevant_docs = os.getenv("RELEVANT_DOCS", RELEVANT_DOCS_DEFAULT) +llm_server_url = os.getenv("LLM_SERVER_URL", "http://localhost:11434/v1") +model_id = os.getenv("MODEL_ID", "llama2") +max_tokens = int(os.getenv("MAX_TOKENS", MAX_TOKENS_DEFAULT)) +model_temperature = float(os.getenv("MODEL_TEMPERATURE", MODEL_TEMPERATURE_DEFAULT)) + +app = setup(relevant_docs, llm_server_url, model_id, max_tokens, model_temperature) + + +@click.command() +@click.option("--host", default="127.0.0.1", help="Host for the FastAPI server (default: 127.0.0.1)") +@click.option("--port", type=int, default=8000, help="Port for the FastAPI server (default: 8000)") +def run(host, port): + # Serve the app using Uvicorn + uvicorn.run("serveragllm_milvus_local:app", host=host, port=port, reload=True) + + +if __name__ == "__main__": + run() \ No newline at end of file From e682dd8531c812304707386453a0905e91e4eea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Mon, 3 Feb 2025 10:21:19 +0000 Subject: [PATCH 07/13] Tmp --- dockers/llm.vdb.service/common.py | 62 +++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/dockers/llm.vdb.service/common.py b/dockers/llm.vdb.service/common.py index ed9880e..088a099 100644 --- a/dockers/llm.vdb.service/common.py +++ b/dockers/llm.vdb.service/common.py @@ -5,7 +5,7 @@ from langchain_community.vectorstores import FAISS from langchain_core.documents import Document from langchain_huggingface import HuggingFaceEmbeddings -from langchain_milvus import Milvus +from langchain_milvus import BM25BuiltInFunction, Milvus def load_jsonl_files_from_directory(directory): @@ -104,18 +104,58 @@ def create_milvus_vectordb_from_data( embeddings = HuggingFaceEmbeddings(model_name=embedding_model_name) print("Convert to Milvus vectorstore") - vectorstore = Milvus( - embedding_function=embeddings, - collection_name=collection_name, - connection_args={"uri": milvus_uri}, - # TODO: these index settings are required for local setup - # make sure it also makes sense for other setups - index_params={ - "metric_type": "L2", - "index_type": "IVF_FLAT", - "params": {"nlist": 1024} + # vectorstore = Milvus( + # embedding_function=embeddings, + # vector_field=["dense", "sparse"], + # builtin_function=BM25BuiltInFunction(), + # collection_name=collection_name, + # connection_args={"uri": milvus_uri}, + # # TODO: these index settings are required for local setup + # # make sure it also makes sense for other setups + # index_params=[ + # { + # "field_name": "dense", + # "index_type": "IVF_FLAT", + # "metric_type": "L2", + # "params": {"nlist": 1024} + # }, + # { + # "field_name": "sparse", + # "index_type": "IVF_FLAT", + # "metric_type": "L2", + # "params": {"nlist": 1024} + # } + # ], + # auto_id=True, + # ) + + from langchain_openai import OpenAIEmbeddings + + vectorstore = Milvus.from_documents( + documents=docs, + embedding=OpenAIEmbeddings(), + builtin_function=BM25BuiltInFunction(), + vector_field=["dense", "sparse"], + connection_args={ + "uri": milvus_uri, }, + index_params=[ + { + "field_name": "dense", + "index_type": "IVF_FLAT", + "metric_type": "L2", + "params": {"nlist": 1024} + }, + { + "field_name": "sparse", + "index_type": "IVF_FLAT", + "metric_type": "L2", + "params": {"nlist": 1024} + } + ], + consistency_level="Strong", auto_id=True, ) + vectorstore.add_documents(documents=docs) return vectorstore From 8bfac8d5282b97ccef428cec1a88ec8ace5a0a4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Tue, 4 Feb 2025 13:09:34 +0000 Subject: [PATCH 08/13] Add scripts for running milvus in docker --- dockers/llm.vdb.service/standalone_embed.sh | 169 ++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 dockers/llm.vdb.service/standalone_embed.sh diff --git a/dockers/llm.vdb.service/standalone_embed.sh b/dockers/llm.vdb.service/standalone_embed.sh new file mode 100644 index 0000000..da0db43 --- /dev/null +++ b/dockers/llm.vdb.service/standalone_embed.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash + +# Licensed to the LF AI & Data foundation under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +run_embed() { + cat << EOF > embedEtcd.yaml +listen-client-urls: http://0.0.0.0:2379 +advertise-client-urls: http://0.0.0.0:2379 +quota-backend-bytes: 4294967296 +auto-compaction-mode: revision +auto-compaction-retention: '1000' +EOF + + cat << EOF > user.yaml +# Extra config to override default milvus.yaml +EOF + + sudo docker run -d \ + --name milvus-standalone \ + --security-opt seccomp:unconfined \ + -e ETCD_USE_EMBED=true \ + -e ETCD_DATA_DIR=/var/lib/milvus/etcd \ + -e ETCD_CONFIG_PATH=/milvus/configs/embedEtcd.yaml \ + -e COMMON_STORAGETYPE=local \ + -v $(pwd)/volumes/milvus:/var/lib/milvus \ + -v $(pwd)/embedEtcd.yaml:/milvus/configs/embedEtcd.yaml \ + -v $(pwd)/user.yaml:/milvus/configs/user.yaml \ + -p 19530:19530 \ + -p 9091:9091 \ + -p 2379:2379 \ + --health-cmd="curl -f http://localhost:9091/healthz" \ + --health-interval=30s \ + --health-start-period=90s \ + --health-timeout=20s \ + --health-retries=3 \ + milvusdb/milvus:v2.5.4 \ + milvus run standalone 1> /dev/null +} + +wait_for_milvus_running() { + echo "Wait for Milvus Starting..." + while true + do + res=`sudo docker ps|grep milvus-standalone|grep healthy|wc -l` + if [ $res -eq 1 ] + then + echo "Start successfully." + echo "To change the default Milvus configuration, add your settings to the user.yaml file and then restart the service." + break + fi + sleep 1 + done +} + +start() { + res=`sudo docker ps|grep milvus-standalone|grep healthy|wc -l` + if [ $res -eq 1 ] + then + echo "Milvus is running." + exit 0 + fi + + res=`sudo docker ps -a|grep milvus-standalone|wc -l` + if [ $res -eq 1 ] + then + sudo docker start milvus-standalone 1> /dev/null + else + run_embed + fi + + if [ $? -ne 0 ] + then + echo "Start failed." + exit 1 + fi + + wait_for_milvus_running +} + +stop() { + sudo docker stop milvus-standalone 1> /dev/null + + if [ $? -ne 0 ] + then + echo "Stop failed." + exit 1 + fi + echo "Stop successfully." + +} + +delete_container() { + res=`sudo docker ps|grep milvus-standalone|wc -l` + if [ $res -eq 1 ] + then + echo "Please stop Milvus service before delete." + exit 1 + fi + sudo docker rm milvus-standalone 1> /dev/null + if [ $? -ne 0 ] + then + echo "Delete milvus container failed." + exit 1 + fi + echo "Delete milvus container successfully." +} + +delete() { + delete_container + sudo rm -rf $(pwd)/volumes + sudo rm -rf $(pwd)/embedEtcd.yaml + sudo rm -rf $(pwd)/user.yaml + echo "Delete successfully." +} + +upgrade() { + read -p "Please confirm if you'd like to proceed with the upgrade. The default will be to the latest version. Confirm with 'y' for yes or 'n' for no. > " check + if [ "$check" == "y" ] ||[ "$check" == "Y" ];then + res=`sudo docker ps -a|grep milvus-standalone|wc -l` + if [ $res -eq 1 ] + then + stop + delete_container + fi + + curl -sfL https://raw.githubusercontent.com/milvus-io/milvus/master/scripts/standalone_embed.sh -o standalone_embed_latest.sh && \ + bash standalone_embed_latest.sh start 1> /dev/null && \ + echo "Upgrade successfully." + else + echo "Exit upgrade" + exit 0 + fi +} + +case $1 in + restart) + stop + start + ;; + start) + start + ;; + stop) + stop + ;; + upgrade) + upgrade + ;; + delete) + delete + ;; + *) + echo "please use bash standalone_embed.sh restart|start|stop|upgrade|delete" + ;; +esac From 19c13ec2ba58908f3237c43e7e77f2a3d9816855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Tue, 4 Feb 2025 13:34:08 +0000 Subject: [PATCH 09/13] Update test and store data for hybrid search --- dockers/llm.vdb.service/.env_local_template | 2 +- dockers/llm.vdb.service/.env_s3_template | 2 +- dockers/llm.vdb.service/common.py | 54 ++----------------- .../llm.vdb.service/createvectordb_test.py | 30 +++++------ dockers/llm.vdb.service/test_data/.env_local | 2 +- .../test_data/.env_local_milvus | 2 +- dockers/llm.vdb.service/test_data/.env_s3 | 2 +- 7 files changed, 24 insertions(+), 70 deletions(-) diff --git a/dockers/llm.vdb.service/.env_local_template b/dockers/llm.vdb.service/.env_local_template index 66c5dd0..495627f 100644 --- a/dockers/llm.vdb.service/.env_local_template +++ b/dockers/llm.vdb.service/.env_local_template @@ -8,5 +8,5 @@ OUTPUT_FILENAME=/path/to/local/output_pickled.obj # EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 # Milvus Vector DB Optional Settings -# MILVUS_URI="./milvus_example.db" +# MILVUS_URI="http://localhost:19530" # MILVUS_COLLECTION_NAME="test_milvus_collection" \ No newline at end of file diff --git a/dockers/llm.vdb.service/.env_s3_template b/dockers/llm.vdb.service/.env_s3_template index 4dde0cf..d0a8542 100644 --- a/dockers/llm.vdb.service/.env_s3_template +++ b/dockers/llm.vdb.service/.env_s3_template @@ -12,5 +12,5 @@ AWS_SECRET_ACCESS_KEY=my-secret-key # EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 # Milvus Vector DB Optional Settings -# MILVUS_URI="./milvus_example.db" +# MILVUS_URI="http://localhost:19530" # MILVUS_COLLECTION_NAME="test_milvus_collection" \ No newline at end of file diff --git a/dockers/llm.vdb.service/common.py b/dockers/llm.vdb.service/common.py index 088a099..9dc01ba 100644 --- a/dockers/llm.vdb.service/common.py +++ b/dockers/llm.vdb.service/common.py @@ -104,56 +104,12 @@ def create_milvus_vectordb_from_data( embeddings = HuggingFaceEmbeddings(model_name=embedding_model_name) print("Convert to Milvus vectorstore") - # vectorstore = Milvus( - # embedding_function=embeddings, - # vector_field=["dense", "sparse"], - # builtin_function=BM25BuiltInFunction(), - # collection_name=collection_name, - # connection_args={"uri": milvus_uri}, - # # TODO: these index settings are required for local setup - # # make sure it also makes sense for other setups - # index_params=[ - # { - # "field_name": "dense", - # "index_type": "IVF_FLAT", - # "metric_type": "L2", - # "params": {"nlist": 1024} - # }, - # { - # "field_name": "sparse", - # "index_type": "IVF_FLAT", - # "metric_type": "L2", - # "params": {"nlist": 1024} - # } - # ], - # auto_id=True, - # ) - - from langchain_openai import OpenAIEmbeddings - - vectorstore = Milvus.from_documents( - documents=docs, - embedding=OpenAIEmbeddings(), - builtin_function=BM25BuiltInFunction(), + vectorstore = Milvus( + embedding_function=embeddings, vector_field=["dense", "sparse"], - connection_args={ - "uri": milvus_uri, - }, - index_params=[ - { - "field_name": "dense", - "index_type": "IVF_FLAT", - "metric_type": "L2", - "params": {"nlist": 1024} - }, - { - "field_name": "sparse", - "index_type": "IVF_FLAT", - "metric_type": "L2", - "params": {"nlist": 1024} - } - ], - consistency_level="Strong", + builtin_function=BM25BuiltInFunction(), + collection_name=collection_name, + connection_args={"uri": milvus_uri}, auto_id=True, ) diff --git a/dockers/llm.vdb.service/createvectordb_test.py b/dockers/llm.vdb.service/createvectordb_test.py index 21726c1..759a8c9 100644 --- a/dockers/llm.vdb.service/createvectordb_test.py +++ b/dockers/llm.vdb.service/createvectordb_test.py @@ -2,10 +2,9 @@ import os import pytest import s3fs +import subprocess from botocore.session import Session -from langchain_huggingface import HuggingFaceEmbeddings -from langchain_milvus import Milvus from moto.moto_server.threaded_moto_server import ThreadedMotoServer from s3fs.core import S3FileSystem @@ -25,26 +24,25 @@ def test_create_faiss_vector_db_using_local_files(): os.remove("test_data/output/output_pickled.obj") -def test_create_milvus_vector_db_using_local_files(): +@pytest.fixture(scope="module") +def standalone_environment(): + # Start the standalone environment before tests + try: + subprocess.run(["bash", "standalone_embed.sh", "start"], check=True) + yield + finally: + # Stop the standalone environment after tests, even if tests fail + subprocess.run(["bash", "standalone_embed.sh", "stop"], check=True) + subprocess.run(["bash", "standalone_embed.sh", "delete"], check=True) + + +def test_create_milvus_vector_db_using_local_files(standalone_environment): ctx = click.Context(run) try: ctx.forward(run, env_file="test_data/.env_local_milvus") except SystemExit as e: assert e.code == 0 - URI = "./milvus_example.db" - embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2") - vector_store_loaded = Milvus( - embeddings, - connection_args={"uri": URI}, - collection_name="langchain_example", - ) - - vector_store_loaded.similarity_search_with_score("Enhance error tracking?", k=2) - - if os.path.exists("milvus_example.db"): - os.remove("milvus_example.db") - @pytest.fixture(scope="module") def s3_base(): diff --git a/dockers/llm.vdb.service/test_data/.env_local b/dockers/llm.vdb.service/test_data/.env_local index 8c3fa71..dd22e3e 100644 --- a/dockers/llm.vdb.service/test_data/.env_local +++ b/dockers/llm.vdb.service/test_data/.env_local @@ -8,5 +8,5 @@ OUTPUT_FILENAME=test_data/output/output_pickled.obj # EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 # Milvus Vector DB Optional Settings -# MILVUS_URI="./milvus_example.db" +# MILVUS_URI="http://localhost:19530" # MILVUS_COLLECTION_NAME="test_milvus_collection" \ No newline at end of file diff --git a/dockers/llm.vdb.service/test_data/.env_local_milvus b/dockers/llm.vdb.service/test_data/.env_local_milvus index ff1abb6..f4a6b45 100644 --- a/dockers/llm.vdb.service/test_data/.env_local_milvus +++ b/dockers/llm.vdb.service/test_data/.env_local_milvus @@ -8,5 +8,5 @@ OUTPUT_FILENAME=test_data/output/output_pickled.obj # EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 # Milvus Vector DB Optional Settings -MILVUS_URI="./milvus_example.db" +MILVUS_URI="http://localhost:19530" MILVUS_COLLECTION_NAME="test_milvus_collection" \ No newline at end of file diff --git a/dockers/llm.vdb.service/test_data/.env_s3 b/dockers/llm.vdb.service/test_data/.env_s3 index bb5eddb..d9b3406 100644 --- a/dockers/llm.vdb.service/test_data/.env_s3 +++ b/dockers/llm.vdb.service/test_data/.env_s3 @@ -13,5 +13,5 @@ AWS_REGION=us-east-2 # EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 # Milvus Vector DB Optional Settings -# MILVUS_URI="./milvus_example.db" +# MILVUS_URI="http://localhost:19530" # MILVUS_COLLECTION_NAME="test_milvus_collection" \ No newline at end of file From f98d5e240964f5f5f3b4ae83e1d57fe1df6ad460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Tue, 4 Feb 2025 13:46:04 +0000 Subject: [PATCH 10/13] Fix settings check --- dockers/llm.vdb.service/createvectordb.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dockers/llm.vdb.service/createvectordb.py b/dockers/llm.vdb.service/createvectordb.py index aa93e27..0c938b3 100644 --- a/dockers/llm.vdb.service/createvectordb.py +++ b/dockers/llm.vdb.service/createvectordb.py @@ -15,11 +15,13 @@ def run(env_file: str): s3_settings, local_settings = try_load_settings(env_file) if s3_settings: + if s3_settings.milvus_uri and s3_settings.milvus_collection_name: + raise "Missing config" service = S3VectorDbCreationService(s3_settings) service.create() elif local_settings: - if local_settings: + if local_settings.milvus_uri and local_settings.milvus_collection_name: service = LocalDirMilvusDbCreationService(local_settings) service.create() else: From 9789698b97b5b7d23b2340998b93a5e2dcdc136a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Wed, 5 Feb 2025 17:47:10 +0000 Subject: [PATCH 11/13] Configure search --- dockers/llm.rag.service/serveragllm_milvus_local.py | 3 ++- dockers/llm.vdb.service/common.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dockers/llm.rag.service/serveragllm_milvus_local.py b/dockers/llm.rag.service/serveragllm_milvus_local.py index 739d4fc..b0d379b 100644 --- a/dockers/llm.rag.service/serveragllm_milvus_local.py +++ b/dockers/llm.rag.service/serveragllm_milvus_local.py @@ -62,7 +62,8 @@ def setup( retriever = vectorstore.as_retriever( search_kwargs={ "k": relevant_docs, - "metric_type": "L2", + "ranker_type": "weighted", + "ranker_params": {"weights": [0.49, 0.51]}, } ) print("Created Vector DB retriever successfully. \n") diff --git a/dockers/llm.vdb.service/common.py b/dockers/llm.vdb.service/common.py index 9dc01ba..e3bcf2c 100644 --- a/dockers/llm.vdb.service/common.py +++ b/dockers/llm.vdb.service/common.py @@ -110,7 +110,7 @@ def create_milvus_vectordb_from_data( builtin_function=BM25BuiltInFunction(), collection_name=collection_name, connection_args={"uri": milvus_uri}, - auto_id=True, + auto_id=True ) vectorstore.add_documents(documents=docs) From 80a9edadbc1f6caa16e76d8e4165fb43d231c190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Fri, 7 Feb 2025 11:40:41 +0000 Subject: [PATCH 12/13] WIP --- dockers/llm.rag.service/common.py | 20 ++++++++- .../serveragllm_milvus_local.py | 42 +++++++++++++------ 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/dockers/llm.rag.service/common.py b/dockers/llm.rag.service/common.py index 4994ab9..f632954 100644 --- a/dockers/llm.rag.service/common.py +++ b/dockers/llm.rag.service/common.py @@ -3,6 +3,9 @@ import logging +logging.basicConfig(level=logging.DEBUG) + + def format_context(results: List[Dict[str, Any]]) -> str: """Format search results into context for the LLM""" context_parts = [] @@ -44,7 +47,22 @@ def trim_answer(generated_answer: str, label_separator: str) -> str: def get_answer_with_settings(question, retriever, client, model_id, max_tokens, model_temperature, system_prompt): - docs = retriever.invoke(input=question) + search_params = { + "param": { + "metric_type": "L2", + "params": {"nprobe": 10}, + }, + "limit": 5, + "field_names": ["page_content", "metadata"], + "vector_field": ["dense", "sparse"], + "weights": [0.7, 0.2] # Weights for dense and sparse vectors + } + + docs = retriever.get_relevant_documents( + query=question, + search_kwargs=search_params + ) + num_of_docs = len(docs) logging.info(f"Number of relevant documents retrieved and that will be used as context for query: {num_of_docs}") diff --git a/dockers/llm.rag.service/serveragllm_milvus_local.py b/dockers/llm.rag.service/serveragllm_milvus_local.py index b0d379b..8b07302 100644 --- a/dockers/llm.rag.service/serveragllm_milvus_local.py +++ b/dockers/llm.rag.service/serveragllm_milvus_local.py @@ -9,6 +9,8 @@ # "uvicorn", # "weaviate", # "langchain_milvus", +# "langchain-openai", +# "pymilvus" # ] # /// @@ -46,26 +48,42 @@ def setup( app = FastAPI() # TODO: move to imports - from langchain_huggingface import HuggingFaceEmbeddings - from langchain_milvus import Milvus + from langchain_milvus.retrievers import MilvusCollectionHybridSearchRetriever + from langchain_milvus.function import ( + BM25BuiltInFunction, + ) + from langchain.embeddings import HuggingFaceEmbeddings + from pymilvus import connections, Collection, utility, WeightedRanker # TODO: pass through settings or params URI = "http://localhost:19530" + collection_name = "test_milvus_collection" embedding_model_name = "sentence-transformers/all-MiniLM-L6-v2" embeddings = HuggingFaceEmbeddings(model_name=embedding_model_name) - vectorstore = Milvus( - embeddings, - connection_args={"uri": URI}, - collection_name="langchain_example", + + connections.connect( + alias="default", + uri=URI ) - retriever = vectorstore.as_retriever( - search_kwargs={ - "k": relevant_docs, - "ranker_type": "weighted", - "ranker_params": {"weights": [0.49, 0.51]}, - } + # Connect to the existing collection + collection = Collection(collection_name) + collection.load() + + # from langchain_openai import OpenAIEmbeddings + # dense_embedding_func = OpenAIEmbeddings() + + # Initialize the hybrid retriever with both vector fields + retriever = MilvusCollectionHybridSearchRetriever( + collection=collection, + content_field="page_content", # Field containing the document text + anns_fields=["dense", "sparse"], # Both vector fields + metadata_fields=["metadata"], # Include all metadata + field_embeddings=[embeddings, BM25BuiltInFunction()], # You might need to specify how sparse embeddings are handled + # Reranking configuration (optional but resolves validation) + rerank=WeightedRanker(0.5, 0.5), # or provide a reranking method if available ) + print("Created Vector DB retriever successfully. \n") print("Creating an OpenAI client to the hosted model at URL: ", llm_server_url) From 157c589a9aed384121b874fb6539689aadd82983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Fri, 7 Feb 2025 13:06:42 +0000 Subject: [PATCH 13/13] Haystack experiment --- dockers/llm.vdb.service/haystack_milvus.py | 98 ++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 dockers/llm.vdb.service/haystack_milvus.py diff --git a/dockers/llm.vdb.service/haystack_milvus.py b/dockers/llm.vdb.service/haystack_milvus.py new file mode 100644 index 0000000..49b5812 --- /dev/null +++ b/dockers/llm.vdb.service/haystack_milvus.py @@ -0,0 +1,98 @@ +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "haystack", +# "fastembed-haystack", +# "milvus-haystack", +# "pymilvus", +# "sentence-transformers>=3.0.0", +# ] +# /// +import os + +from haystack import Document, Pipeline +from haystack.utils import Secret +from haystack.components.embedders import HuggingFaceAPIDocumentEmbedder, HuggingFaceAPITextEmbedder, OpenAIDocumentEmbedder, OpenAITextEmbedder +from haystack.components.writers import DocumentWriter +from haystack.document_stores.types import DuplicatePolicy +from haystack_integrations.components.embedders.fastembed import ( + FastembedSparseDocumentEmbedder, + FastembedSparseTextEmbedder, +) + +from milvus_haystack import MilvusDocumentStore, MilvusHybridRetriever + +token = "xxx" +os.environ["HUGGINGFACE_HUB_TOKEN"] = token + +document_store = MilvusDocumentStore( + connection_args={"uri": "./milvus.db"}, + drop_old=True, + sparse_vector_field="sparse_vector", # Specify a name of the sparse vector field to enable hybrid retrieval. +) + +documents = [ + Document(content="My name is Wolfgang and I live in Berlin"), + Document(content="I saw a black horse running"), + Document(content="Germany has many big cities"), + Document(content="fastembed is supported by and maintained by Milvus."), +] + +writer = DocumentWriter(document_store=document_store, policy=DuplicatePolicy.OVERWRITE) + +# dense_embedder = OpenAIDocumentEmbedder() +dense_embedder = HuggingFaceAPIDocumentEmbedder( + api_type="text_embeddings_inference", + api_params={ + "model": "sentence-transformers/all-MiniLM-L6-v2", + "url": "http://localhost:11434", + }, + token=Secret.from_token(token), +) + +indexing_pipeline = Pipeline() +indexing_pipeline.add_component("sparse_doc_embedder", FastembedSparseDocumentEmbedder()) +indexing_pipeline.add_component("dense_doc_embedder", dense_embedder) +indexing_pipeline.add_component("writer", writer) +indexing_pipeline.connect("sparse_doc_embedder", "dense_doc_embedder") +indexing_pipeline.connect("dense_doc_embedder", "writer") + +indexing_pipeline.run({"sparse_doc_embedder": {"documents": documents}}) + +querying_pipeline = Pipeline() +querying_pipeline.add_component("sparse_text_embedder", + FastembedSparseTextEmbedder(model="prithvida/Splade_PP_en_v1")) + +# dense_text_embedder = OpenAITextEmbedder() +dense_text_embedder = HuggingFaceAPITextEmbedder( + api_type="text_embeddings_inference", + api_params={ + "model": "sentence-transformers/all-MiniLM-L6-v2", + "url": "http://localhost:11434", + }, + token=Secret.from_token(token), +) + +querying_pipeline.add_component("dense_text_embedder", dense_text_embedder) +querying_pipeline.add_component( + "retriever", + MilvusHybridRetriever( + document_store=document_store, + # reranker=WeightedRanker(0.5, 0.5), # Default is RRFRanker() + ) +) + +querying_pipeline.connect("sparse_text_embedder.sparse_embedding", "retriever.query_sparse_embedding") +querying_pipeline.connect("dense_text_embedder.embedding", "retriever.query_embedding") + +question = "Who supports fastembed?" + +results = querying_pipeline.run( + {"dense_text_embedder": {"text": question}, + "sparse_text_embedder": {"text": question}} +) + +print(results) +print(results["retriever"]["documents"][0]) + +# Document(id=..., content: 'fastembed is supported by and maintained by Milvus.', embedding: vector of size 1536, sparse_embedding: vector with 48 non-zero elements)