From b95f7e457d786d3af7ca6d27ec1985f49f4d204e Mon Sep 17 00:00:00 2001 From: John Bencina Date: Sat, 17 May 2025 15:29:26 -0700 Subject: [PATCH 01/12] Update settings path data type annotation --- src/vecsync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vecsync/settings.py b/src/vecsync/settings.py index d499d0f..d259424 100644 --- a/src/vecsync/settings.py +++ b/src/vecsync/settings.py @@ -21,7 +21,7 @@ class SettingData(BaseModel): class Settings: - def __init__(self, path: str | None = None): + def __init__(self, path: Path | None = None): self.file = path or Path(user_config_dir("vecsync")) / "settings.json" if not self.file.exists(): From d9a4658e733b63ee34a4457981132ec00ad70ef7 Mon Sep 17 00:00:00 2001 From: John Bencina Date: Sat, 17 May 2025 15:30:01 -0700 Subject: [PATCH 02/12] Enable settings file path change in constructor --- src/vecsync/chat/clients/openai.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/vecsync/chat/clients/openai.py b/src/vecsync/chat/clients/openai.py index fc14faf..947ac27 100644 --- a/src/vecsync/chat/clients/openai.py +++ b/src/vecsync/chat/clients/openai.py @@ -102,14 +102,17 @@ class OpenAIClient: store_name : str The name of the vector store to use for this client. The named assistant will be created in the form of "vecsync-{store_name}". - + settings_path : str | None + The path to the settings file. If None, the default settings file will be used. + This is used to store the thread ID for the current conversation. """ - def __init__(self, store_name: str): + def __init__(self, store_name: str, settings_path: str | None = None): self.client = OpenAI() self.store_name = store_name self.assistant_name = f"vecsync-{store_name}" self.connected = False + self.settings_path = settings_path def connect(self): """Connect to the OpenAI API and load the assistant and thread. @@ -156,7 +159,7 @@ def _get_thread_id(self) -> str: str The thread ID for the current conversation. """ - settings = Settings() + settings = Settings(path=self.settings_path) # TODO: Ideally we would grab the thread ID from OpenAI but there doesn't seem to be # a way to do that. So we are storing it in the settings file for now. @@ -232,7 +235,7 @@ def _create_assistant(self) -> str: model="gpt-4o-mini", ) - settings = Settings() + settings = Settings(path=self.settings_path) del settings["openai_thread_id"] print(f"🖥️ Assistant created: {assistant.name}") @@ -253,7 +256,7 @@ def _create_thread(self) -> str: thread = self.client.beta.threads.create() print(f"💬 Conversation started: {thread.id}") - settings = Settings() + settings = Settings(path=self.settings_path) settings["openai_thread_id"] = thread.id return thread.id From 31de13a238d66aff81f48ac22d98bc5cc13e31d7 Mon Sep 17 00:00:00 2001 From: John Bencina Date: Sat, 17 May 2025 15:30:17 -0700 Subject: [PATCH 03/12] Update UV lock --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 422d9cc..96655d1 100644 --- a/uv.lock +++ b/uv.lock @@ -278,7 +278,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload_time = "2025-05-10T17:42:51.123Z" } wheels = [ From 538d213be4b12917113280ef47c63c07ae8c131d Mon Sep 17 00:00:00 2001 From: John Bencina Date: Sat, 17 May 2025 15:31:47 -0700 Subject: [PATCH 04/12] Inital commit of OAI tests --- tests/chat/clients/test_openai.py | 231 ++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 tests/chat/clients/test_openai.py diff --git a/tests/chat/clients/test_openai.py b/tests/chat/clients/test_openai.py new file mode 100644 index 0000000..85c8c0c --- /dev/null +++ b/tests/chat/clients/test_openai.py @@ -0,0 +1,231 @@ +from datetime import datetime +from types import SimpleNamespace + +import pytest +from pydantic import BaseModel + +import vecsync.chat.clients.openai as client_mod +from vecsync.chat.clients.openai import OpenAIClient +from vecsync.settings import Settings +from vecsync.store.openai import OpenAiVectorStore + + +class MockAssistant(BaseModel): + id: str + name: str + + +class MockThread(BaseModel): + id: str + + +class MockMessageContentText(BaseModel): + value: str + + +class MockMessageContent(BaseModel): + type: str + text: MockMessageContentText + + +class MockMessageData(BaseModel): + content: MockMessageContent + + +class MockMessage(BaseModel): + created_at: int + data: MockMessageData + thread_id: str + + +class MockVectorStore(BaseModel): + id: str + name: str + + +def mock_vector_store(): + vector_store = [] + file_store = [] + vector_file_store = [] + + def create_vector_store(name): + store = MockVectorStore(id=f"vector_store_{len(vector_store) + 1}", name=name) + vector_store.append(store) + return store + + def list_vector_stores(): + return vector_store + + def list_files(): + return file_store + + def list_vector_store_files(vector_store_id): + return vector_file_store + + def delete_vector_store_file(vector_store_id, file_id): + for vector_file in vector_file_store: + if vector_file.id == file_id: + vector_file_store.remove(vector_file) + + def delete_file(file_id): + for file in file_store: + if file.id == file_id: + file_store.remove(file) + + # attach methods + vs_files_ns = SimpleNamespace() + vs_files_ns.list = list_vector_store_files + vs_files_ns.delete = delete_vector_store_file + + stores_ns = SimpleNamespace() + stores_ns.create = create_vector_store + stores_ns.list = list_vector_stores + stores_ns.files = vs_files_ns + + files_ns = SimpleNamespace() + files_ns.list = list_files + files_ns.delete = delete_file + + # build your “client” + client = SimpleNamespace() + client.vector_stores = stores_ns + client.files = files_ns + + return client + + +def mock_client_backend(): + # our in‐memory store + assistant_store = [] + threads_store = [] + message_store = [] + + def create_assistant(**kwargs): + name = kwargs["name"] + assistant = MockAssistant(id=f"assistant_{name}_{len(assistant_store) + 1}", name=name) + assistant_store.append(assistant) + return assistant + + def list_assistants(): + return assistant_store + + def delete_assistant(assistant_id): + for assistant in assistant_store: + if assistant.id == assistant_id: + assistant_store.remove(assistant) + + def create_thread(**kwargs): + thread = MockThread(id=f"thread_{len(threads_store) + 1}") + threads_store.append(thread) + return thread + + def create_message(**kwargs): + created_at = int(datetime.now().timestamp()) + message = MockMessage( + created_at=created_at, + data=MockMessageData( + content=MockMessageContent(type="text", text=MockMessageContentText(value=kwargs["content"])) + ), + thread_id=kwargs["thread_id"], + ) + message_store.append(message) + return message + + # attach methods + assistants_ns = SimpleNamespace() + assistants_ns.create = create_assistant + assistants_ns.list = list_assistants + assistants_ns.delete = delete_assistant + + threads_ns = SimpleNamespace() + threads_ns.create = create_thread + + # build your “client” + client = SimpleNamespace() + client.beta = SimpleNamespace() + client.beta.assistants = assistants_ns + client.beta.threads = threads_ns + + return client + + +@pytest.fixture +def mocked_vector_store(): + store = OpenAiVectorStore(name="test_store") + store.client = mock_vector_store() + store.create() + return store + + +@pytest.fixture +def mocked_client(tmp_path, mocked_vector_store, monkeypatch): + monkeypatch.setattr(client_mod, "OpenAiVectorStore", lambda store_name: mocked_vector_store) + + settings_path = tmp_path / "settings.json" + client = OpenAIClient(store_name="test_store", settings_path=settings_path) + client.client = mock_client_backend() + + return client + + +def test_list_assistants(mocked_client): + mocked_client.client.beta.assistants.create(name="vecsync-1") + mocked_client.client.beta.assistants.create(name="vecsync-2") + mocked_client.client.beta.assistants.create(name="other-3") + + assistants = mocked_client.list_assistants() + + assert len(assistants) == 2 + + +def test_delete_assistant(mocked_client): + assistant = mocked_client.client.beta.assistants.create(name="vecsync-1") + mocked_client.client.beta.assistants.create(name="vecsync-2") + + mocked_client.delete_assistant(assistant.id) + + assert len(mocked_client.list_assistants()) == 1 + + +def test_create_thread(mocked_client): + thread_id = mocked_client._create_thread() + assert thread_id == "thread_1" + + +def test_get_thread_id_new(mocked_client): + thread_id = mocked_client._get_thread_id() + assert thread_id == "thread_1" + + +def test_get_thread_id_existing(mocked_client): + settings = Settings(path=mocked_client.settings_path) + settings["openai_thread_id"] = "thread_2" + + thread_id = mocked_client._get_thread_id() + assert thread_id == "thread_2" + + +def test_create_assistant(mocked_client, mocked_vector_store): + mocked_client.vector_store = mocked_vector_store + id = mocked_client._create_assistant() + assert id == "assistant_vecsync-test_store_1" + + +def test_get_assistant_id_new(mocked_client, mocked_vector_store): + mocked_client.vector_store = mocked_vector_store + assistant_id = mocked_client._get_assistant_id() + assert assistant_id == "assistant_vecsync-test_store_1" + + +def test_get_assistant_id_existing(mocked_client): + mocked_client.client.beta.assistants.create(name="vecsync-1") + assistant_id = mocked_client._get_assistant_id() + assert assistant_id == "assistant_vecsync-1_1" + + +def test_get_assistant_id_multiple(mocked_client): + mocked_client.client.beta.assistants.create(name="vecsync-1") + mocked_client.client.beta.assistants.create(name="vecsync-2") + assistant_id = mocked_client._get_assistant_id() + assert assistant_id == "assistant_vecsync-1_1" + assert len(mocked_client.client.beta.assistants.list()) == 1 From 5c73fc96500af8b13e80c58ababed660e53388f2 Mon Sep 17 00:00:00 2001 From: John Bencina Date: Sun, 18 May 2025 09:44:22 -0700 Subject: [PATCH 05/12] Add message checks --- tests/chat/clients/test_openai.py | 44 ++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/tests/chat/clients/test_openai.py b/tests/chat/clients/test_openai.py index 85c8c0c..ed60d39 100644 --- a/tests/chat/clients/test_openai.py +++ b/tests/chat/clients/test_openai.py @@ -29,13 +29,20 @@ class MockMessageContent(BaseModel): class MockMessageData(BaseModel): - content: MockMessageContent + content: list[MockMessageContent] + created_at: int # TODO: Check if this is really at both levels + role: str class MockMessage(BaseModel): - created_at: int data: MockMessageData thread_id: str + created_at: int + + +class MockThreadMessageResponse(BaseModel): + thread_id: str + data: list[MockMessageData] class MockVectorStore(BaseModel): @@ -121,24 +128,37 @@ def create_thread(**kwargs): def create_message(**kwargs): created_at = int(datetime.now().timestamp()) + message = MockMessage( created_at=created_at, data=MockMessageData( - content=MockMessageContent(type="text", text=MockMessageContentText(value=kwargs["content"])) + created_at=created_at, + content=[MockMessageContent(type="text", text=MockMessageContentText(value=kwargs["content"]))], + role=kwargs["role"], ), thread_id=kwargs["thread_id"], ) message_store.append(message) return message + def list_messages(**kwargs): + thread_id = kwargs["thread_id"] + messages = [message.data for message in message_store if message.thread_id == thread_id] + return MockThreadMessageResponse(thread_id=thread_id, data=messages) + # attach methods assistants_ns = SimpleNamespace() assistants_ns.create = create_assistant assistants_ns.list = list_assistants assistants_ns.delete = delete_assistant + messages_ns = SimpleNamespace() + messages_ns.create = create_message + messages_ns.list = list_messages + threads_ns = SimpleNamespace() threads_ns.create = create_thread + threads_ns.messages = messages_ns # build your “client” client = SimpleNamespace() @@ -229,3 +249,21 @@ def test_get_assistant_id_multiple(mocked_client): assistant_id = mocked_client._get_assistant_id() assert assistant_id == "assistant_vecsync-1_1" assert len(mocked_client.client.beta.assistants.list()) == 1 + + +def test_load_history_none(mocked_client): + history = mocked_client.load_history() + assert history == [] + + +def test_load_history_valid(mocked_client): + mocked_client.send_message("Hello") + mocked_client.send_message("World") + mocked_client.client.beta.threads.messages.create( + thread_id=mocked_client.thread_id, role="assistant", content="Response" + ) + + history = mocked_client.load_history() + + assert len(history) == 3 + assert [x["role"] for x in history] == ["user", "user", "assistant"] From a364ad9beb457aa17b93de1d4835aedff75474e8 Mon Sep 17 00:00:00 2001 From: John Bencina Date: Sun, 18 May 2025 10:58:22 -0700 Subject: [PATCH 06/12] Update remote_count setting --- src/vecsync/cli.py | 2 +- src/vecsync/cli/sync.py | 2 +- src/vecsync/store/openai.py | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/vecsync/cli.py b/src/vecsync/cli.py index d0b4b47..e4573c2 100644 --- a/src/vecsync/cli.py +++ b/src/vecsync/cli.py @@ -77,7 +77,7 @@ def sync(source: str): f"Saved: {result.files_saved} | Deleted: {result.files_deleted} | Skipped: {result.files_skipped} ", "yellow", ) - cprint(f"Remote count: {result.updated_count}", "yellow") + cprint(f"Remote count: {result.remote_count}", "yellow") cprint(f"Duration: {result.duration:.2f} seconds", "yellow") diff --git a/src/vecsync/cli/sync.py b/src/vecsync/cli/sync.py index 8dfb7a2..dea583d 100644 --- a/src/vecsync/cli/sync.py +++ b/src/vecsync/cli/sync.py @@ -40,5 +40,5 @@ def sync(source: str): f"Saved: {result.files_saved} | Deleted: {result.files_deleted} | Skipped: {result.files_skipped} ", "yellow", ) - cprint(f"Remote count: {result.updated_count}", "yellow") + cprint(f"Remote count: {result.remote_count}", "yellow") cprint(f"Duration: {result.duration:.2f} seconds", "yellow") diff --git a/src/vecsync/store/openai.py b/src/vecsync/store/openai.py index 551d414..d5da208 100644 --- a/src/vecsync/store/openai.py +++ b/src/vecsync/store/openai.py @@ -13,7 +13,7 @@ class SyncOperationResult(BaseModel): files_saved: int files_deleted: int files_skipped: int - updated_count: int + remote_count: int duration: float @@ -88,8 +88,9 @@ def _delete_files(self, files_to_remove: list[str]) -> set[str]: removed_file_ids = [] for file_id in tqdm(files_to_remove): self.client.vector_stores.files.delete(vector_store_id=self.store.id, file_id=file_id) - self.client.files.delete(file_id=file_id) - removed_file_ids.append(file_id) + result = self.client.files.delete(file_id=file_id) + if result.deleted: + removed_file_ids.append(file_id) return set(removed_file_ids) @@ -147,6 +148,6 @@ def sync(self, files: list[Path]): files_saved=len(files_to_upload), files_deleted=len(files_to_remove), files_skipped=len(duplicate_file_names), - updated_count=len(existing_vector_file_ids | files_to_attach), + remote_count=len(existing_vector_file_ids | files_to_attach), duration=duration, ) From c3e095fb9445da53ca50ed36d4abb003b68ec27b Mon Sep 17 00:00:00 2001 From: John Bencina Date: Sun, 18 May 2025 11:00:12 -0700 Subject: [PATCH 07/12] Update test case coverage for open AI --- .gitignore | 1 + .../test_openai.py => openai/conftest.py} | 135 +++++++----------- tests/openai/test_openai_chat.py | 82 +++++++++++ tests/openai/test_openai_store.py | 96 +++++++++++++ 4 files changed, 234 insertions(+), 80 deletions(-) rename tests/{chat/clients/test_openai.py => openai/conftest.py} (64%) create mode 100644 tests/openai/test_openai_chat.py create mode 100644 tests/openai/test_openai_store.py diff --git a/.gitignore b/.gitignore index 583cd7d..7a3c618 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ wheels/ # Development files .coverage +.coverage.* .env *.pdf diff --git a/tests/chat/clients/test_openai.py b/tests/openai/conftest.py similarity index 64% rename from tests/chat/clients/test_openai.py rename to tests/openai/conftest.py index ed60d39..e23d7c6 100644 --- a/tests/chat/clients/test_openai.py +++ b/tests/openai/conftest.py @@ -1,12 +1,13 @@ +import os from datetime import datetime from types import SimpleNamespace +from typing import Any import pytest from pydantic import BaseModel import vecsync.chat.clients.openai as client_mod from vecsync.chat.clients.openai import OpenAIClient -from vecsync.settings import Settings from vecsync.store.openai import OpenAiVectorStore @@ -50,6 +51,24 @@ class MockVectorStore(BaseModel): name: str +class MockFileUpload(BaseModel): + id: str + file: Any + + +class MockFile(BaseModel): + id: str + filename: str + + +class MockFileDeletedResult(BaseModel): + deleted: bool + + +class MockVectorStoreDeletedResult(BaseModel): + deleted: bool + + def mock_vector_store(): vector_store = [] file_store = [] @@ -60,6 +79,13 @@ def create_vector_store(name): vector_store.append(store) return store + def delete_vector_store(vector_store_id): + for store in vector_store: + if store.id == vector_store_id: + vector_store.remove(store) + return MockFileDeletedResult(deleted=True) + return MockFileDeletedResult(deleted=False) + def list_vector_stores(): return vector_store @@ -78,20 +104,39 @@ def delete_file(file_id): for file in file_store: if file.id == file_id: file_store.remove(file) + return MockFileDeletedResult(deleted=True) + return MockFileDeletedResult(deleted=False) + + def create_file(**kwargs): + base_name = os.path.basename(kwargs["file"].name) + file = MockFileUpload(id=f"file_{len(file_store) + 1}", file=kwargs["file"]) + file_store.append(MockFile(id=file.id, filename=base_name)) + return file + + def create_and_poll(vector_store_id, file_id): + for store in vector_store: + if store.id == vector_store_id: + vector_file = MockFile(id=file_id, filename=f"file_{file_id}") + vector_file_store.append(vector_file) + return vector_file + return None # attach methods vs_files_ns = SimpleNamespace() vs_files_ns.list = list_vector_store_files vs_files_ns.delete = delete_vector_store_file + vs_files_ns.create_and_poll = create_and_poll stores_ns = SimpleNamespace() stores_ns.create = create_vector_store + stores_ns.delete = delete_vector_store stores_ns.list = list_vector_stores stores_ns.files = vs_files_ns files_ns = SimpleNamespace() files_ns.list = list_files files_ns.delete = delete_file + files_ns.create = create_file # build your “client” client = SimpleNamespace() @@ -188,82 +233,12 @@ def mocked_client(tmp_path, mocked_vector_store, monkeypatch): return client -def test_list_assistants(mocked_client): - mocked_client.client.beta.assistants.create(name="vecsync-1") - mocked_client.client.beta.assistants.create(name="vecsync-2") - mocked_client.client.beta.assistants.create(name="other-3") - - assistants = mocked_client.list_assistants() - - assert len(assistants) == 2 - - -def test_delete_assistant(mocked_client): - assistant = mocked_client.client.beta.assistants.create(name="vecsync-1") - mocked_client.client.beta.assistants.create(name="vecsync-2") - - mocked_client.delete_assistant(assistant.id) - - assert len(mocked_client.list_assistants()) == 1 - - -def test_create_thread(mocked_client): - thread_id = mocked_client._create_thread() - assert thread_id == "thread_1" - - -def test_get_thread_id_new(mocked_client): - thread_id = mocked_client._get_thread_id() - assert thread_id == "thread_1" - - -def test_get_thread_id_existing(mocked_client): - settings = Settings(path=mocked_client.settings_path) - settings["openai_thread_id"] = "thread_2" - - thread_id = mocked_client._get_thread_id() - assert thread_id == "thread_2" - - -def test_create_assistant(mocked_client, mocked_vector_store): - mocked_client.vector_store = mocked_vector_store - id = mocked_client._create_assistant() - assert id == "assistant_vecsync-test_store_1" - - -def test_get_assistant_id_new(mocked_client, mocked_vector_store): - mocked_client.vector_store = mocked_vector_store - assistant_id = mocked_client._get_assistant_id() - assert assistant_id == "assistant_vecsync-test_store_1" - - -def test_get_assistant_id_existing(mocked_client): - mocked_client.client.beta.assistants.create(name="vecsync-1") - assistant_id = mocked_client._get_assistant_id() - assert assistant_id == "assistant_vecsync-1_1" - - -def test_get_assistant_id_multiple(mocked_client): - mocked_client.client.beta.assistants.create(name="vecsync-1") - mocked_client.client.beta.assistants.create(name="vecsync-2") - assistant_id = mocked_client._get_assistant_id() - assert assistant_id == "assistant_vecsync-1_1" - assert len(mocked_client.client.beta.assistants.list()) == 1 - - -def test_load_history_none(mocked_client): - history = mocked_client.load_history() - assert history == [] - - -def test_load_history_valid(mocked_client): - mocked_client.send_message("Hello") - mocked_client.send_message("World") - mocked_client.client.beta.threads.messages.create( - thread_id=mocked_client.thread_id, role="assistant", content="Response" - ) - - history = mocked_client.load_history() - - assert len(history) == 3 - assert [x["role"] for x in history] == ["user", "user", "assistant"] +@pytest.fixture +def create_test_upload(tmp_path): + files = set() + for i in range(3): + file = tmp_path / f"test_file_{i}.txt" + files.add(file) + with open(file, "w") as f: + f.write(f"This is test file {i}") + return files diff --git a/tests/openai/test_openai_chat.py b/tests/openai/test_openai_chat.py new file mode 100644 index 0000000..8f05a53 --- /dev/null +++ b/tests/openai/test_openai_chat.py @@ -0,0 +1,82 @@ +from vecsync.settings import Settings + + +def test_list_assistants(mocked_client): + mocked_client.client.beta.assistants.create(name="vecsync-1") + mocked_client.client.beta.assistants.create(name="vecsync-2") + mocked_client.client.beta.assistants.create(name="other-3") + + assistants = mocked_client.list_assistants() + + assert len(assistants) == 2 + + +def test_delete_assistant(mocked_client): + assistant = mocked_client.client.beta.assistants.create(name="vecsync-1") + mocked_client.client.beta.assistants.create(name="vecsync-2") + + mocked_client.delete_assistant(assistant.id) + + assert len(mocked_client.list_assistants()) == 1 + + +def test_create_thread(mocked_client): + thread_id = mocked_client._create_thread() + assert thread_id == "thread_1" + + +def test_get_thread_id_new(mocked_client): + thread_id = mocked_client._get_thread_id() + assert thread_id == "thread_1" + + +def test_get_thread_id_existing(mocked_client): + settings = Settings(path=mocked_client.settings_path) + settings["openai_thread_id"] = "thread_2" + + thread_id = mocked_client._get_thread_id() + assert thread_id == "thread_2" + + +def test_create_assistant(mocked_client, mocked_vector_store): + mocked_client.vector_store = mocked_vector_store + id = mocked_client._create_assistant() + assert id == "assistant_vecsync-test_store_1" + + +def test_get_assistant_id_new(mocked_client, mocked_vector_store): + mocked_client.vector_store = mocked_vector_store + assistant_id = mocked_client._get_assistant_id() + assert assistant_id == "assistant_vecsync-test_store_1" + + +def test_get_assistant_id_existing(mocked_client): + mocked_client.client.beta.assistants.create(name="vecsync-1") + assistant_id = mocked_client._get_assistant_id() + assert assistant_id == "assistant_vecsync-1_1" + + +def test_get_assistant_id_multiple(mocked_client): + mocked_client.client.beta.assistants.create(name="vecsync-1") + mocked_client.client.beta.assistants.create(name="vecsync-2") + assistant_id = mocked_client._get_assistant_id() + assert assistant_id == "assistant_vecsync-1_1" + assert len(mocked_client.client.beta.assistants.list()) == 1 + + +def test_load_history_none(mocked_client): + history = mocked_client.load_history() + assert history == [] + + +def test_load_history_valid(mocked_client): + mocked_client.send_message("Hello") + mocked_client.send_message("World") + mocked_client.client.beta.threads.messages.create( + thread_id=mocked_client.thread_id, role="assistant", content="Response" + ) + + history = mocked_client.load_history() + + assert len(history) == 3 + assert [x["role"] for x in history] == ["user", "user", "assistant"] diff --git a/tests/openai/test_openai_store.py b/tests/openai/test_openai_store.py new file mode 100644 index 0000000..b96e936 --- /dev/null +++ b/tests/openai/test_openai_store.py @@ -0,0 +1,96 @@ +import pytest + + +def test_get_files_none(mocked_vector_store): + files = mocked_vector_store.get_files() + assert len(files) == 0 + + +def test_get_valid_store(mocked_vector_store): + store = mocked_vector_store.get() + assert store.name == "test_store" + assert store.id == "vector_store_1" + + +def test_get_invalid_store(mocked_vector_store): + mocked_vector_store.name = "invalid_store" + with pytest.raises(ValueError): + mocked_vector_store.get() + + +def test_get_files_empty(mocked_vector_store): + mocked_vector_store.get() + files = mocked_vector_store.get_files() + assert len(files) == 0 + + +def test_get_files_existing(mocked_vector_store, create_test_upload): + files_uploaded = mocked_vector_store._upload_files(create_test_upload) + + remote_files = mocked_vector_store.get_files() + + assert len(remote_files) == len(files_uploaded) == 3 + + +def test_delete_files(mocked_vector_store, create_test_upload): + files_uploaded = mocked_vector_store._upload_files(create_test_upload) + assert len(files_uploaded) == 3 + + removed_files = mocked_vector_store._delete_files(files_uploaded) + assert len(removed_files) == 3 + + remote_files = mocked_vector_store.get_files() + assert len(remote_files) == 0 + + +def test_delete_files_invalid(mocked_vector_store): + removed_files = mocked_vector_store._delete_files(["test"]) + assert len(removed_files) == 0 + + remote_files = mocked_vector_store.get_files() + assert len(remote_files) == 0 + + +def test_delete_store(mocked_vector_store): + mocked_vector_store.delete() + assert mocked_vector_store.store is None + + +def test_get(mocked_vector_store): + store = mocked_vector_store.get() + assert store.name == "test_store" + assert store.id == "vector_store_1" + + +def test_attach_files(mocked_vector_store, create_test_upload): + files_uploaded = mocked_vector_store._upload_files(create_test_upload) + assert len(files_uploaded) == 3 + + mocked_vector_store._attach_files(files_uploaded) + + remote_files = mocked_vector_store.get_files() + assert len(remote_files) == 3 + + +def test_sync_files(mocked_vector_store, create_test_upload): + result = mocked_vector_store.sync(create_test_upload) + + assert result.files_saved == 3 + assert result.files_deleted == 0 + assert result.files_skipped == 0 + assert result.remote_count == 3 + assert result.duration > 0 + + +def test_sync_files_with_existing_overlap(mocked_vector_store, create_test_upload): + files = list(create_test_upload) + + result1 = mocked_vector_store.sync(files[:2]) + assert result1.files_saved == 2 + + result2 = mocked_vector_store.sync(files) + assert result2.files_saved == 1 + assert result2.files_deleted == 0 + assert result2.files_skipped == 2 + assert result2.remote_count == 3 + assert result2.duration > 0 From 56dea04a89278cd8a396e3794fb64f7db4188c91 Mon Sep 17 00:00:00 2001 From: John Bencina Date: Mon, 19 May 2025 07:40:12 -0700 Subject: [PATCH 08/12] Add chat response test cases --- src/vecsync/chat/clients/openai.py | 3 +- tests/openai/conftest.py | 61 +++++++++++++++++++++++++++++- tests/openai/test_openai_chat.py | 18 +++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/src/vecsync/chat/clients/openai.py b/src/vecsync/chat/clients/openai.py index 947ac27..e736afa 100644 --- a/src/vecsync/chat/clients/openai.py +++ b/src/vecsync/chat/clients/openai.py @@ -63,7 +63,8 @@ def on_message_delta(self, delta, snapshot): def on_message_done(self, message): # Append citations at the end of the response text = self.formatter.get_references(self.annotations, self.files) - self.queue.put(text) + if len(text) > 0: + self.queue.put(text) self.active = False def consume_queue(self, timeout: float = 1.0): diff --git a/tests/openai/conftest.py b/tests/openai/conftest.py index e23d7c6..df68c77 100644 --- a/tests/openai/conftest.py +++ b/tests/openai/conftest.py @@ -7,7 +7,8 @@ from pydantic import BaseModel import vecsync.chat.clients.openai as client_mod -from vecsync.chat.clients.openai import OpenAIClient +from vecsync.chat.clients.openai import OpenAIClient, OpenAIHandler +from vecsync.chat.formatter import ConsoleFormatter from vecsync.store.openai import OpenAiVectorStore @@ -69,6 +70,25 @@ class MockVectorStoreDeletedResult(BaseModel): deleted: bool +class MockStreamResponseAnnotation(BaseModel): + type: str + text: str + + +class MockStreamResponseText(BaseModel): + value: str + annotations: list[MockStreamResponseAnnotation] + + +class MockStreamResponseContent(BaseModel): + type: str + text: MockStreamResponseText + + +class MockStreamResponse(BaseModel): + content: list[MockStreamResponseContent] + + def mock_vector_store(): vector_store = [] file_store = [] @@ -191,6 +211,33 @@ def list_messages(**kwargs): messages = [message.data for message in message_store if message.thread_id == thread_id] return MockThreadMessageResponse(thread_id=thread_id, data=messages) + def stream_response(**kwargs): + class StreamManager: + def __init__(self, handler): + self.handler = handler + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + def until_done(self): + text = """This is a test messsage from the assistant""" + for delta in text.split(): + message = MockStreamResponse( + content=[ + MockStreamResponseContent( + type="text", text=MockStreamResponseText(value=delta, annotations=[]) + ) + ] + ) + self.handler.on_message_delta(delta=message, snapshot=None) + + self.handler.on_message_done(message=None) + + return StreamManager(handler=kwargs["event_handler"]) + # attach methods assistants_ns = SimpleNamespace() assistants_ns.create = create_assistant @@ -201,9 +248,13 @@ def list_messages(**kwargs): messages_ns.create = create_message messages_ns.list = list_messages + runs_ns = SimpleNamespace() + runs_ns.stream = stream_response + threads_ns = SimpleNamespace() threads_ns.create = create_thread threads_ns.messages = messages_ns + threads_ns.runs = runs_ns # build your “client” client = SimpleNamespace() @@ -233,6 +284,14 @@ def mocked_client(tmp_path, mocked_vector_store, monkeypatch): return client +@pytest.fixture +def mocked_client_handler(): + return OpenAIHandler( + files={"file_1", "filename.txt"}, + formatter=ConsoleFormatter(), + ) + + @pytest.fixture def create_test_upload(tmp_path): files = set() diff --git a/tests/openai/test_openai_chat.py b/tests/openai/test_openai_chat.py index 8f05a53..58dd76e 100644 --- a/tests/openai/test_openai_chat.py +++ b/tests/openai/test_openai_chat.py @@ -80,3 +80,21 @@ def test_load_history_valid(mocked_client): assert len(history) == 3 assert [x["role"] for x in history] == ["user", "user", "assistant"] + + +def test_message(mocked_client, mocked_client_handler): + mocked_client.stream_response(thread_id="", assistant_id="", handler=mocked_client_handler) + + items = [] + while not mocked_client_handler.queue.empty(): + items.append(mocked_client_handler.queue.get_nowait()) + + assert items == ["This", "is", "a", "test", "messsage", "from", "the", "assistant"] + + +def test_consume_queue(mocked_client, mocked_client_handler): + mocked_client.stream_response(thread_id="", assistant_id="", handler=mocked_client_handler) + + items = list(mocked_client_handler.consume_queue()) + + assert items == ["This", "is", "a", "test", "messsage", "from", "the", "assistant"] From e7846c8be63b91a7d0d09b2bbb3a50fd352eedeb Mon Sep 17 00:00:00 2001 From: John Bencina Date: Mon, 19 May 2025 07:43:41 -0700 Subject: [PATCH 09/12] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dacd133..993ef4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) - Commands for `assistants list` and `assistants clean` - Automatic cleanup of any extra assistants in user account when initiating chat - Added docstrings for undocumented functions +- Test case coverage for most OpenAI chat and vector store operations ### Changed - Updated CLI chat command to `vs chat` From bc129c768bb620bd33e98b113bef47d013744e3e Mon Sep 17 00:00:00 2001 From: John Bencina Date: Mon, 19 May 2025 07:43:53 -0700 Subject: [PATCH 10/12] Add dummy key for testing --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f88f3d9..f68f7ff 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,6 +25,8 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.10", "3.11", "3.12", "3.13"] runs-on: ${{ matrix.os }} + env: + OPENAI_API_KEY: dummy_value_for_tests # Dummy value steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 From 89f45ba395d5a31f0585f2585f61fd33fc2e3341 Mon Sep 17 00:00:00 2001 From: John Bencina Date: Mon, 19 May 2025 07:46:26 -0700 Subject: [PATCH 11/12] Cleanup changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 993ef4d..1b42743 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) ## [0.6.0] ### Added -- Commands for `assistants list` and `assistants clean` +- Added CLI commands for `assistants list` and `assistants clean` - Automatic cleanup of any extra assistants in user account when initiating chat - Added docstrings for undocumented functions -- Test case coverage for most OpenAI chat and vector store operations +- Test case coverage for most OpenAI chat and vector store operations [#15](https://github.com/jbencina/vecsync/issues/15) ### Changed - Updated CLI chat command to `vs chat` From f1546ed64adb46bc9f694ec944f90268d8e9e8b6 Mon Sep 17 00:00:00 2001 From: John Bencina Date: Mon, 19 May 2025 07:50:40 -0700 Subject: [PATCH 12/12] Fix typo --- tests/openai/conftest.py | 2 +- tests/openai/test_openai_chat.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/openai/conftest.py b/tests/openai/conftest.py index df68c77..6360fe9 100644 --- a/tests/openai/conftest.py +++ b/tests/openai/conftest.py @@ -223,7 +223,7 @@ def __exit__(self, exc_type, exc_value, traceback): return False def until_done(self): - text = """This is a test messsage from the assistant""" + text = """This is a test message from the assistant""" for delta in text.split(): message = MockStreamResponse( content=[ diff --git a/tests/openai/test_openai_chat.py b/tests/openai/test_openai_chat.py index 58dd76e..1527de3 100644 --- a/tests/openai/test_openai_chat.py +++ b/tests/openai/test_openai_chat.py @@ -89,7 +89,7 @@ def test_message(mocked_client, mocked_client_handler): while not mocked_client_handler.queue.empty(): items.append(mocked_client_handler.queue.get_nowait()) - assert items == ["This", "is", "a", "test", "messsage", "from", "the", "assistant"] + assert items == ["This", "is", "a", "test", "message", "from", "the", "assistant"] def test_consume_queue(mocked_client, mocked_client_handler): @@ -97,4 +97,4 @@ def test_consume_queue(mocked_client, mocked_client_handler): items = list(mocked_client_handler.consume_queue()) - assert items == ["This", "is", "a", "test", "messsage", "from", "the", "assistant"] + assert items == ["This", "is", "a", "test", "message", "from", "the", "assistant"]