From 3ee24e18b97b83bb17a22f5fce73110ea01e1978 Mon Sep 17 00:00:00 2001 From: LatekVon Date: Wed, 3 Jul 2024 23:19:24 +0200 Subject: [PATCH 01/32] add initial deepsearcher project --- schedulers/deepsearcher.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 schedulers/deepsearcher.py diff --git a/schedulers/deepsearcher.py b/schedulers/deepsearcher.py new file mode 100644 index 0000000..928ed33 --- /dev/null +++ b/schedulers/deepsearcher.py @@ -0,0 +1,34 @@ +# The first automatic scheduler. +# When launched, looks at the available data, picks interesting topics and requests deeper searches about them. +from core.databases.db_completion_tasks import CompletionTask +from core.tools.utils import sleep_noisy + + +# get finished tasks +# extract interesting talking points +# repeat until a unique one has been found +# create new tasks out of it + +# we can also make a great use of embedding here +# use very broad embeddings to see which topics align the most with others +# to see how well certain topic is developed so far + +# poc: just get a random summary and dispatch a new one from it +# then: grab a random topic from one of the summaries, and focus on it, +# continuously condensing summaries and requesting them on deeper topics + +def get_random_completion() -> CompletionTask: + pass + + +def extract_interesting_topics() -> list[str]: + pass + + +def schedule_new_search() -> str: + pass + + +def start_deepsearch(): + while True: + sleep_noisy(6) From 993514b9add46df8be16e3320d82295bf3d77946 Mon Sep 17 00:00:00 2001 From: LatekVon Date: Wed, 3 Jul 2024 23:21:38 +0200 Subject: [PATCH 02/32] add completion task getter function --- core/databases/db_completion_tasks.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/core/databases/db_completion_tasks.py b/core/databases/db_completion_tasks.py index e408b3e..9da0c6d 100644 --- a/core/databases/db_completion_tasks.py +++ b/core/databases/db_completion_tasks.py @@ -106,6 +106,22 @@ def db_get_incomplete_completion_tasks(amount: int = 1): return results +def db_get_complete_completion_tasks(amount: int = 1): + with Session(engine) as session: + session.expire_on_commit = False + + query = ( + select(CompletionTask) + .where(CompletionTask.completed == True) + .limit(amount) + ) + + results = list(session.scalars(query).all()) + session.expunge_all() + + return results + + def db_release_executing_tasks(uuid_list: list[str]): with Session(engine) as session: session.execute( From c9796139f55216c410751f1b7fd45c906e3b1217 Mon Sep 17 00:00:00 2001 From: LatekVon Date: Wed, 3 Jul 2024 23:22:31 +0200 Subject: [PATCH 03/32] add troubleshooting note to README --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 9e6d120..1b588a1 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,9 @@ Frontend is launched separately to back end, run the following command to start - `environment.yml` is the linux env, but for macOS (silicon) and windows there are other available - Apple intel is not supported anymore, but you can still get it working by manually installing any missing package that comes up during the program execution. +- `pull access denied for X` error: The connection may occasionally get throttled, resulting in this error. + To solve this issue, let all the current downloads finish downloading, and restart the program. + Repeat until every file is downloaded. ### This is a monorepo for both a tool, and apps for it: From 4a81b849020cc18741674cfddb839630b7b5aab8 Mon Sep 17 00:00:00 2001 From: LatekVon Date: Thu, 4 Jul 2024 23:43:46 +0200 Subject: [PATCH 04/32] remove old scheduler.py --- .github/workflows/black.yml | 4 -- core/chainables/web.py | 1 + workers/scheduler.py | 88 ------------------------------------- 3 files changed, 1 insertion(+), 92 deletions(-) delete mode 100644 workers/scheduler.py diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index ae040ad..6bc4398 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -24,7 +24,3 @@ jobs: uses: cytopia/docker-black@0.8 with: path: 'workers/embedder.py' - - name: Python Black (scheduler) - uses: cytopia/docker-black@0.8 - with: - path: 'workers/scheduler.py' diff --git a/core/chainables/web.py b/core/chainables/web.py index 8489f20..6098ad4 100644 --- a/core/chainables/web.py +++ b/core/chainables/web.py @@ -77,3 +77,4 @@ def web_news_lookup_prompt(): ), ] ) + diff --git a/workers/scheduler.py b/workers/scheduler.py deleted file mode 100644 index 526da4a..0000000 --- a/workers/scheduler.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -Thread scheduling algorithms: -(with both of these, i assume 1: search can be interrupted, 2: there will be either recursion or total limit) -DFS: would use up the least memory, but could potentially go too deep into one topic -BFS: it really only has to schedule 1 layer ahead so memory wouldn't really be that bad, - and it could pain a really broad picture, which would allow for better control, as this broad picture - is already a very specific topic, giving us a very balanced response - -So general architecture for now: - -short_research() -> string: - # quick research thread, within context, dispatches other complex operations - # TODO: at context overflow, we can either stop, reduce and continue, or fork into # smaller threads - -each thread can be either a stub, or a node. -- stub: returns with a string, as short as possible, with some key info. Ex: - - none: basic model response - - google: just return a summarized google query - - project-rag: highly specific data set about this project, - rag population will be running in background, or with each node. - Embeddings on this amount of text are relatively cheap, fast, efficient. - - response: basic model response -- node: a thread which schedules one of the following: - - node: recursive thread creation, can be interpreted as a loop with limited iterations - - stub: one of the stubs listed above - -problems: -- error-tolerating function calling has to be implemented, each response HAS to be a function, - otherwise it has to be either: - - treated as a basic response - - redone - - ignored, discarded, to avoid wasting resources 'Response error' will be returned -- how does the LLM actually determine if the task is complex enough to be split / just googled -- how does the LLM know whether to google, rag, or just respond? - - divide data into EXTERNAL, INTERNAL, BASIC - - response may be either INFORMATIVE (lookup), or an OPINION (basic response) -(we have to remember every little evaluation wastes a lot of time) - -A generally good approach to this is to provide sample code to these models, and then ask them to use it. -For example: -``` -You can use one of these functions: -def function_call(): ... -``` -or -``` -You are tasked with picking the best function for this task. -Go through your thought process step by step. -Only use one function. (this line will not really work with any small models, we have to be more creative) -options: -- function_call() - this function will let you... -- other_call() - this one will ... -- another_call() - -It's just important to be very descriptive, often because of how tokenization works, -models will not recognize the name of the function itself. -For function-calling reference see OpenBMB/ChatDev -""" - -import queue -from dataclasses import dataclass - -# auto-dispatching worker -# INPUTS: topics_of_interest, summaries -# TASK: deepens - - -@dataclass -class ThreadObject: - pass - - -@dataclass(order=True) -class ThreadTask: - priority: int - item: ThreadObject - - -thread_queue: queue.PriorityQueue - - -def new_research_thread(prompt: str, remaining_depth: int) -> str: - if remaining_depth <= 0: - return "SYSTEM ERROR cannot perform operation maximum search depth reached" - - -def research_loop() -> str: - pass From 07b3bf62e7ef8dd34bdc75d29463a50f1533b15f Mon Sep 17 00:00:00 2001 From: LatekVon Date: Sat, 6 Jul 2024 02:09:50 +0200 Subject: [PATCH 05/32] add new structured query prompts --- core/chainables/web.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/core/chainables/web.py b/core/chainables/web.py index 6098ad4..50f011f 100644 --- a/core/chainables/web.py +++ b/core/chainables/web.py @@ -78,3 +78,30 @@ def web_news_lookup_prompt(): ] ) + +# TODO: simple question -> answer, not yet required by any scheduler. +def basic_query_prompt(): + pass + + +# some schedulers may require data-extraction capabilities outside data-gathering +def structured_extraction_prompt(): + return ChatPromptTemplate.from_messages( + [ + ( + "system", + "You are a data extraction and analysis specialist. " + "Your job is to respond in a structured way to the question you were given. " + "You are to follow the orders given precisely and intelligently. " + "Satisfy the requested task to the best of your abilities. " + ), + ( + "user", + "Data: " + "```" + "{data}" + "```" + 'User request: "{user_request}"' + ), + ] + ) From 46235296ec24954d17de3f3822f8663deb6ac49f Mon Sep 17 00:00:00 2001 From: LatekVon Date: Sat, 6 Jul 2024 02:24:20 +0200 Subject: [PATCH 06/32] add functional model loader --- core/tools/model_loader.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/core/tools/model_loader.py b/core/tools/model_loader.py index 33370f7..5262cfc 100644 --- a/core/tools/model_loader.py +++ b/core/tools/model_loader.py @@ -3,6 +3,7 @@ from langchain_community.llms.ollama import Ollama from huggingface_hub import hf_hub_download from llama_cpp import Llama +from langchain_experimental.llms.ollama_functions import OllamaFunctions from configurator import get_runtime_config from core.tools import errorlib @@ -11,6 +12,7 @@ llm_config = runtime_configuration.llm_config embedder_config = runtime_configuration.embedder_config + # problem with the current caching: we have to share those singletons across instances # fixme: n_gpu_layers=-1 is a poor approach, it can and will cause crashes. @@ -27,6 +29,11 @@ def load_ollama_llm() -> Ollama: return llm +def load_ollama_functional_llm() -> OllamaFunctions: + # todo: As far as i see OllamaFunctions could be used by default, but old code has to be adapted + return OllamaFunctions(model=llm_config.model_name, base_url="http://ollama:11434") + + def load_ollama_embedder() -> OllamaEmbeddings: cached_embedder = runtime_configuration.embedder_object if cached_embedder: @@ -80,7 +87,7 @@ def load_llm(): errorlib.pretty_error( title="Tried loading LLM without a valid configuration", advice=f"Your worker configuration file is likely missing " - f"a valid {Fore.CYAN}llm_config_name{Fore.RESET} variable", + f"a valid {Fore.CYAN}llm_config_name{Fore.RESET} variable", ) if llm_config.supplier == "hugging_face": @@ -89,12 +96,31 @@ def load_llm(): return load_ollama_llm() +def load_functional_llm(): + # EXPERIMENTAL + if llm_config is None: + errorlib.pretty_error( + title="Tried loading LLM without a valid configuration", + advice=f"Your worker configuration file is likely missing " + f"a valid {Fore.CYAN}llm_config_name{Fore.RESET} variable", + ) + + if llm_config.supplier == "hugging_face": + errorlib.pretty_error( + title="Tried running functional model with a HF configuration.", + advice=f"Functional models are not yet supported with llama.cpp loaders. " + f"Please switch to {Fore.CYAN}Ollama{Fore.RESET} or stop using functional models.", + ) + else: + return load_ollama_llm() + + def load_embedder(): if embedder_config is None: errorlib.pretty_error( title="Tried loading EMBEDDER without a valid configuration", advice=f"Your worker configuration file is likely missing " - f"a valid {Fore.CYAN}embedder_config_name{Fore.RESET} variable", + f"a valid {Fore.CYAN}embedder_config_name{Fore.RESET} variable", ) if embedder_config.supplier == "hugging_face": From 887f7beb8025260ca8d8cf616fb71ec8f6fffef4 Mon Sep 17 00:00:00 2001 From: LatekVon Date: Wed, 10 Jul 2024 23:54:58 +0200 Subject: [PATCH 07/32] main loop and bug fixes --- core/chainables/web.py | 3 ++- core/tools/model_loader.py | 2 +- schedulers/deepsearcher.py | 29 +++++++++++++++++++++++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/core/chainables/web.py b/core/chainables/web.py index 50f011f..5a7b921 100644 --- a/core/chainables/web.py +++ b/core/chainables/web.py @@ -93,7 +93,8 @@ def structured_extraction_prompt(): "You are a data extraction and analysis specialist. " "Your job is to respond in a structured way to the question you were given. " "You are to follow the orders given precisely and intelligently. " - "Satisfy the requested task to the best of your abilities. " + "You are provided with data chunk, use it, to fulfill user's request. " + "Satisfy the requested task to the best of your abilities." ), ( "user", diff --git a/core/tools/model_loader.py b/core/tools/model_loader.py index 5262cfc..e7f4726 100644 --- a/core/tools/model_loader.py +++ b/core/tools/model_loader.py @@ -112,7 +112,7 @@ def load_functional_llm(): f"Please switch to {Fore.CYAN}Ollama{Fore.RESET} or stop using functional models.", ) else: - return load_ollama_llm() + return load_ollama_functional_llm() def load_embedder(): diff --git a/schedulers/deepsearcher.py b/schedulers/deepsearcher.py index 928ed33..9cba954 100644 --- a/schedulers/deepsearcher.py +++ b/schedulers/deepsearcher.py @@ -1,6 +1,9 @@ # The first automatic scheduler. # When launched, looks at the available data, picks interesting topics and requests deeper searches about them. +from langchain_core.output_parsers import StrOutputParser + from core.databases.db_completion_tasks import CompletionTask +from core.tools.model_loader import load_functional_llm from core.tools.utils import sleep_noisy @@ -17,18 +20,40 @@ # then: grab a random topic from one of the summaries, and focus on it, # continuously condensing summaries and requesting them on deeper topics +llm = load_functional_llm() + +output_parser = StrOutputParser() + + +def are_workers_free(): + # check if there are tasks scheduled waiting for execution + return True + + def get_random_completion() -> CompletionTask: pass -def extract_interesting_topics() -> list[str]: +def extract_interesting_topics(text: str) -> list[str]: + # todo: perform semantic grouping of subjects + their context pass -def schedule_new_search() -> str: +def schedule_new_completion(query: str) -> str: pass def start_deepsearch(): while True: + # fixme: replace False with free worker checking + if not are_workers_free(): + continue + + completion = get_random_completion() + completion_text = completion.completion_result + topics = extract_interesting_topics(completion_text) + + for topic in topics: + schedule_new_completion(topic) + sleep_noisy(6) From 07edde03bad8dfa02d1976952bf6ac7e6fca2b1e Mon Sep 17 00:00:00 2001 From: LatekVon Date: Sun, 14 Jul 2024 16:41:22 +0200 Subject: [PATCH 08/32] implement initial core functions --- schedulers/deepsearcher.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/schedulers/deepsearcher.py b/schedulers/deepsearcher.py index 9cba954..4b44974 100644 --- a/schedulers/deepsearcher.py +++ b/schedulers/deepsearcher.py @@ -1,12 +1,14 @@ # The first automatic scheduler. # When launched, looks at the available data, picks interesting topics and requests deeper searches about them. +from __future__ import annotations + from langchain_core.output_parsers import StrOutputParser -from core.databases.db_completion_tasks import CompletionTask +from core.chainables.web import structured_extraction_prompt +from core.databases.db_completion_tasks import CompletionTask, db_get_complete_completion_tasks, db_add_completion_task from core.tools.model_loader import load_functional_llm from core.tools.utils import sleep_noisy - # get finished tasks # extract interesting talking points # repeat until a unique one has been found @@ -20,27 +22,38 @@ # then: grab a random topic from one of the summaries, and focus on it, # continuously condensing summaries and requesting them on deeper topics -llm = load_functional_llm() +# TODO: move all LLM load to summarizer, local for now +llm = load_functional_llm() output_parser = StrOutputParser() +extraction_chain = structured_extraction_prompt() | llm | output_parser + def are_workers_free(): - # check if there are tasks scheduled waiting for execution + # todo: check if there are non-busy workers available return True -def get_random_completion() -> CompletionTask: - pass +def get_random_completion() -> CompletionTask | None: + completions_list = db_get_complete_completion_tasks() + if len(completions_list) > 0: + return completions_list[0] + else: + return None def extract_interesting_topics(text: str) -> list[str]: # todo: perform semantic grouping of subjects + their context + # fixme: instead of TODO, extracting a single topic as a POC for now + extraction_request = "aa" + + extraction_chain({'data': text, 'user_request': extraction_request}) pass def schedule_new_completion(query: str) -> str: - pass + return db_add_completion_task(query, 'info') def start_deepsearch(): @@ -57,3 +70,5 @@ def start_deepsearch(): schedule_new_completion(topic) sleep_noisy(6) + print("DBG: shutting down deepsearcher - only 1 loop scheduled") + return From 0e60669c82299de63ef71890e697c39a8f433eba Mon Sep 17 00:00:00 2001 From: LatekVon Date: Mon, 15 Jul 2024 21:09:44 +0200 Subject: [PATCH 09/32] add deep_searcher as an optional service to docker compose --- docker/deep_searcher/Dockerfile | 16 ++++++++++++++++ docker/docker-compose.yml | 14 ++++++++++++++ main.py | 4 ++++ 3 files changed, 34 insertions(+) create mode 100644 docker/deep_searcher/Dockerfile diff --git a/docker/deep_searcher/Dockerfile b/docker/deep_searcher/Dockerfile new file mode 100644 index 0000000..6d282f4 --- /dev/null +++ b/docker/deep_searcher/Dockerfile @@ -0,0 +1,16 @@ +FROM condaforge/miniforge3 + +WORKDIR /app + +COPY . /app + +RUN apt-get update && apt-get install -y \ + build-essential \ + g++ \ + && rm -rf /var/lib/apt/lists/* + +RUN conda env create -f environment.yml + +SHELL ["conda", "run", "-n", "ResearchChain", "/bin/bash", "-c"] + +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "ResearchChain", "python3", "main.py", "-w", "deep_searcher"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e2861ff..161ae33 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -125,6 +125,20 @@ services: networks: - app-network + deep_searcher: + profiles: + - deep_searcher + image: deep_searcher + depends_on: + postgres: + condition: service_started + build: + context: ../. + dockerfile: ./docker/frontend/Dockerfile + networks: + - app-network + + volumes: ollama: pgdata: diff --git a/main.py b/main.py index cc24e22..bb70bb2 100644 --- a/main.py +++ b/main.py @@ -6,6 +6,7 @@ from configurator import get_runtime_config, args from core.databases import db_base from core.tools import errorlib +from schedulers.deepsearcher import start_deep_searcher from workers.crawler import start_crawler from workers.embedder import start_embedder from workers.summarizer import start_summarizer @@ -31,6 +32,9 @@ start_embedder() if runtime_config.worker_type == "summarizer": start_summarizer() + if runtime_config.worker_type == "deep_searcher": + start_deep_searcher() + except requests.exceptions.ConnectionError: errorlib.pretty_error( title=f"OLLAMA called but not running", From cc92986c81f35ce87ed22bdb504125aba9296afd Mon Sep 17 00:00:00 2001 From: LatekVon Date: Mon, 15 Jul 2024 21:12:38 +0200 Subject: [PATCH 10/32] finish all initial deep searcher flow functions --- schedulers/deepsearcher.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/schedulers/deepsearcher.py b/schedulers/deepsearcher.py index 4b44974..b7e2a60 100644 --- a/schedulers/deepsearcher.py +++ b/schedulers/deepsearcher.py @@ -44,19 +44,22 @@ def get_random_completion() -> CompletionTask | None: def extract_interesting_topics(text: str) -> list[str]: + # todo: separate function for extracting topics and getting one, + # run the extraction one only when lacking topics # todo: perform semantic grouping of subjects + their context # fixme: instead of TODO, extracting a single topic as a POC for now - extraction_request = "aa" + extraction_request = "Find exactly one topic from this text, it must be interesting. Reply in 3 words at most." - extraction_chain({'data': text, 'user_request': extraction_request}) - pass + result = extraction_chain.invoke({'data': text, 'user_request': extraction_request}) + + return [result] def schedule_new_completion(query: str) -> str: return db_add_completion_task(query, 'info') -def start_deepsearch(): +def start_deep_searcher(): while True: # fixme: replace False with free worker checking if not are_workers_free(): @@ -70,5 +73,5 @@ def start_deepsearch(): schedule_new_completion(topic) sleep_noisy(6) - print("DBG: shutting down deepsearcher - only 1 loop scheduled") + print("DBG: shutting down deep_searcher - only 1 loop scheduled") return From bae8aa4a3ad7a8b0ea96bceff194e9fb31146292 Mon Sep 17 00:00:00 2001 From: LatekVon Date: Mon, 15 Jul 2024 21:39:38 +0200 Subject: [PATCH 11/32] black reformat --- core/chainables/web.py | 11 ++--------- core/databases/db_completion_tasks.py | 4 +--- core/tools/model_loader.py | 8 ++++---- schedulers/deepsearcher.py | 10 +++++++--- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/core/chainables/web.py b/core/chainables/web.py index 5a7b921..8614686 100644 --- a/core/chainables/web.py +++ b/core/chainables/web.py @@ -94,15 +94,8 @@ def structured_extraction_prompt(): "Your job is to respond in a structured way to the question you were given. " "You are to follow the orders given precisely and intelligently. " "You are provided with data chunk, use it, to fulfill user's request. " - "Satisfy the requested task to the best of your abilities." - ), - ( - "user", - "Data: " - "```" - "{data}" - "```" - 'User request: "{user_request}"' + "Satisfy the requested task to the best of your abilities.", ), + ("user", "Data: ```{data}``` User request: '{user_request}'"), ] ) diff --git a/core/databases/db_completion_tasks.py b/core/databases/db_completion_tasks.py index 9da0c6d..0a58bbc 100644 --- a/core/databases/db_completion_tasks.py +++ b/core/databases/db_completion_tasks.py @@ -111,9 +111,7 @@ def db_get_complete_completion_tasks(amount: int = 1): session.expire_on_commit = False query = ( - select(CompletionTask) - .where(CompletionTask.completed == True) - .limit(amount) + select(CompletionTask).where(CompletionTask.completed == True).limit(amount) ) results = list(session.scalars(query).all()) diff --git a/core/tools/model_loader.py b/core/tools/model_loader.py index e7f4726..84697b3 100644 --- a/core/tools/model_loader.py +++ b/core/tools/model_loader.py @@ -87,7 +87,7 @@ def load_llm(): errorlib.pretty_error( title="Tried loading LLM without a valid configuration", advice=f"Your worker configuration file is likely missing " - f"a valid {Fore.CYAN}llm_config_name{Fore.RESET} variable", + f"a valid {Fore.CYAN}llm_config_name{Fore.RESET} variable", ) if llm_config.supplier == "hugging_face": @@ -102,14 +102,14 @@ def load_functional_llm(): errorlib.pretty_error( title="Tried loading LLM without a valid configuration", advice=f"Your worker configuration file is likely missing " - f"a valid {Fore.CYAN}llm_config_name{Fore.RESET} variable", + f"a valid {Fore.CYAN}llm_config_name{Fore.RESET} variable", ) if llm_config.supplier == "hugging_face": errorlib.pretty_error( title="Tried running functional model with a HF configuration.", advice=f"Functional models are not yet supported with llama.cpp loaders. " - f"Please switch to {Fore.CYAN}Ollama{Fore.RESET} or stop using functional models.", + f"Please switch to {Fore.CYAN}Ollama{Fore.RESET} or stop using functional models.", ) else: return load_ollama_functional_llm() @@ -120,7 +120,7 @@ def load_embedder(): errorlib.pretty_error( title="Tried loading EMBEDDER without a valid configuration", advice=f"Your worker configuration file is likely missing " - f"a valid {Fore.CYAN}embedder_config_name{Fore.RESET} variable", + f"a valid {Fore.CYAN}embedder_config_name{Fore.RESET} variable", ) if embedder_config.supplier == "hugging_face": diff --git a/schedulers/deepsearcher.py b/schedulers/deepsearcher.py index b7e2a60..e659492 100644 --- a/schedulers/deepsearcher.py +++ b/schedulers/deepsearcher.py @@ -5,7 +5,11 @@ from langchain_core.output_parsers import StrOutputParser from core.chainables.web import structured_extraction_prompt -from core.databases.db_completion_tasks import CompletionTask, db_get_complete_completion_tasks, db_add_completion_task +from core.databases.db_completion_tasks import ( + CompletionTask, + db_get_complete_completion_tasks, + db_add_completion_task, +) from core.tools.model_loader import load_functional_llm from core.tools.utils import sleep_noisy @@ -50,13 +54,13 @@ def extract_interesting_topics(text: str) -> list[str]: # fixme: instead of TODO, extracting a single topic as a POC for now extraction_request = "Find exactly one topic from this text, it must be interesting. Reply in 3 words at most." - result = extraction_chain.invoke({'data': text, 'user_request': extraction_request}) + result = extraction_chain.invoke({"data": text, "user_request": extraction_request}) return [result] def schedule_new_completion(query: str) -> str: - return db_add_completion_task(query, 'info') + return db_add_completion_task(query, "info") def start_deep_searcher(): From 8eeceb72ed5bbf042b3300f8f32910712148125e Mon Sep 17 00:00:00 2001 From: LatekVon Date: Mon, 15 Jul 2024 22:14:33 +0200 Subject: [PATCH 12/32] update misconfiguration --- configurator.py | 1 + docker/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/configurator.py b/configurator.py index c039b10..a44b39c 100644 --- a/configurator.py +++ b/configurator.py @@ -27,6 +27,7 @@ "crawler", "embedder", "summarizer", + "deep_searcher", ], default="none", help="Select one of the ready worker configs to be used", diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 161ae33..8e99cba 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -134,7 +134,7 @@ services: condition: service_started build: context: ../. - dockerfile: ./docker/frontend/Dockerfile + dockerfile: ./docker/deep_searcher/Dockerfile networks: - app-network From 23ac042600eec3191b7a5c367f8f18c532864fff Mon Sep 17 00:00:00 2001 From: LatekVon Date: Mon, 15 Jul 2024 22:36:08 +0200 Subject: [PATCH 13/32] update conda env --- environment.yml | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/environment.yml b/environment.yml index a28ae00..b11ec2f 100644 --- a/environment.yml +++ b/environment.yml @@ -3,27 +3,46 @@ channels: - conda-forge - defaults dependencies: + - _libgcc_mutex=0.1 + - _openmp_mutex=4.5 - beautifulsoup4=4.12.2 - brotli-python=1.0.9 - bzip2=1.0.8 - ca-certificates=2024.3.11 - certifi=2024.6.2 + - cffi=1.16.0 + - charset-normalizer=3.3.2 - colorama=0.4.6 + - cudatoolkit=11.8.0 - filelock=3.13.1 - fsspec=2023.10.0 - googlesearch=3.0.0 + - h2=4.1.0 + - hpack=4.0.0 - huggingface_hub=0.20.3 + - hyperframe=6.0.1 + - idna=3.7 + - keyutils=1.6.1 - krb5=1.20.1 + - ld_impl_linux-64=2.40 - libblas=3.9.0 - libcxx=8.0.0 + - libcxxabi=8.0.0 - libedit=3.1.20230828 - libfaiss=1.7.4 - libffi=3.4.2 + - libgcc-ng=14.1.0 + - libgfortran-ng=11.3.0 - libgfortran5=11.3.0 + - libgomp=14.1.0 - liblapack=3.9.0 + - libnsl=2.0.1 - libopenblas=0.3.23 - libpq=12.17 - libsqlite=3.45.3 + - libstdcxx-ng=14.1.0 + - libuuid=2.38.1 + - libxcrypt=4.4.36 - libzlib=1.2.13 - llvm-openmp=18.1.3 - ncurses=6.4.20240210 @@ -31,6 +50,7 @@ dependencies: - packaging=23.2 - pip=24.0 - psycopg2=2.9.9 + - pycparser=2.22 - pysocks=1.7.1 - python=3.9.19 - python_abi=3.9 @@ -42,12 +62,13 @@ dependencies: - sqlite=3.45.3 - tk=8.6.13 - tqdm=4.65.0 - - typing_extensions=4.9.0 - tzdata=2024a - wheel=0.43.0 - xz=5.2.6 - yaml=0.2.5 - zlib=1.2.13 + - zstandard=0.23.0 + - zstd=1.5.6 - pip: - aiohttp==3.9.5 - aiosignal==1.3.1 @@ -57,7 +78,6 @@ dependencies: - attrs==23.2.0 - black==24.4.2 - chardet==5.2.0 - - charset-normalizer==3.3.2 - click==8.1.7 - dataclasses-json==0.6.4 - diskcache==5.6.3 @@ -68,19 +88,20 @@ dependencies: - fastapi==0.111.0 - fastapi-cli==0.0.4 - frozenlist==1.4.1 + - greenlet==3.0.3 - h11==0.14.0 - httpcore==1.0.5 - httptools==0.6.1 - httpx==0.27.0 - - idna==3.7 - jinja2==3.1.3 - jsonpatch==1.33 - jsonpointer==2.4 - - langchain==0.1.17 - - langchain-community==0.0.36 - - langchain-core==0.1.50 - - langchain-text-splitters==0.0.1 - - langsmith==0.1.48 + - langchain==0.2.8 + - langchain-community==0.2.7 + - langchain-core==0.2.19 + - langchain-experimental==0.0.62 + - langchain-text-splitters==0.2.2 + - langsmith==0.1.85 - llama-cpp-python==0.2.61 - markdown-it-py==3.0.0 - markupsafe==2.1.5 From 8a71e42b856b45c519360414c1c9a735a49cd3da Mon Sep 17 00:00:00 2001 From: LatekVon Date: Mon, 15 Jul 2024 23:43:41 +0200 Subject: [PATCH 14/32] add TEMPORARY worker entry for ds --- configs/deep_searcher.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 configs/deep_searcher.json diff --git a/configs/deep_searcher.json b/configs/deep_searcher.json new file mode 100644 index 0000000..3fef431 --- /dev/null +++ b/configs/deep_searcher.json @@ -0,0 +1,5 @@ +{ + "worker_type": "crawler", + "llm_config_name": "ollama_medium", + "embedder_config_name": "none" +} \ No newline at end of file From c2c0d213784d86f6f7b6846f5a319b7e245f3c7f Mon Sep 17 00:00:00 2001 From: LatekVon Date: Mon, 15 Jul 2024 23:44:12 +0200 Subject: [PATCH 15/32] remove all remaining runtime crashes and bugs from ds --- docker/docker-compose.yml | 6 +++++- main.py | 6 ++++++ schedulers/deepsearcher.py | 11 +++++++++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 8e99cba..fd459c1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -132,12 +132,16 @@ services: depends_on: postgres: condition: service_started + rabbitmq: + condition: service_healthy + ollama: + condition: service_started build: context: ../. dockerfile: ./docker/deep_searcher/Dockerfile networks: - app-network - + - ollama-network volumes: ollama: diff --git a/main.py b/main.py index bb70bb2..5459d2e 100644 --- a/main.py +++ b/main.py @@ -14,6 +14,12 @@ colorama_init() db_base.db_init() +# todo: change deep_searcher into a scheduler instead of worker, -s flag +# this is because workers use cpu, gpu and memory resources in a distributed way, +# while a scheduler only does lightweight management without any significant load, +# and most importantly contrary to worker, doesn't require a config file to run +# theoretically this makes the crawler a scheduler as well, but for now it is a core part + if args.worker_type == "webui": # fixme: this is a workaround, webui should be started from it's folder uvicorn.run("webui.main:app", host="0.0.0.0", port=8000) diff --git a/schedulers/deepsearcher.py b/schedulers/deepsearcher.py index e659492..97df019 100644 --- a/schedulers/deepsearcher.py +++ b/schedulers/deepsearcher.py @@ -10,7 +10,7 @@ db_get_complete_completion_tasks, db_add_completion_task, ) -from core.tools.model_loader import load_functional_llm +from core.tools.model_loader import load_llm from core.tools.utils import sleep_noisy # get finished tasks @@ -28,7 +28,7 @@ # TODO: move all LLM load to summarizer, local for now -llm = load_functional_llm() +llm = load_llm() output_parser = StrOutputParser() extraction_chain = structured_extraction_prompt() | llm | output_parser @@ -67,13 +67,20 @@ def start_deep_searcher(): while True: # fixme: replace False with free worker checking if not are_workers_free(): + sleep_noisy(6) continue completion = get_random_completion() + + if not completion: + sleep_noisy(6) + continue + completion_text = completion.completion_result topics = extract_interesting_topics(completion_text) for topic in topics: + print('dispatching new summaries:', topic) schedule_new_completion(topic) sleep_noisy(6) From 6574008244848534d284c9c206a87ca1e6381b2f Mon Sep 17 00:00:00 2001 From: LatekVon Date: Tue, 16 Jul 2024 01:01:47 +0200 Subject: [PATCH 16/32] refactor small portion of summarizer --- workers/summarizer.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/workers/summarizer.py b/workers/summarizer.py index da8ed31..746b53c 100644 --- a/workers/summarizer.py +++ b/workers/summarizer.py @@ -81,9 +81,11 @@ def summarize(channel): prompt_core=current_task.prompt, query_type=current_task.mode.lower() ) + # fixme: there is no context building here, it's just the first result! context = db_search_for_similar_queries(task_query) - if context is None: + if context is None or len(context) == 0: + print('received context is none') return def interpret_prompt_mode(): @@ -94,26 +96,22 @@ def interpret_prompt_mode(): elif current_task.mode == "wiki": return web_wiki_lookup_prompt() - def get_user_prompt(_: dict): - return current_task.prompt - - def get_context(_: dict): - return context[0].page_content - web_interpret_prompt_mode = interpret_prompt_mode() print("Summarizing task with uuid: ", current_task.uuid) + chain = ( - { - "search_data": RunnableLambda(get_context), - # this has to be a RunnableLambda, it cannot be a string - "user_request": RunnableLambda(get_user_prompt), - } - | web_interpret_prompt_mode + web_interpret_prompt_mode | llm | output_parser ) - summary = chain.invoke(current_task) + + chain_input = { + "search_data": context[0].page_content, + "user_request": current_task.prompt, + } + + summary = chain.invoke(chain_input) db_update_completion_task_after_summarizing(summary, current_task.uuid) print(f"{Fore.CYAN}Completed task with uuid: {Fore.RESET}", current_task.uuid) From a92ae9e652ca6cf4e588b064969752182e52256f Mon Sep 17 00:00:00 2001 From: LatekVon Date: Tue, 16 Jul 2024 01:05:58 +0200 Subject: [PATCH 17/32] add missing llm initialization and missing context compiler --- workers/summarizer.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/workers/summarizer.py b/workers/summarizer.py index 746b53c..bccd2e6 100644 --- a/workers/summarizer.py +++ b/workers/summarizer.py @@ -11,16 +11,16 @@ db_release_executing_tasks, db_required_crawl_tasks_for_uuid, ) -from langchain_core.runnables import RunnableLambda from core.classes.query import WebQuery from core.chainables.web import ( web_docs_lookup_prompt, web_news_lookup_prompt, web_wiki_lookup_prompt, ) -from core.tools.model_loader import load_llm +from core.tools.model_loader import load_llm, runtime_configuration from langchain_core.output_parsers import StrOutputParser +from core.tools.scraper import docs_to_context from core.tools.utils import sleep_noisy from colorama import Fore, Style @@ -29,7 +29,8 @@ output_parser = StrOutputParser() -llm = None +llm_config = runtime_configuration.llm_config +llm = load_llm() # even though a single task takes a long time to complete, # as soon as one task is started, all elements of the queue are released @@ -82,12 +83,14 @@ def summarize(channel): ) # fixme: there is no context building here, it's just the first result! - context = db_search_for_similar_queries(task_query) + context_list = db_search_for_similar_queries(task_query) - if context is None or len(context) == 0: + if context_list is None or len(context_list) == 0: print('received context is none') return + context = docs_to_context(context_list, llm_config.model_token_limit / 2) + def interpret_prompt_mode(): if current_task.mode == "news": return web_news_lookup_prompt() @@ -107,7 +110,7 @@ def interpret_prompt_mode(): ) chain_input = { - "search_data": context[0].page_content, + "search_data": context, "user_request": current_task.prompt, } From 55229b7d1ae1a39f7e49b777e4bd9cf22cd1eb54 Mon Sep 17 00:00:00 2001 From: LatekVon Date: Tue, 16 Jul 2024 01:17:52 +0200 Subject: [PATCH 18/32] bugs, improvements and reformatting --- schedulers/deepsearcher.py | 6 ++++-- workers/summarizer.py | 16 ++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/schedulers/deepsearcher.py b/schedulers/deepsearcher.py index 97df019..83874e6 100644 --- a/schedulers/deepsearcher.py +++ b/schedulers/deepsearcher.py @@ -11,7 +11,7 @@ db_add_completion_task, ) from core.tools.model_loader import load_llm -from core.tools.utils import sleep_noisy +from core.tools.utils import sleep_noisy, remove_characters # get finished tasks # extract interesting talking points @@ -60,7 +60,9 @@ def extract_interesting_topics(text: str) -> list[str]: def schedule_new_completion(query: str) -> str: - return db_add_completion_task(query, "info") + # fixme: regex remove anything not a-zA-Z + pure_query = remove_characters(query, ['"', "'", ':', '?', '!']) + return db_add_completion_task(pure_query, "info") def start_deep_searcher(): diff --git a/workers/summarizer.py b/workers/summarizer.py index bccd2e6..8397e79 100644 --- a/workers/summarizer.py +++ b/workers/summarizer.py @@ -98,21 +98,24 @@ def interpret_prompt_mode(): return web_docs_lookup_prompt() elif current_task.mode == "wiki": return web_wiki_lookup_prompt() + else: + # todo: add info query - plain basic nothing + return web_wiki_lookup_prompt() web_interpret_prompt_mode = interpret_prompt_mode() print("Summarizing task with uuid: ", current_task.uuid) chain = ( - web_interpret_prompt_mode - | llm - | output_parser + web_interpret_prompt_mode + | llm + | output_parser ) chain_input = { - "search_data": context, - "user_request": current_task.prompt, - } + "search_data": context, + "user_request": current_task.prompt, + } summary = chain.invoke(chain_input) db_update_completion_task_after_summarizing(summary, current_task.uuid) @@ -123,6 +126,7 @@ def interpret_prompt_mode(): previous_queued_tasks = 0 + # 1. get a list of available tasks, in the backend they'll be automatically set as executing # 2. parse through all of them, until one that has all it's dependencies resolved appears # 3. once one is found to be ready, release all the other tasks (reset 'executing') From 8d80708747d13c67fedcbaccb9eee688e98f86ee Mon Sep 17 00:00:00 2001 From: LatekVon Date: Tue, 16 Jul 2024 01:47:24 +0200 Subject: [PATCH 19/32] remove resolved fixme --- workers/summarizer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workers/summarizer.py b/workers/summarizer.py index 8397e79..e02cb4d 100644 --- a/workers/summarizer.py +++ b/workers/summarizer.py @@ -82,7 +82,6 @@ def summarize(channel): prompt_core=current_task.prompt, query_type=current_task.mode.lower() ) - # fixme: there is no context building here, it's just the first result! context_list = db_search_for_similar_queries(task_query) if context_list is None or len(context_list) == 0: From b3daa0ac82667ba3d0e0d696fd079f61c8b61ecd Mon Sep 17 00:00:00 2001 From: LatekVon Date: Tue, 16 Jul 2024 17:48:32 +0200 Subject: [PATCH 20/32] remove unnecessary config line --- docker/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index fd459c1..2673c7d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -39,8 +39,6 @@ services: ollama: image: ollama/ollama - expose: - - 11434 ports: - 11434:11434 pull_policy: always From 2ed54707673418b3690a524f41f233274a987115 Mon Sep 17 00:00:00 2001 From: LatekVon Date: Wed, 24 Jul 2024 00:17:12 +0200 Subject: [PATCH 21/32] add basic completion prompt --- core/chainables/web.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/core/chainables/web.py b/core/chainables/web.py index 8614686..73c9527 100644 --- a/core/chainables/web.py +++ b/core/chainables/web.py @@ -79,9 +79,19 @@ def web_news_lookup_prompt(): ) -# TODO: simple question -> answer, not yet required by any scheduler. def basic_query_prompt(): - pass + return ChatPromptTemplate.from_messages( + [ + ( + "system", + "You are a personal assistant. " + "Your job is to respond to the requests given to you by the user. " + "You are to follow the requests given precisely and intelligently. " + "Answer or complete the request to the best of your abilities. ", + ), + ("user", "{user_request}"), + ] + ) # some schedulers may require data-extraction capabilities outside data-gathering @@ -94,7 +104,7 @@ def structured_extraction_prompt(): "Your job is to respond in a structured way to the question you were given. " "You are to follow the orders given precisely and intelligently. " "You are provided with data chunk, use it, to fulfill user's request. " - "Satisfy the requested task to the best of your abilities.", + "Satisfy the requested task to the best of your abilities. ", ), ("user", "Data: ```{data}``` User request: '{user_request}'"), ] From 2ca5d723a4dc3c95e698c0c1d56ace6a87abfade Mon Sep 17 00:00:00 2001 From: LatekVon Date: Wed, 24 Jul 2024 23:55:58 +0200 Subject: [PATCH 22/32] add basic query to summarizer workflow --- workers/summarizer.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/workers/summarizer.py b/workers/summarizer.py index e02cb4d..5494d51 100644 --- a/workers/summarizer.py +++ b/workers/summarizer.py @@ -15,7 +15,7 @@ from core.chainables.web import ( web_docs_lookup_prompt, web_news_lookup_prompt, - web_wiki_lookup_prompt, + web_wiki_lookup_prompt, basic_query_prompt, ) from core.tools.model_loader import load_llm, runtime_configuration from langchain_core.output_parsers import StrOutputParser @@ -30,7 +30,7 @@ output_parser = StrOutputParser() llm_config = runtime_configuration.llm_config -llm = load_llm() +llm = None # even though a single task takes a long time to complete, # as soon as one task is started, all elements of the queue are released @@ -98,7 +98,6 @@ def interpret_prompt_mode(): elif current_task.mode == "wiki": return web_wiki_lookup_prompt() else: - # todo: add info query - plain basic nothing return web_wiki_lookup_prompt() web_interpret_prompt_mode = interpret_prompt_mode() @@ -116,6 +115,17 @@ def interpret_prompt_mode(): "user_request": current_task.prompt, } + if current_task.mode == 'basic': + chain = ( + basic_query_prompt() + | llm + | output_parser + ) + + chain_input = { + "user_request": current_task.prompt, + } + summary = chain.invoke(chain_input) db_update_completion_task_after_summarizing(summary, current_task.uuid) From 2e135b9266c63d26783278d380884c8f282c755f Mon Sep 17 00:00:00 2001 From: LatekVon Date: Fri, 26 Jul 2024 00:15:28 +0200 Subject: [PATCH 23/32] add readme entry on running locally --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 1b588a1..fac5a8d 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,24 @@ Frontend is launched separately to back end, run the following command to start To solve this issue, let all the current downloads finish downloading, and restart the program. Repeat until every file is downloaded. +#### Running locally + +If you intend on running this project locally, whether for development, debugging or performance purposes, +you'll still need to have these three docker containers launched somewhere in the background: `ollama`, `postgres` and `rabbitmq`. + +Here's a full command on how to initialize the conda environment run them all at once: + +- `conda env create -f environment.yml` +- `sudo docker-compose -f docker/docker-compose.yml up ollama postgres rabbitmq` + +Rest of the workers and services can now be launched directly via `main.py`: + +- `conda activate ResearchChain` +- `python main.py -w crawler` + +For list of all available workers and services see: +- `python main.py -h` + ### This is a monorepo for both a tool, and apps for it: #### Research Chain From 1b44c62701a3286521435e89e288f6a4c62fc8e9 Mon Sep 17 00:00:00 2001 From: LatekVon Date: Fri, 26 Jul 2024 00:24:58 +0200 Subject: [PATCH 24/32] fix crashes on start --- core/tools/model_loader.py | 2 +- schedulers/deepsearcher.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/tools/model_loader.py b/core/tools/model_loader.py index 84697b3..0c88ed3 100644 --- a/core/tools/model_loader.py +++ b/core/tools/model_loader.py @@ -100,7 +100,7 @@ def load_functional_llm(): # EXPERIMENTAL if llm_config is None: errorlib.pretty_error( - title="Tried loading LLM without a valid configuration", + title="Tried loading functional LLM without a valid configuration", advice=f"Your worker configuration file is likely missing " f"a valid {Fore.CYAN}llm_config_name{Fore.RESET} variable", ) diff --git a/schedulers/deepsearcher.py b/schedulers/deepsearcher.py index 83874e6..1c61eff 100644 --- a/schedulers/deepsearcher.py +++ b/schedulers/deepsearcher.py @@ -28,12 +28,9 @@ # TODO: move all LLM load to summarizer, local for now -llm = load_llm() +llm = None output_parser = StrOutputParser() -extraction_chain = structured_extraction_prompt() | llm | output_parser - - def are_workers_free(): # todo: check if there are non-busy workers available return True @@ -54,6 +51,11 @@ def extract_interesting_topics(text: str) -> list[str]: # fixme: instead of TODO, extracting a single topic as a POC for now extraction_request = "Find exactly one topic from this text, it must be interesting. Reply in 3 words at most." + global llm + if llm is None: + llm = load_llm() + extraction_chain = structured_extraction_prompt() | llm | output_parser + result = extraction_chain.invoke({"data": text, "user_request": extraction_request}) return [result] From 1ae5daa14bf7e9128412a5f986104a35050db61c Mon Sep 17 00:00:00 2001 From: LatekVon Date: Fri, 26 Jul 2024 21:34:17 +0200 Subject: [PATCH 25/32] fix license --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 261eeb9..a4ed920 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2024 Research Chain Team Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From d7ab5c85525cf569284fe01ac5fe032d4b34c3e0 Mon Sep 17 00:00:00 2001 From: LatekVon Date: Sat, 27 Jul 2024 01:09:19 +0200 Subject: [PATCH 26/32] isolate schedulers from workers, add workaround for schedulers crashing on startup --- configs/deep_searcher.json | 5 ----- configs/none.json | 5 +++++ configurator.py | 16 +++++++++++++--- core/databases/db_completion_tasks.py | 1 - main.py | 24 ++++++++++++++++-------- schedulers/deepsearcher.py | 1 + 6 files changed, 35 insertions(+), 17 deletions(-) delete mode 100644 configs/deep_searcher.json create mode 100644 configs/none.json diff --git a/configs/deep_searcher.json b/configs/deep_searcher.json deleted file mode 100644 index 3fef431..0000000 --- a/configs/deep_searcher.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "worker_type": "crawler", - "llm_config_name": "ollama_medium", - "embedder_config_name": "none" -} \ No newline at end of file diff --git a/configs/none.json b/configs/none.json new file mode 100644 index 0000000..0001f50 --- /dev/null +++ b/configs/none.json @@ -0,0 +1,5 @@ +{ + "worker_type": "none", + "llm_config_name": "none", + "embedder_config_name": "none" +} \ No newline at end of file diff --git a/configurator.py b/configurator.py index a44b39c..5a2c111 100644 --- a/configurator.py +++ b/configurator.py @@ -23,15 +23,25 @@ type=str, dest="worker_type", choices=[ - "webui", "crawler", "embedder", "summarizer", - "deep_searcher", ], default="none", help="Select one of the ready worker configs to be used", ) +parser.add_argument( + "-s", + "--run-scheduler", + type=str, + dest="scheduler_type", + choices=[ + "webui", + "deep_searcher", + ], + default="none", + help="Select one of the available schedulers", +) parser.add_argument( "-c", "--custom-worker-path", @@ -79,7 +89,7 @@ def get_runtime_config(): global runtime_config - fallback_config_path = "configs/crawler.json" + fallback_config_path = "configs/none.json" if args.worker_type == "webui": # fixme: this is a workaround, webui should be started from it's folder diff --git a/core/databases/db_completion_tasks.py b/core/databases/db_completion_tasks.py index 0a58bbc..e20c771 100644 --- a/core/databases/db_completion_tasks.py +++ b/core/databases/db_completion_tasks.py @@ -1,4 +1,3 @@ -from typing import Optional from sqlalchemy import String, TEXT, Integer, Boolean, select, update from sqlalchemy.orm import Mapped, mapped_column, Session, relationship diff --git a/main.py b/main.py index 5459d2e..5172ad6 100644 --- a/main.py +++ b/main.py @@ -20,17 +20,27 @@ # and most importantly contrary to worker, doesn't require a config file to run # theoretically this makes the crawler a scheduler as well, but for now it is a core part -if args.worker_type == "webui": - # fixme: this is a workaround, webui should be started from it's folder - uvicorn.run("webui.main:app", host="0.0.0.0", port=8000) +if args.worker_type == "none" and args.scheduler_type == "none": + errorlib.pretty_error( + title=f"No worker or scheduler was selected to run.", + advice=f"You have to select either worker or a scheduler to run.", + ) -if args.worker_type == "webui": +if args.worker_type != "none" and args.scheduler_type != "none": errorlib.pretty_error( - title=f"No flags were provided", - advice=f"---", + title=f"Both a worker and a scheduler was selected to run.", + advice=f"You may only select either worker or a scheduler to run.", ) + +if args.scheduler_type == "webui": + uvicorn.run("webui.main:app", host="0.0.0.0", port=8000) + try: + # todo: dynamically allocate available-scheduler array + if args.scheduler_type == "deep_searcher": + start_deep_searcher() + runtime_config = get_runtime_config() if runtime_config.worker_type == "crawler": start_crawler() @@ -38,8 +48,6 @@ start_embedder() if runtime_config.worker_type == "summarizer": start_summarizer() - if runtime_config.worker_type == "deep_searcher": - start_deep_searcher() except requests.exceptions.ConnectionError: errorlib.pretty_error( diff --git a/schedulers/deepsearcher.py b/schedulers/deepsearcher.py index 1c61eff..1e4f714 100644 --- a/schedulers/deepsearcher.py +++ b/schedulers/deepsearcher.py @@ -31,6 +31,7 @@ llm = None output_parser = StrOutputParser() + def are_workers_free(): # todo: check if there are non-busy workers available return True From e6eaea08f4924448d50b63f7e437b1cae77df109 Mon Sep 17 00:00:00 2001 From: LatekVon Date: Sat, 27 Jul 2024 01:11:50 +0200 Subject: [PATCH 27/32] fix bug and change naming --- configurator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configurator.py b/configurator.py index 5a2c111..1892b85 100644 --- a/configurator.py +++ b/configurator.py @@ -89,11 +89,11 @@ def get_runtime_config(): global runtime_config - fallback_config_path = "configs/none.json" + empty_config_path = "configs/none.json" - if args.worker_type == "webui": + if args.worker_type == "none": # fixme: this is a workaround, webui should be started from it's folder - return load_runtime_config_from_file(fallback_config_path) + return load_runtime_config_from_file(empty_config_path) # fetch cache if runtime_config: From 7dc836275792a6f82323de77316a292c30df0f6b Mon Sep 17 00:00:00 2001 From: LatekVon Date: Sat, 27 Jul 2024 03:26:47 +0200 Subject: [PATCH 28/32] update scheduler dockerfiles --- docker/deep_searcher/Dockerfile | 2 +- docker/webui/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/deep_searcher/Dockerfile b/docker/deep_searcher/Dockerfile index 6d282f4..c0290bd 100644 --- a/docker/deep_searcher/Dockerfile +++ b/docker/deep_searcher/Dockerfile @@ -13,4 +13,4 @@ RUN conda env create -f environment.yml SHELL ["conda", "run", "-n", "ResearchChain", "/bin/bash", "-c"] -ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "ResearchChain", "python3", "main.py", "-w", "deep_searcher"] +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "ResearchChain", "python3", "main.py", "-s", "deep_searcher"] diff --git a/docker/webui/Dockerfile b/docker/webui/Dockerfile index 8c4a0cb..cd07c01 100644 --- a/docker/webui/Dockerfile +++ b/docker/webui/Dockerfile @@ -16,4 +16,4 @@ EXPOSE 8000 SHELL ["conda", "run", "-n", "ResearchChain", "/bin/bash", "-c"] -ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "ResearchChain", "python3", "main.py", "-w", "webui"] +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "ResearchChain", "python3", "main.py", "-s", "webui"] From dd88785da0d0c1f5617f2725f82a1fbbccb9008a Mon Sep 17 00:00:00 2001 From: LatekVon Date: Sat, 27 Jul 2024 12:08:57 +0200 Subject: [PATCH 29/32] make frontend an optional container --- docker/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2673c7d..add193b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -111,6 +111,8 @@ services: - app-network frontend: + profiles: + - frontend image: frontend depends_on: - postgres From 50287b420d4bf51dec4d9577e6907d604f74db8d Mon Sep 17 00:00:00 2001 From: LatekVon Date: Sat, 27 Jul 2024 12:10:35 +0200 Subject: [PATCH 30/32] make pgadmin an optional container --- docker/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index add193b..c7c7895 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -26,6 +26,8 @@ services: - app-network pgadmin: + profiles: + - pgadmin image: dpage/pgadmin4 depends_on: - postgres From 2e741027a9b7be55239529800c4a4585b2c505b6 Mon Sep 17 00:00:00 2001 From: LatekVon Date: Sat, 27 Jul 2024 12:30:10 +0200 Subject: [PATCH 31/32] add basic task as an available mode on frontend --- webui/frontend/src/app/components/PromptInput.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/webui/frontend/src/app/components/PromptInput.tsx b/webui/frontend/src/app/components/PromptInput.tsx index c1c883f..46b474a 100644 --- a/webui/frontend/src/app/components/PromptInput.tsx +++ b/webui/frontend/src/app/components/PromptInput.tsx @@ -149,6 +149,7 @@ function PromptInput() { +