From 795384e499aeebec29d9ccba8615b1cbd364f20a Mon Sep 17 00:00:00 2001
From: andrea <andrea.valentini.1993@gmail.com>
Date: Wed, 19 Oct 2022 22:33:29 +0200
Subject: [PATCH] add config.yml file

---
 backend/requirements.txt                      | Bin 2854 -> 1379 bytes
 backend/src/app.py                            |  41 +++---------------
 backend/src/config.py                         |  30 +++++++++++++
 .../src/{backend_config.py => config.yaml}    |  13 +++---
 backend/src/executors.py                      |  10 ++---
 backend/src/flows.py                          |   6 +--
 backend/src/helpers.py                        |  19 ++++++--
 7 files changed, 67 insertions(+), 52 deletions(-)
 create mode 100644 backend/src/config.py
 rename backend/src/{backend_config.py => config.yaml} (54%)

diff --git a/backend/requirements.txt b/backend/requirements.txt
index c0f4d79ce3c992a68fa1744da6a1860801a08f92..7e19f862e2dee47c9818f1fa4d86bda5fedbf924 100644
GIT binary patch
literal 1379
zcmZ8h%W~T=47~GS#L+8ua?qjob|%-JIy6ODj46^KAv^NdcR@Mn<0d==EEWqY8z$>a
z9)`A+jjV;TVJ3>h(8*`nv6=1JD>n>{Z1;GkWYjVYwe01SP=2$4(+{!{DsNs_WET@w
z8irohvK5Lb{b`_0B0E_$tkLpCw!P33+l_3VP(sftWkZz@X;IGJO+qT5A>ypQ48t>w
zPUx^K!7nPoQ=<&R!B1;8oLvJbObTm?8&|FYRHrRcI42cn__}8}@zbUqcfdDKKL)K*
zQi#KG+#MH=A4<(hWl|AXYx(?yC{yPGgJQN|qbG&nFl{fpqwK|$!rgdhGuQxZbUdf1
znXpFoN076d)ohfbxze-I7wgaehm)C{(PV@Fip)(@P=aH@8+-J{r0iJFXOx+gHr2eI
zN+OeDN3R%cFYFRSB4<z4Vn~gO;K!fHdDAYeBmYv%?jYXm{!3G0Fu35!nJSNk`i8lG
z3n!FfY^;%sz1bYxm5IZc=;rSIry3ST(FN|ji>4OK>d1~(gCeH672Z*y^eFtg><er=
z>^v%cQ9KuR%o>P_<{XWFjMs=)>s+`E15^bVF$K&vn>AOEj`66Et@eP62!+!+F(y<x
zinyr-62~{jM10ecRam6gvOOwK7L7_mD_*zrf`Y7S*>v1%;~xq%nDPY<nuD)EtXau&
zZ78JuatAX0%6QO{|1Qae2-GP+S+cL&&u>4z7g|IzXY+wF`_(iZ9Pg{iJBAiVA(GWI
z5Vp9)T4Zf6mMUjdDw@G&*tfIEE6c!dJoG_H`8?s;Ou14}hs;MXTBC7M;6%P0MYaF{
z21<QzakWLqJU{LtIYU&ETf7W{X1o$(#5HKb_>ex*C|r!^n+9_z3i9Ae1@>Ro)#jpf
zWbnn%`-s=3mj_<mzi6>OhD%!n^}oyqCgNZB`$NAqIS8#|)GgkpR;*X612cl!{JyPM
z7Xn^8n0)RxMS6Zbh@UQV$~bYnNS`8NNjKv5=|?k&rV|?&;6}WPyN!=0v=P4nz3ibk

literal 2854
zcmai$OK%fF5QOK9#7{AbALMc1z#XJqIU#%fO#EWkj^q4zpuVmh&n5~&t99&5Pjz>7
zb<h0yJx;5%OLKj$(=0XVi=Kg=PpPl>Dt$~Jl5Ljhpl2TzL#^HF9d_0mVJy;C*6Xlg
zKR8{CMhLb1jMFA;uCzYV`?<y0rgOwY6!^H-+i%wzt!-14dcs&~|4eZo^&ML<z#6o+
zL|XqNn@T(Dz#wbqd)4NA7Z{^Jsl-aQAeX+P%+XY7sT}Yy6H-rc?euFCxv$bocrdbg
zbPko6St*0Z@Ctw6n!`#68$CN=fm63uWJOdNf%QXVUaJ=8Kysxri<S1CwT9I~-^cpC
zY%PaTH83;YS@BT$I*`0#JSsmGMitS3V{f*(5r!*ko5u?VHN(cMbM2$*L}9jByA3Y0
zg|)X1U*;z3D_b~gGTJFxtCX4#6LGj+bLFrL?zh1+Ua5gO&KbkVUOA!qygCXLYp1*3
z6K`mBEu^KYOx0)leAMo(xc&crv_21hD`DVs`|hr#qU^_t*|najK<>KNxnqSI&CgPO
zu+ZbEU>uDT{V`2CsHG|^RhCXA1GqMWbk$xl(j6#uA`JSB>P_|XmmT(;y+6nswL8j&
zRePZ}*U?805m)im@2DfM<ya@dRZkV3cJ$xxbiji=VMTPA|1xE$%t>z}t95!(?AA3J
zfK%tWmP$sk`}N$Q@0f68lNrbiS=CfzqfAi*9Ma8JSysJOgaQw32Jm|mOl-9ihD+BD
zf_+bby}YBX@uiaZ9<=(!GZrYZ(KmFM**2r@=zGPqjtow*?unLbwT~WgP0{jESm+u)
zXE9j{Cfpd)yHManJLVVW9Gi0^t12S&R5b5Y1}jH@4iz!K;0;gI#nGF$d!Z~sb6Dlm
z2pd?m+VKPv_$htpb!KJu1yl4}FT5TiTRi)8A~Ghh{es!Gr~dHq6j7pR=UO5t)<qAF
z%6iCM;FFEKex$GId-@iLMNLeIh2H1*|3EqV^!3cQGO+n2<~g6ctPCOpvs(wJ%RuQ1
z!)K@CZN&B}rWR)v(;oK9B%%uD!IrR*t5eP%&N`~Xr{@c2>{!1VX~O3p{Z^{x>Qj%m
zdpQeG@GShm6*1hkeL`K$hxJj|YUFJtZ=B0zeiv={q^ToE?f?ZJW^?~jJKsX4Qj*UU
z`q;_<4Zr3N)pNGt*XO$PbIfFD{;{i-QLX_0e{c0kIOLa40^`^xOdV9|sX74u>A>sw
z%s&3jH?~%eD|v`6aew?&{LgLWvhVI0nsYrW17-)ab$C(0puL3|_o$wrd&$-cat7r#
zhy@dLCvM9$Dffj_Ofi^xed{z%*KVdbnB+`F-?QFMD(Y0`)JayacQ{K=828dSv|nzh
RzJ<a3i}KpHF;kDS@)r_}p1=SA

diff --git a/backend/src/app.py b/backend/src/app.py
index 0632587..98ffba6 100644
--- a/backend/src/app.py
+++ b/backend/src/app.py
@@ -4,42 +4,12 @@
 from docarray import DocumentArray
 from docarray.document.generators import from_csv
 
-from backend_config import papers_data_path, papers_data_url
+from config import config, create_parser
 from flows import index_flow, search_flow
 from helpers import download_csv, log, maximise_csv_field_size_limit
 
-
-# boolean args:
-# https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse/36031646
-def str2bool(v):
-    if isinstance(v, bool):
-        return v
-    if v.lower() in ("yes", "true", "t", "y", "1"):
-        return True
-    elif v.lower() in ("no", "false", "f", "n", "0"):
-        return False
-    else:
-        raise argparse.ArgumentTypeError("Boolean value expected.")
-
-
-def get_args():
-    # Command line arguments definitions
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--index",
-        dest="index",
-        default=False,
-        action="store_true",
-        help="index the available documents",
-    )
-    parser.add_argument(
-        "--n",
-        type=int,
-        default=0,
-        help="when `--index` is used, specifies the number of documnts to index (0 indexes the full dataset)",
-    )
-
-    return parser.parse_args()
+papers_data_path = config["papers_data_path"].get()
+papers_data_url = config["papers_data_url"].get()
 
 
 def index(n):
@@ -70,12 +40,13 @@ def index(n):
         indexer.index(papers, request_size=32)
 
 
-args = get_args()
+args = create_parser()
+config.set_args(args)
 
 if args.index:
     index(args.n)
 
-# running the search/finetuning flow as a service
+    # running the search/finetuning flow as a service
 flow = search_flow()
 flow.expose_endpoint("/finetune", summary="Finetune documents.", tags=["Finetuning"])
 
diff --git a/backend/src/config.py b/backend/src/config.py
new file mode 100644
index 0000000..2eb5fc1
--- /dev/null
+++ b/backend/src/config.py
@@ -0,0 +1,30 @@
+from argparse import ArgumentParser
+from pathlib import Path
+
+from confuse import Configuration
+
+config = Configuration("backend")
+
+file_abs_path = Path(__file__).parent.resolve()
+config.set_file(file_abs_path / "config.yaml")
+
+
+def create_parser():
+
+    parser = ArgumentParser()
+
+    parser.add_argument(
+        "--index",
+        dest="index",
+        default=False,
+        action="store_true",
+        help="index the available documents",
+    )
+    parser.add_argument(
+        "--n",
+        type=int,
+        default=0,
+        help="when `--index` is used, specifies the number of documnts to index (0 indexes the full dataset)",
+    )
+
+    return parser.parse_args()
diff --git a/backend/src/backend_config.py b/backend/src/config.yaml
similarity index 54%
rename from backend/src/backend_config.py
rename to backend/src/config.yaml
index 0d7a5eb..b1ea3b4 100644
--- a/backend/src/backend_config.py
+++ b/backend/src/config.yaml
@@ -1,17 +1,18 @@
+
 # Hugging Face: https://huggingface.co/sentence-transformers/allenai-specter
-embedding_model = "sentence-transformers/allenai-specter"
+embedding_model : "sentence-transformers/allenai-specter"
 
 # dataset link
-papers_data_url = "http://www.lri.fr/owncloud/index.php/s/OO987IvsoKwWI3l/download"
+papers_data_url : "http://www.lri.fr/owncloud/index.php/s/OO987IvsoKwWI3l/download"
 
 # Number of search results to show.
-top_k = 5
+top_k : 5
 
 # Protein file path relative to root.
-papers_data_path = "data/papers.csv"
+papers_data_path : "data/papers.csv"
 
 # Prints logs to command line if true.
-print_logs = True
+print_logs : True
 
 # Search service port
-search_port = 8020
+search_port : 8020
diff --git a/backend/src/executors.py b/backend/src/executors.py
index 5db6203..54b173b 100644
--- a/backend/src/executors.py
+++ b/backend/src/executors.py
@@ -1,14 +1,14 @@
-import re
 import os
 
-from typing import Sequence, List, Tuple
-from sentence_transformers import SentenceTransformer, InputExample, losses
+from jina import DocumentArray, Executor, requests
+from sentence_transformers import InputExample, SentenceTransformer, losses
 from torch.utils.data import DataLoader
-from jina import Executor, requests, Document, DocumentArray
 
-from backend_config import top_k, embedding_model
+from config import config
 from helpers import log
 
+embedding_model = config["embedding_model"].get()
+
 
 def get_model_dir():
     model_dir = f"./models/{embedding_model}"
diff --git a/backend/src/flows.py b/backend/src/flows.py
index 7f9501c..fa89182 100644
--- a/backend/src/flows.py
+++ b/backend/src/flows.py
@@ -1,7 +1,7 @@
 from jina import Flow
 
+from config import config
 from executors import SpecterExecutor
-from backend_config import top_k, search_port
 
 # Using a standard indexer: https://hub.jina.ai/executor/zb38xlt4
 indexer = "jinahub://SimpleIndexer"
@@ -21,14 +21,14 @@ def index_flow():
 
 def search_flow():
     flow = (
-        Flow(port_expose=search_port, protocol="http")
+        Flow(port_expose=config["search_port"].get(), protocol="http")
         .add(uses=SpecterExecutor)
         .add(
             uses=indexer,
             uses_with={
                 "match_args": {
                     "metric": "cosine",
-                    "limit": top_k,
+                    "limit": config["top_k"].get(),
                 },
             },
         )
diff --git a/backend/src/helpers.py b/backend/src/helpers.py
index 4ff6f09..476b5fe 100644
--- a/backend/src/helpers.py
+++ b/backend/src/helpers.py
@@ -1,10 +1,11 @@
 import os
+from argparse import ArgumentTypeError
 from csv import field_size_limit
 from sys import maxsize
 
 import requests
 
-from backend_config import print_logs
+from config import config
 
 
 def download_csv(url, fp):
@@ -15,15 +16,27 @@ def download_csv(url, fp):
 
 
 def log(message):
-    if print_logs:
+    if config["print_logs"].get():
         print(message)
 
 
 def maximise_csv_field_size_limit(maxInt=maxsize):
-
     while True:
         try:
             field_size_limit(maxInt)
             break
         except OverflowError:
             maxInt = int(maxInt / 10)
+
+
+# boolean args:
+# https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse/36031646
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ("yes", "true", "t", "y", "1"):
+        return True
+    elif v.lower() in ("no", "false", "f", "n", "0"):
+        return False
+    else:
+        raise ArgumentTypeError("Boolean value expected.")