updated backend to fully use config.yaml

Author: sacca97

prospector/.env-sample

@@ -19,4 +19,3 @@ POSTGRES_PASSWORD=example
 REDIS_URL=redis://localhost:6379/0
 NVD_API_KEY=yourNvdApiKey
 PYTHONPATH=.
-GITHUB_TOKEN=yourGithubApiToken

prospector/Makefile

@@ -31,13 +31,13 @@ docker-clean:
 	@docker-compose down --rmi all -v 2>/dev/null
 	@echo "$(DONE) Stopped and removed all container and images"
 
-# @echo "$(PROGRESS) Cleaning volumes"
-# @docker volume prune -f
-# @echo "$(DONE) Cleaned volumes"
+	@echo "$(PROGRESS) Cleaning volumes"
+	@docker volume prune -f
+	@echo "$(DONE) Cleaned volumes"
 
-# @echo "$(PROGRESS) Cleaning residue"
-# @docker system prune -a -f
-# @echo "$(DONE) Cleaned residue"
+	@echo "$(PROGRESS) Cleaning residue"
+	@docker system prune -a -f
+	@echo "$(DONE) Cleaned residue"
 
 
 clean:

prospector/api/routers/preprocessed.py

@@ -4,6 +4,10 @@
 from fastapi.responses import JSONResponse
 
 from commitdb.postgres import PostgresCommitDB
+from util.config_parser import parse_config_file
+
+config = parse_config_file()
+
 
 router = APIRouter(
     prefix="/commits",
@@ -18,7 +22,13 @@ async def get_commits(
     repository_url: str,
     commit_id: Optional[str] = None,
 ):
-    db = PostgresCommitDB()
+    db = PostgresCommitDB(
+        config.database.user,
+        config.database.password,
+        config.database.host,
+        config.database.port,
+        config.database.dbname,
+    )
     db.connect()
     data = db.lookup(repository_url, commit_id)
 
@@ -32,7 +42,13 @@ async def get_commits(
 @router.post("/")
 async def upload_preprocessed_commit(payload: List[Dict[str, Any]]):
 
-    db = PostgresCommitDB()
+    db = PostgresCommitDB(
+        config.database.user,
+        config.database.password,
+        config.database.host,
+        config.database.port,
+        config.database.dbname,
+    )
     db.connect()
 
     for commit in payload:

prospector/client/cli/prospector_client_test.py

@@ -2,8 +2,6 @@
 
 import pytest
 
-from commitdb.postgres import PostgresCommitDB
-
 from .prospector_client import prospector
 
 OPENCAST_CVE = "CVE-2021-21318"

prospector/commitdb/commitdb_test.py

@@ -6,7 +6,7 @@
 
 @pytest.fixture
 def setupdb():
-    db = PostgresCommitDB()
+    db = PostgresCommitDB("postgres", "example", "localhost", "5432", "postgres")
     db.connect()
     # db.reset()
     return db

prospector/commitdb/postgres.py

@@ -12,13 +12,13 @@
 from commitdb import CommitDB
 from log.logger import logger
 
-DB_CONNECT_STRING = "postgresql://{}:{}@{}:{}/{}".format(
-    os.getenv("POSTGRES_USER", "postgres"),
-    os.getenv("POSTGRES_PASSWORD", "example"),
-    os.getenv("POSTGRES_HOST", "localhost"),
-    os.getenv("POSTGRES_PORT", "5432"),
-    os.getenv("POSTGRES_DBNAME", "postgres"),
-).lower()
+# DB_CONNECT_STRING = "postgresql://{}:{}@{}:{}/{}".format(
+#     os.getenv("POSTGRES_USER", "postgres"),
+#     os.getenv("POSTGRES_PASSWORD", "example"),
+#     os.getenv("POSTGRES_HOST", "localhost"),
+#     os.getenv("POSTGRES_PORT", "5432"),
+#     os.getenv("POSTGRES_DBNAME", "postgres"),
+# ).lower()
 
 
 class PostgresCommitDB(CommitDB):
@@ -27,12 +27,14 @@ class PostgresCommitDB(CommitDB):
     for PostgreSQL
     """
 
-    def __init__(self):
-        self.connect_string = ""
+    def __init__(self, user, password, host, port, dbname):
+        self.connect_string = "postgresql://{}:{}@{}:{}/{}".format(
+            user, password, host, port, dbname
+        ).lower()
         self.connection = None
 
-    def connect(self, connect_string=DB_CONNECT_STRING):
-        self.connection = psycopg2.connect(connect_string)
+    def connect(self):
+        self.connection = psycopg2.connect(self.connect_string)
 
     def lookup(self, repository: str, commit_id: str = None) -> List[Dict[str, Any]]:
         if not self.connection:

prospector/config-sample.yaml

@@ -20,6 +20,13 @@ use_backend: optional
 # Optional backend info to save/use already preprocessed data
 backend: http://localhost:8000
 
+database:
+  user: postgres
+  password: example
+  host: db
+  port: 5432
+  dbname: postgres
+
 # Report file format: "html", "json", "console" or "all"
 # and the file name
 report:

prospector/docker-compose.yml

@@ -11,15 +11,15 @@ services:
       - redis
       - db
     environment:
-      GIT_CACHE: /tmp
+      GIT_CACHE: /tmp/gitcache
       CVE_DATA_PATH: /app/cve_data
       REDIS_URL: redis://redis:6379/0
-      POSTGRES_HOST: db
-      POSTGRES_PORT: 5432
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: example
-      POSTGRES_DBNAME: postgres
-      NVD_API_KEY: ${NVD_API_KEY}
+      #POSTGRES_HOST: db
+      #POSTGRES_PORT: 5432
+      #POSTGRES_USER: postgres
+      #POSTGRES_PASSWORD: example
+      #POSTGRES_DBNAME: postgres
+      #NVD_API_KEY: ${NVD_API_KEY}
 
   worker:
     build:
@@ -41,7 +41,7 @@ services:
     ports:
       - "5432:5432"
     environment:
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_PASSWORD: example #${POSTGRES_PASSWORD}
     volumes:
       - ./ddl:/docker-entrypoint-initdb.d
 

prospector/docker/api/Dockerfile

@@ -14,4 +14,4 @@ ENV PYTHONPATH .
 RUN rm -rf /app/rules
 RUN mkdir /app/cve_data
 
-CMD ["./start.sh"]
+CMD ["python","./main.py"]

prospector/docker/api/start.sh

@@ -1,6 +1,6 @@
 #! /usr/bin/env sh
 
-python api/routers/nvd_feed_update.py
-echo "NVD feed download complete"
+# python api/routers/nvd_feed_update.py
+# echo "NVD feed download complete"
 
-python main.py
+python main.py