Cache parts information with a database

Author: ubruhin

app.py

@@ -2,17 +2,21 @@
 
 import json
 import os
+import sqlite3
 
 from flask import (Flask, g, make_response, request, send_from_directory,
                    url_for)
 from werkzeug.middleware.proxy_fix import ProxyFix
 
+from database import Database
+from provider_cache import PartsCache
 from provider_partstack import Partstack
 
 app = Flask(__name__)
 app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)
 
 PARTS_MAX_COUNT = 10
+PARTS_CACHE_MAX_AGE = 30*24*3600  # 30 days due to quota limits
 PARTS_QUERY_TIMEOUT = 8.0
 
 
@@ -46,6 +50,20 @@ def _write_status(key_values):
         app.logger.critical(str(e))
 
 
+def _get_db():
+    db = getattr(g, '_db', None)
+    if db is None:
+        db = g._database = sqlite3.connect('/config/db.sqlite')
+    return db
+
+
+@app.teardown_appcontext
+def _close_db(exception):
+    db = getattr(g, '_db', None)
+    if db is not None:
+        db.close()
+
+
 @app.route('/api/v1/parts', methods=['GET'])
 def parts():
     enabled = _get_config('parts_operational', False)
@@ -71,21 +89,37 @@ def parts_query():
     parts = payload['parts'][:PARTS_MAX_COUNT]
     parts = [dict(mpn=p['mpn'], manufacturer=p['manufacturer']) for p in parts]
 
-    # Fetch parts from provider.
+    # Prepare database.
+    db = Database(_get_db(), app.logger)
+
+    # Fetch parts from providers.
     status = dict()
-    provider = Partstack(_get_config('parts_query_url'),
-                         _get_config('parts_query_token'),
-                         PARTS_QUERY_TIMEOUT, app.logger)
-    provider.fetch(parts, status)
+    cache_hits = 0
+    providers = [
+        PartsCache(db, max_age=PARTS_CACHE_MAX_AGE),
+        Partstack(_get_config('parts_query_url'),
+                  _get_config('parts_query_token'),
+                  PARTS_QUERY_TIMEOUT, db, app.logger),
+    ]
+    for provider in providers:
+        cache_hits += provider.fetch(parts, status)
 
     # Handle status changes.
     if len(status):
         _write_status(status)
 
     # Complete parts which were not found.
+    found = 0
     for part in parts:
         if 'results' not in part:
             part['results'] = 0
+        if part['results'] > 0:
+            found += 1
+
+    # Store request in database.
+    app.logger.debug(f"Queried {len(parts)} parts, {cache_hits} from cache: "
+                     f"{found} found, {len(parts) - found} not found")
+    db.add_parts_request(len(parts), cache_hits, found)
 
     # Return response.
     return dict(parts=parts)

database.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+
+from ast import literal_eval
+
+
+class Database:
+    def __init__(self, db, logger):
+        self._db = db
+        self._logger = logger
+
+        with self._db as db:
+            db_version = db.execute('PRAGMA user_version').fetchone()[0]
+            if db_version < 1:
+                logger.info("Migrating database to version 1...")
+                db.execute("""
+                    CREATE TABLE IF NOT EXISTS parts_requests (
+                        id INTEGER PRIMARY KEY NOT NULL,
+                        datetime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                        count INTEGER NOT NULL,
+                        cache_hits INTEGER NOT NULL,
+                        with_result INTEGER NOT NULL
+                    )
+                """)
+                db.execute("""
+                    CREATE TABLE IF NOT EXISTS parts_cache (
+                        id INTEGER PRIMARY KEY NOT NULL,
+                        mpn TEXT NOT NULL,
+                        manufacturer TEXT NOT NULL,
+                        provider TEXT NOT NULL,
+                        datetime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                        part TEXT NOT NULL,
+                        UNIQUE(mpn, manufacturer, provider)
+                    )
+                """)
+                db.execute('PRAGMA user_version=1')
+        self._db.execute("VACUUM")
+
+    def add_parts_request(self, count: int, cache_hits: int,
+                          with_result: int):
+        with self._db as db:
+            db.execute(
+                "INSERT INTO parts_requests "
+                "(count, cache_hits, with_result) "
+                "VALUES (?, ?, ?)",
+                (count, cache_hits, with_result)
+            )
+
+    def add_parts_cache(self, provider: str, part: dict):
+        with self._db as db:
+            db.execute(
+                "INSERT INTO parts_cache "
+                "(mpn, manufacturer, provider, part) "
+                "VALUES (?, ?, ?, ?) "
+                "ON CONFLICT(mpn, manufacturer, provider) DO UPDATE SET "
+                "  datetime = CURRENT_TIMESTAMP, "
+                "  part = excluded.part",
+                (part['mpn'], part['manufacturer'], provider, str(part))
+            )
+
+    def get_parts_cache(self, mpn, manufacturer, max_age):
+        with self._db as db:
+            cur = db.cursor()
+            cur.execute(
+                "SELECT part FROM parts_cache "
+                "WHERE mpn=? AND manufacturer=? "
+                "AND datetime >= datetime('now', ?)"
+                "ORDER BY datetime DESC",
+                (mpn, manufacturer, f"-{max_age} seconds")
+            )
+            row = cur.fetchone()
+            return literal_eval(row[0]) if row is not None else None

provider_cache.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+
+
+class PartsCache:
+    def __init__(self, db, max_age):
+        self._db = db
+        self._max_age = max_age
+
+    def fetch(self, parts, status):
+        hits = 0
+        for part in parts:
+            if 'results' not in part:
+                if self._fetch_part(part):
+                    hits += 1
+        return hits
+
+    def _fetch_part(self, part):
+        result = self._db.get_parts_cache(part['mpn'], part['manufacturer'],
+                                          self._max_age)
+        if result is not None:
+            part.update(result)
+            return True
+        else:
+            return False

provider_dummy.py

@@ -4,17 +4,19 @@
 class DummyProvider:
     ID = 'dummy'
 
-    def __init__(self):
-        pass
+    def __init__(self, db):
+        self._db = db
 
     def fetch(self, parts, status):
         for i in range(len(parts)):
             part = parts[i]
             if 'results' not in part:
                 self._fetch_part(part, i)
+        return 0
 
     def _fetch_part(self, part, i):
         if i % 2:
             part['results'] = 1
             part['status'] = 'Active'
             part['prices'] = [dict(quantity=1, price=13.37)]
+            self._db.add_parts_cache(self.ID, part)

provider_partstack.py

@@ -66,10 +66,11 @@ class Partstack:
     URL = 'https://partstack.com'
     LOGO_FILENAME = 'parts-provider-partstack.png'
 
-    def __init__(self, query_url, query_token, query_timeout, logger):
+    def __init__(self, query_url, query_token, query_timeout, db, logger):
         self._query_url = query_url
         self._query_token = query_token
         self._query_timeout = query_timeout
+        self._db = db
         self._logger = logger
 
     def fetch(self, parts, status):
@@ -111,6 +112,8 @@ def fetch(self, parts, status):
                     self._add_availability(parts[i], summary)
                     self._add_prices(parts[i], summary)
                     self._add_resources(parts[i], product)
+                    self._db.add_parts_cache(self.ID, parts[i])
+        return 0
 
     def _build_headers(self):
         return {