Add QueryLogs and QueryRewritingPath frontend features

client/src/dashboard/QueryRewritingPath.js

@@ -52,7 +52,7 @@ const QueryRewritingPath = NiceModal.create(({ queryId }) => {
   };
 
   // call rewritePath() only once after initial rendering
-  React.useEffect(() => {getRewritingPath({queryId})}, []);
+  React.useEffect(() => {getRewritingPath(queryId)}, []);
 
   return (
     <Dialog

core/data_manager.py

@@ -1,3 +1,4 @@
+import datetime
 import sqlite3
 from sqlite3 import Error
 from pathlib import Path
@@ -86,6 +87,61 @@ def update_rule(self, rule: dict) -> None:
             self.db_conn.commit()
         except Error as e:
             print(e)
+
+    def log_query(self, original_query: str, rewritten_query: str, rewriting_path: list) -> None:
+        try:
+            cur = self.db_conn.cursor()
+            cur.execute('''SELECT IFNULL(MAX(id), 0) + 1 FROM query_logs;''')
+            query_id = cur.fetchone()[0]
+
+            cur.execute('''INSERT INTO query_logs (id, timestamp, latency, original_sql, rewritten_sql) 
+                                       VALUES (?, ?, ?, ?, ?)''', 
+                        [query_id, datetime.datetime.now(), -1, original_query, rewritten_query])
+            seq = 1
+            for rewriting in rewriting_path:
+                cur.execute('''INSERT INTO rewriting_paths (query_id, seq, rule_id, rewritten_sql)
+                                           VALUES (?, ?, ?, ?)''', 
+                            [query_id, seq, rewriting[0], rewriting[1]])
+                seq += 1
+            self.db_conn.commit()
+        except Error as e:
+            print(e)
+
+    def list_queries(self) -> List[Dict]:
+        try:
+            cur = self.db_conn.cursor()
+            cur.execute('''SELECT id, 
+                                  timestamp, 
+                                  latency, 
+                                  original_sql,
+                                  rewritten_sql
+                           FROM query_logs 
+                           ORDER BY id desc''')
+            return cur.fetchall()
+        except Error as e:
+            print(e)
+
+    def get_original_sql(self, query_id: int) -> str:
+        try:
+            cur = self.db_conn.cursor()
+            cur.execute('''SELECT original_sql
+                           FROM query_logs 
+                           WHERE id = ?''', [query_id])
+            return cur.fetchall()[0]
+        except Error as e:
+            print(e)
+
+    def list_rewritings(self, query_id: int) -> List[Dict]:
+        try:
+            cur = self.db_conn.cursor()
+            cur.execute('''SELECT seq, 
+                                  name, 
+                                  rewritten_sql
+                           FROM rewriting_paths LEFT JOIN rules ON rules.id = rewriting_paths.rule_id
+                           WHERE query_id = ?''', [query_id])
+            return cur.fetchall()
+        except Error as e:
+            print(e)
 
 
 if __name__ == '__main__':

core/query_logger.py

@@ -0,0 +1,45 @@
+import sys
+# append the path of the parent directory
+sys.path.append("..")
+from core.data_manager import DataManager
+import json
+
+
+class QueryLogger:
+
+    def __init__(self, dm: DataManager) -> None:
+        self.dm = dm
+
+    def __del__(self):
+        del self.dm
+
+    def log_query(self, original_query: str, rewritten_query: str, rewriting_path: list) -> None:
+        self.dm.log_query(original_query, rewritten_query, rewriting_path)
+
+    def list_queries(self) -> list:
+        queries = self.dm.list_queries()
+        res = []
+        for query in queries:
+            res.append({
+                'id': query[0],
+                'timestamp': query[1],
+                'latency': query[2],
+                'original_sql': query[3],
+                'rewritten_sql': query[4]
+            })
+        return res
+
+    def rewriting_path(self, query_id: str) -> dict:
+        original_sql = self.dm.get_original_sql(query_id)
+        res = {
+            "original_sql": original_sql,
+            "rewritings":[]
+        }
+        rewritings = self.dm.list_rewritings(query_id)
+        for rewriting in rewritings:
+            res["rewritings"].append({
+                "seq": rewriting[0],
+                "rule": rewriting[1],
+                "rewritten_sql": rewriting[2]
+            })
+        return res

core/query_rewriter.py

@@ -33,10 +33,12 @@ def beautify(query: str) -> str:
     #     }
     # 
     @staticmethod
-    def rewrite(query: str, rules: list) -> str:
+    def rewrite(query: str, rules: list) -> tuple[str, list]:
 
         query_ast = parse(query)
 
+        rewriting_path = []
+
         new_query = True
         while new_query is True:
             new_query = False
@@ -45,10 +47,11 @@ def rewrite(query: str, rules: list) -> str:
                 if QueryRewriter.match(query_ast, rule, memo):
                     query_ast = QueryRewriter.take_actions(query_ast, rule, memo)
                     query_ast = QueryRewriter.replace(query_ast, rule, memo)
+                    rewriting_path.append([rule['id'], format(query_ast)])
                     new_query = True
                     break
 
-        return format(query_ast)
+        return format(query_ast), rewriting_path
 
     # Traverse query AST tree, and check if rule->pattern matches any node of in query
     # 

schema/rules.sql

@@ -23,3 +23,21 @@ CREATE TABLE IF NOT EXISTS internal_rules(
     actions_json TEXT,
     FOREIGN KEY (rule_id) REFERENCES rules(id)
 );
+
+CREATE TABLE IF NOT EXISTS query_logs(
+    id INTEGER PRIMARY KEY,
+    timestamp TEXT,
+    latency REAL,
+    original_sql TEXT,
+    rewritten_sql TEXT
+);
+
+CREATE TABLE IF NOT EXISTS rewriting_paths(
+    query_id INTEGER,
+    seq INTEGER,
+    rule_id INTEGER,
+    rewritten_sql TEXT,
+    PRIMARY KEY (query_id, seq),
+    FOREIGN KEY (query_id) REFERENCES query_logs(id),
+    FOREIGN KEY (rule_id) REFERENCES rules(id)
+)

server/server.py

@@ -9,6 +9,7 @@
 from core.query_rewriter import QueryRewriter
 from core.data_manager import DataManager
 from core.rule_manager import RuleManager
+from core.query_logger import QueryLogger
 
 PORT = 8000
 DIRECTORY = "static"
@@ -17,6 +18,7 @@ class MyHTTPRequestHandler(SimpleHTTPRequestHandler):
 
     dm = DataManager()
     rm = RuleManager(dm)
+    ql = QueryLogger(dm)
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, directory=DIRECTORY, **kwargs)
@@ -45,8 +47,11 @@ def post_query(self):
         log_text += "\n--------------------------------------------------"
         logging.info(log_text)
         rules = self.rm.fetch_enabled_rules(database)
-        rewritten_query = QueryRewriter.rewrite(original_query, rules)
+        rewritten_query, rewriting_path = QueryRewriter.rewrite(original_query, rules)
         rewritten_query = QueryPatcher.patch(rewritten_query, database)
+        for rewriting in rewriting_path:
+            rewriting[1] = QueryPatcher.patch(rewriting[1], database)
+        self.ql.log_query(original_query, rewritten_query, rewriting_path)
         log_text = ""
         log_text += "\n=================================================="
         log_text += "\n    Rewritten query"
@@ -96,6 +101,42 @@ def post_switch_rule(self):
         response = BytesIO()
         response.write(str(success).encode('utf-8'))
         self.wfile.write(response.getvalue())
+
+    def post_list_queries(self):
+        content_length = int(self.headers['Content-Length'])
+        body = self.rfile.read(content_length)
+        request = body.decode('utf-8')
+
+        # logging
+        logging.info("\n[/listQueries] request:")
+        logging.info(request)
+
+        queries_json = self.ql.list_queries()
+
+        self.send_response(200)
+        self.end_headers()
+        response = BytesIO()
+        response.write(json.dumps(queries_json).encode('utf-8'))
+        self.wfile.write(response.getvalue())
+
+    def post_rewriting_path(self):
+        content_length = int(self.headers['Content-Length'])
+        body = self.rfile.read(content_length)
+        request = body.decode('utf-8')
+
+        # logging
+        logging.info("\n[/rewritingPath] request:")
+        logging.info(request)
+
+        # enable/disable rule to data manager
+        query_id = json.loads(request)["queryId"]
+        rewriting_path_json = self.ql.rewriting_path(query_id)
+
+        self.send_response(200)
+        self.end_headers()
+        response = BytesIO()
+        response.write(json.dumps(rewriting_path_json).encode('utf-8'))
+        self.wfile.write(response.getvalue())
 
     def do_POST(self):
         if self.path == "/":
@@ -104,6 +145,10 @@ def do_POST(self):
             self.post_list_rules()
         elif self.path == "/switchRule":
             self.post_switch_rule()
+        elif self.path == "/listQueries":
+            self.post_list_queries()
+        elif self.path == "/rewritingPath":
+            self.post_rewriting_path()
 
 
 if __name__ == '__main__':