From 6ce9dd9df2ef309df911dbc709aa71f349b5020a Mon Sep 17 00:00:00 2001 From: Qiushi Bai Date: Wed, 12 Oct 2022 20:46:27 -0700 Subject: [PATCH 1/2] Initial stage of QueryLogger implementation. --- core/data_manager.py | 56 ++++++++++++++++++++++++++++++++++++++++++ core/query_logger.py | 45 +++++++++++++++++++++++++++++++++ core/query_rewriter.py | 7 ++++-- schema/rules.sql | 18 ++++++++++++++ server/server.py | 47 ++++++++++++++++++++++++++++++++++- 5 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 core/query_logger.py diff --git a/core/data_manager.py b/core/data_manager.py index 20885ea..6c0e1f0 100644 --- a/core/data_manager.py +++ b/core/data_manager.py @@ -1,3 +1,4 @@ +import datetime import sqlite3 from sqlite3 import Error from pathlib import Path @@ -86,6 +87,61 @@ def update_rule(self, rule: dict) -> None: self.db_conn.commit() except Error as e: print(e) + + def log_query(self, original_query: str, rewritten_query: str, rewriting_path: list) -> None: + try: + cur = self.db_conn.cursor() + cur.execute('''SELECT MAX(id) + 1 FROM query_logs;''') + query_id = cur.fetchall()[0] + + cur.execute('''INSERT INTO query_logs (id, timestamp, latency, original_sql, rewritten_sql) + VALUES (?, ?, ?, ?, ?)''', + [query_id, datetime.datetime.now(), -1, original_query, rewritten_query]) + seq = 1 + for rewriting in rewriting_path: + cur.execute('''INSERT INTO rewriting_paths (query_id, seq, rule_id, rewritten_sql) + VALUES (?, ?, ?, ?)''', + [query_id, seq, rewriting[0], rewriting[1]]) + seq += 1 + self.db_conn.commit() + except Error as e: + print(e) + + def list_queries(self) -> List[Dict]: + try: + cur = self.db_conn.cursor() + cur.execute('''SELECT id, + timestamp, + latency, + original_sql, + rewritten_sql + FROM query_logs + ORDER BY id desc''') + return cur.fetchall() + except Error as e: + print(e) + + def get_original_sql(self, query_id: int) -> str: + try: + cur = self.db_conn.cursor() + cur.execute('''SELECT original_sql + FROM query_logs + WHERE id = ?''', [query_id]) + return cur.fetchall()[0] + except Error as e: + print(e) + + def list_rewritings(self, query_id: int) -> List[Dict]: + try: + cur = self.db_conn.cursor() + cur.execute('''SELECT seq, + name, + rewritten_sql + FROM rewriting_paths LEFT JOIN rules ON rules.id = rewriting_paths.rule_id + WHERE query_id = ?''', [query_id]) + return cur.fetchall() + except Error as e: + print(e) if __name__ == '__main__': diff --git a/core/query_logger.py b/core/query_logger.py new file mode 100644 index 0000000..be88e6b --- /dev/null +++ b/core/query_logger.py @@ -0,0 +1,45 @@ +import sys +# append the path of the parent directory +sys.path.append("..") +from core.data_manager import DataManager +import json + + +class QueryLogger: + + def __init__(self, dm: DataManager) -> None: + self.dm = dm + + def __del__(self): + del self.dm + + def log_query(self, original_query: str, rewritten_query: str, rewriting_path: list) -> None: + self.dm.log_query(original_query, rewritten_query, rewriting_path) + + def list_queries(self) -> list: + queries = self.dm.list_queries() + res = [] + for query in queries: + res.append({ + 'id': query[0], + 'timestamp': query[1], + 'latency': query[2], + 'original_sql': query[3], + 'rewritten_sql': query[4] + }) + return res + + def rewriting_path(self, query_id: str) -> dict: + original_sql = self.dm.get_original_sql(query_id) + res = { + "original_sql": original_sql, + "rewritings":[] + } + rewritings = self.dm.list_rewritings(query_id) + for rewriting in rewritings: + res["rewritings"].append({ + "seq": rewriting[0], + "rule": rewriting[1], + "rewritten_sql": rewriting[2] + }) + return res diff --git a/core/query_rewriter.py b/core/query_rewriter.py index 2b6eeb2..1208ba2 100644 --- a/core/query_rewriter.py +++ b/core/query_rewriter.py @@ -33,10 +33,12 @@ def beautify(query: str) -> str: # } # @staticmethod - def rewrite(query: str, rules: list) -> str: + def rewrite(query: str, rules: list) -> tuple[str, list]: query_ast = parse(query) + rewriting_path = [] + new_query = True while new_query is True: new_query = False @@ -45,10 +47,11 @@ def rewrite(query: str, rules: list) -> str: if QueryRewriter.match(query_ast, rule, memo): query_ast = QueryRewriter.take_actions(query_ast, rule, memo) query_ast = QueryRewriter.replace(query_ast, rule, memo) + rewriting_path.append((rule['id'], format(query_ast))) new_query = True break - return format(query_ast) + return format(query_ast), rewriting_path # Traverse query AST tree, and check if rule->pattern matches any node of in query # diff --git a/schema/rules.sql b/schema/rules.sql index 4835430..1a9dcba 100644 --- a/schema/rules.sql +++ b/schema/rules.sql @@ -23,3 +23,21 @@ CREATE TABLE IF NOT EXISTS internal_rules( actions_json TEXT, FOREIGN KEY (rule_id) REFERENCES rules(id) ); + +CREATE TABLE IF NOT EXISTS query_logs( + id INTEGER PRIMARY KEY, + timestamp TEXT, + latency REAL, + original_sql TEXT, + rewritten_sql TEXT +); + +CREATE TABLE IF NOT EXISTS rewriting_paths( + query_id INTEGER, + seq INTEGER, + rule_id INTEGER, + rewritten_sql TEXT, + PRIMARY KEY (query_id, seq), + FOREIGN KEY (query_id) REFERENCES query_logs(id), + FOREIGN KEY (rule_id) REFERENCES rules(id) +) diff --git a/server/server.py b/server/server.py index 2c310b5..edb4b2f 100644 --- a/server/server.py +++ b/server/server.py @@ -9,6 +9,7 @@ from core.query_rewriter import QueryRewriter from core.data_manager import DataManager from core.rule_manager import RuleManager +from core.query_logger import QueryLogger PORT = 8000 DIRECTORY = "static" @@ -17,6 +18,7 @@ class MyHTTPRequestHandler(SimpleHTTPRequestHandler): dm = DataManager() rm = RuleManager(dm) + ql = QueryLogger(dm) def __init__(self, *args, **kwargs): super().__init__(*args, directory=DIRECTORY, **kwargs) @@ -45,8 +47,11 @@ def post_query(self): log_text += "\n--------------------------------------------------" logging.info(log_text) rules = self.rm.fetch_enabled_rules(database) - rewritten_query = QueryRewriter.rewrite(original_query, rules) + rewritten_query, rewriting_path = QueryRewriter.rewrite(original_query, rules) rewritten_query = QueryPatcher.patch(rewritten_query, database) + for rewriting in rewriting_path: + rewriting[1] = QueryPatcher.patch(rewriting[1]) + self.ql.log_query(original_query, rewritten_query, rewriting_path) log_text = "" log_text += "\n==================================================" log_text += "\n Rewritten query" @@ -96,6 +101,42 @@ def post_switch_rule(self): response = BytesIO() response.write(str(success).encode('utf-8')) self.wfile.write(response.getvalue()) + + def post_list_queries(self): + content_length = int(self.headers['Content-Length']) + body = self.rfile.read(content_length) + request = body.decode('utf-8') + + # logging + logging.info("\n[/listQueries] request:") + logging.info(request) + + queries_json = self.ql.list_queries() + + self.send_response(200) + self.end_headers() + response = BytesIO() + response.write(json.dumps(queries_json).encode('utf-8')) + self.wfile.write(response.getvalue()) + + def post_rewriting_path(self): + content_length = int(self.headers['Content-Length']) + body = self.rfile.read(content_length) + request = body.decode('utf-8') + + # logging + logging.info("\n[/rewritingPath] request:") + logging.info(request) + + # enable/disable rule to data manager + query_id = json.loads(request)["queryId"] + rewriting_path_json = self.ql.rewriting_path(query_id) + + self.send_response(200) + self.end_headers() + response = BytesIO() + response.write(json.dumps(rewriting_path_json).encode('utf-8')) + self.wfile.write(response.getvalue()) def do_POST(self): if self.path == "/": @@ -104,6 +145,10 @@ def do_POST(self): self.post_list_rules() elif self.path == "/switchRule": self.post_switch_rule() + elif self.path == "/listQueries": + self.post_list_queries() + elif self.path == "/rewritingPath": + self.post_rewriting_path() if __name__ == '__main__': From 675cb91ae565d5cff58e69f523d72268f36088c6 Mon Sep 17 00:00:00 2001 From: Qiushi Bai Date: Thu, 13 Oct 2022 13:20:53 -0700 Subject: [PATCH 2/2] Fix bugs to support QueryLogs and QueryRewritingPath frontend features. --- client/src/dashboard/QueryRewritingPath.js | 2 +- core/data_manager.py | 4 ++-- core/query_rewriter.py | 2 +- server/server.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/src/dashboard/QueryRewritingPath.js b/client/src/dashboard/QueryRewritingPath.js index b33dd83..80139ec 100644 --- a/client/src/dashboard/QueryRewritingPath.js +++ b/client/src/dashboard/QueryRewritingPath.js @@ -52,7 +52,7 @@ const QueryRewritingPath = NiceModal.create(({ queryId }) => { }; // call rewritePath() only once after initial rendering - React.useEffect(() => {getRewritingPath({queryId})}, []); + React.useEffect(() => {getRewritingPath(queryId)}, []); return ( None: def log_query(self, original_query: str, rewritten_query: str, rewriting_path: list) -> None: try: cur = self.db_conn.cursor() - cur.execute('''SELECT MAX(id) + 1 FROM query_logs;''') - query_id = cur.fetchall()[0] + cur.execute('''SELECT IFNULL(MAX(id), 0) + 1 FROM query_logs;''') + query_id = cur.fetchone()[0] cur.execute('''INSERT INTO query_logs (id, timestamp, latency, original_sql, rewritten_sql) VALUES (?, ?, ?, ?, ?)''', diff --git a/core/query_rewriter.py b/core/query_rewriter.py index 1208ba2..c13c8ae 100644 --- a/core/query_rewriter.py +++ b/core/query_rewriter.py @@ -47,7 +47,7 @@ def rewrite(query: str, rules: list) -> tuple[str, list]: if QueryRewriter.match(query_ast, rule, memo): query_ast = QueryRewriter.take_actions(query_ast, rule, memo) query_ast = QueryRewriter.replace(query_ast, rule, memo) - rewriting_path.append((rule['id'], format(query_ast))) + rewriting_path.append([rule['id'], format(query_ast)]) new_query = True break diff --git a/server/server.py b/server/server.py index edb4b2f..b4b09ca 100644 --- a/server/server.py +++ b/server/server.py @@ -50,7 +50,7 @@ def post_query(self): rewritten_query, rewriting_path = QueryRewriter.rewrite(original_query, rules) rewritten_query = QueryPatcher.patch(rewritten_query, database) for rewriting in rewriting_path: - rewriting[1] = QueryPatcher.patch(rewriting[1]) + rewriting[1] = QueryPatcher.patch(rewriting[1], database) self.ql.log_query(original_query, rewritten_query, rewriting_path) log_text = "" log_text += "\n=================================================="