diff --git a/client/src/App.js b/client/src/App.js index 591b8b8..18bb37b 100644 --- a/client/src/App.js +++ b/client/src/App.js @@ -1,9 +1,15 @@ +import React from 'react'; import logo from './logo.svg'; import './App.css'; import Dashboard from './dashboard/Dashboard'; +// user context +import {userContext} from './userContext'; function App() { document.title = 'QueryBooster'; + + const [user, setUser] = React.useState({"id": 1, "email": "alice@ics.uci.edu"}); + return (
{/*
@@ -20,7 +26,9 @@ function App() { Learn React
*/} - + + +
); } diff --git a/client/src/dashboard/ApplicationSelect.js b/client/src/dashboard/ApplicationSelect.js new file mode 100644 index 0000000..9947231 --- /dev/null +++ b/client/src/dashboard/ApplicationSelect.js @@ -0,0 +1,85 @@ +import React, { useState, useCallback } from 'react'; +import Box from '@mui/material/Box'; +import Button from '@mui/material/Button'; +import Dialog from '@mui/material/Dialog'; +import DialogContent from '@mui/material/DialogContent'; +import DialogTitle from '@mui/material/DialogTitle'; +import DialogActions from '@mui/material/DialogActions'; +import NiceModal, { useModal } from '@ebay/nice-modal-react'; +import axios from 'axios'; +import defaultApplicationsData from '../mock-api/listApplications'; + +const ApplicationSelect = NiceModal.create(({ user }) => { + const modal = useModal(); + // Set up a state for list of applications + const [applications, setApplications] = React.useState([]); + // Set up a state for selected application Id + const [selectedAppId, setSelectedAppId] = useState(-1); + + const handleSelectChange = (event) => { + setSelectedAppId(event.target.value); + }; + + // initial loading applications for the current user from server + const listApplications = (user) => { + console.log('[/listApplications] -> request:'); + console.log(' user_id: ' + user.id); + // post listApplications request to server + axios.post('/listApplications', { 'user_id': user.id }) + .then(function (response) { + console.log('[/listApplications] -> response:'); + console.log(response); + // update the state for list of applications + setApplications(response.data); + }) + .catch(function (error) { + console.log('[/listApplications] -> error:'); + console.log(error); + // mock the result + console.log(defaultApplicationsData); + setApplications(defaultApplicationsData); + }); + }; + + // call listApplications() only once after initial rendering + React.useEffect(() => { listApplications(user) }, []); + + const handleSubmit = useCallback(() => { + const selectedApplication = applications.find((app) => app.id == selectedAppId); + console.log("[ApplicationSelect] selectedAppId = " + selectedAppId); + console.log("[ApplicationSelect] applications = "); + console.log(applications); + console.log("[ApplicationSelect] find selected application = "); + console.log(selectedApplication); + modal.resolve(selectedApplication); + modal.hide(); + }, [modal]); + + return ( + modal.hide()} + TransitionProps={{ + onExited: () => modal.remove(), + }} + maxWidth={'sm'} + > + Enable Rule for Application + + + + + + + + + + ); +}); + +export default ApplicationSelect; \ No newline at end of file diff --git a/client/src/dashboard/ApplicationTag.js b/client/src/dashboard/ApplicationTag.js new file mode 100644 index 0000000..48ddf69 --- /dev/null +++ b/client/src/dashboard/ApplicationTag.js @@ -0,0 +1,78 @@ +import * as React from 'react'; +import axios from 'axios'; +import { useModal } from '@ebay/nice-modal-react'; +import ApplicationSelect from './ApplicationSelect'; +import {userContext} from '../userContext'; + +function AppTagCell({ruleId: initialRuleId, tags: initialApps }) { + const [ruleId, setRule] = React.useState(initialRuleId); + const [apps, setApps] = React.useState(initialApps); + // Set up a state for providing forceUpdate function + const [, updateState] = React.useState(); + const forceUpdate = React.useCallback(() => updateState({}), []); + + const applicationSelectModal = useModal(ApplicationSelect); + + const user = React.useContext(userContext); + + function handleSelect(selectedApplication) { + if (selectedApplication) { + // post enableRule request to server + axios.post('/enableRule', {'rule': {'id': ruleId}, 'app': selectedApplication}) + .then(function (response) { + console.log('[/enableRule] -> response:'); + console.log(response); + setApps([...apps, {'app_id': selectedApplication.id, 'app_name': selectedApplication.name}]); + forceUpdate(); + }) + .catch(function (error) { + console.log('[/enableRule] -> error:'); + console.log(error); + // TODO - alter the entered application name doest not exist + }); + } + } + + const handleAddApp = React.useCallback(() => { + applicationSelectModal.show({user}).then((selectedApplication) => { + console.log("[ApplicationTag] selectedApplication = "); + console.log(selectedApplication); + handleSelect(selectedApplication); + }); + }, [applicationSelectModal]); + + function handleRemoveApp(app) { + // post disableRule request to server + axios.post('/disableRule', {'rule': {'id': ruleId}, 'app': {'id': app.app_id, 'name': app.app_name}}) + .then(function (response) { + console.log('[/disableRule] -> response:'); + console.log(response); + const updatedApps = apps.filter((a) => a !== app); + setApps(updatedApps); + forceUpdate(); + }) + .catch(function (error) { + console.log('[/disableRule] -> error:'); + console.log(error); + }); + } + + return ( +
+ {apps.map((app) => ( + + {app.app_name} + + + ))} + +
+ ); +} + +export default AppTagCell; diff --git a/client/src/dashboard/QueryLogs.js b/client/src/dashboard/QueryLogs.js index cb7a633..2942b67 100644 --- a/client/src/dashboard/QueryLogs.js +++ b/client/src/dashboard/QueryLogs.js @@ -12,6 +12,7 @@ import defaultQueriesData from '../mock-api/listQueries'; import QueryRewritingPath from './QueryRewritingPath'; import SyntaxHighlighter from 'react-syntax-highlighter'; import { vs } from 'react-syntax-highlighter/dist/esm/styles/hljs'; +import {userContext} from '../userContext'; export default function QueryLogs() { @@ -21,10 +22,12 @@ export default function QueryLogs() { const [, updateState] = React.useState(); const forceUpdate = React.useCallback(() => updateState({}), []); + const user = React.useContext(userContext); + // initial loading queries from server const listQueries = (_page) => { // post listQueries request to server - axios.post('/listQueries', {page: _page}) + axios.post('/listQueries', {page: _page, 'user_id': user.id}) .then(function (response) { console.log('[/listQueries] -> response:'); console.log(response); @@ -57,8 +60,9 @@ export default function QueryLogs() { ID + App Timestamp - Boosted + Rewritten Before Latency(s) After Latency(s) SQL @@ -70,6 +74,7 @@ export default function QueryLogs() { {queries.map((query) => ( selectQuery(query)}> {query.id} + {query.app_name} {query.timestamp} {query.boosted} {query.before_latency/1000} diff --git a/client/src/dashboard/RewritingRules.js b/client/src/dashboard/RewritingRules.js index f94fc91..03dac57 100644 --- a/client/src/dashboard/RewritingRules.js +++ b/client/src/dashboard/RewritingRules.js @@ -10,13 +10,14 @@ import TableCell from '@mui/material/TableCell'; import TableContainer from '@mui/material/TableContainer'; import TableHead from '@mui/material/TableHead'; import TableRow from '@mui/material/TableRow'; -import Switch from '@mui/material/Switch'; import Title from './Title'; import defaultRulesData from '../mock-api/listRules'; import SyntaxHighlighter from 'react-syntax-highlighter'; import { vs } from 'react-syntax-highlighter/dist/esm/styles/hljs'; import { Box } from '@mui/material'; import AddRewritingRule from './AddRewritingRule'; +import AppTagCell from './ApplicationTag'; +import {userContext} from '../userContext'; export default function RewrittingRules() { @@ -26,10 +27,14 @@ export default function RewrittingRules() { const [, updateState] = React.useState(); const forceUpdate = React.useCallback(() => updateState({}), []); + const user = React.useContext(userContext); + // initial loading rules from server const listRules = () => { + console.log('[/listRules] -> request:'); + console.log(' user_id: ' + user.id); // post listRules request to server - axios.post('/listRules', {}) + axios.post('/listRules', {'user_id': user.id}) .then(function (response) { console.log('[/listRules] -> response:'); console.log(response); @@ -119,7 +124,7 @@ export default function RewrittingRules() { Name Pattern Rewrite - Enabled + Enabled Apps Delete @@ -139,10 +144,11 @@ export default function RewrittingRules() { - handleChange(event, rule)} - inputProps={{ 'aria-label': 'controlled' }} /> + inputProps={{ 'aria-label': 'controlled' }} /> */} + diff --git a/client/src/mock-api/listApplications.js b/client/src/mock-api/listApplications.js new file mode 100644 index 0000000..7e14f46 --- /dev/null +++ b/client/src/mock-api/listApplications.js @@ -0,0 +1,16 @@ +const defaultApplicationsData = [ + { + "id": 1, + "name": "TwitterPg" + }, + { + "id": 2, + "name": "TpchPg" + }, + { + "id": 3, + "name": "TwitterMySQL" + } +]; + +export default defaultApplicationsData; \ No newline at end of file diff --git a/client/src/mock-api/listQueries.js b/client/src/mock-api/listQueries.js index 2b1ca2f..dbfc26b 100644 --- a/client/src/mock-api/listQueries.js +++ b/client/src/mock-api/listQueries.js @@ -2,36 +2,34 @@ const defaultQueriesData = [ { "id": 1, "timestamp": "2022-10-12 16:36:03", - "latency": 3, - "original_sql": `SELECT SUM(1) AS "cnt:tweets_5460F7F804494E7CB9FD188E329004C1:ok", + "rewritten": 'YES', + "before_latency": 35000, + "after_latency": 3200, + "sql": `SELECT SUM(1) AS "cnt:tweets_5460F7F804494E7CB9FD188E329004C1:ok", CAST("tweets"."state_name" AS TEXT) AS "state_name" FROM "public"."tweets" "tweets" WHERE ((CAST(DATE_TRUNC('QUARTER', CAST("tweets"."created_at" AS DATE)) AS DATE) IN ((TIMESTAMP '2016-04-01 00:00:00.000'), (TIMESTAMP '2016-07-01 00:00:00.000'), (TIMESTAMP '2016-10-01 00:00:00.000'), (TIMESTAMP '2017-01-01 00:00:00.000'))) AND (STRPOS(CAST(LOWER(CAST(CAST("tweets"."text" AS TEXT) AS TEXT)) AS TEXT), CAST('iphone' AS TEXT)) > 0)) GROUP BY 2`, - "rewritten_sql": `SELECT SUM(1) AS "cnt:tweets_5460F7F804494E7CB9FD188E329004C1:ok", - tweets.state_name AS state_name - FROM public.tweets AS tweets - WHERE DATE_TRUNC('QUARTER', tweets.created_at) IN ((TIMESTAMP '2016-04-01 00:00:00.000'), (TIMESTAMP '2016-07-01 00:00:00.000'), (TIMESTAMP '2016-10-01 00:00:00.000'), (TIMESTAMP '2017-01-01 00:00:00.000')) - AND tweets.text ILIKE '%iphone%' - GROUP BY 2` + "suggestion": "NO", + "suggested_latency": -1000, + "app_name": "TwitterPg" }, { "id": 0, "timestamp": "2022-10-12 16:31:42", - "latency": 34, - "original_sql": `SELECT SUM(1) AS "cnt:tweets_5460F7F804494E7CB9FD188E329004C1:ok", + "rewritten": 'YES', + "before_latency": 32000, + "after_latency": 2800, + "sql": `SELECT SUM(1) AS "cnt:tweets_5460F7F804494E7CB9FD188E329004C1:ok", CAST("tweets"."state_name" AS TEXT) AS "state_name" FROM "public"."tweets" "tweets" WHERE ((CAST(DATE_TRUNC('QUARTER', CAST("tweets"."created_at" AS DATE)) AS DATE) IN ((TIMESTAMP '2017-10-01 00:00:00.000'), (TIMESTAMP '2018-01-01 00:00:00.000'), (TIMESTAMP '2018-04-01 00:00:00.000'))) AND (STRPOS(CAST(LOWER(CAST(CAST("tweets"."text" AS TEXT) AS TEXT)) AS TEXT), CAST('iphone' AS TEXT)) > 0)) GROUP BY 2`, - "rewritten_sql": `SELECT SUM(1) AS "cnt:tweets_5460F7F804494E7CB9FD188E329004C1:ok", - CAST(tweets.state_name AS TEXT) AS state_name - FROM public.tweets AS tweets - WHERE CAST(DATE_TRUNC('QUARTER', CAST(tweets.created_at AS DATE)) AS DATE) IN ((TIMESTAMP '2017-10-01 00:00:00.000'), (TIMESTAMP '2018-01-01 00:00:00.000'), (TIMESTAMP '2018-04-01 00:00:00.000')) - AND STRPOS(CAST(LOWER(CAST(CAST(tweets.text AS TEXT) AS TEXT)) AS TEXT), CAST('iphone' AS TEXT)) > 0 - GROUP BY 2` + "suggestion": "NO", + "suggested_latency": -1000, + "app_name": "TwitterPg" } ]; diff --git a/client/src/mock-api/listRules.js b/client/src/mock-api/listRules.js index 209573d..132db95 100644 --- a/client/src/mock-api/listRules.js +++ b/client/src/mock-api/listRules.js @@ -7,7 +7,7 @@ const defaultRulesData = [ "constraints": "", "rewrite": "MAX()", "actions": "", - "enabled": true + "enabled_apps": [{"app_id": 1, "app_name": "TwitterPg"}] }, { "id": 10, @@ -17,7 +17,7 @@ const defaultRulesData = [ "constraints": "TYPE(x)=DATE", "rewrite": "", "actions": "", - "enabled": false + "enabled_apps": [{"app_id": 1, "app_name": "TwitterPg"}, {"app_id": 3, "app_name": "TwitterMySQL"}] }, { "id": 11, @@ -27,7 +27,7 @@ const defaultRulesData = [ "constraints": "TYPE(x)=TEXT", "rewrite": "", "actions": "", - "enabled": false + "enabled_apps": [{"app_id": 1, "app_name": "TwitterPg"}, {"app_id": 3, "app_name": "TwitterMySQL"}] }, { "id": 21, @@ -37,7 +37,7 @@ const defaultRulesData = [ "constraints": "IS(y)=CONSTANT and\nTYPE(y)=STRING", "rewrite": " ILIKE '%%'", "actions": "", - "enabled": false + "enabled_apps": [{"app_id": 1, "app_name": "TwitterPg"}, {"app_id": 3, "app_name": "TwitterMySQL"}] }, { "id": 30, @@ -47,7 +47,7 @@ const defaultRulesData = [ "constraints": "UNIQUE(tb1, a1)", "rewrite": "select <> \nfrom \nwhere 1=1 \nand <>\n", "actions": "SUBSTITUTE(s1, t2, t1) and\nSUBSTITUTE(p1, t2, t1)", - "enabled": true + "enabled_apps": [{"app_id": 1, "app_name": "TwitterPg"}, {"app_id": 3, "app_name": "TwitterMySQL"}] }, { "id": 101, @@ -57,7 +57,7 @@ const defaultRulesData = [ "constraints": "", "rewrite": "", "actions": "", - "enabled": true + "enabled_apps": [{"app_id": 3, "app_name": "TwitterMySQL"}] }, { "id": 102, @@ -67,7 +67,7 @@ const defaultRulesData = [ "constraints": "TYPE(x)=STRING", "rewrite": " = ", "actions": "", - "enabled": true + "enabled_apps": [{"app_id": 3, "app_name": "TwitterMySQL"}] } ]; diff --git a/client/src/userContext.js b/client/src/userContext.js new file mode 100644 index 0000000..fd6fd15 --- /dev/null +++ b/client/src/userContext.js @@ -0,0 +1,5 @@ +import React from 'react'; + +const userContext = React.createContext({user: {}}); + +export { userContext }; \ No newline at end of file diff --git a/core/app_manager.py b/core/app_manager.py new file mode 100644 index 0000000..cb44b38 --- /dev/null +++ b/core/app_manager.py @@ -0,0 +1,23 @@ +import sys +# append the path of the parent directory +sys.path.append("..") +from core.data_manager import DataManager + + +class AppManager: + + def __init__(self, dm: DataManager) -> None: + self.dm = dm + + def __del__(self): + del self.dm + + def list_applications(self, userid: int) -> list: + applications = self.dm.list_applications(userid) + res = [] + for app in applications: + res.append({ + 'id': app[0], + 'name': app[1] + }) + return res diff --git a/core/data_manager.py b/core/data_manager.py index cf73eb7..e0fa05b 100644 --- a/core/data_manager.py +++ b/core/data_manager.py @@ -1,8 +1,14 @@ +import sys +# append the path of the parent directory +sys.path.append("..") import datetime +import json import sqlite3 +import traceback from sqlite3 import Error from pathlib import Path from typing import Dict, List +from data.rules import get_rule class DataManager: @@ -10,14 +16,43 @@ def __init__(self) -> None: db_path = Path(__file__).parent / "../" self.db_conn = sqlite3.connect(db_path / 'querybooster.db') self.__init_schema() + self.__init_data() def __init_schema(self) -> None: try: cur = self.db_conn.cursor() schema_path = Path(__file__).parent / "../schema" - with open(schema_path / 'rules.sql') as rules_sql_file: - rules_sql = rules_sql_file.read() - cur.executescript(rules_sql) + with open(schema_path / 'schema.sql') as schema_sql_file: + schema_sql = schema_sql_file.read() + cur.executescript(schema_sql) + except Error as e: + print(e) + + def __init_data(self) -> None: + try: + # create two users: Alice and Bob + # + self.update_user({'id': 1, 'email': 'alice@ics.uci.edu'}) + self.update_user({'id': 2, 'email': 'bob@cs.ucla.edu'}) + # create one app for Alice + # + self.update_application({'id': 1, 'name': 'TwitterPg', 'guid': 'Alice-Tableau-Twitter-Pg', 'user_id': 1}) + # create one app for Bob + # + self.update_application({'id': 2, 'name': 'TpchPg', 'guid': 'Bob-Tableau-Tpch-Pg', 'user_id': 2}) + # create one rule for Alice + # + rule = get_rule('remove_max_distinct') + rule['owner_id'] = 1 + rule['pattern_json'] = json.dumps(rule['pattern_json']) + rule['constraints_json'] = json.dumps(rule['constraints_json']) + rule['rewrite_json'] = json.dumps(rule['rewrite_json']) + rule['actions_json'] = json.dumps(rule['actions_json']) + self.update_rule(rule) + # enable it for its app + # + self.enable_rule(rule_id=rule['id'], app_id=1, app_name='TwitterPg') + except Error as e: print(e) @@ -25,25 +60,29 @@ def __del__(self): if self.db_conn: self.db_conn.close() - def list_rules(self) -> List[Dict]: + def list_rules(self, userid: int) -> List[Dict]: try: cur = self.db_conn.cursor() - cur.execute('''SELECT id, - key, - name, - pattern, - constraints, - rewrite, - actions, - CASE WHEN disabled is NULL THEN 1 ELSE 0 END AS enabled, - database - FROM rules LEFT OUTER JOIN disable_rules - ON rules.id = disable_rules.rule_id''') + cur.execute('''SELECT rules.id, + rules.key, + rules.name, + rules.pattern, + rules.constraints, + rules.rewrite, + rules.actions, + enabled.application_id, + applications.name AS application_name + FROM rules LEFT OUTER JOIN enabled + ON rules.id = enabled.rule_id + LEFT OUTER JOIN applications + ON enabled.application_id = applications.id + WHERE rules.owner_id = ? + AND applications.user_id = ?''', [userid, userid]) return cur.fetchall() except Error as e: print(e) - def enabled_rules(self, database: str) -> List[Dict]: + def enabled_rules(self, appguid: str) -> List[Dict]: try: cur = self.db_conn.cursor() cur.execute('''SELECT id, @@ -53,51 +92,83 @@ def enabled_rules(self, database: str) -> List[Dict]: constraints_json, rewrite_json, actions_json - FROM rules LEFT JOIN disable_rules ON rules.id = disable_rules.rule_id + FROM rules JOIN enabled ON rules.id = enabled.rule_id + JOIN applications ON enabled.application_id = applications.id LEFT JOIN internal_rules ON rules.id = internal_rules.rule_id - WHERE disable_rules.disabled IS NULL AND rules.database = ? - ORDER BY rules.id''', [database]) + WHERE applications.guid = ? + ORDER BY rules.id''', [appguid]) return cur.fetchall() except Error as e: print(e) - def switch_rule(self, rule_id: int, enabled: bool) -> bool: + def enable_rule(self, rule_id: int, app_id: int, app_name: str) -> bool: try: cur = self.db_conn.cursor() - if enabled: - cur.execute('''DELETE FROM disable_rules WHERE rule_id = ?''', [rule_id]) - else: - cur.execute('''INSERT OR IGNORE INTO disable_rules (rule_id, disabled) VALUES (?, 1)''', [rule_id]) + if not app_id: + cur.execute('''SELECT id FROM applications WHERE name = ?''', [app_name]) + app_id = cur.fetchone()[0] + cur.execute('''INSERT OR IGNORE INTO enabled (rule_id, application_id) VALUES (?, ?)''', [rule_id, app_id]) self.db_conn.commit() return True - except Error as e: - print(e) + except Error as er: + print('[Error] in enable_rule:') + print('rule_id: ', rule_id, 'app_id: ', app_id, 'app_name: ', app_name) + print('SQLite error: %s' % (' '.join(er.args))) + print("Exception class is: ", er.__class__) + print('SQLite traceback: ') + exc_type, exc_value, exc_tb = sys.exc_info() + print(traceback.format_exception(exc_type, exc_value, exc_tb)) + return False + + def disable_rule(self, rule_id: int, app_id: int, app_name: str) -> bool: + try: + cur = self.db_conn.cursor() + if not app_id: + cur.execute('''SELECT id FROM applications WHERE name = ?''', [app_name]) + app_id = cur.fetchone()[0] + cur.execute('''DELETE FROM enabled WHERE rule_id = ? AND application_id = ?''', [rule_id, app_id]) + self.db_conn.commit() + return True + except Error as er: + print('[Error] in disable_rule:') + print('rule_id: ', rule_id, 'app_id: ', app_id, 'app_name: ', app_name) + print('SQLite error: %s' % (' '.join(er.args))) + print("Exception class is: ", er.__class__) + print('SQLite traceback: ') + exc_type, exc_value, exc_tb = sys.exc_info() + print(traceback.format_exception(exc_type, exc_value, exc_tb)) return False def update_rule(self, rule: dict) -> None: try: cur = self.db_conn.cursor() - cur.execute('''REPLACE INTO rules (id, key, name, pattern, constraints, rewrite, actions, database) + cur.execute('''REPLACE INTO rules (id, key, name, pattern, constraints, rewrite, actions, owner_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?)''', [rule['id'], rule['key'], rule['name'], rule['pattern'], - rule['constraints'], rule['rewrite'], rule['actions'], rule['database'] + rule['constraints'], rule['rewrite'], rule['actions'], rule['owner_id'] ]) cur.execute('''REPLACE INTO internal_rules (rule_id, pattern_json, constraints_json, rewrite_json, actions_json) VALUES (?, ?, ?, ?, ?)''', [rule['id'], rule['pattern_json'], rule['constraints_json'], rule['rewrite_json'], rule['actions_json']]) self.db_conn.commit() - except Error as e: - print(e) + except Error as er: + print('[Error] in update_rule:') + print(rule) + print('SQLite error: %s' % (' '.join(er.args))) + print("Exception class is: ", er.__class__) + print('SQLite traceback: ') + exc_type, exc_value, exc_tb = sys.exc_info() + print(traceback.format_exception(exc_type, exc_value, exc_tb)) - def add_rule(self, rule: dict) -> bool: + def add_rule(self, rule: dict, user_id: int) -> bool: try: cur = self.db_conn.cursor() cur.execute('''SELECT IFNULL(MAX(id), 0) + 1 FROM rules;''') rule['id'] = cur.fetchone()[0] - cur.execute('''INSERT INTO rules (id, key, name, pattern, constraints, rewrite, actions, database) + cur.execute('''INSERT INTO rules (id, key, name, pattern, constraints, rewrite, actions, owner_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?)''', [rule['id'], rule['key'], rule['name'], rule['pattern'], - rule['constraints'], rule['rewrite'], rule['actions'], rule['database'] + rule['constraints'], rule['rewrite'], rule['actions'], user_id ]) cur.execute('''INSERT INTO internal_rules (rule_id, pattern_json, constraints_json, rewrite_json, actions_json) VALUES (?, ?, ?, ?, ?)''', [rule['id'], rule['pattern_json'], rule['constraints_json'], rule['rewrite_json'], rule['actions_json']]) @@ -122,10 +193,10 @@ def delete_rule(self, rule: dict) -> bool: def log_query(self, appguid: str, guid: str, original_query: str, rewritten_query: str, rewriting_path: list) -> None: try: cur = self.db_conn.cursor() - cur.execute('''SELECT IFNULL(MAX(id), 0) + 1 FROM query_logs;''') + cur.execute('''SELECT IFNULL(MAX(id), 0) + 1 FROM queries;''') query_id = cur.fetchone()[0] - cur.execute('''INSERT INTO query_logs (id, timestamp, appguid, guid, query_time_ms, original_sql, rewritten_sql) + cur.execute('''INSERT INTO queries (id, timestamp, appguid, guid, query_time_ms, original_sql, rewritten_sql) VALUES (?, ?, ?, ?, ?, ?, ?)''', [query_id, datetime.datetime.now(), appguid, guid, -1000, original_query, rewritten_query]) seq = 1 @@ -141,7 +212,7 @@ def log_query(self, appguid: str, guid: str, original_query: str, rewritten_quer def report_query(self, appguid: str, guid: str, query_time_ms: int) -> None: try: cur = self.db_conn.cursor() - cur.execute('''UPDATE query_logs + cur.execute('''UPDATE queries SET query_time_ms = ? WHERE appguid = ? AND guid = ?''', @@ -150,19 +221,20 @@ def report_query(self, appguid: str, guid: str, query_time_ms: int) -> None: except Error as e: print(e) - def list_queries(self) -> List[Dict]: + def list_queries(self, userid: int) -> List[Dict]: try: cur = self.db_conn.cursor() cur.execute('''SELECT id, timestamp, - boosted, + rewritten, before_latency, after_latency, sql, suggestion, - suggested_latency - FROM queries - ORDER BY id desc''') + suggested_latency, + app_name + FROM query_log + WHERE user_id = ?''', [userid]) return cur.fetchall() except Error as e: print(e) @@ -171,7 +243,7 @@ def get_original_sql(self, query_id: int) -> str: try: cur = self.db_conn.cursor() cur.execute('''SELECT original_sql - FROM query_logs + FROM queries WHERE id = ?''', [query_id]) return cur.fetchall()[0] except Error as e: @@ -188,6 +260,39 @@ def list_rewritings(self, query_id: int) -> List[Dict]: return cur.fetchall() except Error as e: print(e) + + def update_user(self, user: dict) -> None: + try: + cur = self.db_conn.cursor() + cur.execute('''REPLACE INTO users (id, email) + VALUES (?, ?)''', + [user['id'], user['email']]) + self.db_conn.commit() + except Error as e: + print('[Error] in update_user:') + print(e) + + def update_application(self, app: dict) -> None: + try: + cur = self.db_conn.cursor() + cur.execute('''REPLACE INTO applications (id, name, guid, user_id) + VALUES (?, ?, ?, ?)''', + [app['id'], app['name'], app['guid'], app['user_id']]) + self.db_conn.commit() + except Error as e: + print('[Error] in update_application:') + print(e) + + def list_applications(self, userid: int) -> List[Dict]: + try: + cur = self.db_conn.cursor() + cur.execute('''SELECT id, + name + FROM applications + WHERE applications.user_id = ?''', [userid]) + return cur.fetchall() + except Error as e: + print(e) if __name__ == '__main__': diff --git a/core/query_logger.py b/core/query_manager.py similarity index 85% rename from core/query_logger.py rename to core/query_manager.py index 005cd84..cecc237 100644 --- a/core/query_logger.py +++ b/core/query_manager.py @@ -5,7 +5,7 @@ import json -class QueryLogger: +class QueryManager: def __init__(self, dm: DataManager) -> None: self.dm = dm @@ -19,19 +19,20 @@ def log_query(self, appguid: str, guid: str, original_query: str, rewritten_quer def report_query(self, appguid: str, guid: str, query_time_ms: int) -> None: self.dm.report_query(appguid, guid, query_time_ms) - def list_queries(self) -> list: - queries = self.dm.list_queries() + def list_queries(self, userid: int) -> list: + queries = self.dm.list_queries(userid) res = [] for query in queries: res.append({ 'id': query[0], 'timestamp': query[1], - 'boosted': query[2], + 'rewritten': query[2], 'before_latency': query[3], 'after_latency': query[4], 'sql': query[5], 'suggestion': query[6], - 'suggested_latency': query[7] + 'suggested_latency': query[7], + 'app_name': query[8] }) return res diff --git a/core/rule_manager.py b/core/rule_manager.py index 94c43fa..1f092e2 100644 --- a/core/rule_manager.py +++ b/core/rule_manager.py @@ -20,18 +20,18 @@ def __init_rules(self) -> None: def __del__(self): del self.dm - def add_rule(self, rule: dict) -> bool: + def add_rule(self, rule: dict, user_id: int) -> bool: rule['key'] = '_'.join([word.lower() for word in str(rule['name']).split(' ')]) rule['pattern_json'], rule['rewrite_json'], rule['mapping'] = RuleParser.parse(rule['pattern'], rule['rewrite']) rule['constraints_json'] = RuleParser.parse_constraints(rule['constraints'], rule['mapping']) rule['actions_json'] = RuleParser.parse_actions(rule['actions'], rule['mapping']) - return self.dm.add_rule(rule) + return self.dm.add_rule(rule, user_id) def delete_rule(self, rule: dict) -> bool: return self.dm.delete_rule(rule) - def fetch_enabled_rules(self, database: str) -> list: - enabled_rules = self.dm.enabled_rules(database) + def fetch_enabled_rules(self, appguid: str) -> list: + enabled_rules = self.dm.enabled_rules(appguid) res = [] for enabled_rule in enabled_rules: res.append({ @@ -45,20 +45,38 @@ def fetch_enabled_rules(self, database: str) -> list: }) return res - def list_rules(self) -> list: - rules = self.dm.list_rules() + def list_rules(self, userid: int) -> list: + rules = self.dm.list_rules(userid) + # group the rule together and + # list its enabled applications + rule_applications = {} + for rule in rules: + rule_id = rule[0] + application_id = rule[7] + application_name = rule[8] + if application_id is not None: + if rule_id in rule_applications: + rule_applications[rule_id].append({"app_id": application_id, "app_name": application_name}) + else: + rule_applications[rule_id] = [{"app_id": application_id, "app_name": application_name}] + else: + rule_applications[rule_id] = None res = [] + visited_rule_ids = [] for rule in rules: - res.append({ - 'id': rule[0], - 'key': rule[1], - 'name': rule[2], - 'pattern': rule[3], - 'constraints': rule[4], - 'rewrite': rule[5], - 'actions': rule[6], - 'enabled': True if rule[7] == 1 else False - }) + if rule[0] not in visited_rule_ids: + res.append({ + 'id': rule[0], + 'key': rule[1], + 'name': rule[2], + 'pattern': rule[3], + 'constraints': rule[4], + 'rewrite': rule[5], + 'actions': rule[6], + 'enabled_apps': rule_applications[rule[0]] + }) + visited_rule_ids.append(rule[0]) + return res def transform_rule_graph(self, root_rule: dict) -> dict: diff --git a/schema/rules.sql b/schema/rules.sql deleted file mode 100644 index 7f4f61f..0000000 --- a/schema/rules.sql +++ /dev/null @@ -1,99 +0,0 @@ -CREATE TABLE IF NOT EXISTS rules( - id INTEGER PRIMARY KEY, - key VARCHAR(255) UNIQUE, - name VARCHAR(2048) NOT NULL, - pattern TEXT, - constraints TEXT, - rewrite TEXT, - actions TEXT, - database VARCHAR(255) NOT NULL -); - -CREATE TABLE IF NOT EXISTS disable_rules( - rule_id INTEGER UNIQUE, - disabled BOOLEAN, - CONSTRAINT fk_rules - FOREIGN KEY (rule_id) - REFERENCES rules(id) - ON DELETE CASCADE -); - -CREATE TABLE IF NOT EXISTS internal_rules( - rule_id INTEGER UNIQUE, - pattern_json TEXT, - constraints_json TEXT, - rewrite_json TEXT, - actions_json TEXT, - CONSTRAINT fk_rules - FOREIGN KEY (rule_id) - REFERENCES rules(id) - ON DELETE CASCADE -); - -CREATE TABLE IF NOT EXISTS query_logs( - id INTEGER PRIMARY KEY, - appguid TEXT, - guid TEXT, - timestamp TEXT, - query_time_ms REAL, - original_sql TEXT, - rewritten_sql TEXT -); - -CREATE TABLE IF NOT EXISTS rewriting_paths( - query_id INTEGER, - seq INTEGER, - rule_id INTEGER, - rewritten_sql TEXT, - PRIMARY KEY (query_id, seq), - CONSTRAINT fk_queries - FOREIGN KEY (query_id) - REFERENCES query_logs(id) - ON DELETE CASCADE, - CONSTRAINT fk_rules - FOREIGN KEY (rule_id) - REFERENCES rules(id) - ON DELETE CASCADE -); - -CREATE TABLE IF NOT EXISTS suggestions( - query_id INTEGER UNIQUE, - query_time_ms REAL, - rewritten_sql TEXT, - CONSTRAINT fk_queries - FOREIGN KEY (query_id) - REFERENCES query_logs(id) - ON DELETE CASCADE -); - -CREATE TABLE IF NOT EXISTS suggestion_rewriting_paths( - query_id INTEGER, - seq INTEGER, - rule_id INTEGER, - rewritten_sql TEXT, - PRIMARY KEY (query_id, seq), - CONSTRAINT fk_queries - FOREIGN KEY (query_id) - REFERENCES suggestions(query_id) - ON DELETE CASCADE, - CONSTRAINT fk_rules - FOREIGN KEY (rule_id) - REFERENCES rules(id) - ON DELETE CASCADE -); - -CREATE VIEW IF NOT EXISTS queries AS -SELECT ql.id AS id, - ql.timestamp AS timestamp, - (CASE WHEN ql.original_sql = ql.rewritten_sql THEN 'NO' - WHEN ql.original_sql != ql.rewritten_sql THEN 'YES' - END) AS boosted, - (SELECT AVG(ql1.query_time_ms) FROM query_logs ql1 WHERE ql1.rewritten_sql=ql.original_sql) AS before_latency, - ql.query_time_ms AS after_latency, - ql.original_sql AS sql, - (CASE WHEN s.query_id IS NOT NULL THEN 'YES' ELSE 'NO' - END) AS suggestion, - (CASE WHEN s.query_id IS NOT NULL THEN s.query_time_ms ELSE -1000 - END) AS suggested_latency - FROM query_logs ql LEFT OUTER JOIN suggestions s ON ql.id = s.query_id - ORDER BY ql.timestamp DESC; \ No newline at end of file diff --git a/schema/schema.sql b/schema/schema.sql new file mode 100644 index 0000000..d8ec2d2 --- /dev/null +++ b/schema/schema.sql @@ -0,0 +1,144 @@ +CREATE TABLE IF NOT EXISTS rules( + id INTEGER PRIMARY KEY, + key VARCHAR(255) UNIQUE, + name VARCHAR(2048) NOT NULL, + pattern TEXT, + constraints TEXT, + rewrite TEXT, + actions TEXT, + owner_id INTEGER, + CONSTRAINT fk_rules + FOREIGN KEY (owner_id) + REFERENCES users(id) +); + +CREATE TABLE IF NOT EXISTS internal_rules( + rule_id INTEGER UNIQUE, + pattern_json TEXT, + constraints_json TEXT, + rewrite_json TEXT, + actions_json TEXT, + CONSTRAINT fk_rules + FOREIGN KEY (rule_id) + REFERENCES rules(id) + ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS users( + id INTEGER PRIMARY KEY, + email TEXT +); + +CREATE TABLE IF NOT EXISTS applications( + id INTEGER PRIMARY KEY, + name TEXT, + guid TEXT, + user_id INTEGER, + CONSTRAINT fk_users + FOREIGN KEY (user_id) + REFERENCES users(id) +); + +CREATE TABLE IF NOT EXISTS enabled( + application_id INTEGER, + rule_id INTEGER, + PRIMARY KEY (application_id, rule_id), + CONSTRAINT fk_applications + FOREIGN KEY (application_id) + REFERENCES applications(id) + ON DELETE CASCADE, + CONSTRAINT fk_rules + FOREIGN KEY (rule_id) + REFERENCES rules(id) + ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS queries( + id INTEGER PRIMARY KEY, + guid TEXT, + appguid TEXT, + timestamp TEXT, + query_time_ms REAL, + original_sql TEXT, + sql TEXT +); + +CREATE TABLE IF NOT EXISTS rewriting_paths( + query_id INTEGER, + seq INTEGER, + rule_id INTEGER, + rewritten_sql TEXT, + PRIMARY KEY (query_id, seq), + CONSTRAINT fk_queries + FOREIGN KEY (query_id) + REFERENCES queries(id), + CONSTRAINT fk_rules + FOREIGN KEY (rule_id) + REFERENCES rules(id) +); + +CREATE TABLE IF NOT EXISTS suggestions( + query_id INTEGER UNIQUE, + query_time_ms REAL, + rewritten_sql TEXT, + CONSTRAINT fk_queries + FOREIGN KEY (query_id) + REFERENCES queries(id) + ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS suggestion_rewriting_paths( + query_id INTEGER, + seq INTEGER, + rule_id INTEGER, + rewritten_sql TEXT, + PRIMARY KEY (query_id, seq), + CONSTRAINT fk_queries + FOREIGN KEY (query_id) + REFERENCES suggestions(query_id), + CONSTRAINT fk_rules + FOREIGN KEY (rule_id) + REFERENCES rules(id) +); + +CREATE TABLE IF NOT EXISTS tables( + id INTEGER PRIMARY KEY, + application_id INTEGER, + name TEXT, + CONSTRAINT fk_applications + FOREIGN KEY (application_id) + REFERENCES applications(id) + ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS columns( + id INTEGER PRIMARY KEY, + table_id INTEGER, + name TEXT, + type TEXT, + CONSTRAINT fk_tables + FOREIGN KEY (table_id) + REFERENCES tables(id) + ON DELETE CASCADE +); + +DROP VIEW IF EXISTS query_log; +CREATE VIEW query_log AS +SELECT q.id AS id, + q.timestamp AS timestamp, + (CASE WHEN q.sql = q.original_sql THEN 'NO' + WHEN q.sql != q.original_sql THEN 'YES' + END) AS rewritten, + (SELECT AVG(q1.query_time_ms) FROM queries q1 WHERE q1.sql=q.original_sql) AS before_latency, + q.query_time_ms AS after_latency, + q.original_sql AS sql, + (CASE WHEN s.query_id IS NOT NULL THEN 'YES' ELSE 'NO' + END) AS suggestion, + (CASE WHEN s.query_id IS NOT NULL THEN s.query_time_ms ELSE -1000 + END) AS suggested_latency, + a.user_id AS user_id, + a.name AS app_name + FROM queries q + JOIN applications a ON q.appguid = a.guid + LEFT OUTER JOIN suggestions s ON q.id = s.query_id + ORDER BY q.timestamp DESC; \ No newline at end of file diff --git a/server/server.py b/server/server.py index e2c88e6..d6b22eb 100644 --- a/server/server.py +++ b/server/server.py @@ -11,7 +11,8 @@ from core.data_manager import DataManager from core.rule_generator import RuleGenerator from core.rule_manager import RuleManager -from core.query_logger import QueryLogger +from core.query_manager import QueryManager +from core.app_manager import AppManager PORT = 8000 DIRECTORY = "static" @@ -20,7 +21,8 @@ class MyHTTPRequestHandler(SimpleHTTPRequestHandler): dm = DataManager() rm = RuleManager(dm) - ql = QueryLogger(dm) + qm = QueryManager(dm) + am = AppManager(dm) def __init__(self, *args, **kwargs): super().__init__(*args, directory=DIRECTORY, **kwargs) @@ -51,21 +53,24 @@ def post_query(self): log_text += "\n Original query" log_text += "\n--------------------------------------------------" log_text += "\n appguid: " + appguid + log_text += "\n guid: " + guid + log_text += "\n db: " + database log_text += "\n" + QueryRewriter.beautify(original_query) log_text += "\n--------------------------------------------------" logging.info(log_text) - rules = self.rm.fetch_enabled_rules(database) + rules = self.rm.fetch_enabled_rules(appguid) rewritten_query, rewriting_path = QueryRewriter.rewrite(original_query, rules) rewritten_query = QueryPatcher.patch(rewritten_query, database) for rewriting in rewriting_path: rewriting[1] = QueryPatcher.patch(rewriting[1], database) - self.ql.log_query(appguid, guid, QueryPatcher.patch(QueryRewriter.reformat(original_query), database), rewritten_query, rewriting_path) + self.qm.log_query(appguid, guid, QueryPatcher.patch(QueryRewriter.reformat(original_query), database), rewritten_query, rewriting_path) log_text = "" log_text += "\n==================================================" log_text += "\n Rewritten query" log_text += "\n--------------------------------------------------" log_text += "\n appguid: " + appguid log_text += "\n guid: " + guid + log_text += "\n db: " + database log_text += "\n" + QueryRewriter.beautify(rewritten_query) log_text += "\n--------------------------------------------------" logging.info(log_text) @@ -83,10 +88,11 @@ def post_query(self): log_text += "\n--------------------------------------------------" log_text += "\n appguid: " + appguid log_text += "\n guid: " + guid + log_text += "\n db: " + database log_text += "\n query_time_ms: " + str(query_time_ms) log_text += "\n--------------------------------------------------" logging.info(log_text) - self.ql.report_query(appguid, guid, query_time_ms) + self.qm.report_query(appguid, guid, query_time_ms) self.send_response(200) self.end_headers() response = BytesIO() @@ -102,7 +108,10 @@ def post_list_rules(self): logging.info("\n[/listRules] request:") logging.info(request) - rules_json = self.rm.list_rules() + request = json.loads(request, strict=False) + user_id = request['user_id'] + + rules_json = self.rm.list_rules(user_id) self.send_response(200) self.end_headers() @@ -110,18 +119,41 @@ def post_list_rules(self): response.write(json.dumps(rules_json).encode('utf-8')) self.wfile.write(response.getvalue()) - def post_switch_rule(self): + def post_enable_rule(self): content_length = int(self.headers['Content-Length']) body = self.rfile.read(content_length) request = body.decode('utf-8') # logging - logging.info("\n[/switcheRule] request:") + logging.info("\n[/enableRule] request:") logging.info(request) - # enable/disable rule to data manager - rule = json.loads(request) - success = self.dm.switch_rule(rule['id'], rule['enabled']) + # enable rule for the given app to data manager + request = json.loads(request, strict=False) + rule = request['rule'] + app = request['app'] + success = self.dm.enable_rule(rule['id'], app['id'], app['name']) + + self.send_response(200) + self.end_headers() + response = BytesIO() + response.write(str(success).encode('utf-8')) + self.wfile.write(response.getvalue()) + + def post_disable_rule(self): + content_length = int(self.headers['Content-Length']) + body = self.rfile.read(content_length) + request = body.decode('utf-8') + + # logging + logging.info("\n[/disableRule] request:") + logging.info(request) + + # enable rule for the given app to data manager + request = json.loads(request, strict=False) + rule = request['rule'] + app = request['app'] + success = self.dm.disable_rule(rule['id'], app['id'], app['name']) self.send_response(200) self.end_headers() @@ -139,8 +171,10 @@ def post_add_rule(self): logging.info(request) # add rule to rule manager - rule = json.loads(request) - success = self.rm.add_rule(rule) + request = json.loads(request, strict=False) + rule = request['rule'] + user_id = request['user_id'] + success = self.rm.add_rule(rule, user_id) self.send_response(200) self.end_headers() @@ -176,7 +210,10 @@ def post_list_queries(self): logging.info("\n[/listQueries] request:") logging.info(request) - queries_json = self.ql.list_queries() + request = json.loads(request, strict=False) + user_id = request['user_id'] + + queries_json = self.qm.list_queries(user_id) self.send_response(200) self.end_headers() @@ -195,7 +232,7 @@ def post_rewriting_path(self): # fetch rewriting path from query logger query_id = json.loads(request)["queryId"] - rewriting_path_json = self.ql.rewriting_path(query_id) + rewriting_path_json = self.qm.rewriting_path(query_id) self.send_response(200) self.end_headers() @@ -353,14 +390,36 @@ def post_recommend_rules(self): response = BytesIO() response.write(json.dumps(recommend_rules_json).encode('utf-8')) self.wfile.write(response.getvalue()) + + def post_list_applications(self): + content_length = int(self.headers['Content-Length']) + body = self.rfile.read(content_length) + request = body.decode('utf-8') + + # logging + logging.info("\n[/listApplications] request:") + logging.info(request) + + request = json.loads(request, strict=False) + user_id = request['user_id'] + + applications_json = self.am.list_applications(user_id) + + self.send_response(200) + self.end_headers() + response = BytesIO() + response.write(json.dumps(applications_json).encode('utf-8')) + self.wfile.write(response.getvalue()) def do_POST(self): if self.path == "/": self.post_query() elif self.path == "/listRules": self.post_list_rules() - elif self.path == "/switchRule": - self.post_switch_rule() + elif self.path == "/enableRule": + self.post_enable_rule() + elif self.path == "/disableRule": + self.post_disable_rule() elif self.path == "/listQueries": self.post_list_queries() elif self.path == "/rewritingPath": @@ -379,6 +438,8 @@ def do_POST(self): self.post_add_rule() elif self.path == "/deleteRule": self.post_delete_rule() + elif self.path == "/listApplications": + self.post_list_applications() if __name__ == '__main__':