From f0172d360e821e214d02aa12e66f4fab4a4f7db8 Mon Sep 17 00:00:00 2001 From: Jyotsnarajan Date: Tue, 17 Oct 2023 16:06:28 +0530 Subject: [PATCH] [tools/migrations] Script to remove non-working workshop-demos --- tools/migrations/remove_workshop_demos.py | 189 ++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100755 tools/migrations/remove_workshop_demos.py diff --git a/tools/migrations/remove_workshop_demos.py b/tools/migrations/remove_workshop_demos.py new file mode 100755 index 00000000..e87d9929 --- /dev/null +++ b/tools/migrations/remove_workshop_demos.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 + +import datetime +import json +import os +import sys +import requests +import argparse +import sqlite3 as lite + + +# parse the arguments +ap = argparse.ArgumentParser() +ap.add_argument("--demoinfo_db", "-demoinfodb", required=True, help="databse directory containing 'demoinfo.db' file") +ap.add_argument("--blob_db", "-blobsdb", required=True, help="databse directory containing 'blobs.db' file") +ap.add_argument("--blob_dir", "-blobsdir", required=True, help="staticData directory containing blobs") +ap.add_argument("-r", '--remove', help="Remove Demo", action="store_true") +args = ap.parse_args() +demoinfo_db = args.demoinfo_db +blob_db = args.blob_db +blob_dir = args.blob_dir + +def connect_to_db(db_file): + try: + return lite.connect(db_file) + except Exception as ex: + print("Couldn't connect to the DB {}. Error: {}".format(db_file, ex)) + raise + +def main(): + di_conn = None + DI_DB = os.path.join(demoinfo_db, 'demoinfo.db') + BL_DB = os.path.join(blob_db, 'blobs.db') + try: + di_conn = connect_to_db(DI_DB) + di_cursor = di_conn.cursor() + + bl_conn = connect_to_db(BL_DB) + bl_cursor = bl_conn.cursor() + + demoinfo_demo_list = get_workshop_demos_from_demoinfo(di_cursor) + + workshop_demo_list, old_workshop_demo_list, non_working_demos = ([] for i in range(3)) + for workshop_demos in demoinfo_demo_list: + demo_id = workshop_demos['id'] + workshop_demo_list.append(demo_id) + + parameter = get_params(di_cursor, demo_id) + + #input blob required to run the demo + blob = get_blob(bl_cursor, demo_id) + if not blob: + continue + blob_path = os.path.join(blob_dir, blob[:1], blob[1:2], blob) + + #check if blob exists + if not os.path.isfile(blob_path): + raise FileNotFoundError(blob) + + for demo_id in workshop_demo_list: + clientData = {"demo_id": demo_id, "params": parameter, "origin": "upload"} + + files = {'file_0': blob_path} + + params = { + 'clientData': json.dumps(clientData), + + } + + #Execute the demos + r = requests.post('https://ipolcore.ipol.im/api/core/run', files=files, params=params) + + response = r.json() + if response ['status'] != 'OK': + non_working_demos.append(demo_id) + #print(f'Failed to run the demo: {demo_id}') + + ''' + #Remove workshop demos that's not working + if args.remove: + params={ + 'demo_id': demo_id + } + response = requests.post('http:/ipolcore.ipol.im/api/core/delete_demo', params=params).json() + if response ['status'] != 'OK': + print("Failed to delete demo ID: {}".format(demo_id)) + ''' + + print("List of All workshop Demos:", len(workshop_demo_list), "\n" ,workshop_demo_list, "\n") + print("List of 6 months old workshop Demos:", len(old_workshop_demo_list), "\n" ,old_workshop_demo_list, "\n") + print("List of All non working Demos:", len(non_working_demos), "\n" ,non_working_demos, "\n") + + + except Exception as ex: + print("Failure in the migration. Error: {}".format(ex)) + + finally: + if di_conn is not None: + di_conn.close() + + +def get_workshop_demos_from_demoinfo(demoinfo_cursor): + """ + Get the list of all workshop demos, their authors and editors + """ + workshop_demos = [] + + demoinfo_cursor.execute(""" + SELECT d.editor_demo_id, d.modification + FROM demo AS d, state AS s + WHERE s.name = "workshop" AND d.stateID = s.ID + """) + workshop_demos = [{'id': i[0], 'modification': i[1], 'a_name': '', 'a_mail': '', 'e_name': '', 'e_mail': ''} for i in demoinfo_cursor.fetchall()] + for workshop_demo in workshop_demos: + demoinfo_cursor.execute(""" + SELECT a.name, a.mail + FROM author as a, demo_author as da + WHERE da.demoID = ? AND a.ID = da.authorId + """, (workshop_demo['id'],)) + author_info = demoinfo_cursor.fetchone() + if author_info: + workshop_demo['a_name'] = author_info[0] + workshop_demo['a_mail'] = author_info[1] + for workshop_demo in workshop_demos: + demoinfo_cursor.execute(""" + SELECT e.name, e.mail + FROM editor as e, demo_editor as de + WHERE de.demoID = ? AND e.ID = de.editorId + """, (workshop_demo['id'],)) + editor_info = demoinfo_cursor.fetchone() + if editor_info: + workshop_demo['e_name'] = editor_info[0] + workshop_demo['e_mail'] = editor_info[1] + return workshop_demos + + +def get_params(demoinfo_cursor, editor_demo_id): + """ + Get the list of parameters for given demo ID + """ + param_dict = {} + + demoinfo_cursor.execute(""" + SELECT demod.DDL + FROM demo AS d, demo_demodescription AS dd, demodescription AS demod + WHERE d.editor_demo_id = ? AND dd.demoID = d.ID AND dd.demodescriptionId = demod.ID + ORDER BY demod.creation DESC LIMIT 1 + """, (editor_demo_id, )) + + param_infos = demoinfo_cursor.fetchone() + + if param_infos: + param_info = json.loads(param_infos[0]) + if 'params' in param_info: + for param in param_info['params']: + if 'id' not in param.keys(): + continue + if 'default_value' in param.keys(): + param_dict[param['id']] = param['default_value'] + elif 'values' in param.keys(): + if 'default' in param['values'].keys(): + param_dict[param['id']] = param['values']['default'] + else: + param_dict[param['id']] = param['values'][list(param['values'].keys())[0]] + else: + pass + return param_dict + + +def get_blob(blobs_cursor, editor_demo_id): + """ + Get the first blob for given demo ID + """ + param_info = [] + + blobs_cursor.execute(""" + SELECT b.hash, b.extension + FROM demos AS d, demos_blobs AS db, blobs AS b + WHERE d.editor_demo_id = ? AND d.id = db.demo_id AND db.blob_id = b.ID + """, (editor_demo_id, )) + blob_info = blobs_cursor.fetchone() + + if blob_info: + return blob_info[0] + blob_info[1] + + return None + + +main() \ No newline at end of file