From ef0bce2af201d420ceec0367e070ce031c46dd7f Mon Sep 17 00:00:00 2001 From: "lunga.mthembu" Date: Fri, 9 Dec 2022 09:56:20 +0200 Subject: [PATCH 1/9] dev --- litemigration-cli.py | 79 +++++++ litemigration/database.py | 457 +++++++++++++++++++++++++++----------- 2 files changed, 412 insertions(+), 124 deletions(-) create mode 100644 litemigration-cli.py diff --git a/litemigration-cli.py b/litemigration-cli.py new file mode 100644 index 0000000..db1ef3b --- /dev/null +++ b/litemigration-cli.py @@ -0,0 +1,79 @@ +import argparse +import importlib + +def check_settings(): + """ + Looks for database.py file with the list of migrations + List of migrations variable should be migration_changes + Returns the module file + """ + try: + mod = importlib.import_module('database') + return mod + #print(type(mod)) + #print(dir(mod)) + #print(mod.migration_changes) + + except ModuleNotFoundError: + print('Unable to find database file') + exit() + except AttributeError: + print('Unable to find migration_changes') + exit() + + + +def show_migrations(): + """ + Get + """ + module = check_settings() + db = module.db + db.show_migrations(module.migration_changes) + +def migration(direction: str): + print('Run forward or reverse migration') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Manage migrations') + subparsers = parser.add_subparsers(help='sub-command help') + + show_migration = subparsers.add_parser('showmigrations', help='show current migrations') + show_migration.set_defaults(func=show_migrations) + + migrate = subparsers.add_parser('migrate', help='Run forward or reverse migrations') + migrate.set_defaults(func=migration) + migrate.add_argument('direction', choices=['up', 'down'], help='forward or reverse migration') + + args = parser.parse_args() + args.func() + + +# showmigrations - show all migrations, including unapplied +# migrate up - apply migrations +# migrate down - reverse migrations upto a particular version +# +# from litemigration.database import SqliteDatabase, Migration +# +# +# db = SqliteDatabase('example.db') +# db.initialize() +# +# change = [ +# Migration( +# version=2, +# up='CREATE TABLE player(name VARCHAR NOT NULL,score INTEGER)', +# down='DROP TABLE player' +# ), +# Migration( +# version=3, +# up='INSERT INTO player(name,score) VALUES("Menzi", 10)', +# down='DELETE FROM PLAYER where name="Menzi"' +# ) +# ] +# +# +# db.add_migration(change) +# #db.reverse_migration(2, change) + diff --git a/litemigration/database.py b/litemigration/database.py index 39ec139..f82935b 100755 --- a/litemigration/database.py +++ b/litemigration/database.py @@ -1,135 +1,344 @@ -#!/usr/bin/python3 - -import datetime as dt import logging -import sys -from typing import List, Tuple +import sqlite3 + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from datetime import datetime +from typing import List + +from terminaltables import AsciiTable +from colorclass import Color log = logging.getLogger(__name__) -class Database: - "Create migration control" - def __init__(self, db_type, host=None, port=None, user=None, - password=None, database=None): - self.db_type = db_type - self.host = host - self.port = port - self.user = user - self.password = password - self.database = database - self.details = "" - self.connect = self._get_connector() - self.cursor = self.connect.cursor() - - def _get_connector(self): - """' - Return database connection from specified database from user - """ - supported_databases = {'postgresql': self._postgresql, - 'sqlite': self._sqlite} - try: - connect = supported_databases[self.db_type]() - return connect - except KeyError: - log.critical("Unknown database or not supported") - exit() +@dataclass +class Migration: + version: int + up: str + down: str - def _get_initail_sql_migration(self) -> Tuple[str, str]: - """ - Return 2 sql commands: - 1) Create migration - 2) Insert into migration table - """ - sqlite_create = ("CREATE TABLE migration(" - 'id INTEGER PRIMARY KEY NOT NULL,' - 'version INTEGER UNIQUE NOT NULL,' - 'date TIMESTAMP NOT NULL)', - "INSERT INTO migration(version,date) VALUES(0,?)") - - pg_create = ("CREATE TABLE migration(" - 'id SERIAL PRIMARY KEY,' - 'version INTEGER NOT NULL,' - 'date DATE NOT NULL)', - "INSERT INTO migration(version,date) VALUES(0,%s)") - - all_sql = {'postgresql': pg_create, - 'sqlite': sqlite_create} - - return all_sql[self.db_type] - - def initialise(self): - "Create new database and add initial migration" - create_table, initial_insert = self._get_initail_sql_migration() + +class MigrationException(Exception): + pass + + +class Database(ABC): + @abstractmethod + def connection(self): + pass + + @abstractmethod + def initialize(self): + pass + + @abstractmethod + def add_migration(self, change: List[Migration]): + pass + + @abstractmethod + def show_migrations(self, change: List[Migration]): + print("Showing the database migrations") + + +class SqliteDatabase(Database): + def __init__(self, name): + self.name = name + self.connect = self.connection() + + def connection(self): + connect = sqlite3.connect(self.name) + return connect + + def initialize(self): + query = ( + 'CREATE TABLE migration(' + 'id INTEGER PRIMARY KEY NOT NULL,' + 'version INTEGER UNIQUE NOT NULL,' + 'date TIMESTAMP NOT NULL)' + ) + + cur = self.connect.cursor() try: - self.cursor.execute(create_table) - self.cursor.execute(initial_insert, - (dt.datetime.now(),)) + cur.execute(query) + cur.execute("INSERT INTO migration(version,date) VALUES(1,?)", (datetime.now(),)) self.connect.commit() - log.info("Database has been created") - except Exception as e: - log.error("Unable to add migration table") - log.exception(e) - sys.exit() + except sqlite3.OperationalError: + log.info('Database already exists') - def add_schema(self, change_list: List[Tuple[int, str]]): - """ - The first migration change should be version 1 - """ - if self.db_type == 'postgresql': - insert_sql = "INSERT INTO migration(version,date) VALUES(%s,%s)" - elif self.db_type == 'sqlite': - insert_sql = "INSERT INTO migration(version,date) VALUES(?,?)" - - for change_id, sql_statement in change_list: - self.cursor.execute('SELECT max(version) from migration') - (max_id,) = self.cursor.fetchone() - if max_id >= change_id: - log.info("schema change id {} is smaller than the latest" - "change".format(change_id)) - log.info("or schema change id has already been applied ") - else: - try: - self.cursor.execute(sql_statement) - self.cursor.execute(insert_sql, - (change_id, dt.datetime.now(),)) - self.connect.commit() - log.info("new schema added") - except Exception: - log.error("Unable to add schema {}".format(change_id), - exc_info=True) - sys.exit() - - def _postgresql(self): - "create postgresql connection and return the connection object" - try: - import psycopg2 - connect = psycopg2.connect(database=self.database, - host=self.host, - user=self.user, - password=self.password, - port=self.port) - return connect - except ImportError: - log.error("Unable to find python3 postgresql module") - sys.exit() - except psycopg2.Error as e: - log.error("Unable to connect to postgresql") - log.exception(e) - sys.exit() - except psycopg2.OperationalError as e: - log.exception(e) - sys.exit() - - def _sqlite(self): + def add_migration(self, change: List[Migration]): + cur = self.connect.cursor() + cur.execute('SELECT max(version) from migration') + (max_version,) = cur.fetchone() + for migration in change: + print(f'Current migration {migration} ') + if max_version >= migration.version: + log.info(f'migration {migration.version} already applied') + continue + + if migration.version - max_version != 1: + log.error(f'missing migration version before {migration.version}') + raise MigrationException('missing migration version before {}'.format(migration.version)) + try: + cur.execute(migration.up) + cur.execute("INSERT INTO migration(version,date) VALUES(?,?)", (migration.version, datetime.now())) + self.connect.commit() + max_version = migration.version + except sqlite3.OperationalError as error: + log.error(f'unable to apply migration {migration.version}') + raise MigrationException(error) + + self.connect.close() + + def reverse_migration(self, version: int, change: List[Migration]): """ - Create an sqlite connection and return the connection object + Migration version to revert to from max version. + So if max version is 10 and version choosen is 5. + Version 10 - 6 will be reverted """ - import sqlite3 - try: - connect = sqlite3.connect(self.database) - return connect - except sqlite3.OperationalError: - log.error("unable to connect to sqlite database", - exc_info=True) - sys.exit() + cur = self.connect.cursor() + cur.execute('SELECT max(version) from migration') + (max_id,) = cur.fetchone() + if version > max_id: + raise MigrationException('version greater than max version unbale to reverse') + + for migration in reversed(change): + if migration.version == version: + break + elif migration.version > version: + cur.execute(migration.down) + cur.execute('DELETE FROM migration where version=?', (migration.version,)) + self.connect.commit() + self.connect.close() + + def show_migrations(self, change: List[Migration]): + version = 0 + data = [] + data.append(['Applied', 'Version', 'Date']) + + cur = self.connect.cursor() + cur.execute('SELECT version, date FROM migration') + applied = cur.fetchall() + for m in applied: + data.append([Color('{autogreen}Yes{/autogreen}'), m[0], m[1]]) + version = m[0] + + for m in change: + if version < m.version: + data.append([Color('{autored}No{/autored}'), m.version]) + + table = AsciiTable(data) + print(table.table) + + + +# class PostgresqlDatabase(Database): +# def __init__(self, host, port, user, password, name): +# self.host = host +# self.port = port +# self.user = user +# self.password = password +# self.name = name +# self.connect = self.connection() +# +# def connection(self): +# connect = psycopg2.connect(database=self.database, +# host=self.host, +# user=self.user,s +# password=self.password, +# port=self.port) +# return connect +# +# def initialize(self): +# create_query = ("CREATE TABLE migration(" +# 'id SERIAL PRIMARY KEY,' +# 'version INTEGER NOT NULL,' +# 'date DATE NOT NULL)') +# insert_query = "INSERT INTO migration(version,date) VALUES(1,%s)" +# try: +# cur = self.connect.cursor() +# cur.execute(create_query) +# cur.execute(insert_query, dt.datetime.now()) +# except psycopg2.OperationalError as error: +# log.exception(error) +# exit() + + # def add_migration(self, change: List[Migration]): + # """ + # list of database changes + # [1, (insert into)] + # """ + # cur = self.connect.cursor() + # cur.execute('SELECT max(version) from migration') + # (max_id, _) = cur.fetchone() + # + # for migration in change: + # if max_id > migration.version: + # log.info(f'schema migration {migration.version} has already been applied') + # continue + # else: + # if max_id - migration != 1: + # log.error(f'migration version {migration.version} not continouus') + # raise MigrationException('migration version not continous') + # else: + # try: + # cur.execute(migration.up) + # cur.execute() + # self.connect.commit() + # log.info(f'migration {migration.version} added') + # max_id = migration.version + # except Exception: + # pass + # + # + # for change_id, sql_statement in change_list: + # self.cursor.execute('SELECT max(version) from migration') + # (max_id,) = self.cursor.fetchone() + # if max_id >= change_id: + # log.info("schema change id {} is smaller than the latest" + # "change".format(change_id)) + # log.info("or schema change id has already been applied ") + # else: + # try: + # self.cursor.execute(sql_statement) + # self.cursor.execute(insert_sql, + # (change_id, dt.datetime.now(),)) + # self.connect.commit() + # log.info("new schema added") + # except Exception: + # log.error("Unable to add schema {}".format(change_id), + # exc_info=True) + # sys.exit() + + + + + + + + +# class Database: +# "Create migration control" +# def __init__(self, db_type, host=None, port=None, user=None, +# password=None, database=None): +# self.db_type = db_type +# self.host = host +# self.port = port +# self.user = user +# self.password = password +# self.database = database +# self.details = "" +# self.connect = self._get_connector() +# self.cursor = self.connect.cursor() +# +# def _get_connector(self): +# """' +# Return database connection from specified database from user +# """ +# supported_databases = {'postgresql': self._postgresql, +# 'sqlite': self._sqlite} +# try: +# connect = supported_databases[self.db_type]() +# return connect +# except KeyError: +# log.critical("Unknown database or not supported") +# exit() +# +# def _get_initail_sql_migration(self) -> Tuple[str, str]: +# """ +# Return 2 sql commands: +# 1) Create migration +# 2) Insert into migration table +# """ +# sqlite_create = ("CREATE TABLE migration(" +# 'id INTEGER PRIMARY KEY NOT NULL,' +# 'version INTEGER UNIQUE NOT NULL,' +# 'date TIMESTAMP NOT NULL)', +# "INSERT INTO migration(version,date) VALUES(0,?)") +# +# pg_create = ("CREATE TABLE migration(" +# 'id SERIAL PRIMARY KEY,' +# 'version INTEGER NOT NULL,' +# 'date DATE NOT NULL)', +# "INSERT INTO migration(version,date) VALUES(0,%s)") +# +# all_sql = {'postgresql': pg_create, +# 'sqlite': sqlite_create} +# +# return all_sql[self.db_type] +# +# def initialise(self): +# "Create new database and add initial migration" +# create_table, initial_insert = self._get_initail_sql_migration() +# try: +# self.cursor.execute(create_table) +# self.cursor.execute(initial_insert, +# (dt.datetime.now(),)) +# self.connect.commit() +# log.info("Database has been created") +# except Exception as e: +# log.error("Unable to add migration table") +# log.exception(e) +# sys.exit() +# +# def add_schema(self, change_list: List[Tuple[int, str]]): +# """ +# The first migration change should be version 1 +# """ +# if self.db_type == 'postgresql': +# insert_sql = "INSERT INTO migration(version,date) VALUES(%s,%s)" +# elif self.db_type == 'sqlite': +# insert_sql = "INSERT INTO migration(version,date) VALUES(?,?)" +# +# for change_id, sql_statement in change_list: +# self.cursor.execute('SELECT max(version) from migration') +# (max_id,) = self.cursor.fetchone() +# if max_id >= change_id: +# log.info("schema change id {} is smaller than the latest" +# "change".format(change_id)) +# log.info("or schema change id has already been applied ") +# else: +# try: +# self.cursor.execute(sql_statement) +# self.cursor.execute(insert_sql, +# (change_id, dt.datetime.now(),)) +# self.connect.commit() +# log.info("new schema added") +# except Exception: +# log.error("Unable to add schema {}".format(change_id), +# exc_info=True) +# sys.exit() +# +# def _postgresql(self): +# "create postgresql connection and return the connection object" +# try: +# import psycopg2 +# connect = psycopg2.connect(database=self.database, +# host=self.host, +# user=self.user, +# password=self.password, +# port=self.port) +# return connect +# except ImportError: +# log.error("Unable to find python3 postgresql module") +# sys.exit() +# except psycopg2.Error as e: +# log.error("Unable to connect to postgresql") +# log.exception(e) +# sys.exit() +# except psycopg2.OperationalError as e: +# log.exception(e) +# sys.exit() +# +# def _sqlite(self): +# """ +# Create an sqlite connection and return the connection object +# """ +# import sqlite3 +# try: +# connect = sqlite3.connect(self.database) +# return connect +# except sqlite3.OperationalError: +# log.error("unable to connect to sqlite database", +# exc_info=True) +# sys.exit() From 1e12a6831153c888d9e5088dcb940603195ede42 Mon Sep 17 00:00:00 2001 From: ebsuku Date: Sun, 11 Dec 2022 22:05:22 +0200 Subject: [PATCH 2/9] dev --- litemigration-cli.py | 92 +++++++-------- litemigration/database.py | 237 ++++---------------------------------- 2 files changed, 71 insertions(+), 258 deletions(-) diff --git a/litemigration-cli.py b/litemigration-cli.py index db1ef3b..9542a06 100644 --- a/litemigration-cli.py +++ b/litemigration-cli.py @@ -1,7 +1,9 @@ import argparse import importlib -def check_settings(): +from litemigration.database import Database + +def check_settings() -> dict: """ Looks for database.py file with the list of migrations List of migrations variable should be migration_changes @@ -9,30 +11,52 @@ def check_settings(): """ try: mod = importlib.import_module('database') - return mod - #print(type(mod)) - #print(dir(mod)) - #print(mod.migration_changes) + return { + 'database': mod.db, + 'changes': mod.MIGRATION_CHANGES + } - except ModuleNotFoundError: - print('Unable to find database file') + except ModuleNotFoundError as error: + print(f'Unable to find database file: {error}') exit() - except AttributeError: - print('Unable to find migration_changes') + except AttributeError as error: + print(f'Unable to find migration_changes: {error}') exit() -def show_migrations(): +def show_migrations(params): """ - Get + Show the status of the current migrations """ - module = check_settings() - db = module.db - db.show_migrations(module.migration_changes) + settings = check_settings() + db = settings['database'] + changes = settings['changes'] + db.show_migrations(changes) + -def migration(direction: str): - print('Run forward or reverse migration') +def migration(params): + """ + * Add new migrations + * Reverse existing migrations + """ + settings = check_settings() + db: Database = settings['database'] + changes = settings['changes'] + if params.direction == 'up': + db.add_migrations(changes) + elif params.direction == 'down' and params.dry: + if params.version == 0: + print("migration version needed") + exit() + else: + db.dry_run_reverse(params.version, changes) + elif params.direction == 'down': + if params.version == 0: + print("migration version needed") + exit() + else: + db.reverse_migrations(params.version, changes) if __name__ == '__main__': @@ -43,37 +67,13 @@ def migration(direction: str): show_migration.set_defaults(func=show_migrations) migrate = subparsers.add_parser('migrate', help='Run forward or reverse migrations') + migrate.add_argument('direction', choices=['up', 'down'], help='forward [up] or reverse migration [down]') + migrate.add_argument('version', help='Version number at which to stop the migration', nargs='?', type=int, default=0) + migrate.add_argument('--dry', help='Show migrations to be reversed or applied', action='store_const', const=True) migrate.set_defaults(func=migration) - migrate.add_argument('direction', choices=['up', 'down'], help='forward or reverse migration') args = parser.parse_args() - args.func() - - -# showmigrations - show all migrations, including unapplied -# migrate up - apply migrations -# migrate down - reverse migrations upto a particular version -# -# from litemigration.database import SqliteDatabase, Migration -# -# -# db = SqliteDatabase('example.db') -# db.initialize() -# -# change = [ -# Migration( -# version=2, -# up='CREATE TABLE player(name VARCHAR NOT NULL,score INTEGER)', -# down='DROP TABLE player' -# ), -# Migration( -# version=3, -# up='INSERT INTO player(name,score) VALUES("Menzi", 10)', -# down='DELETE FROM PLAYER where name="Menzi"' -# ) -# ] -# -# -# db.add_migration(change) -# #db.reverse_migration(2, change) + args.func(args) + + diff --git a/litemigration/database.py b/litemigration/database.py index f82935b..0dd1c07 100755 --- a/litemigration/database.py +++ b/litemigration/database.py @@ -33,13 +33,17 @@ def initialize(self): pass @abstractmethod - def add_migration(self, change: List[Migration]): + def add_migrations(self, change: List[Migration]): pass @abstractmethod def show_migrations(self, change: List[Migration]): print("Showing the database migrations") + @abstractmethod + def reverse_migrations(self, version: int, change: List[Migration]): + pass + class SqliteDatabase(Database): def __init__(self, name): @@ -66,7 +70,7 @@ def initialize(self): except sqlite3.OperationalError: log.info('Database already exists') - def add_migration(self, change: List[Migration]): + def add_migrations(self, change: List[Migration]): cur = self.connect.cursor() cur.execute('SELECT max(version) from migration') (max_version,) = cur.fetchone() @@ -90,7 +94,7 @@ def add_migration(self, change: List[Migration]): self.connect.close() - def reverse_migration(self, version: int, change: List[Migration]): + def reverse_migrations(self, version: int, change: List[Migration]): """ Migration version to revert to from max version. So if max version is 10 and version choosen is 5. @@ -130,215 +134,24 @@ def show_migrations(self, change: List[Migration]): table = AsciiTable(data) print(table.table) + def dry_run_reverse(self, version: int, change: List[Migration]): + """ + Show which changes will be reversed (--dry) + """ + data = [] + data.append(['Reverse', 'Version']) + cur = self.connect.cursor() + cur.execute('SELECT max(version) from migration') + (max_id,) = cur.fetchone() + if version > max_id: + raise MigrationException('version greater than max version unable to reverse') -# class PostgresqlDatabase(Database): -# def __init__(self, host, port, user, password, name): -# self.host = host -# self.port = port -# self.user = user -# self.password = password -# self.name = name -# self.connect = self.connection() -# -# def connection(self): -# connect = psycopg2.connect(database=self.database, -# host=self.host, -# user=self.user,s -# password=self.password, -# port=self.port) -# return connect -# -# def initialize(self): -# create_query = ("CREATE TABLE migration(" -# 'id SERIAL PRIMARY KEY,' -# 'version INTEGER NOT NULL,' -# 'date DATE NOT NULL)') -# insert_query = "INSERT INTO migration(version,date) VALUES(1,%s)" -# try: -# cur = self.connect.cursor() -# cur.execute(create_query) -# cur.execute(insert_query, dt.datetime.now()) -# except psycopg2.OperationalError as error: -# log.exception(error) -# exit() - - # def add_migration(self, change: List[Migration]): - # """ - # list of database changes - # [1, (insert into)] - # """ - # cur = self.connect.cursor() - # cur.execute('SELECT max(version) from migration') - # (max_id, _) = cur.fetchone() - # - # for migration in change: - # if max_id > migration.version: - # log.info(f'schema migration {migration.version} has already been applied') - # continue - # else: - # if max_id - migration != 1: - # log.error(f'migration version {migration.version} not continouus') - # raise MigrationException('migration version not continous') - # else: - # try: - # cur.execute(migration.up) - # cur.execute() - # self.connect.commit() - # log.info(f'migration {migration.version} added') - # max_id = migration.version - # except Exception: - # pass - # - # - # for change_id, sql_statement in change_list: - # self.cursor.execute('SELECT max(version) from migration') - # (max_id,) = self.cursor.fetchone() - # if max_id >= change_id: - # log.info("schema change id {} is smaller than the latest" - # "change".format(change_id)) - # log.info("or schema change id has already been applied ") - # else: - # try: - # self.cursor.execute(sql_statement) - # self.cursor.execute(insert_sql, - # (change_id, dt.datetime.now(),)) - # self.connect.commit() - # log.info("new schema added") - # except Exception: - # log.error("Unable to add schema {}".format(change_id), - # exc_info=True) - # sys.exit() - - - - - - - + for migration in reversed(change): + if migration.version == version: + break + elif migration.version > version: + data.append([Color('{autogreen}Yes{/autogreen}'), migration.version]) -# class Database: -# "Create migration control" -# def __init__(self, db_type, host=None, port=None, user=None, -# password=None, database=None): -# self.db_type = db_type -# self.host = host -# self.port = port -# self.user = user -# self.password = password -# self.database = database -# self.details = "" -# self.connect = self._get_connector() -# self.cursor = self.connect.cursor() -# -# def _get_connector(self): -# """' -# Return database connection from specified database from user -# """ -# supported_databases = {'postgresql': self._postgresql, -# 'sqlite': self._sqlite} -# try: -# connect = supported_databases[self.db_type]() -# return connect -# except KeyError: -# log.critical("Unknown database or not supported") -# exit() -# -# def _get_initail_sql_migration(self) -> Tuple[str, str]: -# """ -# Return 2 sql commands: -# 1) Create migration -# 2) Insert into migration table -# """ -# sqlite_create = ("CREATE TABLE migration(" -# 'id INTEGER PRIMARY KEY NOT NULL,' -# 'version INTEGER UNIQUE NOT NULL,' -# 'date TIMESTAMP NOT NULL)', -# "INSERT INTO migration(version,date) VALUES(0,?)") -# -# pg_create = ("CREATE TABLE migration(" -# 'id SERIAL PRIMARY KEY,' -# 'version INTEGER NOT NULL,' -# 'date DATE NOT NULL)', -# "INSERT INTO migration(version,date) VALUES(0,%s)") -# -# all_sql = {'postgresql': pg_create, -# 'sqlite': sqlite_create} -# -# return all_sql[self.db_type] -# -# def initialise(self): -# "Create new database and add initial migration" -# create_table, initial_insert = self._get_initail_sql_migration() -# try: -# self.cursor.execute(create_table) -# self.cursor.execute(initial_insert, -# (dt.datetime.now(),)) -# self.connect.commit() -# log.info("Database has been created") -# except Exception as e: -# log.error("Unable to add migration table") -# log.exception(e) -# sys.exit() -# -# def add_schema(self, change_list: List[Tuple[int, str]]): -# """ -# The first migration change should be version 1 -# """ -# if self.db_type == 'postgresql': -# insert_sql = "INSERT INTO migration(version,date) VALUES(%s,%s)" -# elif self.db_type == 'sqlite': -# insert_sql = "INSERT INTO migration(version,date) VALUES(?,?)" -# -# for change_id, sql_statement in change_list: -# self.cursor.execute('SELECT max(version) from migration') -# (max_id,) = self.cursor.fetchone() -# if max_id >= change_id: -# log.info("schema change id {} is smaller than the latest" -# "change".format(change_id)) -# log.info("or schema change id has already been applied ") -# else: -# try: -# self.cursor.execute(sql_statement) -# self.cursor.execute(insert_sql, -# (change_id, dt.datetime.now(),)) -# self.connect.commit() -# log.info("new schema added") -# except Exception: -# log.error("Unable to add schema {}".format(change_id), -# exc_info=True) -# sys.exit() -# -# def _postgresql(self): -# "create postgresql connection and return the connection object" -# try: -# import psycopg2 -# connect = psycopg2.connect(database=self.database, -# host=self.host, -# user=self.user, -# password=self.password, -# port=self.port) -# return connect -# except ImportError: -# log.error("Unable to find python3 postgresql module") -# sys.exit() -# except psycopg2.Error as e: -# log.error("Unable to connect to postgresql") -# log.exception(e) -# sys.exit() -# except psycopg2.OperationalError as e: -# log.exception(e) -# sys.exit() -# -# def _sqlite(self): -# """ -# Create an sqlite connection and return the connection object -# """ -# import sqlite3 -# try: -# connect = sqlite3.connect(self.database) -# return connect -# except sqlite3.OperationalError: -# log.error("unable to connect to sqlite database", -# exc_info=True) -# sys.exit() + table = AsciiTable(data) + print(table.table) From 232448236c142f89469b3ff33b598a4665937e3b Mon Sep 17 00:00:00 2001 From: ebsuku Date: Sun, 11 Dec 2022 22:06:01 +0200 Subject: [PATCH 3/9] add pip requirements --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8fb25e9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +colorclass==2.2.2 +-e git+ssh://git@github.com/knightebsuku/python3-litemigration.git@ef0bce2af201d420ceec0367e070ce031c46dd7f#egg=litemigration +terminaltables==3.1.10 From c7d57585d0f936302923a0ed73355e9b1026a77a Mon Sep 17 00:00:00 2001 From: "lunga.mthembu" Date: Mon, 12 Dec 2022 17:15:07 +0200 Subject: [PATCH 4/9] dev --- litemigration-cli.py | 7 +-- litemigration/database.py | 94 +++++++++++++++++++-------------------- requirements.txt | 4 +- tests/__init__.py | 0 tests/test.py | 60 +++++++++++++++++++++++++ 5 files changed, 114 insertions(+), 51 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/test.py diff --git a/litemigration-cli.py b/litemigration-cli.py index 9542a06..e142bee 100644 --- a/litemigration-cli.py +++ b/litemigration-cli.py @@ -24,7 +24,6 @@ def check_settings() -> dict: exit() - def show_migrations(params): """ Show the status of the current migrations @@ -50,13 +49,15 @@ def migration(params): print("migration version needed") exit() else: - db.dry_run_reverse(params.version, changes) + table = db.dry_run_reverse(params.version, changes) + print(table) elif params.direction == 'down': if params.version == 0: print("migration version needed") exit() else: - db.reverse_migrations(params.version, changes) + table = db.reverse_migrations(params.version, changes) + print(table) if __name__ == '__main__': diff --git a/litemigration/database.py b/litemigration/database.py index 0dd1c07..8d8e41b 100755 --- a/litemigration/database.py +++ b/litemigration/database.py @@ -24,6 +24,9 @@ class MigrationException(Exception): class Database(ABC): + def __init__(self): + self.connect = self.connection() + @abstractmethod def connection(self): pass @@ -36,19 +39,56 @@ def initialize(self): def add_migrations(self, change: List[Migration]): pass - @abstractmethod - def show_migrations(self, change: List[Migration]): - print("Showing the database migrations") - @abstractmethod def reverse_migrations(self, version: int, change: List[Migration]): pass + def show_migrations(self, change: List[Migration]): + version = 0 + data = [] + data.append(['Applied', 'Version', 'Date']) + + cur = self.connect.cursor() + cur.execute('SELECT version, date FROM migration') + applied = cur.fetchall() + for m in applied: + data.append([Color('{autogreen}Yes{/autogreen}'), m[0], m[1]]) + version = m[0] + + for m in change: + if version < m.version: + data.append([Color('{autored}No{/autored}'), m.version]) + + table = AsciiTable(data) + return table + + def dry_run_reverse(self, version: int, change: List[Migration]): + """ + Show which changes will be reversed (--dry) + """ + data = [] + data.append(['Reverse', 'Version']) + + cur = self.connect.cursor() + cur.execute('SELECT max(version) from migration') + (max_id,) = cur.fetchone() + if version > max_id: + raise MigrationException('version greater than max version unable to reverse') + + for migration in reversed(change): + if migration.version == version: + break + elif migration.version > version: + data.append([Color('{autogreen}Yes{/autogreen}'), migration.version]) + + table = AsciiTable(data) + return table + class SqliteDatabase(Database): def __init__(self, name): self.name = name - self.connect = self.connection() + super().__init__() def connection(self): connect = sqlite3.connect(self.name) @@ -75,7 +115,6 @@ def add_migrations(self, change: List[Migration]): cur.execute('SELECT max(version) from migration') (max_version,) = cur.fetchone() for migration in change: - print(f'Current migration {migration} ') if max_version >= migration.version: log.info(f'migration {migration.version} already applied') continue @@ -87,8 +126,10 @@ def add_migrations(self, change: List[Migration]): cur.execute(migration.up) cur.execute("INSERT INTO migration(version,date) VALUES(?,?)", (migration.version, datetime.now())) self.connect.commit() + print(f'Migration {migration.version} applied....' + Color('{autogreen}Ok{/autogreen}')) max_version = migration.version except sqlite3.OperationalError as error: + print(f'Migration {migration.version} applied....' + Color('{autored}Error{/autored}')) log.error(f'unable to apply migration {migration.version}') raise MigrationException(error) @@ -114,44 +155,3 @@ def reverse_migrations(self, version: int, change: List[Migration]): cur.execute('DELETE FROM migration where version=?', (migration.version,)) self.connect.commit() self.connect.close() - - def show_migrations(self, change: List[Migration]): - version = 0 - data = [] - data.append(['Applied', 'Version', 'Date']) - - cur = self.connect.cursor() - cur.execute('SELECT version, date FROM migration') - applied = cur.fetchall() - for m in applied: - data.append([Color('{autogreen}Yes{/autogreen}'), m[0], m[1]]) - version = m[0] - - for m in change: - if version < m.version: - data.append([Color('{autored}No{/autored}'), m.version]) - - table = AsciiTable(data) - print(table.table) - - def dry_run_reverse(self, version: int, change: List[Migration]): - """ - Show which changes will be reversed (--dry) - """ - data = [] - data.append(['Reverse', 'Version']) - - cur = self.connect.cursor() - cur.execute('SELECT max(version) from migration') - (max_id,) = cur.fetchone() - if version > max_id: - raise MigrationException('version greater than max version unable to reverse') - - for migration in reversed(change): - if migration.version == version: - break - elif migration.version > version: - data.append([Color('{autogreen}Yes{/autogreen}'), migration.version]) - - table = AsciiTable(data) - print(table.table) diff --git a/requirements.txt b/requirements.txt index 8fb25e9..d1b0be2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ colorclass==2.2.2 --e git+ssh://git@github.com/knightebsuku/python3-litemigration.git@ef0bce2af201d420ceec0367e070ce031c46dd7f#egg=litemigration +freezegun==1.2.2 +python-dateutil==2.8.2 +six==1.16.0 terminaltables==3.1.10 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test.py b/tests/test.py new file mode 100644 index 0000000..f7665ff --- /dev/null +++ b/tests/test.py @@ -0,0 +1,60 @@ +import unittest +import os +from datetime import datetime +from freezegun import freeze_time + +from colorclass import Color +from litemigration.database import SqliteDatabase, Migration + + +@freeze_time('2022-01-01 00:00:00') +class TestMigration(unittest.TestCase): + + def setUp(self) -> None: + self.db = SqliteDatabase('test.db') + self.db.initialize() + self.migration_changes = [ + Migration( + version=2, + up='CREATE TABLE player(name VARCHAR NOT NULL,score INTEGER)', + down='DROP TABLE player' + ), + Migration( + version=3, + up='INSERT INTO player(name,score) VALUES("User", 10)', + down='DELETE FROM PLAYER where name="User"' + ) + ] + + def test_add_migration(self): + self.db.add_migrations(self.migration_changes) + + cur = self.db.connection().cursor() + cur.execute('SELECT max(version) FROM migration') + (max_id,) = cur.fetchone() + self.assertEqual(max_id, 3) + + def test_show_migrations(self): + data = [ + ['Applied', 'Version', 'Date'], + [Color('{autogreen}Yes{/autogreen}'), 1, datetime.now()], + [Color('{autored}No{/autored}'), 2], + [Color('{autored}No{/autored}'), 3] + ] + table = self.db.show_migrations(self.migration_changes) + print(table.table_data) + self.assertEqual(data, table.table_data) + + def test_reverse_migrations(self): + pass + + def test_dry_reverse_migrations(self): + pass + + def tearDown(self) -> None: + os.remove('test.db') + + + +if __name__ == '__main__': + unittest.main() From 979ee3d077286c96cab9bdbbde49d253cf15d448 Mon Sep 17 00:00:00 2001 From: "lunga.mthembu" Date: Tue, 13 Dec 2022 16:50:56 +0200 Subject: [PATCH 5/9] add full tests --- tests/test.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/test.py b/tests/test.py index f7665ff..007d3e3 100644 --- a/tests/test.py +++ b/tests/test.py @@ -37,19 +37,40 @@ def test_add_migration(self): def test_show_migrations(self): data = [ ['Applied', 'Version', 'Date'], - [Color('{autogreen}Yes{/autogreen}'), 1, datetime.now()], + [Color('{autogreen}Yes{/autogreen}'), 1, str(datetime.now())], [Color('{autored}No{/autored}'), 2], [Color('{autored}No{/autored}'), 3] ] table = self.db.show_migrations(self.migration_changes) - print(table.table_data) self.assertEqual(data, table.table_data) def test_reverse_migrations(self): - pass + self.db.add_migrations(self.migration_changes) + + self.db.connect = self.db.connection() + cur = self.db.connect.cursor() + cur.execute('SELECT max(version) FROM migration') + (max_id,) = cur.fetchone() + self.assertEqual(max_id, 3) + + self.db.connect = self.db.connection() + self.db.reverse_migrations(1, self.migration_changes) + + cur = self.db.connection().cursor() + cur.execute('SELECT max(version) FROM migration') + (max_id,) = cur.fetchone() + self.assertEqual(max_id, 1) def test_dry_reverse_migrations(self): - pass + self.db.add_migrations(self.migration_changes) + data = [ + ['Reverse', 'Version'], + [Color('{autogreen}Yes{/autogreen}'), 3] + ] + # Reverse the last migration Version 3 + self.db.connect = self.db.connection() + table = self.db.dry_run_reverse(2, self.migration_changes) + self.assertEqual(data, table.table_data) def tearDown(self) -> None: os.remove('test.db') From 59bcc58b3d04338cf60e234494955b3640837dd3 Mon Sep 17 00:00:00 2001 From: "lunga.mthembu" Date: Tue, 13 Dec 2022 16:51:09 +0200 Subject: [PATCH 6/9] dev --- litemigration-cli.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litemigration-cli.py b/litemigration-cli.py index e142bee..b29268a 100644 --- a/litemigration-cli.py +++ b/litemigration-cli.py @@ -31,7 +31,8 @@ def show_migrations(params): settings = check_settings() db = settings['database'] changes = settings['changes'] - db.show_migrations(changes) + table = db.show_migrations(changes) + print(table) def migration(params): @@ -50,14 +51,14 @@ def migration(params): exit() else: table = db.dry_run_reverse(params.version, changes) - print(table) + print(table.table) elif params.direction == 'down': if params.version == 0: print("migration version needed") exit() else: table = db.reverse_migrations(params.version, changes) - print(table) + print(table.table) if __name__ == '__main__': From 68741a2aade106342decf8077cd0da4da71a197e Mon Sep 17 00:00:00 2001 From: ebsuku Date: Tue, 20 Dec 2022 01:21:29 +0200 Subject: [PATCH 7/9] add postgresql support --- litemigration-cli.py | 2 +- litemigration/database.py | 119 +++++++++++++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/litemigration-cli.py b/litemigration-cli.py index b29268a..5021a1c 100644 --- a/litemigration-cli.py +++ b/litemigration-cli.py @@ -32,7 +32,7 @@ def show_migrations(params): db = settings['database'] changes = settings['changes'] table = db.show_migrations(changes) - print(table) + print(table.table) def migration(params): diff --git a/litemigration/database.py b/litemigration/database.py index 8d8e41b..78a79bf 100755 --- a/litemigration/database.py +++ b/litemigration/database.py @@ -9,6 +9,12 @@ from terminaltables import AsciiTable from colorclass import Color +try: + import psycopg2 +except ImportError: + pass + + log = logging.getLogger(__name__) @@ -67,7 +73,7 @@ def dry_run_reverse(self, version: int, change: List[Migration]): Show which changes will be reversed (--dry) """ data = [] - data.append(['Reverse', 'Version']) + data.append(['Reversed (Dummy)', 'Version']) cur = self.connect.cursor() cur.execute('SELECT max(version) from migration') @@ -107,8 +113,9 @@ def initialize(self): cur.execute(query) cur.execute("INSERT INTO migration(version,date) VALUES(1,?)", (datetime.now(),)) self.connect.commit() - except sqlite3.OperationalError: - log.info('Database already exists') + except sqlite3.OperationalError as error: + log.info(f'Error creating migration table: {error}') + raise MigrationException(error) def add_migrations(self, change: List[Migration]): cur = self.connect.cursor() @@ -155,3 +162,109 @@ def reverse_migrations(self, version: int, change: List[Migration]): cur.execute('DELETE FROM migration where version=?', (migration.version,)) self.connect.commit() self.connect.close() + + +class Postgresql(Database): + + def __init__(self, name, user, password, host, port=5432): + self.name = name + self.user = user + self.password = password + self.host = host + self.port = port + super().__init__() + + def connection(self): + """ + Check if psycopg2 is installed if using postgres database + """ + try: + import psycopg2 + except ImportError: + raise MigrationException('postgresql driver not installed') + + try: + conn = psycopg2.connect( + host=self.host, + database=self.name, + user=self.user, + password=self.password, + port=self.port + ) + except psycopg2.OperationalError as error: + log.error(f'Unable to connect to database: {error}') + raise MigrationException(f'Unable to connect to database: {error}') + else: + return conn + + def initialize(self): + query = ( + 'CREATE TABLE migration(' + 'id SERIAL PRIMARY KEY NOT NULL,' + 'version INTEGER UNIQUE NOT NULL,' + 'date TIMESTAMP NOT NULL)' + ) + + cur = self.connect.cursor() + try: + cur.execute(query) + cur.execute("INSERT INTO migration(version,date) VALUES(1, %s)", (datetime.now(),)) + self.connect.commit() + except (psycopg2.OperationalError, psycopg2.DatabaseError) as error: + log.info(f'Error creating migration table: {error}') + raise MigrationException(error) + + def add_migrations(self, change: List[Migration]): + cur = self.connect.cursor() + cur.execute('SELECT max(version) from migration') + (max_version,) = cur.fetchone() + for migration in change: + if max_version >= migration.version: + log.info(f'migration {migration.version} already applied') + continue + + if migration.version - max_version != 1: + log.error(f'missing migration version before {migration.version}') + raise MigrationException('missing migration version before {}'.format(migration.version)) + try: + cur.execute(migration.up) + cur.execute("INSERT INTO migration(version,date) VALUES(%s,%s)", (migration.version, datetime.now())) + self.connect.commit() + print(f'Migration {migration.version} applied....' + Color('{autogreen}Ok{/autogreen}')) + max_version = migration.version + except (psycopg2.OperationalError, psycopg2.DatabaseError) as error: + print(f'Migration {migration.version} applied....' + Color('{autored}Error{/autored}')) + log.error(f'unable to apply migration {migration.version}') + raise MigrationException(error) + + self.connect.close() + + def reverse_migrations(self, version: int, change: List[Migration]): + """ + Migration version to revert to from max version. + So if max version is 10 and version chosen is 5. + Version 10 - 6 will be reverted + """ + data = [] + data.append(['Reversed', 'Version']) + + cur = self.connect.cursor() + cur.execute('SELECT max(version) from migration') + (max_id,) = cur.fetchone() + if version > max_id: + raise MigrationException('version greater than max version unable to reverse') + + for migration in reversed(change): + if migration.version == version: + break + elif migration.version > version: + try: + cur.execute(migration.down) + cur.execute('DELETE FROM migration where version=%s', (migration.version,)) + self.connect.commit() + data.append([Color('{autogreen}Yes{/autogreen}'), migration.version]) + except (psycopg2.DatabaseError, psycopg2.OperationalError): + data.append([Color('{autored}Error{/autored}'), migration.version]) + self.connect.close() + table = AsciiTable(data) + return table From 36861238ba9d92d02a190e098af03f1f5093d848 Mon Sep 17 00:00:00 2001 From: ebsuku Date: Tue, 20 Dec 2022 01:22:05 +0200 Subject: [PATCH 8/9] format --- tests/test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index 007d3e3..7bcb714 100644 --- a/tests/test.py +++ b/tests/test.py @@ -76,6 +76,5 @@ def tearDown(self) -> None: os.remove('test.db') - if __name__ == '__main__': unittest.main() From aa14738a52ab12146d7050d55b44a528f941f338 Mon Sep 17 00:00:00 2001 From: ebsuku Date: Wed, 15 Feb 2023 17:53:16 +0200 Subject: [PATCH 9/9] dev --- litemigration-cli.py | 2 ++ setup.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/litemigration-cli.py b/litemigration-cli.py index 5021a1c..41a247a 100644 --- a/litemigration-cli.py +++ b/litemigration-cli.py @@ -1,3 +1,5 @@ +#!/usr/bin/python3 + import argparse import importlib diff --git a/setup.py b/setup.py index 44430d5..3c6dc01 100755 --- a/setup.py +++ b/setup.py @@ -4,11 +4,11 @@ if __name__ == '__main__': setup(name='litemigration', - version='1.1.1', - description='Simple simple module to help modify database changes in sqlite', + version='2.0.0', + description='Super simple module to help modify database changes in sqlite', author='Lunga Mthembu', - author_email='stumenz.complex@gmail.com', - url='https://github.com/stumenz/python3-litemigration', + author_email='midnight.complex@protonmail.com', + url='https://github.com/knightebsuku/python3-litemigration', license='GPL', packages=['litemigration'], )