From 242769dedb0ada56d5a7fef28525c696a9b5f726 Mon Sep 17 00:00:00 2001 From: aleksspevak Date: Sat, 30 Dec 2023 14:45:26 +0300 Subject: [PATCH 1/4] feat(db): Add meme_raw_upload table and model --- src/database.py | 30 +++++++++++++++++++----------- src/storage/schemas.py | 17 +++++++++++++++++ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/src/database.py b/src/database.py index 479a26a9..1f162f44 100644 --- a/src/database.py +++ b/src/database.py @@ -33,7 +33,6 @@ metadata = MetaData(naming_convention=DB_NAMING_CONVENTION) - language = Table( "language", metadata, @@ -43,7 +42,6 @@ # TODO: flag: show in language selector UI or not ) - meme_source = Table( "meme_source", metadata, @@ -60,7 +58,6 @@ Column("updated_at", DateTime, onupdate=func.now()), ) - meme_raw_telegram = Table( "meme_raw_telegram", metadata, @@ -87,13 +84,27 @@ UniqueConstraint("meme_source_id", "post_id", name=MEME_SOURCE_POST_UNIQUE_CONSTRAINT), ) +meme_raw_upload = Table( + "meme_raw_upload", + metadata, + Column("id", Integer, Identity(), primary_key=True), + Column("message_id", Integer, nullable=False), + Column("chat", JSONB, nullable=False), -# meme_raw_upload = Table( -# "meme_raw_upload", -# metadata, -# TODO: columns TBD, probably also JSONBs to store all raw data -# ) + Column("content", String), + Column("date", DateTime, nullable=False), + Column("out_links", JSONB), + Column("mentions", JSONB), + Column("hashtags", JSONB), + Column("forwarded", JSONB), + + Column("image", JSONB), + Column("video", JSONB), + + Column("created_at", DateTime, server_default=func.now(), nullable=False), + Column("updated_at", DateTime, onupdate=func.now()) +) meme = Table( "meme", @@ -117,7 +128,6 @@ UniqueConstraint("meme_source_id", "raw_meme_id", name=MEME_SOURCE_RAW_MEME_UNIQUE_CONSTRAINT), ) - user_tg = Table( "user_tg", metadata, @@ -135,7 +145,6 @@ Column("updated_at", DateTime, onupdate=func.now()), ) - user = Table( "user", metadata, @@ -147,7 +156,6 @@ Column("blocked_bot_at", DateTime), ) - user_language = Table( "user_language", metadata, diff --git a/src/storage/schemas.py b/src/storage/schemas.py index 56227070..4df1b76d 100644 --- a/src/storage/schemas.py +++ b/src/storage/schemas.py @@ -17,3 +17,20 @@ class OcrResult(CustomModel): model: str result: dict calculated_at: datetime = Field(default_factory=datetime.utcnow) + + +class MemeUserUpload(CustomModel): + message_id: int + chat: dict + + content: str | None = None + date: datetime + + out_links: list[str] | None = None + mentions: list[str] | None = None # mentioned usernames + hashtags: list[str] | None = None + forwarded: dict | None = None + + image: list[dict] | None = None + video: list[dict] | None = None + From 2781d3964af916d5266d2077c183cd8f3f829b41 Mon Sep 17 00:00:00 2001 From: aleksspevak Date: Sat, 30 Dec 2023 14:57:03 +0300 Subject: [PATCH 2/4] feat(db): Add meme_raw_upload table and model --- src/database.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/database.py b/src/database.py index 1f162f44..85a87fdf 100644 --- a/src/database.py +++ b/src/database.py @@ -33,6 +33,7 @@ metadata = MetaData(naming_convention=DB_NAMING_CONVENTION) + language = Table( "language", metadata, @@ -42,6 +43,7 @@ # TODO: flag: show in language selector UI or not ) + meme_source = Table( "meme_source", metadata, @@ -58,6 +60,7 @@ Column("updated_at", DateTime, onupdate=func.now()), ) + meme_raw_telegram = Table( "meme_raw_telegram", metadata, @@ -84,6 +87,7 @@ UniqueConstraint("meme_source_id", "post_id", name=MEME_SOURCE_POST_UNIQUE_CONSTRAINT), ) + meme_raw_upload = Table( "meme_raw_upload", metadata, @@ -106,6 +110,7 @@ Column("updated_at", DateTime, onupdate=func.now()) ) + meme = Table( "meme", metadata, @@ -128,6 +133,7 @@ UniqueConstraint("meme_source_id", "raw_meme_id", name=MEME_SOURCE_RAW_MEME_UNIQUE_CONSTRAINT), ) + user_tg = Table( "user_tg", metadata, @@ -145,6 +151,7 @@ Column("updated_at", DateTime, onupdate=func.now()), ) + user = Table( "user", metadata, @@ -156,6 +163,7 @@ Column("blocked_bot_at", DateTime), ) + user_language = Table( "user_language", metadata, From 296f83ea4ffe046bc8db9d545a47affde6c389cb Mon Sep 17 00:00:00 2001 From: aleksspevak Date: Sat, 30 Dec 2023 16:24:05 +0300 Subject: [PATCH 3/4] feat(db): Combined video and image to media column --- src/database.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/database.py b/src/database.py index 85a87fdf..50ebad76 100644 --- a/src/database.py +++ b/src/database.py @@ -102,9 +102,7 @@ Column("mentions", JSONB), Column("hashtags", JSONB), Column("forwarded", JSONB), - - Column("image", JSONB), - Column("video", JSONB), + Column("media", JSONB), Column("created_at", DateTime, server_default=func.now(), nullable=False), Column("updated_at", DateTime, onupdate=func.now()) From c19e82d9d175631d280b06e12d3372b969df6f2f Mon Sep 17 00:00:00 2001 From: aleksspevak Date: Sat, 30 Dec 2023 19:25:07 +0300 Subject: [PATCH 4/4] feat(db): Combined video and image to media column --- src/storage/schemas.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/storage/schemas.py b/src/storage/schemas.py index 4df1b76d..66a4dc96 100644 --- a/src/storage/schemas.py +++ b/src/storage/schemas.py @@ -30,7 +30,5 @@ class MemeUserUpload(CustomModel): mentions: list[str] | None = None # mentioned usernames hashtags: list[str] | None = None forwarded: dict | None = None - - image: list[dict] | None = None - video: list[dict] | None = None + media: list[dict] | None = None