From 53d6cd3f3879acc692c6c2d63e276cb28ba86b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danii=C5=82=20M=2E?= Date: Wed, 18 Feb 2026 11:46:23 +0100 Subject: [PATCH 1/6] chore: Gentle push to trigger deployment --- backend/core/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/pyproject.toml b/backend/core/pyproject.toml index 45c26bdf..2afe96e5 100644 --- a/backend/core/pyproject.toml +++ b/backend/core/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "core" -version = "1.0.0" +version = "1.0.1" description = "Access Community Tool – Core package" authors = [ { name="danoctua", email="danoctua@gmail.com" } From 848eddbada5c9fb8a2383267d95248f50a5c38a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danii=C5=82=20M=2E?= Date: Wed, 18 Feb 2026 11:59:39 +0100 Subject: [PATCH 2/6] chore: Gentle push to trigger deployment --- backend/core/src/core/actions/jetton.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/src/core/actions/jetton.py b/backend/core/src/core/actions/jetton.py index 852e961b..8e3c2651 100644 --- a/backend/core/src/core/actions/jetton.py +++ b/backend/core/src/core/actions/jetton.py @@ -63,7 +63,7 @@ async def refresh(self, address_raw: str) -> JettonDTO: f"refresh_details_{address_raw}", "1", ex=3600, nx=True ): logger.warning( - f"Refresh details for {address_raw} was triggered already. Please wait for an hour to do it again." + f"Refresh details for {address_raw} was triggered already. Skipping." ) raise HTTPException( status_code=HTTP_409_CONFLICT, From 5d804b2819d829c9ad7fa9cd52b7ad0add40a2d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danii=C5=82=20M=2E?= Date: Wed, 18 Feb 2026 13:34:03 +0100 Subject: [PATCH 3/6] fix: improve task handling and allow address reuse for TCP server --- backend/community_manager/entrypoint.py | 3 ++- backend/core/src/core/utils/probe.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/community_manager/entrypoint.py b/backend/community_manager/entrypoint.py index 12230e32..974de054 100644 --- a/backend/community_manager/entrypoint.py +++ b/backend/community_manager/entrypoint.py @@ -73,12 +73,13 @@ def main() -> None: telethon_service.client.loop.run_until_complete(telethon_service.client.catch_up()) # Start the Gateway Service as a background task on the same loop - telethon_service.client.loop.create_task(gateway_service.start()) + gateway_task = telethon_service.client.loop.create_task(gateway_service.start()) try: telethon_service.client.run_until_disconnected() finally: gateway_service.stop() + telethon_service.client.loop.run_until_complete(gateway_task) if __name__ == "__main__": diff --git a/backend/core/src/core/utils/probe.py b/backend/core/src/core/utils/probe.py index 636042fc..96275f50 100644 --- a/backend/core/src/core/utils/probe.py +++ b/backend/core/src/core/utils/probe.py @@ -32,6 +32,9 @@ def do_GET(self): self.send_response(404) self.end_headers() - with socketserver.TCPServer(("", 8080), HealthCheckHandler) as httpd: + class ReusableTCPServer(socketserver.TCPServer): + allow_reuse_address = True + + with ReusableTCPServer(("", 8080), HealthCheckHandler) as httpd: logger.info("Health check server running on port 8080") httpd.serve_forever() From 976839d492e5097258a3f88bf382b71f07f5f773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danii=C5=82=20M=2E?= Date: Wed, 18 Feb 2026 14:04:23 +0100 Subject: [PATCH 4/6] chore: add logging for Telegram client catch-up process --- backend/community_manager/entrypoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/community_manager/entrypoint.py b/backend/community_manager/entrypoint.py index 974de054..3e5e9e01 100644 --- a/backend/community_manager/entrypoint.py +++ b/backend/community_manager/entrypoint.py @@ -70,6 +70,7 @@ def main() -> None: health_thread.start() telethon_service.start_sync() + logger.info("Telegram client catching up...") telethon_service.client.loop.run_until_complete(telethon_service.client.catch_up()) # Start the Gateway Service as a background task on the same loop From afec8bb36e40752ee3a2f721906f9644b41f9cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danii=C5=82=20M=2E?= Date: Mon, 23 Feb 2026 22:57:50 +0100 Subject: [PATCH 5/6] feat: add non-managed user filtering to yield_all_for_chat method --- backend/core/src/core/services/chat/user.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/backend/core/src/core/services/chat/user.py b/backend/core/src/core/services/chat/user.py index 3a27dd3b..b13fe3c8 100644 --- a/backend/core/src/core/services/chat/user.py +++ b/backend/core/src/core/services/chat/user.py @@ -171,7 +171,7 @@ def get_all( return query.all() def yield_all_for_chat( - self, chat_id: int, batch_size: int = 100 + self, chat_id: int, non_managed_only: bool = False, batch_size: int = 100 ) -> Iterable[list[TelegramChatUser]]: """ Yields all users for a given chat in batches, using keyset pagination. @@ -179,12 +179,16 @@ def yield_all_for_chat( """ last_seen_user_id = 0 while True: + filters = [ + TelegramChatUser.chat_id == chat_id, + TelegramChatUser.user_id > last_seen_user_id, + ] + if non_managed_only: + filters.append(TelegramChatUser.is_managed.is_(False)) + stmt = ( select(TelegramChatUser) - .where( - TelegramChatUser.chat_id == chat_id, - TelegramChatUser.user_id > last_seen_user_id, - ) + .where(*filters) .order_by(TelegramChatUser.user_id.asc()) .limit(batch_size) .options( @@ -193,7 +197,7 @@ def yield_all_for_chat( ) ) ) - users = self.db_session.execute(stmt).scalars().unique().all() + users = list(self.db_session.execute(stmt).scalars().unique().all()) if not users: break From 49387de0e67f5c4c6d71b2551d0ad7ffbaef5892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danii=C5=82=20M=2E?= Date: Wed, 4 Mar 2026 15:44:04 +0100 Subject: [PATCH 6/6] feat: migrate gift collection indexing from slug to integer ID - Update GiftCollection model: replace slug PK with integer id, add options (JSONB) and blockchain_address fields - Update GiftUnique model: replace collection_slug FK with collection_id FK - Update TelegramChatGiftCollection model: replace collection_slug FK with collection_id FK - Create Alembic migration with schema changes, data migration from JSON mapping, and cleanup - Refactor DTOs, services, actions, and API routes to use integer IDs - Update chat eligibility logic (find_relevant_gift_items) to use collection_id - Temporarily disable gift indexer tasks and comment out slug-dependent indexer methods --- backend/api/pos/chat.py | 6 +- backend/api/pos/gift.py | 3 +- backend/api/routes/admin/chat/rule/gift.py | 4 +- backend/api/routes/gift.py | 6 +- .../core/src/core/actions/chat/rule/gift.py | 40 +-- backend/core/src/core/actions/gift.py | 14 +- backend/core/src/core/dtos/chat/rule/gift.py | 11 +- backend/core/src/core/dtos/gift/collection.py | 28 +- backend/core/src/core/dtos/gift/item.py | 8 +- .../data/gift-collection-ids-to-names.json | 111 ++++++++ ...20b1dfcfb-migrate_gift_collection_to_id.py | 251 ++++++++++++++++++ backend/core/src/core/models/gift.py | 18 +- backend/core/src/core/models/rule.py | 10 +- .../core/src/core/services/gift/collection.py | 26 +- backend/core/src/core/services/gift/item.py | 42 +-- backend/core/src/core/utils/gift.py | 7 +- backend/indexer_gifts/actions/item.py | 243 +++++++++-------- backend/indexer_gifts/indexers/item.py | 63 +++-- backend/indexer_gifts/tasks.py | 4 + backend/scheduler/celery_app.py | 11 +- 20 files changed, 632 insertions(+), 274 deletions(-) create mode 100644 backend/core/src/core/migrations/data/gift-collection-ids-to-names.json create mode 100644 backend/core/src/core/migrations/versions/1772632462-abd20b1dfcfb-migrate_gift_collection_to_id.py diff --git a/backend/api/pos/chat.py b/backend/api/pos/chat.py index 863d6e4d..56af5edd 100644 --- a/backend/api/pos/chat.py +++ b/backend/api/pos/chat.py @@ -254,15 +254,15 @@ def validate_category_or_collection(self) -> Self: class TelegramChatGiftRuleCPO(BaseTelegramChatQuantityRuleCPO): - collection_slug: str | None + collection_id: int | None model: str | None = None backdrop: str | None = None pattern: str | None = None @model_validator(mode="after") def validate_category_or_collection(self) -> Self: - if not self.category and not self.collection_slug: - raise ValueError("At least category of collection must be specified") + if not self.category and not self.collection_id: + raise ValueError("At least category or collection must be specified") return self diff --git a/backend/api/pos/gift.py b/backend/api/pos/gift.py index d7582401..0a268ad6 100644 --- a/backend/api/pos/gift.py +++ b/backend/api/pos/gift.py @@ -40,7 +40,8 @@ def from_dto(cls, dto: GiftCollectionsMetadataDTO) -> Self: class GiftFilterPO(GiftFilterDTO): @classmethod def from_query_string(cls, value: str) -> Self: - return cls.model_validate_json(unquote(value)) + data = cls.model_validate_json(unquote(value)) + return data class GiftUniqueInfoFDO(BaseFDO): diff --git a/backend/api/routes/admin/chat/rule/gift.py b/backend/api/routes/admin/chat/rule/gift.py index f2125b15..f7391e5f 100644 --- a/backend/api/routes/admin/chat/rule/gift.py +++ b/backend/api/routes/admin/chat/rule/gift.py @@ -39,7 +39,7 @@ async def add_chat_gift_rule( ) new_rule = await action.create( group_id=rule.group_id, - collection_slug=rule.collection_slug, + collection_id=rule.collection_id, model=rule.model, backdrop=rule.backdrop, pattern=rule.pattern, @@ -64,7 +64,7 @@ async def update_chat_gift_rule( ) updated_rule = await action.update( rule_id=rule_id, - collection_slug=rule.collection_slug, + collection_id=rule.collection_id, model=rule.model, backdrop=rule.backdrop, pattern=rule.pattern, diff --git a/backend/api/routes/gift.py b/backend/api/routes/gift.py index 79ecf44c..98c2cd82 100644 --- a/backend/api/routes/gift.py +++ b/backend/api/routes/gift.py @@ -47,7 +47,7 @@ async def get_gifts_owners( @gift_router.get( - "/{collection_slug}", + "/{collection_id}", description="Returns an object of gift with their holders (telegram ID and blockchain address)", responses={ HTTP_200_OK: {"model": GiftUniqueItemsFDO}, @@ -58,11 +58,11 @@ async def get_gifts_owners( }, ) async def get_collection_holders( - collection_slug: str, + collection_id: int, db_session: Session = Depends(get_db_session), ) -> GiftUniqueItemsFDO: gift_unique_action = GiftUniqueAction(db_session=db_session) - items = gift_unique_action.get_all(collection_slug=collection_slug) + items = gift_unique_action.get_all(collection_id=collection_id) return GiftUniqueItemsFDO( items=[GiftUniqueInfoFDO.from_dto(item) for item in items] ) diff --git a/backend/core/src/core/actions/chat/rule/gift.py b/backend/core/src/core/actions/chat/rule/gift.py index b193a887..c4994e37 100644 --- a/backend/core/src/core/actions/chat/rule/gift.py +++ b/backend/core/src/core/actions/chat/rule/gift.py @@ -36,7 +36,7 @@ def check_duplicates( self, chat_id: int, group_id: int, - collection_slug: str | None, + collection_id: int | None, category: str | None, entity_id: int | None = None, ) -> None: @@ -47,7 +47,7 @@ def check_duplicates( :param chat_id: The unique identifier for the chat where the rule applies. :param group_id: The unique identifier for the group where the rule applies. - :param collection_slug: The slug identifying the collection; can be None if not applicable. + :param collection_id: The id identifying the collection; can be None if not applicable. :param category: The category to which the rule applies; can be None if not applicable. :param entity_id: Optional identifier for the specific entity to exclude from duplicate checks. @@ -56,7 +56,7 @@ def check_duplicates( existing_rules = self.service.find( chat_id=chat_id, group_id=group_id, - collection_slug=collection_slug, + collection_id=collection_id, category=category, ) if next(filter(lambda rule: rule.id != entity_id, existing_rules), None): @@ -67,41 +67,41 @@ def check_duplicates( def validate_params( self, - collection_slug: str | None, + collection_id: int | None, model: str | None, backdrop: str | None, pattern: str | None, ) -> None: - # If the collection slug is not set or attributes are not selected – no need to validate them - if not collection_slug or not any((model, backdrop, pattern)): + # If the collection id is not set or attributes are not selected – no need to validate them + if not collection_id or not any((model, backdrop, pattern)): return - options = self.gift_unique_service.get_unique_options( - collection_slug=collection_slug - ) + # FIXME: Rewrite disabled for now since it needs refactoring + # options = self.gift_unique_service.get_unique_options("...") + options = {} if model and model not in options.get("models", []): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f"Model {model!r} is not available for the collection {collection_slug!r}.", + detail=f"Model {model!r} is not available for the collection {collection_id!r}.", ) if backdrop and backdrop not in options.get("backdrops", []): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f"Backdrop {backdrop!r} is not available for the collection {collection_slug!r}.", + detail=f"Backdrop {backdrop!r} is not available for the collection {collection_id!r}.", ) if pattern and pattern not in options.get("patterns", []): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f"Pattern {pattern!r} is not available for the collection {collection_slug!r}.", + detail=f"Pattern {pattern!r} is not available for the collection {collection_id!r}.", ) async def create( self, group_id: int | None, - collection_slug: str | None, + collection_id: int | None, model: str | None, backdrop: str | None, pattern: str | None, @@ -113,16 +113,16 @@ async def create( self.check_duplicates( chat_id=self.chat.id, group_id=group_id, - collection_slug=collection_slug, + collection_id=collection_id, category=category, ) - self.validate_params(collection_slug, model, backdrop, pattern) + self.validate_params(collection_id, model, backdrop, pattern) new_rule = self.service.create( CreateTelegramChatGiftCollectionRuleDTO( chat_id=self.chat.id, group_id=group_id, - collection_slug=collection_slug, + collection_id=collection_id, model=model, backdrop=backdrop, pattern=pattern, @@ -139,7 +139,7 @@ async def create( async def update( self, rule_id: int, - collection_slug: str | None, + collection_id: int | None, category: str | None, model: str | None, backdrop: str | None, @@ -155,16 +155,16 @@ async def update( self.check_duplicates( chat_id=self.chat.id, group_id=rule.group_id, - collection_slug=collection_slug, + collection_id=collection_id, category=category, entity_id=rule_id, ) - self.validate_params(collection_slug, model, backdrop, pattern) + self.validate_params(collection_id, model, backdrop, pattern) updated_rule = self.service.update( rule=rule, dto=UpdateTelegramChatGiftCollectionRuleDTO( - collection_slug=collection_slug, + collection_id=collection_id, category=category, threshold=threshold, is_enabled=is_enabled, diff --git a/backend/core/src/core/actions/gift.py b/backend/core/src/core/actions/gift.py index 5c1945c0..0f134d7b 100644 --- a/backend/core/src/core/actions/gift.py +++ b/backend/core/src/core/actions/gift.py @@ -43,10 +43,10 @@ def get_metadata(self) -> GiftCollectionsMetadataDTO: collections_with_options = [] for collection in all_collections: - options = self.service.get_unique_options(collection_slug=collection.slug) + options = collection.options collections_with_options.append( GiftCollectionMetadataDTO( - slug=collection.slug, + id=collection.id, title=collection.title, preview_url=collection.preview_url, supply=collection.supply, @@ -89,7 +89,7 @@ def __construct_filter_options_query( for option in options: # Basic filtering logic (collection, model, backdrop, pattern) base_filter = and_( - GiftUnique.collection_slug == option.collection, + GiftUnique.collection_id == option.collection_id, GiftUnique.telegram_owner_id.isnot(None), *filter( None.__ne__, @@ -144,18 +144,18 @@ def get_collections_holders(self, options: list[GiftFilterDTO]) -> Sequence[int] result = self.db_session.execute(query).scalars().all() return result - def get_all(self, collection_slug: str) -> Sequence[GiftUniqueDTO]: + def get_all(self, collection_id: int) -> Sequence[GiftUniqueDTO]: """ Fetches all unique items in a given collection. """ try: - self.collection_service.get(slug=collection_slug) + self.collection_service.get(id=collection_id) except NoResultFound: raise HTTPException( status_code=HTTP_404_NOT_FOUND, - detail=f"Collection {collection_slug!r} not found", + detail=f"Collection {collection_id!r} not found", ) return [ GiftUniqueDTO.from_orm(gift) - for gift in self.service.get_all(collection_slug=collection_slug) + for gift in self.service.get_all(collection_id=collection_id) ] diff --git a/backend/core/src/core/dtos/chat/rule/gift.py b/backend/core/src/core/dtos/chat/rule/gift.py index ed843793..0d32a563 100644 --- a/backend/core/src/core/dtos/chat/rule/gift.py +++ b/backend/core/src/core/dtos/chat/rule/gift.py @@ -12,17 +12,17 @@ class BaseTelegramChatGiftCollectionRuleDTO(BaseModel): threshold: int is_enabled: bool - collection_slug: str | None + collection_id: int | None model: str | None backdrop: str | None pattern: str | None category: str | None @model_validator(mode="after") - def validate_slug_or_category(self) -> Self: - if (self.category is None) == (self.collection_slug is None): + def validate_id_or_category(self) -> Self: + if (self.category is None) == (self.collection_id is None): raise ValueError( - "Either category or collection_slug must be provided and not both." + "Either category or collection_id must be provided and not both." ) return self @@ -73,9 +73,8 @@ def promote_url(self) -> str | None: # FIXME: Turn on when market is released # if self.collection: # return PROMOTE_GIFT_COLLECTION_TEMPLATE.format( - # collection_slug=self.collection.slug + # collection_id=self.collection.id # ) - return None @classmethod diff --git a/backend/core/src/core/dtos/gift/collection.py b/backend/core/src/core/dtos/gift/collection.py index 5d1b5cf9..eefe7377 100644 --- a/backend/core/src/core/dtos/gift/collection.py +++ b/backend/core/src/core/dtos/gift/collection.py @@ -8,7 +8,7 @@ class GiftCollectionDTO(BaseModel): - slug: str + id: int title: str preview_url: str | None supply: int @@ -18,7 +18,7 @@ class GiftCollectionDTO(BaseModel): @classmethod def from_orm(cls, obj: GiftCollection) -> Self: return cls( - slug=obj.slug, + id=obj.id, title=obj.title, preview_url=obj.preview_url, supply=obj.supply, @@ -27,9 +27,9 @@ def from_orm(cls, obj: GiftCollection) -> Self: ) @classmethod - def from_telethon(cls, slug: str, obj: StarGiftUnique, preview_url: str) -> Self: + def from_telethon(cls, id: int, obj: StarGiftUnique, preview_url: str) -> Self: return cls( - slug=slug, + id=id, title=obj.title, preview_url=preview_url, supply=obj.availability_total, @@ -39,7 +39,7 @@ def from_telethon(cls, slug: str, obj: StarGiftUnique, preview_url: str) -> Self class GiftCollectionMetadataDTO(BaseModel): - slug: str + id: int title: str preview_url: str | None supply: int @@ -54,7 +54,7 @@ class GiftCollectionsMetadataDTO(BaseModel): class GiftFilterDTO(BaseModel): - collection: str + collection_id: int model: str | None = None backdrop: str | None = None pattern: str | None = None @@ -68,26 +68,28 @@ class GiftFiltersDTO(BaseModel): def validate_with_context( cls, objs: list[GiftFilterDTO], context: GiftCollectionsMetadataDTO ) -> Self: - context_by_slug = { - collection.slug: collection for collection in context.collections + context_by_id = { + collection.id: collection for collection in context.collections } for obj in objs: - if not (collection_metadata := context_by_slug.get(obj.collection)): - raise ValueError(f"Collection {obj.collection} not found in metadata") + if not (collection_metadata := context_by_id.get(obj.collection_id)): + raise ValueError( + f"Collection {obj.collection_id} not found in metadata" + ) if obj.model and obj.model not in collection_metadata.models: raise ValueError( - f"Model {obj.model} not found in collection {obj.collection}" + f"Model {obj.model} not found in collection {obj.collection_id}" ) if obj.backdrop and obj.backdrop not in collection_metadata.backdrops: raise ValueError( - f"Backdrop {obj.backdrop} not found in collection {obj.collection}" + f"Backdrop {obj.backdrop} not found in collection {obj.collection_id}" ) if obj.pattern and obj.pattern not in collection_metadata.patterns: raise ValueError( - f"Pattern {obj.pattern} not found in collection {obj.collection}" + f"Pattern {obj.pattern} not found in collection {obj.collection_id}" ) return cls(filters=objs) diff --git a/backend/core/src/core/dtos/gift/item.py b/backend/core/src/core/dtos/gift/item.py index 95133d87..aeff7656 100644 --- a/backend/core/src/core/dtos/gift/item.py +++ b/backend/core/src/core/dtos/gift/item.py @@ -14,7 +14,7 @@ class GiftUniqueDTO(BaseModel): slug: str - collection_slug: str + collection_id: int telegram_owner_id: int | None number: int blockchain_address: str | None @@ -28,7 +28,7 @@ class GiftUniqueDTO(BaseModel): def from_orm(cls, obj: GiftUnique) -> Self: return cls( slug=obj.slug, - collection_slug=obj.collection_slug, + collection_id=obj.collection_id, telegram_owner_id=obj.telegram_owner_id, number=obj.number, blockchain_address=obj.blockchain_address, @@ -40,7 +40,7 @@ def from_orm(cls, obj: GiftUnique) -> Self: ) @classmethod - def from_telethon(cls, collection_slug: str, obj: StarGiftUnique) -> Self: + def from_telethon(cls, collection_id: int, obj: StarGiftUnique) -> Self: model_attribute = next( ( attribute @@ -73,7 +73,7 @@ def from_telethon(cls, collection_slug: str, obj: StarGiftUnique) -> Self: return cls( slug=obj.slug, - collection_slug=collection_slug, + collection_id=collection_id, telegram_owner_id=getattr(obj.owner_id, "user_id", None), number=obj.num, blockchain_address=obj.gift_address, diff --git a/backend/core/src/core/migrations/data/gift-collection-ids-to-names.json b/backend/core/src/core/migrations/data/gift-collection-ids-to-names.json new file mode 100644 index 00000000..a81ce01f --- /dev/null +++ b/backend/core/src/core/migrations/data/gift-collection-ids-to-names.json @@ -0,0 +1,111 @@ +{ + "5983471780763796287": "Santa Hat", + "5936085638515261992": "Signet Ring", + "5933671725160989227": "Precious Peach", + "5936013938331222567": "Plush Pepe", + "5913442287462908725": "Spiced Wine", + "5915502858152706668": "Jelly Bunny", + "5915521180483191380": "Durov's Cap", + "5913517067138499193": "Perfume Bottle", + "5882125812596999035": "Eternal Rose", + "5882252952218894938": "Berry Box", + "5857140566201991735": "Vintage Cigar", + "5846226946928673709": "Magic Potion", + "5845776576658015084": "Kissed Frog", + "5825801628657124140": "Hex Pot", + "5825480571261813595": "Evil Eye", + "5841689550203650524": "Sharp Tongue", + "5841391256135008713": "Trapped Heart", + "5839038009193792264": "Skull Flower", + "5837059369300132790": "Scared Cat", + "5821261908354794038": "Spy Agaric", + "5783075783622787539": "Homemade Cake", + "5933531623327795414": "Genie Lamp", + "6028426950047957932": "Lunar Snake", + "6003643167683903930": "Party Sparkler", + "5933590374185435592": "Jester Hat", + "5821384757304362229": "Witch Hat", + "5915733223018594841": "Hanging Star", + "5915550639663874519": "Love Candle", + "6001538689543439169": "Cookie Heart", + "5782988952268964995": "Desk Calendar", + "6001473264306619020": "Jingle Bells", + "5980789805615678057": "Snow Mittens", + "5836780359634649414": "Voodoo Doll", + "5841632504448025405": "Mad Pumpkin", + "5825895989088617224": "Hypno Lollipop", + "5782984811920491178": "B-Day Candle", + "5935936766358847989": "Bunny Muffin", + "5933629604416717361": "Astral Shard", + "5837063436634161765": "Flying Broom", + "5841336413697606412": "Crystal Ball", + "5821205665758053411": "Eternal Candle", + "5936043693864651359": "Swiss Watch", + "5983484377902875708": "Ginger Cookie", + "5879737836550226478": "Mini Oscar", + "5170594532177215681": "Lol Pop", + "5843762284240831056": "Ion Gem", + "5936017773737018241": "Star Notepad", + "5868659926187901653": "Loot Bag", + "5868348541058942091": "Love Potion", + "5868220813026526561": "Toy Bear", + "5868503709637411929": "Diamond Ring", + "5167939598143193218": "Sakura Flower", + "5981026247860290310": "Sleigh Bell", + "5897593557492957738": "Top Hat", + "5856973938650776169": "Record Player", + "5983259145522906006": "Winter Wreath", + "5981132629905245483": "Snow Globe", + "5846192273657692751": "Electric Skull", + "6023752243218481939": "Tama Gadget", + "6003373314888696650": "Candy Cane", + "5933793770951673155": "Neko Helmet", + "6005659564635063386": "Jack-in-the-Box", + "5773668482394620318": "Easter Egg", + "5870661333703197240": "Bonded Ring", + "6023917088358269866": "Pet Snake", + "6023679164349940429": "Snake Box", + "6003767644426076664": "Xmas Stocking", + "6028283532500009446": "Big Year", + "6003735372041814769": "Holiday Drink", + "5859442703032386168": "Gem Signet", + "5897581235231785485": "Light Sword", + "5870784783948186838": "Restless Jar", + "5870720080265871962": "Nail Bracelet", + "5895328365971244193": "Heroic Helmet", + "5895544372761461960": "Bow Tie", + "5868455043362980631": "Heart Locket", + "5871002671934079382": "Lush Bouquet", + "5933543975653737112": "Whip Cupcake", + "5870862540036113469": "Joyful Bundle", + "5868561433997870501": "Cupid Charm", + "5868595669182186720": "Valentine Box", + "6014591077976114307": "Snoop Dogg", + "6012607142387778152": "Swag Bag", + "6012435906336654262": "Snoop Cigar", + "6014675319464657779": "Low Rider", + "6014697240977737490": "Westside Sign", + "6042113507581755979": "Stellar Rocket", + "6005880141270483700": "Jolly Chimp", + "5998981470310368313": "Moon Pendant", + "5933937398953018107": "Ionic Dryer", + "5870972044522291836": "Input Key", + "5895518353849582541": "Mighty Arm", + "6005797617768858105": "Artisan Brick", + "5960747083030856414": "Clover Pin", + "5870947077877400011": "Sky Stilettos", + "5895603153683874485": "Fresh Socks", + "6006064678835323371": "Happy Brownie", + "5900177027566142759": "Ice Cream", + "5773725897517433693": "Spring Basket", + "6005564615793050414": "Instant Ramen", + "6003456431095808759": "Faith Amulet", + "5935877878062253519": "Mousse Cake", + "5902339509239940491": "Bling Binky", + "5963238670868677492": "Money Pot", + "5933737850477478635": "Pretty Posy", + "5839094187366024301": "Khabib's Papakha", + "5882260270843168924": "UFC Strike", + "5830340739074097859": "Victory Medal", + "5999116401002939514": "Rare Bird" +} \ No newline at end of file diff --git a/backend/core/src/core/migrations/versions/1772632462-abd20b1dfcfb-migrate_gift_collection_to_id.py b/backend/core/src/core/migrations/versions/1772632462-abd20b1dfcfb-migrate_gift_collection_to_id.py new file mode 100644 index 00000000..9ad3893c --- /dev/null +++ b/backend/core/src/core/migrations/versions/1772632462-abd20b1dfcfb-migrate_gift_collection_to_id.py @@ -0,0 +1,251 @@ +"""migrate_gift_collection_to_id + +Revision ID: abd20b1dfcfb +Revises: 6c6b45ffe090 +Create Date: 2026-03-04 13:54:22.662223 + +""" +from typing import Sequence, Union +import json +from pathlib import Path + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "abd20b1dfcfb" +down_revision: Union[str, None] = "6c6b45ffe090" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +GIFT_COLLECTION_IDS_MAPPING_PATH = ( + Path(__file__).parent / "../data/gift-collection-ids-to-names.json" +) + + +def upgrade_schema() -> None: + # 1. Add new columns + op.add_column( + "gift_collection", + sa.Column("id", sa.BigInteger(), autoincrement=False, nullable=True), + ) + op.add_column( + "gift_collection", + sa.Column("blockchain_address", sa.String(length=255), nullable=True), + ) + op.add_column( + "gift_collection", + sa.Column("options", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + + op.add_column( + "gift_unique", sa.Column("collection_id", sa.BigInteger(), nullable=True) + ) + op.add_column( + "telegram_chat_gift_collection", + sa.Column("collection_id", sa.BigInteger(), nullable=True), + ) + + # 2. Drop foreign keys referencing `gift_collection.slug` + op.drop_constraint( + "gift_unique_collection_slug_fkey", "gift_unique", type_="foreignkey" + ) + op.drop_constraint( + "telegram_chat_gift_collection_collection_slug_fkey", + "telegram_chat_gift_collection", + type_="foreignkey", + ) + + +def migrate_data_upgrade() -> None: + # 3. Data Migration + with open(GIFT_COLLECTION_IDS_MAPPING_PATH, "r") as f: + mapping = json.load(f) + + connection = op.get_bind() + for col_id, name in mapping.items(): + connection.execute( + sa.text( + 'UPDATE gift_collection SET id = :id, options = \'{"models": [], "backdrops": [], "patterns": []}\'::jsonb WHERE title = :name' + ), + {"id": int(col_id), "name": name}, + ) + + # Update new fields in referencing tables + connection.execute( + sa.text( + "UPDATE gift_unique SET collection_id = gc.id FROM gift_collection gc WHERE gift_unique.collection_slug = gc.slug" + ) + ) + connection.execute( + sa.text( + "UPDATE telegram_chat_gift_collection SET collection_id = gc.id FROM gift_collection gc WHERE telegram_chat_gift_collection.collection_slug = gc.slug" + ) + ) + + +def cleanup_schema_upgrade() -> None: + connection = op.get_bind() + # Delete unmapped collections and their relations + connection.execute( + sa.text( + "DELETE FROM gift_unique WHERE collection_slug IN (SELECT slug FROM gift_collection WHERE id IS NULL)" + ) + ) + connection.execute( + sa.text( + "DELETE FROM telegram_chat_gift_collection WHERE collection_slug IN (SELECT slug FROM gift_collection WHERE id IS NULL)" + ) + ) + connection.execute(sa.text("DELETE FROM gift_collection WHERE id IS NULL")) + + # 4. Remove old PK & slug columns + op.drop_constraint("gift_collection_pkey", "gift_collection", type_="primary") + op.drop_column("gift_collection", "slug") + op.drop_column("gift_unique", "collection_slug") + op.drop_column("telegram_chat_gift_collection", "collection_slug") + + # 5. Alter columns non-nullable where needed + op.alter_column("gift_collection", "id", nullable=False) + op.alter_column("gift_collection", "options", nullable=False) + op.alter_column("gift_unique", "collection_id", nullable=False) + + # 6. Add new PK and foreign keys + op.create_primary_key("gift_collection_pkey", "gift_collection", ["id"]) + + op.create_foreign_key( + "gift_unique_collection_id_fkey", + "gift_unique", + "gift_collection", + ["collection_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_foreign_key( + "telegram_chat_gift_collection_collection_id_fkey", + "telegram_chat_gift_collection", + "gift_collection", + ["collection_id"], + ["id"], + ondelete="CASCADE", + ) + + +BACKUP_TABLES = [ + ("gift_collection", "_backup_gift_collection"), + ("gift_unique", "_backup_gift_unique"), + ("telegram_chat_gift_collection", "_backup_telegram_chat_gift_collection"), +] + + +def backup_tables() -> None: + connection = op.get_bind() + for source, backup in BACKUP_TABLES: + connection.execute(sa.text(f"DROP TABLE IF EXISTS {backup}")) + connection.execute(sa.text(f"CREATE TABLE {backup} AS SELECT * FROM {source}")) + + +def drop_backup_tables() -> None: + connection = op.get_bind() + for _, backup in BACKUP_TABLES: + connection.execute(sa.text(f"DROP TABLE IF EXISTS {backup}")) + + +def upgrade() -> None: + backup_tables() + upgrade_schema() + migrate_data_upgrade() + cleanup_schema_upgrade() + + +def downgrade_schema() -> None: + # 1. Add back columns + op.add_column( + "telegram_chat_gift_collection", + sa.Column( + "collection_slug", + sa.VARCHAR(length=255), + autoincrement=False, + nullable=True, + ), + ) + op.add_column( + "gift_unique", + sa.Column( + "collection_slug", + sa.VARCHAR(length=255), + autoincrement=False, + nullable=True, + ), + ) + op.add_column( + "gift_collection", + sa.Column("slug", sa.VARCHAR(length=255), autoincrement=False, nullable=True), + ) + + # 2. Drop FKs and new PK + op.drop_constraint( + "telegram_chat_gift_collection_collection_id_fkey", + "telegram_chat_gift_collection", + type_="foreignkey", + ) + op.drop_constraint( + "gift_unique_collection_id_fkey", "gift_unique", type_="foreignkey" + ) + op.drop_constraint("gift_collection_pkey", "gift_collection", type_="primary") + + +def migrate_data_downgrade() -> None: + # 3. Data migration + connection = op.get_bind() + connection.execute( + sa.text("UPDATE gift_collection SET slug = REPLACE(LOWER(title), ' ', '-')") + ) + + connection.execute( + sa.text( + "UPDATE gift_unique SET collection_slug = gc.slug FROM gift_collection gc WHERE gift_unique.collection_id = gc.id" + ) + ) + connection.execute( + sa.text( + "UPDATE telegram_chat_gift_collection SET collection_slug = gc.slug FROM gift_collection gc WHERE telegram_chat_gift_collection.collection_id = gc.id" + ) + ) + + +def cleanup_schema_downgrade() -> None: + op.alter_column("gift_collection", "slug", nullable=False) + op.alter_column("gift_unique", "collection_slug", nullable=False) + + op.create_primary_key("gift_collection_pkey", "gift_collection", ["slug"]) + op.create_foreign_key( + "telegram_chat_gift_collection_collection_slug_fkey", + "telegram_chat_gift_collection", + "gift_collection", + ["collection_slug"], + ["slug"], + ondelete="CASCADE", + ) + op.create_foreign_key( + "gift_unique_collection_slug_fkey", + "gift_unique", + "gift_collection", + ["collection_slug"], + ["slug"], + ondelete="CASCADE", + ) + + op.drop_column("telegram_chat_gift_collection", "collection_id") + op.drop_column("gift_unique", "collection_id") + op.drop_column("gift_collection", "id") + op.drop_column("gift_collection", "blockchain_address") + op.drop_column("gift_collection", "options") + + +def downgrade() -> None: + downgrade_schema() + migrate_data_downgrade() + cleanup_schema_downgrade() + drop_backup_tables() diff --git a/backend/core/src/core/models/gift.py b/backend/core/src/core/models/gift.py index e9fb6b8a..feab0225 100644 --- a/backend/core/src/core/models/gift.py +++ b/backend/core/src/core/models/gift.py @@ -1,6 +1,7 @@ import datetime from sqlalchemy import Integer, String, DateTime, ForeignKey, BigInteger +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import mapped_column from core.db import Base @@ -10,11 +11,17 @@ class GiftCollection(PricedEntityMixin): __tablename__ = "gift_collection" - slug = mapped_column(String(255), primary_key=True) + id = mapped_column(BigInteger, primary_key=True, autoincrement=False) title = mapped_column(String(255), nullable=False, unique=True) preview_url = mapped_column(String(255), nullable=True) supply = mapped_column(Integer, nullable=False, default=0) upgraded_count = mapped_column(Integer, nullable=False, default=0) + blockchain_address = mapped_column(String(255), nullable=True) + options = mapped_column( + JSONB, + nullable=False, + default=lambda: {"models": [], "backdrops": [], "patterns": []}, + ) last_updated = mapped_column( DateTime, nullable=False, @@ -27,8 +34,13 @@ class GiftUnique(Base): __tablename__ = "gift_unique" slug = mapped_column(String(255), primary_key=True) - collection_slug = mapped_column( - ForeignKey("gift_collection.slug", ondelete="CASCADE"), nullable=False + collection_id = mapped_column( + ForeignKey( + "gift_collection.id", + ondelete="CASCADE", + name="gift_unique_collection_id_fkey", + ), + nullable=False, ) telegram_owner_id = mapped_column( BigInteger, diff --git a/backend/core/src/core/models/rule.py b/backend/core/src/core/models/rule.py index f29f2786..832856a1 100644 --- a/backend/core/src/core/models/rule.py +++ b/backend/core/src/core/models/rule.py @@ -284,10 +284,14 @@ class TelegramChatStickerCollection(TelegramChatThresholdRuleBase): class TelegramChatGiftCollection(TelegramChatThresholdRuleBase): __tablename__ = "telegram_chat_gift_collection" - collection_slug = mapped_column( - ForeignKey("gift_collection.slug", ondelete="CASCADE"), + collection_id = mapped_column( + ForeignKey( + "gift_collection.id", + ondelete="CASCADE", + name="telegram_chat_gift_collection_collection_id_fkey", + ), nullable=True, - doc="Collection slug that will be used to check eligibility for the collection.", + doc="Collection ID that will be used to check eligibility for the collection.", ) collection: Mapped["GiftCollection"] = relationship( "GiftCollection", diff --git a/backend/core/src/core/services/gift/collection.py b/backend/core/src/core/services/gift/collection.py index 887e5ceb..5ea788be 100644 --- a/backend/core/src/core/services/gift/collection.py +++ b/backend/core/src/core/services/gift/collection.py @@ -3,37 +3,35 @@ class GiftCollectionService(BaseService): - def get(self, slug: str) -> GiftCollection: + def get(self, id: int) -> GiftCollection: return ( - self.db_session.query(GiftCollection) - .filter(GiftCollection.slug == slug) - .one() + self.db_session.query(GiftCollection).filter(GiftCollection.id == id).one() ) - def get_all(self, slugs: list[str] | None = None) -> list[GiftCollection]: + def get_all(self, ids: list[int] | None = None) -> list[GiftCollection]: query = self.db_session.query(GiftCollection) - if slugs: - query = query.filter(GiftCollection.slug.in_(slugs)) - result = query.order_by(GiftCollection.slug).all() + if ids: + query = query.filter(GiftCollection.id.in_(ids)) + result = query.order_by(GiftCollection.id).all() return result - def find(self, slug: str) -> GiftCollection | None: + def find(self, id: int) -> GiftCollection | None: return ( self.db_session.query(GiftCollection) - .filter(GiftCollection.slug == slug) + .filter(GiftCollection.id == id) .first() ) def create( self, - slug: str, + id: int, title: str, preview_url: str | None, supply: int | None, upgraded_count: int | None, ) -> GiftCollection: new_collection = GiftCollection( - slug=slug, + id=id, title=title, preview_url=preview_url, supply=supply, @@ -46,13 +44,13 @@ def create( def update( self, - slug: str, + id: int, title: str, preview_url: str | None, supply: int | None, upgraded_count: int | None, ) -> GiftCollection: - collection = self.get(slug) + collection = self.get(id) collection.title = title collection.preview_url = preview_url collection.supply = supply diff --git a/backend/core/src/core/services/gift/item.py b/backend/core/src/core/services/gift/item.py index bb8d221a..46b5480b 100644 --- a/backend/core/src/core/services/gift/item.py +++ b/backend/core/src/core/services/gift/item.py @@ -1,6 +1,5 @@ from collections import namedtuple -from sqlalchemy import select, func from core.models.gift import GiftUnique from core.services.base import BaseService @@ -10,19 +9,19 @@ class GiftUniqueService(BaseService): - def get(self, slug: str) -> GiftUnique: - return self.db_session.query(GiftUnique).filter(GiftUnique.slug == slug).one() + def get(self, id: int) -> GiftUnique: + return self.db_session.query(GiftUnique).filter(GiftUnique.id == id).one() def get_all( self, - collection_slug: str | None = None, + collection_id: int | None = None, telegram_user_id: int | None = None, number_ge: int | None = None, number_le: int | None = None, ) -> list[GiftUnique]: query = self.db_session.query(GiftUnique) - if collection_slug: - query = query.filter(GiftUnique.collection_slug == collection_slug) + if collection_id: + query = query.filter(GiftUnique.collection_id == collection_id) if telegram_user_id: query = query.filter(GiftUnique.telegram_owner_id == telegram_user_id) if number_ge: @@ -32,27 +31,12 @@ def get_all( return query.order_by(GiftUnique.number).all() - def get_unique_options(self, collection_slug: str): - query = select( - func.array_agg(func.distinct(GiftUnique.model)).label("models"), - func.array_agg(func.distinct(GiftUnique.backdrop)).label("backdrops"), - func.array_agg(func.distinct(GiftUnique.pattern)).label("patterns"), - ).where(GiftUnique.collection_slug == collection_slug) - - result = self.db_session.execute(query).first() - - return { - "models": [m for m in (result.models or []) if m is not None], - "backdrops": [b for b in (result.backdrops or []) if b is not None], - "patterns": [p for p in (result.patterns or []) if p is not None], - } - - def find(self, slug: str) -> GiftUnique | None: - return self.db_session.query(GiftUnique).filter(GiftUnique.slug == slug).first() + def find(self, id: int) -> GiftUnique | None: + return self.db_session.query(GiftUnique).filter(GiftUnique.id == id).first() def create( self, - slug: str, + id: int, number: int, model: str, backdrop: str, @@ -62,7 +46,7 @@ def create( owner_address: str | None, ) -> GiftUnique: new_unique = GiftUnique( - slug=slug, + id=id, number=number, model=model, backdrop=backdrop, @@ -77,7 +61,7 @@ def create( def update( self, - slug: str, + id: int, number: int, model: str, backdrop: str, @@ -86,7 +70,7 @@ def update( blockchain_address: str | None, owner_address: str | None, ) -> GiftUnique: - unique = self.get(slug) + unique = self.get(id) unique.number = number unique.model = model unique.backdrop = backdrop @@ -100,12 +84,12 @@ def update( def update_ownership( self, - slug: str, + id: int, telegram_owner_id: int, blockchain_address: str | None, owner_address: str | None, ) -> GiftUnique: - unique = self.get(slug) + unique = self.get(id) unique.telegram_owner_id = telegram_owner_id unique.blockchain_address = blockchain_address unique.owner_address = owner_address diff --git a/backend/core/src/core/utils/gift.py b/backend/core/src/core/utils/gift.py index f5b99d3b..49660940 100644 --- a/backend/core/src/core/utils/gift.py +++ b/backend/core/src/core/utils/gift.py @@ -15,7 +15,7 @@ def find_relevant_gift_items( This function iterates over a list of gift items and filters them according to the attributes defined in the provided rule object. Attributes such as - collection_slug, model, backdrop, and pattern are used to determine the + collection_id, model, backdrop, and pattern are used to determine the relevance of each item. Items that match the defined criteria are appended to a resulting list, which is then returned. @@ -32,10 +32,7 @@ def find_relevant_gift_items( logger.warning(f"Trying to get gifts by category {rule.category!r}.") continue - if ( - rule.collection_slug is not None - and rule.collection_slug != item.collection_slug - ): + if rule.collection_id is not None and rule.collection_id != item.collection_id: continue if rule.model is not None and rule.model != item.model: diff --git a/backend/indexer_gifts/actions/item.py b/backend/indexer_gifts/actions/item.py index 60fa7950..0dd95c0a 100644 --- a/backend/indexer_gifts/actions/item.py +++ b/backend/indexer_gifts/actions/item.py @@ -1,4 +1,3 @@ -import datetime import logging from pathlib import Path from typing import AsyncGenerator @@ -6,8 +5,6 @@ from sqlalchemy.orm import Session from core.actions.base import BaseAction -from core.constants import GIFT_COLLECTIONS_METADATA_KEY -from core.models.gift import GiftUnique, GiftCollection from core.services.gift.collection import GiftCollectionService from core.services.gift.item import GiftUniqueService from core.services.superredis import RedisService @@ -36,123 +33,123 @@ async def index_all(self) -> AsyncGenerator[set[int], None]: yield await self._index(collection, start=1, stop=collection.upgraded_count) logger.info("Finished indexing all unique items.") - async def _index( - self, collection: GiftCollection, start: int | None, stop: int | None - ) -> set[int]: - """ - Indexes and processes a collection of unique items while managing creation, - updates, and targeted owners. - This function interacts with a database to synchronize the collection data - and updates Redis caching as necessary. - - - :param collection: The gift collection object to be indexed. - :param start: The starting index for the indexing process. Defaults to 1. - :param stop: The ending index for the indexing process. Defaults to the number of upgraded items. - :return: A set of Telegram owner IDs that require targeted actions due to - ownership changes or updates. - """ - if start is None: - start = 1 - - if stop is None: - stop = collection.upgraded_count - - existing_items = { - item.slug: item - for item in self.service.get_all( - collection_slug=collection.slug, - number_ge=start, - number_le=stop, - ) - } - logger.info( - f"Found existing {len(existing_items)} unique items " - f"for collection {collection.slug!r} and starting from index {start} to {stop}..." - ) - targeted_telegram_owner_ids = set() - - # Iterate over batches and process items - async for batch in self.indexer.index_collection_items( - collection_slug=collection.slug, - start=start, - stop=stop, - ): - to_create = [] - to_update = [] - for item in batch: - if existing_item := existing_items.get(item.slug): - # Update record only if necessary - if any( - ( - item.telegram_owner_id != existing_item.telegram_owner_id, - item.owner_address != existing_item.owner_address, - ) - ): - # Ignore if the owner was previously hidden - if isinstance(existing_item.telegram_owner_id, int): - # Store Telegram ID of the previous owner to perform and actions on losing ownership if needed - targeted_telegram_owner_ids.add( - existing_item.telegram_owner_id - ) - to_update.append( - { - "slug": item.slug, - "telegram_owner_id": item.telegram_owner_id, - "owner_address": item.owner_address, - "blockchain_address": item.blockchain_address, - "last_updated": datetime.datetime.now(tz=datetime.UTC), - } - ) - else: - logger.debug( - f"No changes detected for item {item.slug!r} in collection {collection.slug!r}. Skipping." - ) - else: - to_create.append( - GiftUnique( - slug=item.slug, - collection_slug=collection.slug, - model=item.model, - backdrop=item.backdrop, - number=item.number, - pattern=item.pattern, - telegram_owner_id=item.telegram_owner_id, - owner_address=item.owner_address, - blockchain_address=item.blockchain_address, - last_updated=datetime.datetime.now(tz=datetime.UTC), - ) - ) - if to_create: - # Cache has to be cleared as new metadata could appear. - # It could be extended to clear only when new metadata options appear. - self.redis_service.delete(GIFT_COLLECTIONS_METADATA_KEY) - - self.db_session.bulk_save_objects(to_create) - logger.info( - f"Created {len(to_create)} new unique items for collection {collection.slug!r}." - ) - self.db_session.bulk_update_mappings(GiftUnique, to_update) - logger.info( - f"Updated {len(to_update)} existing unique items for collection {collection.slug!r}." - ) - self.db_session.commit() - - return targeted_telegram_owner_ids - - async def index( - self, slug: str, start: int | None = None, stop: int | None = None - ) -> set[int]: - """ - Processes a collection of items by indexing, updating, or creating unique records. It also - tracks and returns Telegram IDs of previous owners of updated items for further actions. - - :param slug: A unique identifier for the collection being processed. - :param start: The starting index for the indexing process. Defaults to 1. - :param stop: The ending index for the indexing process. Defaults to the number of upgraded items. - - :return: A set of Telegram IDs corresponding to previous owners of the updated - items who need further actions. - """ - collection = self.collection_service.get(slug) - return await self._index(collection, start=start, stop=stop) + # async def _index( + # self, collection: GiftCollection, start: int | None, stop: int | None + # ) -> set[int]: + # """ + # Indexes and processes a collection of unique items while managing creation, + # updates, and targeted owners. + # This function interacts with a database to synchronize the collection data + # and updates Redis caching as necessary. + # + # + # :param collection: The gift collection object to be indexed. + # :param start: The starting index for the indexing process. Defaults to 1. + # :param stop: The ending index for the indexing process. Defaults to the number of upgraded items. + # :return: A set of Telegram owner IDs that require targeted actions due to + # ownership changes or updates. + # """ + # if start is None: + # start = 1 + # + # if stop is None: + # stop = collection.upgraded_count + # + # existing_items = { + # item.slug: item + # for item in self.service.get_all( + # collection_slug=collection.slug, + # number_ge=start, + # number_le=stop, + # ) + # } + # logger.info( + # f"Found existing {len(existing_items)} unique items " + # f"for collection {collection.slug!r} and starting from index {start} to {stop}..." + # ) + # targeted_telegram_owner_ids = set() + # + # # Iterate over batches and process items + # async for batch in self.indexer.index_collection_items( + # collection_slug=collection.slug, + # start=start, + # stop=stop, + # ): + # to_create = [] + # to_update = [] + # for item in batch: + # if existing_item := existing_items.get(item.slug): + # # Update record only if necessary + # if any( + # ( + # item.telegram_owner_id != existing_item.telegram_owner_id, + # item.owner_address != existing_item.owner_address, + # ) + # ): + # # Ignore if the owner was previously hidden + # if isinstance(existing_item.telegram_owner_id, int): + # # Store Telegram ID of the previous owner to perform and actions on losing ownership if needed + # targeted_telegram_owner_ids.add( + # existing_item.telegram_owner_id + # ) + # to_update.append( + # { + # "slug": item.slug, + # "telegram_owner_id": item.telegram_owner_id, + # "owner_address": item.owner_address, + # "blockchain_address": item.blockchain_address, + # "last_updated": datetime.datetime.now(tz=datetime.UTC), + # } + # ) + # else: + # logger.debug( + # f"No changes detected for item {item.slug!r} in collection {collection.slug!r}. Skipping." + # ) + # else: + # to_create.append( + # GiftUnique( + # slug=item.slug, + # collection_id=collection.id, + # model=item.model, + # backdrop=item.backdrop, + # number=item.number, + # pattern=item.pattern, + # telegram_owner_id=item.telegram_owner_id, + # owner_address=item.owner_address, + # blockchain_address=item.blockchain_address, + # last_updated=datetime.datetime.now(tz=datetime.UTC), + # ) + # ) + # if to_create: + # # Cache has to be cleared as new metadata could appear. + # # It could be extended to clear only when new metadata options appear. + # self.redis_service.delete(GIFT_COLLECTIONS_METADATA_KEY) + # + # self.db_session.bulk_save_objects(to_create) + # logger.info( + # f"Created {len(to_create)} new unique items for collection {collection.slug!r}." + # ) + # self.db_session.bulk_update_mappings(GiftUnique, to_update) + # logger.info( + # f"Updated {len(to_update)} existing unique items for collection {collection.slug!r}." + # ) + # self.db_session.commit() + # + # return targeted_telegram_owner_ids + # + # async def index( + # self, slug: str, start: int | None = None, stop: int | None = None + # ) -> set[int]: + # """ + # Processes a collection of items by indexing, updating, or creating unique records. It also + # tracks and returns Telegram IDs of previous owners of updated items for further actions. + # + # :param slug: A unique identifier for the collection being processed. + # :param start: The starting index for the indexing process. Defaults to 1. + # :param stop: The ending index for the indexing process. Defaults to the number of upgraded items. + # + # :return: A set of Telegram IDs corresponding to previous owners of the updated + # items who need further actions. + # """ + # collection = self.collection_service.get(slug) + # return await self._index(collection, start=start, stop=stop) diff --git a/backend/indexer_gifts/indexers/item.py b/backend/indexer_gifts/indexers/item.py index 9ad39e92..c6d85876 100644 --- a/backend/indexer_gifts/indexers/item.py +++ b/backend/indexer_gifts/indexers/item.py @@ -7,7 +7,6 @@ from core.dtos.gift.item import GiftUniqueDTO from core.services.supertelethon import TelethonService from indexer_gifts.settings import gifts_indexer_settings -from indexer_gifts.utils import parse_collection_slug_from_gift_slug logger = logging.getLogger(__name__) @@ -77,34 +76,34 @@ async def index_collection_items( # Free session for the next process await self.telethon_service.stop() - async def index_user_gifts( - self, - telegram_user_id: int, - ) -> list[GiftUniqueDTO]: - """ - Indexes and retrieves a list of unique gifts associated with a specific Telegram user. - - The function initiates a Telethon service session, fetches the gifts for a given - Telegram user, and processes them to extract and organize unique gift details. - If a gift's collection slug cannot be parsed from its slug, it logs a warning - message and skips processing for that specific gift. - - :param telegram_user_id: The unique ID of the Telegram user whose gifts are - being indexed. - :return: A list of GiftUniqueDTO objects representing unique gifts associated - with the given Telegram user. - """ - await self.telethon_service.start() - gifts = await self.telethon_service.index_user_gifts(telegram_user_id) - entities = [] - for gift in gifts: - if not (collection_slug := parse_collection_slug_from_gift_slug(gift.slug)): - logger.warning( - f"Can't parse collection slug from gift slug {gift.slug!r}. Skipping. " - ) - continue - entities.append( - GiftUniqueDTO.from_telethon(collection_slug=collection_slug, obj=gift) - ) - await self.telethon_service.stop() - return entities + # async def index_user_gifts( + # self, + # telegram_user_id: int, + # ) -> list[GiftUniqueDTO]: + # """ + # Indexes and retrieves a list of unique gifts associated with a specific Telegram user. + # + # The function initiates a Telethon service session, fetches the gifts for a given + # Telegram user, and processes them to extract and organize unique gift details. + # If a gift's collection slug cannot be parsed from its slug, it logs a warning + # message and skips processing for that specific gift. + # + # :param telegram_user_id: The unique ID of the Telegram user whose gifts are + # being indexed. + # :return: A list of GiftUniqueDTO objects representing unique gifts associated + # with the given Telegram user. + # """ + # await self.telethon_service.start() + # gifts = await self.telethon_service.index_user_gifts(telegram_user_id) + # entities = [] + # for gift in gifts: + # if not (collection_slug := parse_collection_slug_from_gift_slug(gift.slug)): + # logger.warning( + # f"Can't parse collection slug from gift slug {gift.slug!r}. Skipping. " + # ) + # continue + # entities.append( + # GiftUniqueDTO.from_telethon(collection_slug=collection_slug, obj=gift) + # ) + # await self.telethon_service.stop() + # return entities diff --git a/backend/indexer_gifts/tasks.py b/backend/indexer_gifts/tasks.py index bd400689..7b8f75c0 100644 --- a/backend/indexer_gifts/tasks.py +++ b/backend/indexer_gifts/tasks.py @@ -36,6 +36,8 @@ async def index_whitelisted_gift_collections() -> list[GiftCollectionDTO]: :return: A list of `GiftCollectionDTO` objects representing the gift collections retrieved and indexed from the database. """ + return [] + with DBService().db_session() as db_session: gift_collection_service = GiftCollectionService(db_session) collections = gift_collection_service.get_all( @@ -164,6 +166,8 @@ def fetch_gift_ownership_details(): Tasks will be retried automatically on session or phone-number-related errors up to a maximum defined limit. """ + return + collections = asyncio.run(index_whitelisted_gift_collections()) for collection in collections: for i in range( diff --git a/backend/scheduler/celery_app.py b/backend/scheduler/celery_app.py index 5b9b06b7..ea10c27f 100644 --- a/backend/scheduler/celery_app.py +++ b/backend/scheduler/celery_app.py @@ -5,7 +5,6 @@ CELERY_NOTICED_WALLETS_UPLOAD_QUEUE_NAME, CELERY_SYSTEM_QUEUE_NAME, CELERY_STICKER_FETCH_QUEUE_NAME, - CELERY_GIFT_FETCH_QUEUE_NAME, CELERY_INDEX_PRICES_QUEUE_NAME, ) from core.settings import core_settings @@ -48,11 +47,11 @@ def create_app() -> Celery: "schedule": crontab(minute="*/10"), # Every 10 minutes "options": {"queue": CELERY_STICKER_FETCH_QUEUE_NAME}, }, - "fetch-gift-ownerships": { - "task": "fetch-gift-ownership-details", - "schedule": crontab(hour="*/1", minute="0"), # Every hour - "options": {"queue": CELERY_GIFT_FETCH_QUEUE_NAME}, - }, + # "fetch-gift-ownerships": { + # "task": "fetch-gift-ownership-details", + # "schedule": crontab(hour="*/1", minute="0"), # Every hour + # "options": {"queue": CELERY_GIFT_FETCH_QUEUE_NAME}, + # }, "refresh-prices": { "task": "refresh-prices", "schedule": crontab(hour="*/1", minute="*/30"), # Every 30 minutes