diff --git a/app/containers.py b/app/containers.py index 961c3f1..3c0f502 100644 --- a/app/containers.py +++ b/app/containers.py @@ -6,7 +6,7 @@ class AppContainer(containers.DeclarativeContainer): wiring_config = containers.WiringConfiguration( - packages=(".features.content",) + packages=(".features.content", ".features.twitch") ) config = providers.Configuration() resources = providers.Container( diff --git a/app/features/__init__.py b/app/features/__init__.py index e843286..95faf77 100644 --- a/app/features/__init__.py +++ b/app/features/__init__.py @@ -1,6 +1,7 @@ from dependency_injector import containers, providers from .content.containers import ContentContainer +from .twitch.containers import TwitchContainer class FeaturesContainer(containers.DeclarativeContainer): @@ -9,3 +10,6 @@ class FeaturesContainer(containers.DeclarativeContainer): content = providers.Container( ContentContainer, resources=resources, config=config.content ) + twitch = providers.Container( + TwitchContainer, resources=resources, config=config.bot + ) diff --git a/app/features/admin/__init__.py b/app/features/admin/__init__.py index e6646ca..742cceb 100644 --- a/app/features/admin/__init__.py +++ b/app/features/admin/__init__.py @@ -4,9 +4,9 @@ # /admin/ depends on token # /admin/login POST -> redis as store backend -# /admin/logout POST -> redis as tore backend +# /admin/logout POST -> redis as store backend # /admin// GET -> list of items with pagination # -> GET /:id -> get item for edit # -> PATCH /:id -> update item if exists # -> POST /:id -> create item -# -> DELETE /:id -> delete item \ No newline at end of file +# -> DELETE /:id -> delete item diff --git a/app/features/content/models.py b/app/features/content/models.py index ad5caaf..7555d6a 100644 --- a/app/features/content/models.py +++ b/app/features/content/models.py @@ -1,5 +1,6 @@ import typing as T +from datetime import datetime from pydantic import BaseModel diff --git a/app/features/twitch/__init__.py b/app/features/twitch/__init__.py new file mode 100644 index 0000000..3ce88cb --- /dev/null +++ b/app/features/twitch/__init__.py @@ -0,0 +1,3 @@ +from fastapi import APIRouter + +API = APIRouter() diff --git a/app/features/twitch/api.py b/app/features/twitch/api.py new file mode 100644 index 0000000..59b4e8a --- /dev/null +++ b/app/features/twitch/api.py @@ -0,0 +1,48 @@ +from fastapi import Depends +from dependency_injector.wiring import inject, Provide + +from ...resources.services import ModelDataService +from ...containers import AppContainer + +from . import API +from .models import Bot, BotIn, Channel, ChannelIn + +CONTAINER = AppContainer.features.twitch + + +@API.get("/bots", response_model=list[Bot]) +@inject +async def get_bot_many( + query=Depends(Provide[CONTAINER.bot.query]) +): + return await query.all() + + +@API.post("/bot", response_model=Bot) +@inject +async def create_bot( + bot_in: BotIn, + query: ModelDataService = Depends(Provide[CONTAINER.bot.query]) +): + bot: Bot = await query.create(bot_in) + await query.commit() + return bot + + +@API.get("/channels", response_model=list[Channel]) +@inject +async def get_channel_many( + query=Depends(Provide[CONTAINER.channel.query]) +): + return await query.all() + + +@API.post("/channel", response_model=Channel) +@inject +async def create_channel( + channel_in: ChannelIn, + query: ModelDataService = Depends(Provide[CONTAINER.channel.query]) +): + channel: Channel = await query.create(channel_in) + await query.commit() + return channel diff --git a/app/features/twitch/containers.py b/app/features/twitch/containers.py new file mode 100644 index 0000000..0f05f90 --- /dev/null +++ b/app/features/twitch/containers.py @@ -0,0 +1,28 @@ +from dependency_injector import containers, providers + +from ...resources.containers import ModelDataContainer + +from . import API +from .models import Bot, Channel +from .tables import BOT_TABLE, CHANNEL_TABLE + + +class TwitchContainer(containers.DeclarativeContainer): + config = providers.Configuration() + resources = providers.DependenciesContainer( + db=providers.DependenciesContainer() + ) + + api = providers.Object(API) + bot = providers.Container( + ModelDataContainer, + db=resources.db, + model=Bot, + table=BOT_TABLE, + ) + channel = providers.Container( + ModelDataContainer, + db=resources.db, + model=Channel, + table=CHANNEL_TABLE + ) diff --git a/app/features/twitch/models.py b/app/features/twitch/models.py new file mode 100644 index 0000000..dcea853 --- /dev/null +++ b/app/features/twitch/models.py @@ -0,0 +1,56 @@ +import typing as T + +from datetime import datetime +from pydantic import BaseModel + + +class Bot(BaseModel): + id: int + secret_id: str + client_id: str + nickname: str + load: int = 0 + max_load: int = 40 + + +class BotIn(BaseModel): + secret_id: str + client_id: str + nickname: str + + +class User(BaseModel): + id: int + nicknames: list[str] + ext_id: int + + +class Message(BaseModel): + id: int + content: str + user_id: int + channel_id: int + + +class ChannelIn(BaseModel): + url: str + + +class Channel(ChannelIn): + id: int + url: str + start_listen_at: T.Optional[datetime] + ping_listen_at: T.Optional[datetime] + stop_listen_at: T.Optional[datetime] + listen_bot_id: T.Optional[int] + ignore: bool = False + active: bool = False + + +class ChannelAcquire(BaseModel): + size: int + + +class BotWithChannels(BaseModel): + bot: Bot + channels: list[Channel] diff --git a/app/features/twitch/tables.py b/app/features/twitch/tables.py new file mode 100644 index 0000000..0b60b69 --- /dev/null +++ b/app/features/twitch/tables.py @@ -0,0 +1,71 @@ +import sqlalchemy as sa + +from ...resources.providers import METADATA + +BOT_TABLE = sa.Table( + "bot", + METADATA, + sa.Column("id", sa.Integer, primary_key=True, nullable=False), + sa.Column("secret_id", sa.String(255), nullable=False), + sa.Column("client_id", sa.String(255), nullable=False), + sa.Column("nickname", sa.String(255), nullable=False), + sa.Column( + "load", + sa.Integer, + nullable=False, + default=0, + ), + sa.Column( + "max_load", + sa.Integer, + nullable=False, + default=40, + ), +) + +USER_TABLE = sa.Table( + "user", + METADATA, + sa.Column("id", sa.Integer, primary_key=True, nullable=False), + sa.Column("nicknames", sa.ARRAY(sa.String(255)), nullable=False), + sa.Column("ext_id", sa.Integer, nullable=False), +) + +MESSAGE_TABLE = sa.Table( + "message", + METADATA, + sa.Column("id", sa.Integer, primary_key=True, nullable=False), + sa.Column("content", sa.String(255), nullable=False), + sa.Column("user_id", sa.Integer, sa.ForeignKey("user.id"), nullable=False), + sa.Column( + "channel_id", + sa.Integer, + sa.ForeignKey("channel.id"), + nullable=False, + ), +) + +CHANNEL_TABLE = sa.Table( + "channel", + METADATA, + sa.Column("id", sa.Integer, primary_key=True, nullable=False), + sa.Column("url", sa.String(255), nullable=False), + sa.Column("start_listen_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("ping_listen_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("stop_listen_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "listen_bot_id", sa.Integer, sa.ForeignKey("bot.id"), nullable=True + ), + sa.Column( + "ignore", + sa.Boolean, + nullable=False, + default=False, + ), + sa.Column( + "active", + sa.Boolean, + nullable=False, + default=False, + ), +) diff --git a/app/features/twitch/tasks.py b/app/features/twitch/tasks.py new file mode 100644 index 0000000..f5ce712 --- /dev/null +++ b/app/features/twitch/tasks.py @@ -0,0 +1,32 @@ +import asyncio + +import httpx + +from celery import shared_task +from celery.utils.log import get_task_logger +from dependency_injector.wiring import Provide, inject + +from ...resources.services import ModelDataService +from ...containers import AppContainer + +logger = get_task_logger(__name__) + + +@inject +async def fetch_content_async( + content_id, + query: ModelDataService = Provide[ + AppContainer.features.content.data.query + ], +): + content = await query.get(content_id) + content.body = httpx.get(content.url).text + await query.update(content) + await query.commit() + logger.info([content.id, len(content.body)]) + + +@shared_task(autoretry_for=(Exception,), name="fetch_content") +def fetch_content(content_id: int): + loop = asyncio.get_event_loop() + loop.run_until_complete(fetch_content_async(content_id)) diff --git a/app/resources/services.py b/app/resources/services.py index 8f28da9..e4c022d 100644 --- a/app/resources/services.py +++ b/app/resources/services.py @@ -67,7 +67,7 @@ async def filter( query = query.limit(limit) if offset: query = query.offset(offset) - if order_by is None: + if order_by is not None: query = query.order_by(order_by) return ( await self.cursor.one(query) @@ -91,5 +91,7 @@ async def update(self, model, exclude: tuple = ("id",), query=None): ) async def delete(self, id_: int = None, query=None): - query = query or (self.table.c.id == id_) - await self.cursor.one(self.table.delete(query)) + query = query or sa.delete(self.table)\ + .where(self.table.c.id == id_)\ + .returning(self.table) + return await self.cursor.one(query) diff --git a/migrations/versions/0001_initial_migration.py b/migrations/versions/0001_initial_migration.py index 7d2c835..ed4063b 100644 --- a/migrations/versions/0001_initial_migration.py +++ b/migrations/versions/0001_initial_migration.py @@ -19,17 +19,65 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table( - "content", + "bot", sa.Column("id", sa.Integer(), nullable=False), - sa.Column("url", sa.String(length=1024), nullable=False), - sa.Column("body", sa.Text(), nullable=True), + sa.Column("secret_id", sa.String(length=255), nullable=False), + sa.Column("client_id", sa.String(length=255), nullable=False), + sa.Column("nickname", sa.String(length=255), nullable=False), + sa.Column("load", sa.Integer(), nullable=False), + sa.Column("max_load", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "user", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column( + "nicknames", sa.ARRAY(sa.String(length=255)), nullable=False + ), + sa.Column("ext_id", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "channel", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("url", sa.String(length=255), nullable=False), + sa.Column( + "start_listen_at", sa.DateTime(timezone=True), nullable=True + ), + sa.Column("ping_listen_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("stop_listen_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("listen_bot_id", sa.Integer(), nullable=True), + sa.Column("ignore", sa.Boolean(), nullable=False), + sa.Column("active", sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint( + ["listen_bot_id"], + ["bot.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "message", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("content", sa.String(length=255), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("channel_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["channel_id"], + ["channel.id"], + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["user.id"], + ), sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint("url"), ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("content") - # ### end Alembic commands ### + op.drop_table("message") + op.drop_table("channel") + op.drop_table("user") + op.drop_table("bot") + # ### end Alembic commands ### \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 8999d9f..88dcd2f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -100,7 +100,7 @@ test = ["pycodestyle (>=2.7.0,<2.8.0)", "flake8 (>=3.9.2,<3.10.0)", "uvloop (>=0 name = "atomicwrites" version = "1.4.0" description = "Atomic file writes." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -108,7 +108,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "attrs" version = "21.4.0" description = "Classes Without Boilerplate" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -459,7 +459,7 @@ python-versions = ">=3.5" name = "iniconfig" version = "1.1.1" description = "iniconfig: brain-dead simple config-ini parsing" -category = "dev" +category = "main" optional = false python-versions = "*" @@ -617,7 +617,7 @@ test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -656,7 +656,7 @@ wcwidth = "*" name = "py" version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -706,7 +706,7 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pytest" version = "6.2.5" description = "pytest: simple powerful testing with Python" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -737,6 +737,17 @@ pytest = ">=6.1.0" [package.extras] testing = ["coverage (==6.2)", "hypothesis (>=5.7.1)", "flaky (>=3.5.0)", "mypy (==0.931)"] +[[package]] +name = "pytest-order" +version = "1.0.1" +description = "pytest plugin to run your tests in a specific order" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pytest = {version = ">=6.2.4", markers = "python_version >= \"3.10\""} + [[package]] name = "pytz" version = "2021.3" @@ -872,7 +883,7 @@ full = ["itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" @@ -1489,6 +1500,10 @@ pytest-asyncio = [ {file = "pytest-asyncio-0.17.2.tar.gz", hash = "sha256:6d895b02432c028e6957d25fc936494e78c6305736e785d9fee408b1efbc7ff4"}, {file = "pytest_asyncio-0.17.2-py3-none-any.whl", hash = "sha256:e0fe5dbea40516b661ef1bcfe0bd9461c2847c4ef4bb40012324f2454fb7d56d"}, ] +pytest-order = [ + {file = "pytest-order-1.0.1.tar.gz", hash = "sha256:5dd6b929fbd7eaa6d0ee07586f65c623babb0afe72b4843c5f15055d6b3b1b1f"}, + {file = "pytest_order-1.0.1-py3-none-any.whl", hash = "sha256:bbe6e63a8e23741ab3e810d458d1ea7317e797b70f9550512d77d6e9e8fd1bbb"}, +] pytz = [ {file = "pytz-2021.3-py2.py3-none-any.whl", hash = "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c"}, {file = "pytz-2021.3.tar.gz", hash = "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326"}, diff --git a/tests/conftest.py b/tests/conftest.py index c747194..b8ed5e8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,24 @@ import pytest_asyncio from httpx import AsyncClient +from fastapi import APIRouter +from sqlalchemy.exc import ResourceClosedError from app import APP, APP_CONTAINER +test_router = APIRouter() + + +@test_router.get("/raise404") +async def raise_ex(): + raise ResourceClosedError + @pytest_asyncio.fixture async def client(): - async with AsyncClient(app=APP, base_url="http://test.com") as cl: + APP.include_router(test_router) + async_cl = AsyncClient(app=APP, base_url="http://test.com") + async with async_cl as cl: yield cl diff --git a/tests/test_bot.py b/tests/test_bot.py new file mode 100644 index 0000000..e5b45eb --- /dev/null +++ b/tests/test_bot.py @@ -0,0 +1,150 @@ +import pytest +import asyncio +import random +import sqlalchemy as sa + +from app.features.twitch.models import BotIn, Bot +'''Tests must be run together''' + +# @pytest.fixture() +# async def delete + + +@pytest.mark.asyncio +async def test_create_delete(client, container): + create_one = await client.post("/bot", json={"secret_id": "sercret123", + "client_id": "client123", "nickname": "Cooper123"}) + assert create_one.status_code == 200 + + query = await container.features.twitch.bot.query() + id = create_one.json()["id"] + await query.delete(id_=id) + await query.commit() + + query = await container.features.twitch.bot.query() + deleted_bot = await query.get(id) + assert deleted_bot is None + + create_many = await asyncio.gather( + *[client.post("/bot", json={"secret_id": "sercret123", "client_id": + "client123", "nickname": "to_filter"}) + for _ in range(10)] + ) + query = await container.features.twitch.bot.query() + for item in create_many: + assert item.status_code == 200 + item_json = item.json() + + id = item_json["id"] + item_from_db = await query.get(id) + assert item_from_db.dict() == item_json + + await query.delete(id_=id) + await query.commit() + + +@pytest.mark.asyncio +async def test_filter(client, container): + to_filter = await asyncio.gather( + *[client.post("/bot", json={"secret_id": "sercret123", "client_id": + "client123", "nickname": "to_filter"}) + for _ in range(10)] + ) + + query = await container.features.twitch.bot.query() + filtered_bots = await query.filter( + query.table.c.nickname == "to_filter" + ) + assert len(list(filtered_bots)) >= 10 + + for item in to_filter: + id = item.json()["id"] + await query.delete(id_=id) + await query.commit() + + +@pytest.mark.asyncio +async def test_update(client, container): + await asyncio.gather( + *[client.post("/bot", json={"secret_id": "sercret123", "client_id": + "client123", "nickname": "to_filter"}) + for _ in range(10)] + ) + query = await container.features.twitch.bot.query() + bots = list(await query.filter( + query.table.c.nickname == "to_filter" + )) + + for bot in bots: + bot = bot.dict() + id = bot["id"] + secret_id = bot["secret_id"] + client_id = bot["client_id"] + nickname = bot["nickname"] + load = random.randint(1, 40) + max_load = bot["max_load"] + updated_bot = Bot( + id=id, secret_id=secret_id, client_id=client_id, + nickname=nickname, load=load, max_load=max_load + ) + await query.update(updated_bot) + await query.commit() + + query = await container.features.twitch.bot.query() + filtered_bots = await query.filter( + query=sa.and_( + query.table.c.nickname == "to_filter", + sa.not_(query.table.c.load == 0) + ) + ) + assert len(list(filtered_bots)) >= 10 + + for bot in bots: + id = bot.dict()["id"] + await query.delete(id_=id) + await query.commit() + + query = await container.features.twitch.bot.query() + check_for_delete = await query.filter( + query=sa.and_( + query.table.c.nickname == "to_filter", + sa.not_(query.table.c.load == 0) + ) + ) + assert len(list(check_for_delete)) == 0 + + +@pytest.mark.asyncio +async def test_rollback(client, container): + query = await container.features.twitch.bot.query() + + bot_1 = BotIn( + secret_id='test_raise1', + client_id='test', + nickname='testn', + load=4 + ) + bot_2 = BotIn( + secret_id='test_raise2', + client_id='test', + nickname='testn', + load=4 + ) + + create_1 = await query.create(bot_1) + create_2 = await query.create(bot_2) + assert create_1 is not None + assert create_2 is not None + + await client.get('/raise404') + await query.commit() + + query = await container.features.twitch.bot.query() + check_bot_1 = await query.filter( + query=(query.table.c.secret_id == "test_raise1") + ) + check_bot_2 = await query.filter( + query=(query.table.c.secret_id == "test_raise2") + ) + assert list(check_bot_1) == [] + assert list(check_bot_2) == []