diff --git a/.env.example b/.env.example index 82abc58..1de9fb0 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,7 @@ DATABASE_URL=postgresql+asyncpg://fitness:fitness@db:5432/fitness OPENAI_API_KEY="your_openai_key_here" GOOGLE_API_KEY="your_google_key_here" TELEGRAM_BOT_TOKEN="your_telegram_bot_token_here" -REDIS_HOST=redis +REDIS_HOST=localhost REDIS_PORT=6379 REDIS_DB=0 LLM_CONFIG_PATH=llm/llm_config.yaml \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0e884e..1780c35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,10 @@ jobs: POSTGRES_DB: fitness ports: - 5432:5432 + redis: + image: redis:7 + ports: + - 6379:6379 steps: - name: ๐Ÿงพ Checkout repository @@ -87,3 +91,6 @@ jobs: - name: โœ… Run Onboarding Test run: poetry run pytest tests/test_onboarding.py + + - name: โœ… Run Redis Test + run: poetry run pytest tests/test_redis.py diff --git a/llm/chat_memory.py b/llm/chat_memory.py index 19c26e8..48f0351 100644 --- a/llm/chat_memory.py +++ b/llm/chat_memory.py @@ -3,14 +3,17 @@ from sqlalchemy.ext.asyncio import AsyncSession from src.database.models.message import Message from redis.asyncio import Redis +from src.database.models import SubSummary from llm.config_loader import CONFIG from llm.models import get_llm +from src.database.connection import session_maker import logging logger = logging.getLogger(__name__) MAX_RECENT_MESSAGES = CONFIG.get("max_recent_messages", 20) SUMMARY_SIZE = CONFIG.get("summary_chunk_size", 10) +MAX_SUMMARY_STACK = CONFIG.get("max_summary_stack", 10) async def save_message_to_redis(telegram_id: int, role: str, content: str, redis_client: Redis | None = None): @@ -74,9 +77,18 @@ async def create_sub_summary(telegram_id: int, redis_client: Redis | None = None response = await llm.ainvoke(prompt) summary_key = f"user:{telegram_id}:sub_summaries" + + # Check if stack of sub-summaries is full + current_stack = await redis_client.llen(summary_key) + if current_stack >= MAX_SUMMARY_STACK: + # Save to Postgres + await save_sub_summaries_to_db(telegram_id, redis_client) + await redis_client.delete(summary_key) # Clear stack in Redis + + # Push new summary await redis_client.rpush(summary_key, response.content) - # Drop the first SUMMARY_SIZE messages (the oldest ones) + # Trim recent history await redis_client.ltrim(key, SUMMARY_SIZE, -1) @@ -90,4 +102,18 @@ async def get_latest_sub_summary(telegram_id: int, redis_client: Redis | None = redis_client = redis_client or await get_redis() summary_key = f"user:{telegram_id}:sub_summaries" latest = await redis_client.lindex(summary_key, -1) - return latest \ No newline at end of file + return latest + + +async def save_sub_summaries_to_db(telegram_id: int, redis_client: Redis): + summary_key = f"user:{telegram_id}:sub_summaries" + summaries = await redis_client.lrange(summary_key, 0, -1) + + if not summaries: + return + + async with session_maker() as session: + for s in summaries: + sub_summary = SubSummary(telegram_id=telegram_id, summary=s) + session.add(sub_summary) + await session.commit() \ No newline at end of file diff --git a/llm/llm_config.yaml b/llm/llm_config.yaml index 3aaa3ae..44947e6 100644 --- a/llm/llm_config.yaml +++ b/llm/llm_config.yaml @@ -12,4 +12,5 @@ yandex: iam_token: "IAM_TOKEN" max_recent_messages: 6 -summary_chunk_size: 3 \ No newline at end of file +summary_chunk_size: 3 +max_summary_stack: 2 \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index c0943d2..3a5e5a7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -211,7 +211,7 @@ version = "4.8.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "anyio-4.8.0-py3-none-any.whl", hash = "sha256:b5011f270ab5eb0abf13385f851315585cc37ef330dd88e27ec3d34d651fd47a"}, {file = "anyio-4.8.0.tar.gz", hash = "sha256:1d9fe889df5212298c0c0723fa20479d1b94883a2df44bd3897aa91083316f7a"}, @@ -227,6 +227,21 @@ doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1) ; python_version >= \"3.10\"", "uvloop (>=0.21) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\" and python_version < \"3.14\""] trio = ["trio (>=0.26.1)"] +[[package]] +name = "asgi-lifespan" +version = "2.1.0" +description = "Programmatic startup/shutdown of ASGI apps." +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "asgi-lifespan-2.1.0.tar.gz", hash = "sha256:5e2effaf0bfe39829cf2d64e7ecc47c7d86d676a6599f7afba378c31f5e3a308"}, + {file = "asgi_lifespan-2.1.0-py3-none-any.whl", hash = "sha256:ed840706680e28428c01e14afb3875d7d76d3206f3d5b2f2294e059b5c23804f"}, +] + +[package.dependencies] +sniffio = "*" + [[package]] name = "asyncio" version = "3.4.3" @@ -343,7 +358,7 @@ version = "2025.1.31" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"}, {file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"}, @@ -1053,7 +1068,7 @@ version = "0.14.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = false python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, @@ -1065,7 +1080,7 @@ version = "1.0.7" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"}, {file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"}, @@ -1143,7 +1158,7 @@ version = "0.28.1" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, @@ -1183,7 +1198,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -2775,12 +2790,11 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] -markers = {dev = "python_version < \"3.13\""} [[package]] name = "urllib3" @@ -3105,4 +3119,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "7515c062153863c5c78692512d22a1e9b9ea0af82e8a2758f0299ac6aee43c3e" +content-hash = "902b4234311fe2a33d2eed8a0bff7206efee515474ff558da5160e6216f162b3" diff --git a/pyproject.toml b/pyproject.toml index f9a5ffb..ac14090 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,13 +38,13 @@ alembic = "^1.15.1" pytest = "^8.3.5" redis = "^5.2.1" asyncio = "^3.4.3" - +httpx = "0.28.1" [tool.poetry.group.dev.dependencies] pytest = "^8.3.5" -httpx = "^0.28.1" pre-commit = "^4.1.0" pytest-asyncio = "^0.25.3" +asgi-lifespan = "^2.1.0" [build-system] requires = ["poetry-core"] diff --git a/src/database/models/__init__.py b/src/database/models/__init__.py index e311ee0..dc4a727 100644 --- a/src/database/models/__init__.py +++ b/src/database/models/__init__.py @@ -3,6 +3,7 @@ from .activity import Activity from .train_program import Program from .message import Message +from .sub_summary import SubSummary -__all__ = ["User", "Action", "Activity", "Program", "Message"] +__all__ = ["User", "Action", "Activity", "Program", "Message", "SubSummary"] diff --git a/src/database/models/sub_summary.py b/src/database/models/sub_summary.py new file mode 100644 index 0000000..613cfb7 --- /dev/null +++ b/src/database/models/sub_summary.py @@ -0,0 +1,12 @@ +from sqlalchemy import Column, DateTime, Integer, String, ForeignKey, func +from src.database.models.base import Base + +__all__ = ["SubSummary"] + +class SubSummary(Base): + __tablename__ = "sub_summaries" + + id = Column(Integer, primary_key=True, index=True) + telegram_id = Column(Integer, ForeignKey("users.telegram_id"), nullable=False, index=True) + summary = Column(String, nullable=False) + timestamp = Column(DateTime, server_default=func.now(), nullable=False) diff --git a/src/logging_config.py b/src/logging_config.py new file mode 100644 index 0000000..2719343 --- /dev/null +++ b/src/logging_config.py @@ -0,0 +1,12 @@ +import logging +import sys + +def setup_logging(level: int = logging.INFO): + """Set up centralized logging configuration.""" + logging.basicConfig( + level=level, + format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + handlers=[ + logging.StreamHandler(sys.stdout), + ] + ) diff --git a/src/migrations/versions/55abc90e848f_add_subsummary.py b/src/migrations/versions/55abc90e848f_add_subsummary.py new file mode 100644 index 0000000..7295e50 --- /dev/null +++ b/src/migrations/versions/55abc90e848f_add_subsummary.py @@ -0,0 +1,41 @@ +"""Add subSummary + +Revision ID: 55abc90e848f +Revises: d2f8b0a76127 +Create Date: 2025-03-30 17:30:54.302965 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '55abc90e848f' +down_revision: Union[str, None] = 'd2f8b0a76127' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('sub_summaries', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('telegram_id', sa.Integer(), nullable=False), + sa.Column('summary', sa.String(), nullable=False), + sa.Column('timestamp', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['telegram_id'], ['users.telegram_id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_sub_summaries_id'), 'sub_summaries', ['id'], unique=False) + op.create_index(op.f('ix_sub_summaries_telegram_id'), 'sub_summaries', ['telegram_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_sub_summaries_telegram_id'), table_name='sub_summaries') + op.drop_index(op.f('ix_sub_summaries_id'), table_name='sub_summaries') + op.drop_table('sub_summaries') + # ### end Alembic commands ### diff --git a/src/views/redis.py b/src/views/redis.py index 7773103..ad1ed45 100644 --- a/src/views/redis.py +++ b/src/views/redis.py @@ -2,7 +2,7 @@ from src.dependencies.redis import get_redis import json -router = APIRouter(prefix="/redis", tags=["redis"]) +router = APIRouter(tags=["redis"]) @router.get("/ping") diff --git a/tests/test_redis.py b/tests/test_redis.py new file mode 100644 index 0000000..1b9cfc0 --- /dev/null +++ b/tests/test_redis.py @@ -0,0 +1,22 @@ +import pytest +from httpx import AsyncClient, ASGITransport +from api.app import get_application + + +@pytest.mark.asyncio +async def test_redis_ping(): + app = get_application() + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + # manually trigger Redis startup + await app.router.startup() + + res = await client.get("/api/v1/redis/ping") + assert res.status_code == 200 + data = res.json() + assert data["status"] == "OK" + assert data["ping"] == "pong" + + # manually trigger Redis shutdown + await app.router.shutdown()