Private api init

.env.example

@@ -1,8 +1,9 @@
 # To get an api key please use our referral code (they also have free tier)
 # https://www.webshare.io/?referral_code=qvpjdwxqsblt
 PROXY_API_KEY = ""
-KAFKA_BOOTSTRAP_SERVERS = "kafka:9092"
+KAFKA_BOOTSTRAP_SERVERS = "kafka:9092" # public api, scraper and worker 
 SESSION_TIMEOUT = "60"
 DATABASE_URL = "mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata"
 DEBUG = "False"
-PYTHONDONTWRITEBYTECODE=1
+PYTHONDONTWRITEBYTECODE = 1
+ENV = "DEV" # private api 

Makefile

@@ -51,6 +51,15 @@ restart-%: ## Restart a docker service by name, eg make restart-api_public
 	docker compose build $*
 	docker compose up -d $*
 
+info:
+	uv run poly info
+
+libs:
+	uv run poly libs
+
+checks: info libs
+	uv run poly check
+
 setup:
 	uv sync
 

bases/bot_detector/api_private/src/api/__init__.py

@@ -0,0 +1,8 @@
+from fastapi import APIRouter
+
+from . import v2, v3, v4
+
+router = APIRouter()
+router.include_router(v2.router, prefix="/v2")
+router.include_router(v3.router, prefix="/v3")
+router.include_router(v4.router, prefix="/v4")

bases/bot_detector/api_private/src/api/readme.md

@@ -0,0 +1 @@
+the api folder can be considered the controller in the MVC approach

bases/bot_detector/api_private/src/api/v2/__init__.py

@@ -0,0 +1,6 @@
+from fastapi import APIRouter
+from . import player, highscore
+
+router = APIRouter()
+router.include_router(player.router)
+router.include_router(highscore.router)

bases/bot_detector/api_private/src/api/v2/highscore.py

@@ -0,0 +1,64 @@
+import logging
+
+from bot_detector.api_private.src.app.repositories import (
+    PlayerActivityRepo,
+    PlayerSkillsRepo,
+    ScraperDataRepo,
+)
+from bot_detector.api_private.src.app.views.response.highscore import (
+    PlayerHiscoreData,
+)
+
+# from src.app.repositories.highscore import HighscoreRepo
+from bot_detector.api_private.src.core.fastapi.dependencies.session import (
+    get_session,
+)
+from fastapi import APIRouter, Depends, Query
+
+logger = logging.getLogger(__name__)
+
+
+router = APIRouter()
+
+
+@router.get("/highscore/latest")
+async def get_highscore_latest_v2(
+    player_id: int,
+    player_name: str = None,
+    label_id: int = None,
+    many: bool = False,
+    limit: int = Query(default=10, ge=0, le=10_000),
+    session=Depends(get_session),
+):
+    repo = ScraperDataRepo(session=session)
+    repo_skills = PlayerSkillsRepo(session=session)
+    repo_activities = PlayerActivityRepo(session=session)
+
+    data = await repo.select(
+        player_name=player_name,
+        player_id=player_id,
+        label_id=label_id,
+        many=many,
+        limit=limit,
+        history=False,
+    )
+
+    logger.info(data[0])
+    for d in data:
+        scraper_id = d.pop("scraper_id")
+        d["Player_id"] = d.pop("player_id")
+        d["id"] = scraper_id
+        d["timestamp"] = d.pop("created_at")
+        d["ts_date"] = d.pop("record_date")
+
+        skills = await repo_skills.select(scraper_id=scraper_id)
+        activities = await repo_activities.select(scraper_id=scraper_id)
+
+        for skill in skills:
+            d[skill.get("skill_name")] = skill.get("skill_value")
+
+        for activity in activities:
+            d[activity.get("activity_name")] = activity.get("activity_value")
+
+    data = [{k: v for k, v in d.items() if v} for d in data]
+    return [PlayerHiscoreData(**d).model_dump(mode="json") for d in data]

bases/bot_detector/api_private/src/api/v2/player.py

@@ -0,0 +1,29 @@
+from bot_detector.api_private.src.app.repositories.player import PlayerRepo
+from bot_detector.api_private.src.core.fastapi.dependencies.session import (
+    get_session,
+)
+from fastapi import APIRouter, Depends, Query
+
+router = APIRouter()
+
+
+@router.get("/player")
+async def get_player(
+    player_id: str = None,
+    player_name: str = None,
+    label_id: int = None,
+    greater_than: bool = False,
+    limit: int = Query(default=1_000, ge=0, le=100_000),
+    session=Depends(get_session),
+):
+    # TODO: make use of abstract base class
+    repo = PlayerRepo(session=session)
+
+    data = await repo.select(
+        player_id=player_id,
+        player_name=player_name,
+        greater_than=greater_than,
+        label_id=label_id,
+        limit=limit,
+    )
+    return data

bases/bot_detector/api_private/src/api/v3/__init__.py

@@ -0,0 +1,6 @@
+from fastapi import APIRouter
+
+from . import highscore
+
+router = APIRouter()
+router.include_router(highscore.router)

bases/bot_detector/api_private/src/api/v3/highscore.py

@@ -0,0 +1,78 @@
+import logging
+from collections import defaultdict
+
+from bot_detector.api_private.src.app.repositories import ScraperDataRepo
+from bot_detector.api_private.src.app.views.response import (
+    ActivityView,
+    ScraperDataView,
+    SkillView,
+)
+from bot_detector.api_private.src.core.fastapi.dependencies.session import (
+    get_session,
+)
+from fastapi import APIRouter, Depends, Query
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+def convert_to_scraper_data_view(result_list: list[dict]) -> list[ScraperDataView]:
+    # Dictionary to hold grouped data by scraper_id
+    scraper_data_map = defaultdict(lambda: {"skills": [], "activities": []})
+
+    for row in result_list:
+        scraper_id = row["scrape_id"]
+        scraper_data = scraper_data_map[scraper_id]
+
+        # Set shared attributes only once per scraper_id
+        if "created_at" not in scraper_data:
+            scraper_data["created_at"] = row["scrape_ts"]
+            scraper_data["record_date"] = row["scrape_date"]
+            scraper_data["scraper_id"] = scraper_id
+            scraper_data["player_id"] = row["player_id"]
+            scraper_data["player_name"] = row["player_name"]
+
+        # Append to skills or activities based on hs_type
+        if row["hs_type"] == "skill":
+            scraper_data["skills"].append(
+                SkillView(skill_name=row["hs_name"], skill_value=row["hs_value"])
+            )
+        elif row["hs_type"] == "activity":
+            scraper_data["activities"].append(
+                ActivityView(
+                    activity_name=row["hs_name"], activity_value=row["hs_value"]
+                )
+            )
+
+    # Convert the grouped data into ScraperDataView instances
+    return [
+        ScraperDataView(
+            created_at=data["created_at"],
+            record_date=data["record_date"],
+            scraper_id=data["scraper_id"],
+            player_id=data["player_id"],
+            player_name=data["player_name"],
+            skills=data["skills"],
+            activities=data["activities"],
+        )
+        for data in scraper_data_map.values()
+    ]
+
+
+@router.get("/highscore/latest", response_model=list[ScraperDataView])
+async def get_highscore_latest(
+    player_id: int,
+    label_id: int = None,
+    many: bool = False,
+    limit: int = Query(default=10, ge=0, le=10_000),
+    session=Depends(get_session),
+):
+    repo = ScraperDataRepo(session=session)
+    data = await repo.select_latest_scraper_data_v3(
+        player_id=player_id,
+        label_id=label_id,
+        many=many,
+        limit=limit,
+    )
+    return convert_to_scraper_data_view(result_list=data)

bases/bot_detector/api_private/src/api/v4/__init__.py

@@ -0,0 +1,6 @@
+from fastapi import APIRouter
+
+from . import highscore
+
+router = APIRouter()
+router.include_router(highscore.router)

bases/bot_detector/api_private/src/api/v4/highscore.py

@@ -0,0 +1,61 @@
+import logging
+from collections import defaultdict
+from datetime import datetime
+
+from bot_detector.api_private.src.app.repositories import ScraperDataRepo
+from bot_detector.api_private.src.app.views.response import (
+    ActivityView,
+    ScraperDataView,
+    SkillView,
+)
+from bot_detector.api_private.src.core.fastapi.dependencies.session import (
+    get_session,
+)
+from bot_detector.database.repositories.hiscore import HighscoreDataLatestRepo
+from bot_detector.structs import HighscoreDataLatestStruct
+from fastapi import APIRouter, Depends, Query
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+def convert_latest_struct_to_scraper_data_view(
+    records: list[HighscoreDataLatestStruct],
+) -> list[ScraperDataView]:
+    return [
+        ScraperDataView(
+            created_at=record.created_at
+            or datetime.combine(record.scrape_date, datetime.min.time()),
+            record_date=record.scrape_date,
+            scraper_id=record.player_id,
+            player_id=record.player_id,
+            player_name=record.player_name or "Unknown",
+            skills=[
+                SkillView(skill_name=k, skill_value=v)
+                for k, v in (record.skills or {}).items()
+            ],
+            activities=[
+                ActivityView(activity_name=k, activity_value=v)
+                for k, v in (record.activities or {}).items()
+            ],
+        )
+        for record in records
+    ]
+
+
+@router.get("/highscore/latest", response_model=list[ScraperDataView])
+async def get_highscore_latest(
+    player_id: int,
+    label_id: int = None,
+    many: bool = False,
+    limit: int = Query(default=10, ge=0, le=10_000),
+    session=Depends(get_session),
+):
+    repo = HighscoreDataLatestRepo()
+    if many:
+        rows = await repo.select_highscore_list(session, player_id, label_id, limit)
+    else:
+        row = await repo.select_highscore(session, player_id, label_id)
+        rows = [row] if row else []
+    return convert_latest_struct_to_scraper_data_view(rows)

bases/bot_detector/api_private/src/app/readme.md

@@ -0,0 +1,6 @@
+the model is responsible for all the data handeling
+- getting data from the database
+- handles data logic
+
+the view is responsible for the data representation
+- return format etc

bases/bot_detector/api_private/src/app/repositories/__init__.py

@@ -0,0 +1,15 @@
+from .abstract_repo import AbstractAPI
+from .highscore import HighscoreRepo
+from .player import PlayerRepo
+from .player_activities import PlayerActivityRepo
+from .player_skills import PlayerSkillsRepo
+from .scraper_data import ScraperDataRepo
+
+__all__ = [
+    "HighscoreRepo",
+    "PlayerRepo",
+    "PlayerSkillsRepo",
+    "ScraperDataRepo",
+    "AbstractAPI",
+    "PlayerActivityRepo",
+]

bases/bot_detector/api_private/src/app/repositories/abstract_repo.py

@@ -0,0 +1,19 @@
+from abc import ABC, abstractmethod
+
+
+class AbstractAPI(ABC):
+    @abstractmethod
+    def insert(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def select(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def update(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def delete(self):
+        raise NotImplementedError

bases/bot_detector/api_private/src/app/repositories/highscore.py

@@ -0,0 +1,56 @@
+import logging
+
+from bot_detector.api_private.src.app.repositories.abstract_repo import (
+    AbstractAPI,
+)
+from bot_detector.api_private.src.core.database.models import (  # playerHiscoreData,; PlayerHiscoreDataXPChange,
+    PlayerHiscoreDataLatest,
+)
+from bot_detector.api_private.src.core.database.models.player import Player
+from fastapi.encoders import jsonable_encoder
+from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession
+from sqlalchemy.orm import aliased
+from sqlalchemy.sql.expression import Select
+
+logger = logging.getLogger(__name__)
+
+
+class HighscoreRepo(AbstractAPI):
+    def __init__(self, session) -> None:
+        super().__init__()
+        self.session: AsyncSession = session
+
+    def insert(self):
+        raise NotImplementedError
+
+    async def select(
+        self, player_id: int, label_id: int, limit: int, many: bool
+    ) -> dict:
+        table = aliased(PlayerHiscoreDataLatest, name="phd")
+        player = aliased(Player, name="pl")
+
+        sql = Select(player.name, table)
+        sql = sql.join(target=player, onclause=table.Player_id == player.id)
+
+        if player_id:
+            if many:
+                sql = sql.where(table.Player_id >= player_id)
+            else:
+                sql = sql.where(table.Player_id == player_id)
+
+        if label_id:
+            sql = sql.where(player.label_id == label_id)
+
+        sql = sql.limit(limit)
+
+        async with self.session:
+            result: AsyncResult = await self.session.execute(sql)
+            result = result.fetchall()
+        data = [{"name": name, **jsonable_encoder(hs)} for name, hs in result]
+        return data
+
+    async def update(self):
+        raise NotImplementedError
+
+    async def delete(self):
+        raise NotImplementedError