From df6d505af52d7b042a78d17a3bfc5749702cdbf7 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Fri, 6 Jun 2025 21:39:24 -0400 Subject: [PATCH 01/39] init private_api --- bases/bot_detector/api_private/Dockerfile | 31 ++ bases/bot_detector/api_private/LICENSE | 24 ++ bases/bot_detector/api_private/Makefile | 103 ++++++ bases/bot_detector/api_private/README.md | 44 +++ .../api_private/docker-compose-standalone.yml | 65 ++++ .../api_private/docker-compose.yml | 67 ++++ .../api_private/kafka_setup/.dockerignore | 1 + .../api_private/kafka_setup/Dockerfile | 13 + .../kafka_setup/kafka_data/.gitkeep | 0 .../api_private/kafka_setup/requirements.txt | 1 + .../api_private/kafka_setup/setup_kafka.py | 92 +++++ .../bot_detector/api_private/mysql/Dockerfile | 3 + .../docker-entrypoint-initdb.d/00_init.sql | 1 + .../docker-entrypoint-initdb.d/01_tables.sql | 254 +++++++++++++ .../docker-entrypoint-initdb.d/02_data.sql | 73 ++++ .../api_private/mysql_setup/Dockerfile | 10 + .../api_private/mysql_setup/__init__.py | 0 .../api_private/mysql_setup/models.py | 179 +++++++++ .../api_private/mysql_setup/requirements.txt | 8 + .../api_private/mysql_setup/setup_mysql.py | 108 ++++++ bases/bot_detector/api_private/notes.md | 40 ++ .../bot_detector/api_private/requirements.txt | 33 ++ .../api_private/src/api/__init__.py | 7 + .../api_private/src/api/readme.md | 1 + .../api_private/src/api/v1/health.py | 0 .../api_private/src/api/v2/__init__.py | 6 + .../api_private/src/api/v2/highscore.py | 74 ++++ .../api_private/src/api/v2/player.py | 28 ++ .../api_private/src/api/v3/__init__.py | 6 + .../api_private/src/api/v3/highscore.py | 73 ++++ .../bot_detector/api_private/src/app/.gitkeep | 0 .../api_private/src/app/readme.md | 6 + .../api_private/src/app/repositories/.gitkeep | 0 .../src/app/repositories/__init__.py | 15 + .../src/app/repositories/abstract_repo.py | 19 + .../src/app/repositories/highscore.py | 55 +++ .../src/app/repositories/player.py | 41 ++ .../src/app/repositories/player_activities.py | 49 +++ .../src/app/repositories/player_skills.py | 49 +++ .../src/app/repositories/scraper_data.py | 129 +++++++ .../src/app/views/input/__init__.py | 0 .../src/app/views/response/__init__.py | 3 + .../src/app/views/response/highscore.py | 109 ++++++ .../api_private/src/app/views/response/ok.py | 4 + .../src/app/views/response/scraper_data.py | 23 ++ .../api_private/src/core/__init__.py | 4 + .../api_private/src/core/config.py | 11 + .../api_private/src/core/database/.gitkeep | 0 .../api_private/src/core/database/database.py | 22 ++ .../src/core/database/models/.gitkeep | 0 .../src/core/database/models/__init__.py | 26 ++ .../src/core/database/models/activities.py | 27 ++ .../src/core/database/models/highscore.py | 350 ++++++++++++++++++ .../src/core/database/models/player.py | 19 + .../src/core/database/models/prediction.py | 39 ++ .../src/core/database/models/report.py | 29 ++ .../src/core/database/models/scraper_data.py | 22 ++ .../core/database/models/scraper_data_v3.py | 107 ++++++ .../src/core/database/models/skills.py | 27 ++ .../src/core/fastapi/dependencies/.gitkeep | 0 .../src/core/fastapi/dependencies/session.py | 9 + .../fastapi/dependencies/to_jagex_name.py | 3 + .../src/core/fastapi/middleware/logging.py | 27 ++ .../api_private/src/core/kafka/engine.py | 72 ++++ .../api_private/src/core/logging_config.py | 42 +++ .../api_private/src/core/server.py | 51 +++ bases/bot_detector/api_private/tests/.gitkeep | 0 .../tests/_test_highscore_benchmark.py | 44 +++ .../api_private/tests/benchmark.py | 39 ++ .../api_private/tests/conftest.py | 57 +++ .../bot_detector/api_private/tests/pytest.ini | 3 + .../api_private/tests/test_highscore.py | 45 +++ .../tests/test_highscore_benchmark_custom.py | 81 ++++ .../api_private/tests/test_highscore_v2.py | 263 +++++++++++++ .../api_private/tests/test_highscore_v3.py | 65 ++++ .../api_private/tests/test_player.py | 76 ++++ 76 files changed, 3407 insertions(+) create mode 100644 bases/bot_detector/api_private/Dockerfile create mode 100644 bases/bot_detector/api_private/LICENSE create mode 100644 bases/bot_detector/api_private/Makefile create mode 100644 bases/bot_detector/api_private/README.md create mode 100644 bases/bot_detector/api_private/docker-compose-standalone.yml create mode 100644 bases/bot_detector/api_private/docker-compose.yml create mode 100644 bases/bot_detector/api_private/kafka_setup/.dockerignore create mode 100644 bases/bot_detector/api_private/kafka_setup/Dockerfile create mode 100644 bases/bot_detector/api_private/kafka_setup/kafka_data/.gitkeep create mode 100644 bases/bot_detector/api_private/kafka_setup/requirements.txt create mode 100644 bases/bot_detector/api_private/kafka_setup/setup_kafka.py create mode 100644 bases/bot_detector/api_private/mysql/Dockerfile create mode 100644 bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/00_init.sql create mode 100644 bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/01_tables.sql create mode 100644 bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/02_data.sql create mode 100644 bases/bot_detector/api_private/mysql_setup/Dockerfile create mode 100644 bases/bot_detector/api_private/mysql_setup/__init__.py create mode 100644 bases/bot_detector/api_private/mysql_setup/models.py create mode 100644 bases/bot_detector/api_private/mysql_setup/requirements.txt create mode 100644 bases/bot_detector/api_private/mysql_setup/setup_mysql.py create mode 100644 bases/bot_detector/api_private/notes.md create mode 100644 bases/bot_detector/api_private/requirements.txt create mode 100644 bases/bot_detector/api_private/src/api/__init__.py create mode 100644 bases/bot_detector/api_private/src/api/readme.md create mode 100644 bases/bot_detector/api_private/src/api/v1/health.py create mode 100644 bases/bot_detector/api_private/src/api/v2/__init__.py create mode 100644 bases/bot_detector/api_private/src/api/v2/highscore.py create mode 100644 bases/bot_detector/api_private/src/api/v2/player.py create mode 100644 bases/bot_detector/api_private/src/api/v3/__init__.py create mode 100644 bases/bot_detector/api_private/src/api/v3/highscore.py create mode 100644 bases/bot_detector/api_private/src/app/.gitkeep create mode 100644 bases/bot_detector/api_private/src/app/readme.md create mode 100644 bases/bot_detector/api_private/src/app/repositories/.gitkeep create mode 100644 bases/bot_detector/api_private/src/app/repositories/__init__.py create mode 100644 bases/bot_detector/api_private/src/app/repositories/abstract_repo.py create mode 100644 bases/bot_detector/api_private/src/app/repositories/highscore.py create mode 100644 bases/bot_detector/api_private/src/app/repositories/player.py create mode 100644 bases/bot_detector/api_private/src/app/repositories/player_activities.py create mode 100644 bases/bot_detector/api_private/src/app/repositories/player_skills.py create mode 100644 bases/bot_detector/api_private/src/app/repositories/scraper_data.py create mode 100644 bases/bot_detector/api_private/src/app/views/input/__init__.py create mode 100644 bases/bot_detector/api_private/src/app/views/response/__init__.py create mode 100644 bases/bot_detector/api_private/src/app/views/response/highscore.py create mode 100644 bases/bot_detector/api_private/src/app/views/response/ok.py create mode 100644 bases/bot_detector/api_private/src/app/views/response/scraper_data.py create mode 100644 bases/bot_detector/api_private/src/core/__init__.py create mode 100644 bases/bot_detector/api_private/src/core/config.py create mode 100644 bases/bot_detector/api_private/src/core/database/.gitkeep create mode 100644 bases/bot_detector/api_private/src/core/database/database.py create mode 100644 bases/bot_detector/api_private/src/core/database/models/.gitkeep create mode 100644 bases/bot_detector/api_private/src/core/database/models/__init__.py create mode 100644 bases/bot_detector/api_private/src/core/database/models/activities.py create mode 100644 bases/bot_detector/api_private/src/core/database/models/highscore.py create mode 100644 bases/bot_detector/api_private/src/core/database/models/player.py create mode 100644 bases/bot_detector/api_private/src/core/database/models/prediction.py create mode 100644 bases/bot_detector/api_private/src/core/database/models/report.py create mode 100644 bases/bot_detector/api_private/src/core/database/models/scraper_data.py create mode 100644 bases/bot_detector/api_private/src/core/database/models/scraper_data_v3.py create mode 100644 bases/bot_detector/api_private/src/core/database/models/skills.py create mode 100644 bases/bot_detector/api_private/src/core/fastapi/dependencies/.gitkeep create mode 100644 bases/bot_detector/api_private/src/core/fastapi/dependencies/session.py create mode 100644 bases/bot_detector/api_private/src/core/fastapi/dependencies/to_jagex_name.py create mode 100644 bases/bot_detector/api_private/src/core/fastapi/middleware/logging.py create mode 100644 bases/bot_detector/api_private/src/core/kafka/engine.py create mode 100644 bases/bot_detector/api_private/src/core/logging_config.py create mode 100644 bases/bot_detector/api_private/src/core/server.py create mode 100644 bases/bot_detector/api_private/tests/.gitkeep create mode 100644 bases/bot_detector/api_private/tests/_test_highscore_benchmark.py create mode 100644 bases/bot_detector/api_private/tests/benchmark.py create mode 100644 bases/bot_detector/api_private/tests/conftest.py create mode 100644 bases/bot_detector/api_private/tests/pytest.ini create mode 100644 bases/bot_detector/api_private/tests/test_highscore.py create mode 100644 bases/bot_detector/api_private/tests/test_highscore_benchmark_custom.py create mode 100644 bases/bot_detector/api_private/tests/test_highscore_v2.py create mode 100644 bases/bot_detector/api_private/tests/test_highscore_v3.py create mode 100644 bases/bot_detector/api_private/tests/test_player.py diff --git a/bases/bot_detector/api_private/Dockerfile b/bases/bot_detector/api_private/Dockerfile new file mode 100644 index 0000000..516a9ed --- /dev/null +++ b/bases/bot_detector/api_private/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.11-slim as base + +ARG api_port +ENV UVICORN_PORT ${api_port} + +ARG root_path +ENV UVICORN_ROOT_PATH ${root_path} + +# Keeps Python from generating .pyc files in the container +ENV PYTHONDONTWRITEBYTECODE=1 + +# Turns off buffering for easier container logging +ENV PYTHONUNBUFFERED=1 + +# set the working directory +WORKDIR /project + +# install dependencies +COPY ./requirements.txt /project +RUN pip install --no-cache-dir -r requirements.txt + +# copy the scripts to the folder +COPY ./src /project/src + +# production image +FROM base as production +# Creates a non-root user with an explicit UID and adds permission to access the /project folder +RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /project +USER appuser + +CMD ["uvicorn", "src.core.server:app", "--proxy-headers", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/bases/bot_detector/api_private/LICENSE b/bases/bot_detector/api_private/LICENSE new file mode 100644 index 0000000..f75bfcc --- /dev/null +++ b/bases/bot_detector/api_private/LICENSE @@ -0,0 +1,24 @@ +BSD 2-Clause License + +Copyright (c) 2023, OSRS Bot-Detector + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bases/bot_detector/api_private/Makefile b/bases/bot_detector/api_private/Makefile new file mode 100644 index 0000000..4b473ba --- /dev/null +++ b/bases/bot_detector/api_private/Makefile @@ -0,0 +1,103 @@ +.PHONY: clean clean-test clean-pyc clean-build build help +.DEFAULT_GOAL := help + +define PRINT_HELP_PYSCRIPT +import re, sys + +for line in sys.stdin: + match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) + if match: + target, help = match.groups() + print("%-20s %s" % (target, help)) +endef +export PRINT_HELP_PYSCRIPT + +help: + @python3 -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + +clean-pyc: ## clean python cache files + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + find . -name '.pytest_cache' -exec rm -fr {} + + +clean-test: ## cleanup pytests leftovers + rm -f .coverage + rm -fr htmlcov/ + rm -fr test_results/ + rm -f *report.html + rm -f log.html + rm -f test-results.html + rm -f output.xml + +test: clean ## Run pytest unit tests + python3 -m pytest + +test-debug: ## Run unit tests with debugging enabled + python3 -m pytest --pdb + +test-coverage: clean ## Run unit tests and check code coverage + PYTHONPATH=src python3 -m pytest --cov=src tests/ --disable-warnings + +docker-up: ## Startup docker + docker-compose --verbose up + +docker-build: ## Startup docker + docker-compose --verbose up --build + +setup: requirements pre-commit-setup docker-build test-setup api-setup ## setup & run after downloaded repo + +pre-commit-setup: ## Install pre-commit + python3 -m pip install pre-commit + pre-commit --version + +pre-commit: ## Run pre-commit + pre-commit run --all-files + +test-setup: + python3 -m pip install pytest + python3 -m pip install pytest-benchmark + +list-benchmarks: + pytest-benchmark list + +benchmark: + pytest tests/test_highscore_benchmark.py --benchmark-min-rounds=1000 + +create-venv: + python3 -m venv .venv + source .venv/bin/activate + +requirements: + python3 -m pip install -r requirements.txt + python3 -m pip install pytest-asyncio==0.23.6 + python3 -m pip install httpx==0.27.0 + python3 -m pip install pre-commit==3.6.2 + python3 -m pip install ruff==0.1.15 + pre-commit install + +docker-restart: + docker compose down + docker compose up --build -d + +docker-test: + docker compose down + docker compose up --build -d + pytest + +api-setup: + python3 -m pip install "fastapi[all]" + +env-setup: + touch .env + echo "KAFKA_HOST= 'localhost:9092'" >> .env + echo "DATABASE_URL= 'mysql+aiomysql://root:root_bot_buster@localhost:3307/playerdata'" >> .env + echo "ENV='DEV'" >> .env + echo "POOL_RECYCLE='60'" >> .env + echo "POOL_TIMEOUT='30'" >> .env + +docs: + open http://localhost:5000/docs + xdg-open http://localhost:5000/docs + . http://localhost:5000/docs diff --git a/bases/bot_detector/api_private/README.md b/bases/bot_detector/api_private/README.md new file mode 100644 index 0000000..3fc29b2 --- /dev/null +++ b/bases/bot_detector/api_private/README.md @@ -0,0 +1,44 @@ +# public-api +structure: +- src: contains all program files +- src.api: contains the api +- src.app: contains the database logic +- src.core: contains the core componetns +- src.core.database: contains all the database functionality + +# setup +## windows +creating a python venv to work in and install the project requirements +```sh +python -m venv .venv +.venv\Scripts\activate +python -m pip install --upgrade pip +pip install -r requirements.txt +``` +## linux +make sure to have pip +```sh +sudo apt update -y && sudo apt upgrade -y +sudo apt install python3.10-venv -y +sudo apt install python3-pip -y +``` +```sh +python3 -m venv .venv +source .venv/bin/activate +python3 -m pip install --upgrade pip +pip install -r requirements.txt +``` +# for admin purposes saving & upgrading +when you added some dependancies update the requirements +```sh +.venv\Scripts\activate +call pip freeze > requirements.txt +``` +when you want to upgrade the dependancies +```sh +.venv\Scripts\activate +powershell "(Get-Content requirements.txt) | ForEach-Object { $_ -replace '==', '>=' } | Set-Content requirements.txt" +call pip install -r requirements.txt --upgrade +call pip freeze > requirements.txt +powershell "(Get-Content requirements.txt) | ForEach-Object { $_ -replace '>=', '==' } | Set-Content requirements.txt" +``` \ No newline at end of file diff --git a/bases/bot_detector/api_private/docker-compose-standalone.yml b/bases/bot_detector/api_private/docker-compose-standalone.yml new file mode 100644 index 0000000..a5b5303 --- /dev/null +++ b/bases/bot_detector/api_private/docker-compose-standalone.yml @@ -0,0 +1,65 @@ +version: '3' +services: + mysql: + container_name: database + build: + context: ./mysql + image: bot-detector/mysql:latest + environment: + - MYSQL_ROOT_PASSWORD=root_bot_buster + volumes: + - ./mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + # - ./mysql/mount:/var/lib/mysql # creates persistence + ports: + - 3307:3306 + networks: + - botdetector-network + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + timeout: 20s + retries: 10 + + mysql_setup: + container_name: mysql_setup + image: bot-detector/mysql_setup + build: + context: ./mysql_setup + command: ["python", "-u","setup_mysql.py"] + networks: + - botdetector-network + depends_on: + mysql: + condition: service_healthy + + private_api: + container_name: private_api + build: + context: . + dockerfile: Dockerfile + target: base + args: + root_path: / + api_port: 5000 + # command: bash -c "apt update && apt install -y curl && sleep infinity" + command: uvicorn src.core.server:app --host 0.0.0.0 --reload --reload-include src/* + ports: + - 5000:5000 + networks: + - botdetector-network + # this overrides the env_file for the specific variable + environment: + - KAFKA_HOST=kafka:9092 + - DATABASE_URL=mysql+aiomysql://root:root_bot_buster@mysql:3306/playerdata + - ENV=DEV + - POOL_RECYCLE=60 + - POOL_TIMEOUT=30 + # env_file: + # - .env + volumes: + - ./src:/project/src + depends_on: + - mysql + - mysql_setup + +networks: + botdetector-network: diff --git a/bases/bot_detector/api_private/docker-compose.yml b/bases/bot_detector/api_private/docker-compose.yml new file mode 100644 index 0000000..0451218 --- /dev/null +++ b/bases/bot_detector/api_private/docker-compose.yml @@ -0,0 +1,67 @@ +services: + mysql: + container_name: database + build: + context: ./mysql + image: bot-detector/mysql:latest + environment: + - MYSQL_ROOT_PASSWORD=root_bot_buster + volumes: + - ./mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + # - ./mysql/mount:/var/lib/mysql # creates persistence + ports: + - 3307:3306 + networks: + - botdetector-network + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + timeout: 20s + retries: 10 + + mysql_setup: + container_name: mysql_setup + image: bot-detector/mysql_setup + build: + context: ./mysql_setup + command: ["python", "-u","setup_mysql.py"] + networks: + - botdetector-network + depends_on: + mysql: + condition: service_healthy + + private_api: + container_name: private_api + build: + context: . + dockerfile: Dockerfile + target: base + args: + root_path: / + api_port: 5000 + # command: bash -c "apt update && apt install -y curl && sleep infinity" + command: uvicorn src.core.server:app --host 0.0.0.0 --reload --reload-include src/* + ports: + - 5000:5000 + networks: + - botdetector-network + # this overrides the env_file for the specific variable + environment: + - KAFKA_HOST=kafka:9092 + - DATABASE_URL=mysql+aiomysql://root:root_bot_buster@mysql:3306/playerdata + - ENV=DEV + - POOL_RECYCLE=60 + - POOL_TIMEOUT=30 + # env_file: + # - .env + volumes: + - ./src:/project/src + depends_on: + mysql: + condition: service_healthy + mysql_setup: + condition: service_completed_successfully + + +networks: + botdetector-network: diff --git a/bases/bot_detector/api_private/kafka_setup/.dockerignore b/bases/bot_detector/api_private/kafka_setup/.dockerignore new file mode 100644 index 0000000..121473d --- /dev/null +++ b/bases/bot_detector/api_private/kafka_setup/.dockerignore @@ -0,0 +1 @@ +kafka_data/*.json \ No newline at end of file diff --git a/bases/bot_detector/api_private/kafka_setup/Dockerfile b/bases/bot_detector/api_private/kafka_setup/Dockerfile new file mode 100644 index 0000000..9fec847 --- /dev/null +++ b/bases/bot_detector/api_private/kafka_setup/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Set the KAFKA_BROKER environment variable during container runtime +ENV KAFKA_BROKER=localhost:9094 + +CMD ["python", "setup_kafka.py"] \ No newline at end of file diff --git a/bases/bot_detector/api_private/kafka_setup/kafka_data/.gitkeep b/bases/bot_detector/api_private/kafka_setup/kafka_data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/bases/bot_detector/api_private/kafka_setup/requirements.txt b/bases/bot_detector/api_private/kafka_setup/requirements.txt new file mode 100644 index 0000000..34aa892 --- /dev/null +++ b/bases/bot_detector/api_private/kafka_setup/requirements.txt @@ -0,0 +1 @@ +kafka-python \ No newline at end of file diff --git a/bases/bot_detector/api_private/kafka_setup/setup_kafka.py b/bases/bot_detector/api_private/kafka_setup/setup_kafka.py new file mode 100644 index 0000000..54ce59d --- /dev/null +++ b/bases/bot_detector/api_private/kafka_setup/setup_kafka.py @@ -0,0 +1,92 @@ +# setup_kafka.py +import json +from kafka.admin import NewTopic, KafkaAdminClient +from kafka import KafkaProducer +import os +import zipfile + + +def create_topics(): + # Get the Kafka broker address from the environment variable + kafka_broker = os.environ.get("KAFKA_BROKER", "localhost:9094") + + # Create Kafka topics + admin_client = KafkaAdminClient(bootstrap_servers=kafka_broker) + + topics = admin_client.list_topics() + print("existing topics", topics) + + if not topics == []: + admin_client.delete_topics(topics) + + res = admin_client.create_topics( + [ + NewTopic( + name="player", + num_partitions=3, + replication_factor=1, + ), + NewTopic( + name="scraper", + num_partitions=4, + replication_factor=1, + ), + NewTopic( + name="reports", + num_partitions=4, + replication_factor=1, + ), + ] + ) + + print("created_topic", res) + + topics = admin_client.list_topics() + print("all topics", topics) + return + + +def send_json_to_kafka(file_path, producer, topic): + with open(file_path) as file: + data = json.load(file) + + for record in data: + # record = json.dumps(record).encode("utf-8") + producer.send(topic, value=record) + return + + +def insert_data(): + # Get the Kafka broker address from the environment variable + kafka_broker = os.environ.get("KAFKA_BROKER", "localhost:9094") + + zip_file_path = "kafka_data/kafka_data.zip" + extracted_folder = "kafka_data" + + print("Extracting data from the zip archive...") + # Extract the files from the zip archive + with zipfile.ZipFile(zip_file_path, "r") as zip_ref: + zip_ref.extractall(extracted_folder) + + # Create the Kafka producer + producer = KafkaProducer( + bootstrap_servers=kafka_broker, + value_serializer=lambda x: json.dumps(x).encode(), + ) + + for file_name in os.listdir(extracted_folder): + if file_name.endswith(".json"): + file_path = os.path.join(extracted_folder, file_name) + print(f"Processing file: {file_path}") + send_json_to_kafka(file_path, producer, "player") + + print("Data insertion completed.") + + +def setup_kafka(): + create_topics() + insert_data() + + +if __name__ == "__main__": + setup_kafka() \ No newline at end of file diff --git a/bases/bot_detector/api_private/mysql/Dockerfile b/bases/bot_detector/api_private/mysql/Dockerfile new file mode 100644 index 0000000..659f748 --- /dev/null +++ b/bases/bot_detector/api_private/mysql/Dockerfile @@ -0,0 +1,3 @@ +FROM mysql:latest + +EXPOSE 3306 \ No newline at end of file diff --git a/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/00_init.sql b/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/00_init.sql new file mode 100644 index 0000000..7d82066 --- /dev/null +++ b/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/00_init.sql @@ -0,0 +1 @@ +CREATE DATABASE playerdata; \ No newline at end of file diff --git a/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/01_tables.sql b/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/01_tables.sql new file mode 100644 index 0000000..1ae2a3f --- /dev/null +++ b/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/01_tables.sql @@ -0,0 +1,254 @@ +USE playerdata; + +CREATE TABLE Labels ( + id int NOT NULL AUTO_INCREMENT, + label varchar(50) NOT NULL, + PRIMARY KEY (id), + UNIQUE KEY Unique_label (label) USING BTREE +) +; + +CREATE TABLE Players ( + id INT PRIMARY KEY AUTO_INCREMENT, + name TEXT, + created_at TIMESTAMP, + updated_at TIMESTAMP, + possible_ban BOOLEAN, + confirmed_ban BOOLEAN, + confirmed_player BOOLEAN, + label_id INTEGER, + label_jagex INTEGER, + ironman BOOLEAN, + hardcore_ironman BOOLEAN, + ultimate_ironman BOOLEAN, + normalized_name TEXT +); + +CREATE TABLE Reports ( + ID BIGINT PRIMARY KEY AUTO_INCREMENT, + created_at TIMESTAMP, + reportedID INT, + reportingID INT, + region_id INT, + x_coord INT, + y_coord INT, + z_coord INT, + timestamp TIMESTAMP, + manual_detect SMALLINT, + on_members_world INT, + on_pvp_world SMALLINT, + world_number INT, + equip_head_id INT, + equip_amulet_id INT, + equip_torso_id INT, + equip_legs_id INT, + equip_boots_id INT, + equip_cape_id INT, + equip_hands_id INT, + equip_weapon_id INT, + equip_shield_id INT, + equip_ge_value BIGINT, + CONSTRAINT FK_Reported_Players_id FOREIGN KEY (reportedID) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT, + CONSTRAINT FK_Reporting_Players_id FOREIGN KEY (reportingID) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT +); + + +CREATE TABLE Predictions ( + id INT PRIMARY KEY AUTO_INCREMENT, + name VARCHAR(12), + prediction VARCHAR(50), + created TIMESTAMP, + predicted_confidence DECIMAL(5, 2), + real_player DECIMAL(5, 2) DEFAULT 0, + pvm_melee_bot DECIMAL(5, 2) DEFAULT 0, + smithing_bot DECIMAL(5, 2) DEFAULT 0, + magic_bot DECIMAL(5, 2) DEFAULT 0, + fishing_bot DECIMAL(5, 2) DEFAULT 0, + mining_bot DECIMAL(5, 2) DEFAULT 0, + crafting_bot DECIMAL(5, 2) DEFAULT 0, + pvm_ranged_magic_bot DECIMAL(5, 2) DEFAULT 0, + pvm_ranged_bot DECIMAL(5, 2) DEFAULT 0, + hunter_bot DECIMAL(5, 2) DEFAULT 0, + fletching_bot DECIMAL(5, 2) DEFAULT 0, + clue_scroll_bot DECIMAL(5, 2) DEFAULT 0, + lms_bot DECIMAL(5, 2) DEFAULT 0, + agility_bot DECIMAL(5, 2) DEFAULT 0, + wintertodt_bot DECIMAL(5, 2) DEFAULT 0, + runecrafting_bot DECIMAL(5, 2) DEFAULT 0, + zalcano_bot DECIMAL(5, 2) DEFAULT 0, + woodcutting_bot DECIMAL(5, 2) DEFAULT 0, + thieving_bot DECIMAL(5, 2) DEFAULT 0, + soul_wars_bot DECIMAL(5, 2) DEFAULT 0, + cooking_bot DECIMAL(5, 2) DEFAULT 0, + vorkath_bot DECIMAL(5, 2) DEFAULT 0, + barrows_bot DECIMAL(5, 2) DEFAULT 0, + herblore_bot DECIMAL(5, 2) DEFAULT 0, + unknown_bot DECIMAL(5, 2) DEFAULT 0 +); + +CREATE TABLE playerHiscoreData ( + id bigint NOT NULL AUTO_INCREMENT, + timestamp datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + ts_date date DEFAULT NULL, + Player_id int NOT NULL, + total bigint DEFAULT '0', + attack int DEFAULT '0', + defence int DEFAULT '0', + strength int DEFAULT '0', + hitpoints int DEFAULT '0', + ranged int DEFAULT '0', + prayer int DEFAULT '0', + magic int DEFAULT '0', + cooking int DEFAULT '0', + woodcutting int DEFAULT '0', + fletching int DEFAULT '0', + fishing int DEFAULT '0', + firemaking int DEFAULT '0', + crafting int DEFAULT '0', + smithing int DEFAULT '0', + mining int DEFAULT '0', + herblore int DEFAULT '0', + agility int DEFAULT '0', + thieving int DEFAULT '0', + slayer int DEFAULT '0', + farming int DEFAULT '0', + runecraft int DEFAULT '0', + hunter int DEFAULT '0', + construction int DEFAULT '0', + league int DEFAULT '0', + bounty_hunter_hunter int DEFAULT '0', + bounty_hunter_rogue int DEFAULT '0', + cs_all int DEFAULT '0', + cs_beginner int DEFAULT '0', + cs_easy int DEFAULT '0', + cs_medium int DEFAULT '0', + cs_hard int DEFAULT '0', + cs_elite int DEFAULT '0', + cs_master int DEFAULT '0', + lms_rank int DEFAULT '0', + soul_wars_zeal int DEFAULT '0', + abyssal_sire int DEFAULT '0', + alchemical_hydra int DEFAULT '0', + barrows_chests int DEFAULT '0', + bryophyta int DEFAULT '0', + callisto int DEFAULT '0', + cerberus int DEFAULT '0', + chambers_of_xeric int DEFAULT '0', + chambers_of_xeric_challenge_mode int DEFAULT '0', + chaos_elemental int DEFAULT '0', + chaos_fanatic int DEFAULT '0', + commander_zilyana int DEFAULT '0', + corporeal_beast int DEFAULT '0', + crazy_archaeologist int DEFAULT '0', + dagannoth_prime int DEFAULT '0', + dagannoth_rex int DEFAULT '0', + dagannoth_supreme int DEFAULT '0', + deranged_archaeologist int DEFAULT '0', + general_graardor int DEFAULT '0', + giant_mole int DEFAULT '0', + grotesque_guardians int DEFAULT '0', + hespori int DEFAULT '0', + kalphite_queen int DEFAULT '0', + king_black_dragon int DEFAULT '0', + kraken int DEFAULT '0', + kreearra int DEFAULT '0', + kril_tsutsaroth int DEFAULT '0', + mimic int DEFAULT '0', + nex int DEFAULT '0', + nightmare int DEFAULT '0', + phosanis_nightmare int DEFAULT '0', + obor int DEFAULT '0', + phantom_muspah int DEFAULT '0', + sarachnis int DEFAULT '0', + scorpia int DEFAULT '0', + skotizo int DEFAULT '0', + tempoross int DEFAULT '0', + the_gauntlet int DEFAULT '0', + the_corrupted_gauntlet int DEFAULT '0', + theatre_of_blood int DEFAULT '0', + theatre_of_blood_hard int DEFAULT '0', + thermonuclear_smoke_devil int DEFAULT '0', + tombs_of_amascut int DEFAULT '0', + tombs_of_amascut_expert int DEFAULT '0', + tzkal_zuk int DEFAULT '0', + tztok_jad int DEFAULT '0', + venenatis int DEFAULT '0', + vetion int DEFAULT '0', + vorkath int DEFAULT '0', + wintertodt int DEFAULT '0', + zalcano int DEFAULT '0', + zulrah int DEFAULT '0', + rifts_closed int DEFAULT '0', + artio int DEFAULT '0', + calvarion int DEFAULT '0', + duke_sucellus int DEFAULT '0', + spindel int DEFAULT '0', + the_leviathan int DEFAULT '0', + the_whisperer int DEFAULT '0', + vardorvis int DEFAULT '0', + PRIMARY KEY (id), + UNIQUE KEY idx_playerHiscoreData_Player_id_timestamp (Player_id,timestamp), + UNIQUE KEY Unique_player_date (Player_id,ts_date), + CONSTRAINT FK_Players_id FOREIGN KEY (Player_id) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT +); + +/* +-- V3 +*/ +CREATE TABLE skill ( + skill_id tinyint unsigned NOT NULL AUTO_INCREMENT, + skill_name varchar(50) NOT NULL, + PRIMARY KEY (skill_id), + UNIQUE KEY unique_skill_name (skill_name) +); + +CREATE TABLE activity ( + activity_id tinyint unsigned NOT NULL AUTO_INCREMENT, + activity_name varchar(50) NOT NULL, + PRIMARY KEY (activity_id), + UNIQUE KEY unique_activity_name (activity_name) +); + +CREATE TABLE player_skill ( + player_skill_id BIGINT unsigned NOT NULL AUTO_INCREMENT, + skill_id tinyint unsigned NOT NULL, + skill_value int unsigned NOT NULL DEFAULT '0', + PRIMARY KEY (player_skill_id), + UNIQUE KEY unique_skill_value (skill_id, skill_value) +); + +CREATE TABLE player_activity ( + player_activity_id bigint unsigned NOT NULL AUTO_INCREMENT, + activity_id tinyint unsigned NOT NULL, + activity_value int unsigned NOT NULL DEFAULT '0', + PRIMARY KEY (player_activity_id), + UNIQUE KEY unique_activity_value (activity_id, activity_value) +); + +CREATE TABLE scraper_data_v3 ( + scrape_id bigint unsigned NOT NULL AUTO_INCREMENT, + scrape_ts DATETIME NOT NULL, + scrape_date DATE NOT NULL, + player_id INT NOT NULL, + PRIMARY KEY (scrape_id), + UNIQUE KEY unique_player_scrape (player_id, scrape_date), + INDEX idx_scrape_ts (scrape_ts) +); + +CREATE TABLE scraper_player_skill ( + scrape_id BIGINT unsigned NOT NULL, + player_skill_id BIGINT unsigned NOT NULL, + PRIMARY KEY (scrape_id, player_skill_id), + KEY idx_scrape_id (scrape_id), + KEY idx_player_skill_id (player_skill_id) +) +PARTITION BY HASH (scrape_id) PARTITIONS 10; + +CREATE TABLE scraper_player_activity ( + scrape_id BIGINT unsigned NOT NULL, + player_activity_id BIGINT unsigned NOT NULL, + PRIMARY KEY (scrape_id, player_activity_id), + KEY idx_scrape_id (scrape_id), + KEY idx_player_activity_id (player_activity_id) +) +PARTITION BY HASH (scrape_id) PARTITIONS 10; diff --git a/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/02_data.sql b/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/02_data.sql new file mode 100644 index 0000000..6f088ea --- /dev/null +++ b/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/02_data.sql @@ -0,0 +1,73 @@ +USE playerdata; + +INSERT INTO Labels (id, label) +VALUES + (1, 'Real_Player'), + (4, 'Wintertodt_bot'), + (5, 'Mining_bot'), + (7, 'Hunter_bot'), + (8, 'Herblore_bot'), + (9, 'Fletching_bot'), + (10, 'Fishing_bot'), + (11, 'Crafting_bot'), + (12, 'Cooking_bot'), + (13, 'Woodcutting_bot'), + (15, 'Smithing_bot'), + (17, 'Magic_bot'), + (19, 'PVM_Ranged_Magic_bot'), + (21, 'Agility_bot'), + (27, 'Zalcano_bot'), + (38, 'Runecrafting_bot'), + (40, 'PVM_Ranged_bot'), + (41, 'PVM_Melee_bot'), + (42, 'Thieving_bot'), + (52, 'LMS_bot'), + (56, 'Fishing_Cooking_bot'), + (57, 'mort_myre_fungus_bot'), + (59, 'temp_real_player'), + (61, 'Soul_Wars_bot'), + (64, 'Construction_Magic_bot'), + (65, 'Vorkath_bot'), + (66, 'Clue_Scroll_bot'), + (67, 'Barrows_bot'), + (76, 'Woodcutting_Mining_bot'), + ( + 77, 'Woodcutting_Firemaking_bot' + ), + (84, 'Mage_Guild_Store_bot'), + (87, 'Phosani_bot'), + (89, 'Unknown_bot'), + (90, 'Blast_mine_bot'), + (91, 'Zulrah_bot'), + (92, 'test_label'), + (109, 'Nex_bot'), + (110, 'Gauntlet_bot'); + +INSERT INTO Labels (label) VALUES ("Unkown"); +UPDATE Labels set id=0 where label="Unkown"; + + +INSERT INTO skill (skill_name) VALUES +('attack'), ('defence'), ('strength'), ('hitpoints'), ('ranged'), ('prayer'), +('magic'), ('cooking'), ('woodcutting'), ('fletching'), ('fishing'), ('firemaking'), +('crafting'), ('smithing'), ('mining'), ('herblore'), ('agility'), ('thieving'), +('slayer'), ('farming'), ('runecraft'), ('hunter'), ('construction') +; + +INSERT INTO activity (activity_name) VALUES +('league'), ('bounty_hunter_hunter'), ('bounty_hunter_rogue'), ('cs_all'), ('cs_beginner'), +('cs_easy'), ('cs_medium'), ('cs_hard'), ('cs_elite'), ('cs_master'), ('lms_rank'), +('soul_wars_zeal'), ('abyssal_sire'), ('alchemical_hydra'), ('barrows_chests'), ('bryophyta'), +('callisto'), ('cerberus'), ('chambers_of_xeric'), ('chambers_of_xeric_challenge_mode'), +('chaos_elemental'), ('chaos_fanatic'), ('commander_zilyana'), ('corporeal_beast'), +('crazy_archaeologist'), ('dagannoth_prime'), ('dagannoth_rex'), ('dagannoth_supreme'), +('deranged_archaeologist'), ('general_graardor'), ('giant_mole'), ('grotesque_guardians'), +('hespori'), ('kalphite_queen'), ('king_black_dragon'), ('kraken'), ('kreearra'), +('kril_tsutsaroth'), ('mimic'), ('nightmare'), ('nex'), ('phosanis_nightmare'), ('obor'), +('phantom_muspah'), ('sarachnis'), ('scorpia'), ('skotizo'), ('tempoross'), ('the_gauntlet'), +('the_corrupted_gauntlet'), ('theatre_of_blood'), ('theatre_of_blood_hard'), +('thermonuclear_smoke_devil'), ('tombs_of_amascut'), ('tombs_of_amascut_expert'), ('tzkal_zuk'), +('tztok_jad'), ('venenatis'), ('vetion'), ('vorkath'), ('wintertodt'), ('zalcano'), ('zulrah'), +('rifts_closed'), ('artio'), ('calvarion'), ('duke_sucellus'), ('spindel'), ('the_leviathan'), +('the_whisperer'), ('vardorvis') +; diff --git a/bases/bot_detector/api_private/mysql_setup/Dockerfile b/bases/bot_detector/api_private/mysql_setup/Dockerfile new file mode 100644 index 0000000..014cd75 --- /dev/null +++ b/bases/bot_detector/api_private/mysql_setup/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD ["python", "setup_mysql.py"] diff --git a/bases/bot_detector/api_private/mysql_setup/__init__.py b/bases/bot_detector/api_private/mysql_setup/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bases/bot_detector/api_private/mysql_setup/models.py b/bases/bot_detector/api_private/mysql_setup/models.py new file mode 100644 index 0000000..a755e20 --- /dev/null +++ b/bases/bot_detector/api_private/mysql_setup/models.py @@ -0,0 +1,179 @@ +# script to insert all the data we need +import random +from datetime import datetime + +from sqlalchemy import ( + TIMESTAMP, + BigInteger, + Boolean, + Column, + Date, + DateTime, + ForeignKey, + Index, + Integer, + SmallInteger, + String, + UniqueConstraint, + create_engine, +) +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +Base = declarative_base() +random.seed(42) + +# Define other SQLAlchemy models for remaining tables in a similar manner + +# Create an engine and bind the base +engine = create_engine("mysql+pymysql://root:root_bot_buster@mysql:3306/playerdata") +Base.metadata.create_all(engine) + +# Create a session +Session = sessionmaker(bind=engine) +session = Session() + + +class Labels(Base): + __tablename__ = "Labels" + + id = Column(Integer, primary_key=True) + label = Column(String) + + +class Players(Base): + __tablename__ = "Players" + + id = Column(Integer, primary_key=True) + name = Column(String) + created_at = Column(DateTime, default=datetime.utcnow) + updated_at = Column(DateTime, default=datetime.utcnow) + possible_ban = Column(Boolean, default=True) + confirmed_ban = Column(Boolean, default=False) + confirmed_player = Column(Boolean, default=False) + label_id = Column(Integer) + label_jagex = Column(Integer) + # ironman = Column(Boolean) + # hardcore_ironman = Column(Boolean) + # ultimate_ironman = Column(Boolean) + normalized_name = Column(String) + + +class Report(Base): + __tablename__ = "Reports" + + ID = Column(BigInteger, primary_key=True, autoincrement=True) + created_at = Column(TIMESTAMP) + reportedID = Column(Integer) + reportingID = Column(Integer) + region_id = Column(Integer) + x_coord = Column(Integer) + y_coord = Column(Integer) + z_coord = Column(Integer) + timestamp = Column(TIMESTAMP) + manual_detect = Column(SmallInteger) + on_members_world = Column(Integer) + on_pvp_world = Column(SmallInteger) + world_number = Column(Integer) + equip_head_id = Column(Integer) + equip_amulet_id = Column(Integer) + equip_torso_id = Column(Integer) + equip_legs_id = Column(Integer) + equip_boots_id = Column(Integer) + equip_cape_id = Column(Integer) + equip_hands_id = Column(Integer) + equip_weapon_id = Column(Integer) + equip_shield_id = Column(Integer) + equip_ge_value = Column(BigInteger) + + +class Skill(Base): + __tablename__ = "skill" + + skill_id = Column(SmallInteger(), primary_key=True, autoincrement=True) + skill_name = Column(String(50), nullable=False, unique=True) + + +class Activity(Base): + __tablename__ = "activity" + + activity_id = Column(SmallInteger(), primary_key=True, autoincrement=True) + activity_name = Column(String(50), nullable=False, unique=True) + + +class PlayerSkill(Base): + __tablename__ = "player_skill" + + player_skill_id = Column(BigInteger(), primary_key=True, autoincrement=True) + skill_id = Column(SmallInteger(), nullable=False) + skill_value = Column(Integer(), nullable=False, default=0) + + __table_args__ = ( + UniqueConstraint("skill_id", "skill_value", name="unique_skill_value"), + ) + + +class PlayerActivity(Base): + __tablename__ = "player_activity" + + player_activity_id = Column(BigInteger(), primary_key=True, autoincrement=True) + activity_id = Column(SmallInteger(), nullable=False) + activity_value = Column(Integer(), nullable=False, default=0) + + __table_args__ = ( + UniqueConstraint("activity_id", "activity_value", name="unique_activity_value"), + ) + + +class ScraperDataV3(Base): + __tablename__ = "scraper_data_v3" + + scrape_id = Column(BigInteger(), primary_key=True, autoincrement=True) + scrape_ts = Column(DateTime, nullable=False) + scrape_date = Column(Date, nullable=False) + player_id = Column(Integer, nullable=False) + + __table_args__ = ( + UniqueConstraint("player_id", "scrape_date", name="unique_player_scrape"), + Index("idx_scrape_ts", "scrape_ts"), + ) + + +class ScraperPlayerSkill(Base): + __tablename__ = "scraper_player_skill" + + scrape_id = Column( + BigInteger(), + ForeignKey("scraper_data_v3.scrape_id"), + primary_key=True, + ) + player_skill_id = Column( + BigInteger(), + ForeignKey("player_skill.player_skill_id"), + primary_key=True, + ) + + __table_args__ = ( + Index("idx_scrape_id", "scrape_id"), + Index("idx_player_skill_id", "player_skill_id"), + ) + + +class ScraperPlayerActivity(Base): + __tablename__ = "scraper_player_activity" + + scrape_id = Column( + BigInteger(), + ForeignKey("scraper_data_v3.scrape_id"), + primary_key=True, + ) + player_activity_id = Column( + BigInteger(), + ForeignKey("player_activity.player_activity_id"), + primary_key=True, + ) + + __table_args__ = ( + Index("idx_scrape_id", "scrape_id"), + Index("idx_player_activity_id", "player_activity_id"), + ) diff --git a/bases/bot_detector/api_private/mysql_setup/requirements.txt b/bases/bot_detector/api_private/mysql_setup/requirements.txt new file mode 100644 index 0000000..1ab8080 --- /dev/null +++ b/bases/bot_detector/api_private/mysql_setup/requirements.txt @@ -0,0 +1,8 @@ +aiomysql==0.2.0 +cffi==1.16.0 +cryptography==42.0.5 +greenlet==3.0.3 +pycparser==2.22 +PyMySQL==1.1.0 +SQLAlchemy==2.0.28 +typing_extensions==4.10.0 diff --git a/bases/bot_detector/api_private/mysql_setup/setup_mysql.py b/bases/bot_detector/api_private/mysql_setup/setup_mysql.py new file mode 100644 index 0000000..30e5931 --- /dev/null +++ b/bases/bot_detector/api_private/mysql_setup/setup_mysql.py @@ -0,0 +1,108 @@ +import random +from datetime import datetime, timedelta + +from models import ( + Labels, + PlayerActivity, + Players, + PlayerSkill, + Report, + ScraperDataV3, + ScraperPlayerActivity, + ScraperPlayerSkill, + session, +) +from sqlalchemy.exc import IntegrityError + + +def random_date(): + return datetime.utcnow() - timedelta(days=random.randint(0, 365)) + + +def get_labels(): + # Query the labels table to get all id values + label_ids = session.query(Labels.id).all() + label_ids = [id[0] for id in label_ids] # Convert list of tuples to list of ids + return label_ids + + +def insert_players(len_players, label_ids: list): + # Insert data into Players table + for i in range(len_players): + print(f"Player_{i}") + # Check if the player already exists + existing_player = session.query(Players).filter_by(name=f"Player_{i}").first() + if not existing_player: + player = Players( + name=f"Player_{i}", + created_at=random_date(), + updated_at=random_date(), + possible_ban=random.choice([True, False]), + confirmed_ban=random.choice([True, False]), + confirmed_player=random.choice([True, False]), + label_id=random.choice(label_ids), # Select a random id from label_ids + label_jagex=random.randint(0, 2), + normalized_name=f"Player_{i}", + ) + session.add(player) + session.commit() + return + + +def insert_reports(len_reports, len_players): + for i in range(1, len_reports + 1): + print(f"Report_{i}") + # pick random player + reporter = random.randint(1, len_players) + reported = random.randint(1, len_players) + + if reporter == reported: + reported = random.randint(1, len_players) + + try: + session.add( + Report( + created_at=random_date(), + reportedID=reporter, + reportingID=reported, + region_id=random.randint(1, 30000), + x_coord=random.randint(1, 30000), + y_coord=random.randint(1, 30000), + z_coord=random.randint(1, 30000), + timestamp=random_date(), + manual_detect=random.choice([0, 1]), + on_members_world=random.choice([0, 1]), + on_pvp_world=random.choice([0, 1]), + world_number=random.randint(1, 300), + equip_head_id=random.randint(1, 30000), + equip_amulet_id=random.randint(1, 30000), + equip_torso_id=random.randint(1, 30000), + equip_legs_id=random.randint(1, 30000), + equip_boots_id=random.randint(1, 30000), + equip_cape_id=random.randint(1, 30000), + equip_hands_id=random.randint(1, 30000), + equip_weapon_id=random.randint(1, 30000), + equip_shield_id=random.randint(1, 30000), + equip_ge_value=random.randint(1, 2000000000), + ) + ) + except IntegrityError: + session.rollback() # Rollback the transaction if a duplicate entry is encountered + finally: + session.commit() + + +def generate_random_scraper_data(len_scrapers, len_players, skill_ids, activity_ids): + # TODO: + ... + + +def main(): + len_players = 250 + label_ids = get_labels() + insert_players(len_players, label_ids) + insert_reports(len_reports=10_000, len_players=len_players) + + +if __name__ == "__main__": + main() diff --git a/bases/bot_detector/api_private/notes.md b/bases/bot_detector/api_private/notes.md new file mode 100644 index 0000000..9d5da08 --- /dev/null +++ b/bases/bot_detector/api_private/notes.md @@ -0,0 +1,40 @@ +# kubectl +```sh +kubectl port-forward -n kafka svc/bd-prd-kafka-service 9094:9094 +kubectl port-forward -n database svc/mysql 3306:3306 +kubectl port-forward -n bd-prd svc/private-api-svc 5000:5000 +``` + +```sh +python -m venv .venv +.venv\Scripts\activate +python -m pip install --upgrade pip +pip install -r requirements.txt +``` + +```sh +.venv\Scripts\activate +pip freeze > requirements.txt +``` + +# linux / wsl +to open vscode in wsl just open vs code, type `wsl` in the terminal than type `code .` + +tip: you can trim your command line path with +```sh +nano ~/.bashrc +``` +add at the botom, exit nano with ctrl + x then press y to save +```sh +PROMPT_DIRTRIM=3 +``` +restart your shell + +```sh +sudo apt update -y && sudo apt upgrade -y +sudo apt install python3.10-venv -y +``` +```sh +python3 -m venv .venv +touch .venv\bin\activate +``` diff --git a/bases/bot_detector/api_private/requirements.txt b/bases/bot_detector/api_private/requirements.txt new file mode 100644 index 0000000..669de9a --- /dev/null +++ b/bases/bot_detector/api_private/requirements.txt @@ -0,0 +1,33 @@ +aiokafka==0.10.0 +aiomysql==0.2.0 +annotated-types==0.6.0 +anyio==4.3.0 +async-timeout==4.0.3 +asyncmy==0.2.9 +cffi==1.16.0 +click==8.1.7 +colorama==0.4.6 +cryptography==42.0.5 +databases==0.9.0 +exceptiongroup==1.2.0 +fastapi==0.110.0 +greenlet==3.0.3 +h11==0.14.0 +httptools==0.6.1 +idna==3.7 +kafka-python==2.0.2 +packaging==24.0 +pycparser==2.21 +pydantic==2.6.4 +pydantic-settings==2.2.1 +pydantic_core==2.16.3 +PyMySQL==1.1.0 +python-dotenv==1.0.1 +PyYAML==6.0.1 +sniffio==1.3.1 +SQLAlchemy==2.0.28 +starlette==0.36.3 +typing_extensions==4.10.0 +uvicorn==0.29.0 +watchfiles==0.21.0 +websockets==12.0 diff --git a/bases/bot_detector/api_private/src/api/__init__.py b/bases/bot_detector/api_private/src/api/__init__.py new file mode 100644 index 0000000..3e8e7d0 --- /dev/null +++ b/bases/bot_detector/api_private/src/api/__init__.py @@ -0,0 +1,7 @@ +from fastapi import APIRouter + +from . import v2, v3 + +router = APIRouter() +router.include_router(v2.router, prefix="/v2") +router.include_router(v3.router, prefix="/v3") diff --git a/bases/bot_detector/api_private/src/api/readme.md b/bases/bot_detector/api_private/src/api/readme.md new file mode 100644 index 0000000..0ff400d --- /dev/null +++ b/bases/bot_detector/api_private/src/api/readme.md @@ -0,0 +1 @@ +the api folder can be considered the controller in the MVC approach \ No newline at end of file diff --git a/bases/bot_detector/api_private/src/api/v1/health.py b/bases/bot_detector/api_private/src/api/v1/health.py new file mode 100644 index 0000000..e69de29 diff --git a/bases/bot_detector/api_private/src/api/v2/__init__.py b/bases/bot_detector/api_private/src/api/v2/__init__.py new file mode 100644 index 0000000..b8424c4 --- /dev/null +++ b/bases/bot_detector/api_private/src/api/v2/__init__.py @@ -0,0 +1,6 @@ +from fastapi import APIRouter +from . import player, highscore + +router = APIRouter() +router.include_router(player.router) +router.include_router(highscore.router) diff --git a/bases/bot_detector/api_private/src/api/v2/highscore.py b/bases/bot_detector/api_private/src/api/v2/highscore.py new file mode 100644 index 0000000..8a509ae --- /dev/null +++ b/bases/bot_detector/api_private/src/api/v2/highscore.py @@ -0,0 +1,74 @@ +import logging + +from fastapi import APIRouter, Depends, Query + +from src.app.repositories import PlayerActivityRepo, PlayerSkillsRepo, ScraperDataRepo +from src.app.views.response.highscore import PlayerHiscoreData + +# from src.app.repositories.highscore import HighscoreRepo +from src.core.fastapi.dependencies.session import get_session + +logger = logging.getLogger(__name__) + + +router = APIRouter() + + +# @router.get("/highscore/latest", response_model=list[PlayerHiscoreData]) +# async def get_highscore_latest( +# player_id: int, +# label_id: int = None, +# many: bool = False, +# limit: int = Query(default=10, ge=0, le=10_000), +# session=Depends(get_session), +# ): +# repo = HighscoreRepo(session=session) +# data: list[dict] = await repo.select( +# player_id=player_id, label_id=label_id, many=many, limit=limit +# ) + +# data = [{k: v for k, v in d.items() if v} for d in data] +# return [PlayerHiscoreData(**d).model_dump(mode="json") for d in data] + + +@router.get("/highscore/latest") +async def get_highscore_latest_v2( + player_id: int, + player_name: str = None, + label_id: int = None, + many: bool = False, + limit: int = Query(default=10, ge=0, le=10_000), + session=Depends(get_session), +): + repo = ScraperDataRepo(session=session) + repo_skills = PlayerSkillsRepo(session=session) + repo_activities = PlayerActivityRepo(session=session) + + data = await repo.select( + player_name=player_name, + player_id=player_id, + label_id=label_id, + many=many, + limit=limit, + history=False, + ) + + logger.info(data[0]) + for d in data: + scraper_id = d.pop("scraper_id") + d["Player_id"] = d.pop("player_id") + d["id"] = scraper_id + d["timestamp"] = d.pop("created_at") + d["ts_date"] = d.pop("record_date") + + skills = await repo_skills.select(scraper_id=scraper_id) + activities = await repo_activities.select(scraper_id=scraper_id) + + for skill in skills: + d[skill.get("skill_name")] = skill.get("skill_value") + + for activity in activities: + d[activity.get("activity_name")] = activity.get("activity_value") + + data = [{k: v for k, v in d.items() if v} for d in data] + return [PlayerHiscoreData(**d).model_dump(mode="json") for d in data] diff --git a/bases/bot_detector/api_private/src/api/v2/player.py b/bases/bot_detector/api_private/src/api/v2/player.py new file mode 100644 index 0000000..743a58b --- /dev/null +++ b/bases/bot_detector/api_private/src/api/v2/player.py @@ -0,0 +1,28 @@ +from fastapi import APIRouter, Depends, Query + +from src.app.repositories.player import PlayerRepo +from src.core.fastapi.dependencies.session import get_session + +router = APIRouter() + + +@router.get("/player") +async def get_player( + player_id: str = None, + player_name: str = None, + label_id: int = None, + greater_than: bool = False, + limit: int = Query(default=1_000, ge=0, le=100_000), + session=Depends(get_session), +): + # TODO: make use of abstract base class + repo = PlayerRepo(session=session) + + data = await repo.select( + player_id=player_id, + player_name=player_name, + greater_than=greater_than, + label_id=label_id, + limit=limit, + ) + return data diff --git a/bases/bot_detector/api_private/src/api/v3/__init__.py b/bases/bot_detector/api_private/src/api/v3/__init__.py new file mode 100644 index 0000000..8d6dbdc --- /dev/null +++ b/bases/bot_detector/api_private/src/api/v3/__init__.py @@ -0,0 +1,6 @@ +from fastapi import APIRouter + +from . import highscore + +router = APIRouter() +router.include_router(highscore.router) diff --git a/bases/bot_detector/api_private/src/api/v3/highscore.py b/bases/bot_detector/api_private/src/api/v3/highscore.py new file mode 100644 index 0000000..466446f --- /dev/null +++ b/bases/bot_detector/api_private/src/api/v3/highscore.py @@ -0,0 +1,73 @@ +import logging +from collections import defaultdict + +from fastapi import APIRouter, Depends, Query + +from src.app.repositories import ScraperDataRepo +from src.app.views.response import ActivityView, ScraperDataView, SkillView +from src.core.fastapi.dependencies.session import get_session + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +def convert_to_scraper_data_view(result_list: list[dict]) -> list[ScraperDataView]: + # Dictionary to hold grouped data by scraper_id + scraper_data_map = defaultdict(lambda: {"skills": [], "activities": []}) + + for row in result_list: + scraper_id = row["scrape_id"] + scraper_data = scraper_data_map[scraper_id] + + # Set shared attributes only once per scraper_id + if "created_at" not in scraper_data: + scraper_data["created_at"] = row["scrape_ts"] + scraper_data["record_date"] = row["scrape_date"] + scraper_data["scraper_id"] = scraper_id + scraper_data["player_id"] = row["player_id"] + scraper_data["player_name"] = row["player_name"] + + # Append to skills or activities based on hs_type + if row["hs_type"] == "skill": + scraper_data["skills"].append( + SkillView(skill_name=row["hs_name"], skill_value=row["hs_value"]) + ) + elif row["hs_type"] == "activity": + scraper_data["activities"].append( + ActivityView( + activity_name=row["hs_name"], activity_value=row["hs_value"] + ) + ) + + # Convert the grouped data into ScraperDataView instances + return [ + ScraperDataView( + created_at=data["created_at"], + record_date=data["record_date"], + scraper_id=data["scraper_id"], + player_id=data["player_id"], + player_name=data["player_name"], + skills=data["skills"], + activities=data["activities"], + ) + for data in scraper_data_map.values() + ] + + +@router.get("/highscore/latest", response_model=list[ScraperDataView]) +async def get_highscore_latest( + player_id: int, + label_id: int = None, + many: bool = False, + limit: int = Query(default=10, ge=0, le=10_000), + session=Depends(get_session), +): + repo = ScraperDataRepo(session=session) + data = await repo.select_latest_scraper_data_v3( + player_id=player_id, + label_id=label_id, + many=many, + limit=limit, + ) + return convert_to_scraper_data_view(result_list=data) diff --git a/bases/bot_detector/api_private/src/app/.gitkeep b/bases/bot_detector/api_private/src/app/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/bases/bot_detector/api_private/src/app/readme.md b/bases/bot_detector/api_private/src/app/readme.md new file mode 100644 index 0000000..64b97ea --- /dev/null +++ b/bases/bot_detector/api_private/src/app/readme.md @@ -0,0 +1,6 @@ +the model is responsible for all the data handeling +- getting data from the database +- handles data logic + +the view is responsible for the data representation +- return format etc \ No newline at end of file diff --git a/bases/bot_detector/api_private/src/app/repositories/.gitkeep b/bases/bot_detector/api_private/src/app/repositories/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/bases/bot_detector/api_private/src/app/repositories/__init__.py b/bases/bot_detector/api_private/src/app/repositories/__init__.py new file mode 100644 index 0000000..f55ad84 --- /dev/null +++ b/bases/bot_detector/api_private/src/app/repositories/__init__.py @@ -0,0 +1,15 @@ +from .abstract_repo import AbstractAPI +from .highscore import HighscoreRepo +from .player import PlayerRepo +from .player_activities import PlayerActivityRepo +from .player_skills import PlayerSkillsRepo +from .scraper_data import ScraperDataRepo + +__all__ = [ + "HighscoreRepo", + "PlayerRepo", + "PlayerSkillsRepo", + "ScraperDataRepo", + "AbstractAPI", + "PlayerActivityRepo", +] diff --git a/bases/bot_detector/api_private/src/app/repositories/abstract_repo.py b/bases/bot_detector/api_private/src/app/repositories/abstract_repo.py new file mode 100644 index 0000000..ebc0ca4 --- /dev/null +++ b/bases/bot_detector/api_private/src/app/repositories/abstract_repo.py @@ -0,0 +1,19 @@ +from abc import ABC, abstractmethod + + +class AbstractAPI(ABC): + @abstractmethod + def insert(self): + raise NotImplementedError + + @abstractmethod + def select(self): + raise NotImplementedError + + @abstractmethod + def update(self): + raise NotImplementedError + + @abstractmethod + def delete(self): + raise NotImplementedError diff --git a/bases/bot_detector/api_private/src/app/repositories/highscore.py b/bases/bot_detector/api_private/src/app/repositories/highscore.py new file mode 100644 index 0000000..9dadedb --- /dev/null +++ b/bases/bot_detector/api_private/src/app/repositories/highscore.py @@ -0,0 +1,55 @@ +import logging + +from fastapi.encoders import jsonable_encoder +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import Select + +from src.app.repositories.abstract_repo import AbstractAPI +from src.core.database.models import ( # playerHiscoreData,; PlayerHiscoreDataXPChange, + PlayerHiscoreDataLatest, +) +from src.core.database.models.player import Player + +logger = logging.getLogger(__name__) + + +class HighscoreRepo(AbstractAPI): + def __init__(self, session) -> None: + super().__init__() + self.session: AsyncSession = session + + def insert(self): + raise NotImplementedError + + async def select( + self, player_id: int, label_id: int, limit: int, many: bool + ) -> dict: + table = aliased(PlayerHiscoreDataLatest, name="phd") + player = aliased(Player, name="pl") + + sql = Select(player.name, table) + sql = sql.join(target=player, onclause=table.Player_id == player.id) + + if player_id: + if many: + sql = sql.where(table.Player_id >= player_id) + else: + sql = sql.where(table.Player_id == player_id) + + if label_id: + sql = sql.where(player.label_id == label_id) + + sql = sql.limit(limit) + + async with self.session: + result: AsyncResult = await self.session.execute(sql) + result = result.fetchall() + data = [{"name": name, **jsonable_encoder(hs)} for name, hs in result] + return data + + async def update(self): + raise NotImplementedError + + async def delete(self): + raise NotImplementedError diff --git a/bases/bot_detector/api_private/src/app/repositories/player.py b/bases/bot_detector/api_private/src/app/repositories/player.py new file mode 100644 index 0000000..32e0823 --- /dev/null +++ b/bases/bot_detector/api_private/src/app/repositories/player.py @@ -0,0 +1,41 @@ +from fastapi.encoders import jsonable_encoder +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.sql.expression import Select + +from src.core.database.models.player import Player + + +class PlayerRepo: + def __init__(self, session: AsyncSession) -> None: + self.session = session + + async def select( + self, + player_id: int, + player_name: str, + label_id: int, + greater_than: bool, + limit: int = 1_000, + ): + table = Player + sql = Select(table) + + if player_name: + sql = sql.where(table.name == player_name) + + if label_id: + sql = sql.where(table.label_id == label_id) + + if player_id: + if greater_than: + sql = sql.where(table.id >= player_id) + else: + sql = sql.where(table.id == player_id) + + sql = sql.order_by(table.id.asc()) + sql = sql.limit(limit) + + async with self.session: + result: AsyncResult = await self.session.execute(sql) + result = result.scalars().all() + return jsonable_encoder(result) diff --git a/bases/bot_detector/api_private/src/app/repositories/player_activities.py b/bases/bot_detector/api_private/src/app/repositories/player_activities.py new file mode 100644 index 0000000..b6c0993 --- /dev/null +++ b/bases/bot_detector/api_private/src/app/repositories/player_activities.py @@ -0,0 +1,49 @@ +from fastapi.encoders import jsonable_encoder +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import Select + +from src.app.repositories.abstract_repo import AbstractAPI +from src.core.database.models import Activities, PlayerActivities + + +class PlayerActivityRepo(AbstractAPI): + def __init__(self, session: AsyncSession) -> None: + super().__init__() + self.session = session + + async def insert(self, id): + raise NotImplementedError + + async def select( + self, + scraper_id: int = None, + activity_id: int = None, + limit: int = None, + ): + table = aliased(PlayerActivities, name="pa") + + sql = Select(Activities.activity_name, table) + + if scraper_id: + sql = sql.where(table.scraper_id == scraper_id) + + if activity_id: + sql = sql.where(table.activity_id == activity_id) + + if limit: + sql = sql.limit(limit) + + sql = sql.join(Activities, table.activity_id == Activities.activity_id) + + async with self.session: + result: AsyncResult = await self.session.execute(sql) + result = result.fetchall() + data = [{"activity_name": name, **jsonable_encoder(hs)} for name, hs in result] + return data + + async def update(self): + raise NotImplementedError + + async def delete(self): + raise NotImplementedError diff --git a/bases/bot_detector/api_private/src/app/repositories/player_skills.py b/bases/bot_detector/api_private/src/app/repositories/player_skills.py new file mode 100644 index 0000000..ea9705d --- /dev/null +++ b/bases/bot_detector/api_private/src/app/repositories/player_skills.py @@ -0,0 +1,49 @@ +from fastapi.encoders import jsonable_encoder +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import Select + +from src.app.repositories.abstract_repo import AbstractAPI +from src.core.database.models import PlayerSkills, Skills + + +class PlayerSkillsRepo(AbstractAPI): + def __init__(self, session: AsyncSession) -> None: + super().__init__() + self.session = session + + async def insert(self, id): + raise NotImplementedError + + async def select( + self, + scraper_id: int = None, + skill_id: int = None, + limit: int = None, + ): + table = aliased(PlayerSkills, name="ps") + + sql = Select(Skills.skill_name, table) + + if scraper_id: + sql = sql.where(table.scraper_id == scraper_id) + + if skill_id: + sql = sql.where(table.skill_id == skill_id) + + if limit: + sql = sql.limit(limit) + + sql = sql.join(Skills, table.skill_id == Skills.skill_id) + + async with self.session: + result: AsyncResult = await self.session.execute(sql) + result = result.fetchall() + data = [{"skill_name": name, **jsonable_encoder(hs)} for name, hs in result] + return data + + async def update(self): + raise NotImplementedError + + async def delete(self): + raise NotImplementedError diff --git a/bases/bot_detector/api_private/src/app/repositories/scraper_data.py b/bases/bot_detector/api_private/src/app/repositories/scraper_data.py new file mode 100644 index 0000000..56ba093 --- /dev/null +++ b/bases/bot_detector/api_private/src/app/repositories/scraper_data.py @@ -0,0 +1,129 @@ +from sqlalchemy import func, literal, select, union_all +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import aliased + +from src.core.database.models.player import Player +from src.core.database.models.scraper_data_v3 import ( + Activity, + PlayerActivity, + PlayerSkill, + ScraperDataV3, + ScraperPlayerActivity, + ScraperPlayerSkill, + Skill, +) + + +class ScraperDataRepo: + def __init__(self, session: AsyncSession) -> None: + self.session = session + + async def select_latest_scraper_data_v3( + self, + player_id: int = None, + label_id: int = None, + many: bool = True, + limit: int = 1000, + ): + # Aliases for tables + SDV = aliased(ScraperDataV3) + P = aliased(Player) + + # skill specific + SPS = aliased(ScraperPlayerSkill) + PS = aliased(PlayerSkill) + S = aliased(Skill) + + # activity specific + SPA = aliased(ScraperPlayerActivity) + PA = aliased(PlayerActivity) + A = aliased(Activity) + + # Subquery to get the latest scrape date for each player + subquery = ( + select(func.max(SDV.scrape_date).label("max_scrape_date"), SDV.player_id) + .join(P, SDV.player_id == P.id) + .group_by(SDV.player_id) + ) + + if player_id: + if many: + subquery = subquery.where(P.id >= player_id) + else: + subquery = subquery.where(P.id == player_id) + if label_id: + subquery = subquery.where(P.label_id == label_id) + + subquery = subquery.limit(limit) + subquery = subquery.subquery() + + # Skill query + skill_query = ( + select( + SDV.scrape_id, + SDV.scrape_ts, + SDV.scrape_date, + SDV.player_id, + P.name.label("player_name"), + S.skill_id.label("hs_id"), + S.skill_name.label("hs_name"), + PS.skill_value.label("hs_value"), + literal("skill").label("hs_type"), + ) + .select_from(SDV) + .join( + subquery, + (subquery.c.max_scrape_date == SDV.scrape_date) + & (subquery.c.player_id == SDV.player_id), + ) + .join(P, SDV.player_id == P.id) + .join(SPS, SDV.scrape_id == SPS.scrape_id) + .join(PS, SPS.player_skill_id == PS.player_skill_id) + .join(S, PS.skill_id == S.skill_id) + ) + + # Activity query + activity_query = ( + select( + SDV.scrape_id, + SDV.scrape_ts, + SDV.scrape_date, + SDV.player_id, + P.name.label("player_name"), + A.activity_id.label("hs_id"), + A.activity_name.label("hs_name"), + PA.activity_value.label("hs_value"), + literal("activity").label("hs_type"), + ) + .select_from(SDV) + .join( + subquery, + (subquery.c.max_scrape_date == SDV.scrape_date) + & (subquery.c.player_id == SDV.player_id), + ) + .join(P, SDV.player_id == P.id) + .join(SPA, SDV.scrape_id == SPA.scrape_id) + .join(PA, SPA.player_activity_id == PA.player_activity_id) + .join(A, PA.activity_id == A.activity_id) + ) + + # Combine skill and activity queries using union_all + combined_query = union_all(skill_query, activity_query) + + # Wrap the combined_query in a new select statement to apply additional filters + final_query = select( + combined_query.c.scrape_id, + combined_query.c.scrape_ts, + combined_query.c.scrape_date, + combined_query.c.player_id, + combined_query.c.player_name, + combined_query.c.hs_id, + combined_query.c.hs_name, + combined_query.c.hs_value, + combined_query.c.hs_type, + ).select_from(combined_query) + + # Execute the final query + result = await self.session.execute(final_query) + result_list = result.mappings().all() + return result_list diff --git a/bases/bot_detector/api_private/src/app/views/input/__init__.py b/bases/bot_detector/api_private/src/app/views/input/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bases/bot_detector/api_private/src/app/views/response/__init__.py b/bases/bot_detector/api_private/src/app/views/response/__init__.py new file mode 100644 index 0000000..31ca380 --- /dev/null +++ b/bases/bot_detector/api_private/src/app/views/response/__init__.py @@ -0,0 +1,3 @@ +from .scraper_data import ActivityView, ScraperDataView, SkillView + +__all__ = ["ScraperDataView", "SkillView", "ActivityView"] diff --git a/bases/bot_detector/api_private/src/app/views/response/highscore.py b/bases/bot_detector/api_private/src/app/views/response/highscore.py new file mode 100644 index 0000000..2c475b9 --- /dev/null +++ b/bases/bot_detector/api_private/src/app/views/response/highscore.py @@ -0,0 +1,109 @@ +from datetime import date, datetime +from typing import Optional + +from pydantic import BaseModel, ConfigDict + + +class PlayerHiscoreData(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: Optional[int] = None + timestamp: datetime = None + ts_date: Optional[date] = None + name: str + Player_id: int + total: int = 0 + attack: int = 0 + defence: int = 0 + strength: int = 0 + hitpoints: int = 0 + ranged: int = 0 + prayer: int = 0 + magic: int = 0 + cooking: int = 0 + woodcutting: int = 0 + fletching: int = 0 + fishing: int = 0 + firemaking: int = 0 + crafting: int = 0 + smithing: int = 0 + mining: int = 0 + herblore: int = 0 + agility: int = 0 + thieving: int = 0 + slayer: int = 0 + farming: int = 0 + runecraft: int = 0 + hunter: int = 0 + construction: int = 0 + league: int = 0 + bounty_hunter_hunter: int = 0 + bounty_hunter_rogue: int = 0 + cs_all: int = 0 + cs_beginner: int = 0 + cs_easy: int = 0 + cs_medium: int = 0 + cs_hard: int = 0 + cs_elite: int = 0 + cs_master: int = 0 + lms_rank: int = 0 + soul_wars_zeal: int = 0 + abyssal_sire: int = 0 + alchemical_hydra: int = 0 + barrows_chests: int = 0 + bryophyta: int = 0 + callisto: int = 0 + cerberus: int = 0 + chambers_of_xeric: int = 0 + chambers_of_xeric_challenge_mode: int = 0 + chaos_elemental: int = 0 + chaos_fanatic: int = 0 + commander_zilyana: int = 0 + corporeal_beast: int = 0 + crazy_archaeologist: int = 0 + dagannoth_prime: int = 0 + dagannoth_rex: int = 0 + dagannoth_supreme: int = 0 + deranged_archaeologist: int = 0 + general_graardor: int = 0 + giant_mole: int = 0 + grotesque_guardians: int = 0 + hespori: int = 0 + kalphite_queen: int = 0 + king_black_dragon: int = 0 + kraken: int = 0 + kreearra: int = 0 + kril_tsutsaroth: int = 0 + mimic: int = 0 + nightmare: int = 0 + nex: int = 0 + phosanis_nightmare: int = 0 + obor: int = 0 + phantom_muspah: int = 0 + sarachnis: int = 0 + scorpia: int = 0 + skotizo: int = 0 + tempoross: int = 0 + the_gauntlet: int = 0 + the_corrupted_gauntlet: int = 0 + theatre_of_blood: int = 0 + theatre_of_blood_hard: int = 0 + thermonuclear_smoke_devil: int = 0 + tombs_of_amascut: int = 0 + tombs_of_amascut_expert: int = 0 + tzkal_zuk: int = 0 + tztok_jad: int = 0 + venenatis: int = 0 + vetion: int = 0 + vorkath: int = 0 + wintertodt: int = 0 + zalcano: int = 0 + zulrah: int = 0 + rifts_closed: int = 0 + artio: int = 0 + calvarion: int = 0 + duke_sucellus: int = 0 + spindel: int = 0 + the_leviathan: int = 0 + the_whisperer: int = 0 + vardorvis: int = 0 diff --git a/bases/bot_detector/api_private/src/app/views/response/ok.py b/bases/bot_detector/api_private/src/app/views/response/ok.py new file mode 100644 index 0000000..9824fc0 --- /dev/null +++ b/bases/bot_detector/api_private/src/app/views/response/ok.py @@ -0,0 +1,4 @@ +from pydantic import BaseModel + +class Ok(BaseModel): + detail: str = "ok" \ No newline at end of file diff --git a/bases/bot_detector/api_private/src/app/views/response/scraper_data.py b/bases/bot_detector/api_private/src/app/views/response/scraper_data.py new file mode 100644 index 0000000..a8840df --- /dev/null +++ b/bases/bot_detector/api_private/src/app/views/response/scraper_data.py @@ -0,0 +1,23 @@ +from datetime import date, datetime + +from pydantic import BaseModel + + +class SkillView(BaseModel): + skill_name: str + skill_value: int + + +class ActivityView(BaseModel): + activity_name: str + activity_value: int + + +class ScraperDataView(BaseModel): + created_at: datetime + record_date: date + scraper_id: int + player_id: int + player_name: str + skills: list[SkillView] + activities: list[ActivityView] diff --git a/bases/bot_detector/api_private/src/core/__init__.py b/bases/bot_detector/api_private/src/core/__init__.py new file mode 100644 index 0000000..574e129 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/__init__.py @@ -0,0 +1,4 @@ +# needed for log formatting +from . import logging_config + +__all__ = ["logging_config"] diff --git a/bases/bot_detector/api_private/src/core/config.py b/bases/bot_detector/api_private/src/core/config.py new file mode 100644 index 0000000..d3dc92c --- /dev/null +++ b/bases/bot_detector/api_private/src/core/config.py @@ -0,0 +1,11 @@ +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + ENV: str + DATABASE_URL: str + KAFKA_HOST: str + POOL_RECYCLE: int = 25 + POOL_TIMEOUT: int = 25 + +settings = Settings() \ No newline at end of file diff --git a/bases/bot_detector/api_private/src/core/database/.gitkeep b/bases/bot_detector/api_private/src/core/database/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/bases/bot_detector/api_private/src/core/database/database.py b/bases/bot_detector/api_private/src/core/database/database.py new file mode 100644 index 0000000..b02fb3f --- /dev/null +++ b/bases/bot_detector/api_private/src/core/database/database.py @@ -0,0 +1,22 @@ +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.orm import declarative_base, sessionmaker + +from src.core.config import settings + +# Create an async SQLAlchemy engine +engine = create_async_engine( + settings.DATABASE_URL, + pool_timeout=settings.POOL_TIMEOUT, + pool_recycle=settings.POOL_RECYCLE, + echo=(settings.ENV != "PRD"), + pool_pre_ping=True, +) + +# Create a session factory +SessionFactory = sessionmaker( + bind=engine, + expire_on_commit=False, + class_=AsyncSession, # Use AsyncSession for asynchronous operations +) + +Base = declarative_base() diff --git a/bases/bot_detector/api_private/src/core/database/models/.gitkeep b/bases/bot_detector/api_private/src/core/database/models/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/bases/bot_detector/api_private/src/core/database/models/__init__.py b/bases/bot_detector/api_private/src/core/database/models/__init__.py new file mode 100644 index 0000000..58d7b55 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/database/models/__init__.py @@ -0,0 +1,26 @@ +from .activities import Activities, PlayerActivities +from .highscore import ( + PlayerHiscoreDataLatest, + PlayerHiscoreDataXPChange, + playerHiscoreData, +) +from .player import Player +from .prediction import Prediction +from .report import Report +from .scraper_data import ScraperData, ScraperDataLatest +from .skills import PlayerSkills, Skills + +__all__ = [ + "Activities", + "PlayerActivities", + "playerHiscoreData", + "PlayerHiscoreDataLatest", + "PlayerHiscoreDataXPChange", + "Player", + "Prediction", + "Report", + "ScraperData", + "ScraperDataLatest", + "PlayerSkills", + "Skills", +] diff --git a/bases/bot_detector/api_private/src/core/database/models/activities.py b/bases/bot_detector/api_private/src/core/database/models/activities.py new file mode 100644 index 0000000..e252c72 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/database/models/activities.py @@ -0,0 +1,27 @@ +from sqlalchemy import Column, ForeignKey, Integer, String +from sqlalchemy.dialects.mysql import BIGINT, TINYINT + +from src.core.database.database import Base + + +class Activities(Base): + __tablename__ = "activities" + + activity_id = Column(TINYINT, primary_key=True, autoincrement=True) + activity_name = Column(String(50), nullable=False) + + +class PlayerActivities(Base): + __tablename__ = "player_activities" + + scraper_id = Column( + BIGINT, + ForeignKey("scraper_data.scraper_id", ondelete="CASCADE"), + primary_key=True, + ) + activity_id = Column( + TINYINT, + ForeignKey("activities.activity_id", ondelete="CASCADE"), + primary_key=True, + ) + activity_value = Column(Integer, nullable=False, default=0) diff --git a/bases/bot_detector/api_private/src/core/database/models/highscore.py b/bases/bot_detector/api_private/src/core/database/models/highscore.py new file mode 100644 index 0000000..e28bcb4 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/database/models/highscore.py @@ -0,0 +1,350 @@ +from sqlalchemy import ( + BigInteger, + Column, + Date, + DateTime, + ForeignKey, + Index, + Integer, + text, +) + +from src.core.database.database import Base + + +class playerHiscoreData(Base): + __tablename__ = "playerHiscoreData" + __table_args__ = ( + Index("Unique_player_time", "Player_id", "timestamp", unique=True), + Index("Unique_player_date", "Player_id", "ts_date", unique=True), + ) + + id = Column(Integer, primary_key=True, autoincrement=True) + timestamp = Column( + DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP") + ) + ts_date = Column(Date) + Player_id = Column( + ForeignKey("Players.id", ondelete="RESTRICT", onupdate="RESTRICT"), + nullable=False, + ) + total = Column(BigInteger) + attack = Column(Integer) + defence = Column(Integer) + strength = Column(Integer) + hitpoints = Column(Integer) + ranged = Column(Integer) + prayer = Column(Integer) + magic = Column(Integer) + cooking = Column(Integer) + woodcutting = Column(Integer) + fletching = Column(Integer) + fishing = Column(Integer) + firemaking = Column(Integer) + crafting = Column(Integer) + smithing = Column(Integer) + mining = Column(Integer) + herblore = Column(Integer) + agility = Column(Integer) + thieving = Column(Integer) + slayer = Column(Integer) + farming = Column(Integer) + runecraft = Column(Integer) + hunter = Column(Integer) + construction = Column(Integer) + league = Column(Integer) + bounty_hunter_hunter = Column(Integer) + bounty_hunter_rogue = Column(Integer) + cs_all = Column(Integer) + cs_beginner = Column(Integer) + cs_easy = Column(Integer) + cs_medium = Column(Integer) + cs_hard = Column(Integer) + cs_elite = Column(Integer) + cs_master = Column(Integer) + lms_rank = Column(Integer) + soul_wars_zeal = Column(Integer) + abyssal_sire = Column(Integer) + alchemical_hydra = Column(Integer) + barrows_chests = Column(Integer) + bryophyta = Column(Integer) + callisto = Column(Integer) + cerberus = Column(Integer) + chambers_of_xeric = Column(Integer) + chambers_of_xeric_challenge_mode = Column(Integer) + chaos_elemental = Column(Integer) + chaos_fanatic = Column(Integer) + commander_zilyana = Column(Integer) + corporeal_beast = Column(Integer) + crazy_archaeologist = Column(Integer) + dagannoth_prime = Column(Integer) + dagannoth_rex = Column(Integer) + dagannoth_supreme = Column(Integer) + deranged_archaeologist = Column(Integer) + general_graardor = Column(Integer) + giant_mole = Column(Integer) + grotesque_guardians = Column(Integer) + hespori = Column(Integer) + kalphite_queen = Column(Integer) + king_black_dragon = Column(Integer) + kraken = Column(Integer) + kreearra = Column(Integer) + kril_tsutsaroth = Column(Integer) + mimic = Column(Integer) + nightmare = Column(Integer) + nex = Column(Integer) + phosanis_nightmare = Column(Integer) + obor = Column(Integer) + phantom_muspah = Column(Integer) + sarachnis = Column(Integer) + scorpia = Column(Integer) + skotizo = Column(Integer) + tempoross = Column(Integer) + the_gauntlet = Column(Integer) + the_corrupted_gauntlet = Column(Integer) + theatre_of_blood = Column(Integer) + theatre_of_blood_hard = Column(Integer) + thermonuclear_smoke_devil = Column(Integer) + tombs_of_amascut = Column(Integer) + tombs_of_amascut_expert = Column(Integer) + tzkal_zuk = Column(Integer) + tztok_jad = Column(Integer) + venenatis = Column(Integer) + vetion = Column(Integer) + vorkath = Column(Integer) + wintertodt = Column(Integer) + zalcano = Column(Integer) + zulrah = Column(Integer) + + # New columns added + rifts_closed = Column(Integer, default=0) + artio = Column(Integer, default=0) + calvarion = Column(Integer, default=0) + duke_sucellus = Column(Integer, default=0) + spindel = Column(Integer, default=0) + the_leviathan = Column(Integer, default=0) + the_whisperer = Column(Integer, default=0) + vardorvis = Column(Integer, default=0) + + +class PlayerHiscoreDataLatest(Base): + __tablename__ = "playerHiscoreDataLatest" + + id = Column(Integer, primary_key=True) + timestamp = Column( + DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP") + ) + ts_date = Column(Date) + Player_id = Column( + ForeignKey("Players.id", ondelete="RESTRICT", onupdate="RESTRICT"), + nullable=False, + unique=True, + ) + total = Column(BigInteger) + attack = Column(Integer) + defence = Column(Integer) + strength = Column(Integer) + hitpoints = Column(Integer) + ranged = Column(Integer) + prayer = Column(Integer) + magic = Column(Integer) + cooking = Column(Integer) + woodcutting = Column(Integer) + fletching = Column(Integer) + fishing = Column(Integer) + firemaking = Column(Integer) + crafting = Column(Integer) + smithing = Column(Integer) + mining = Column(Integer) + herblore = Column(Integer) + agility = Column(Integer) + thieving = Column(Integer) + slayer = Column(Integer) + farming = Column(Integer) + runecraft = Column(Integer) + hunter = Column(Integer) + construction = Column(Integer) + league = Column(Integer) + bounty_hunter_hunter = Column(Integer) + bounty_hunter_rogue = Column(Integer) + cs_all = Column(Integer) + cs_beginner = Column(Integer) + cs_easy = Column(Integer) + cs_medium = Column(Integer) + cs_hard = Column(Integer) + cs_elite = Column(Integer) + cs_master = Column(Integer) + lms_rank = Column(Integer) + soul_wars_zeal = Column(Integer) + abyssal_sire = Column(Integer) + alchemical_hydra = Column(Integer) + barrows_chests = Column(Integer) + bryophyta = Column(Integer) + callisto = Column(Integer) + cerberus = Column(Integer) + chambers_of_xeric = Column(Integer) + chambers_of_xeric_challenge_mode = Column(Integer) + chaos_elemental = Column(Integer) + chaos_fanatic = Column(Integer) + commander_zilyana = Column(Integer) + corporeal_beast = Column(Integer) + crazy_archaeologist = Column(Integer) + dagannoth_prime = Column(Integer) + dagannoth_rex = Column(Integer) + dagannoth_supreme = Column(Integer) + deranged_archaeologist = Column(Integer) + general_graardor = Column(Integer) + giant_mole = Column(Integer) + grotesque_guardians = Column(Integer) + hespori = Column(Integer) + kalphite_queen = Column(Integer) + king_black_dragon = Column(Integer) + kraken = Column(Integer) + kreearra = Column(Integer) + kril_tsutsaroth = Column(Integer) + mimic = Column(Integer) + nightmare = Column(Integer) + nex = Column(Integer) + phosanis_nightmare = Column(Integer) + obor = Column(Integer) + phantom_muspah = Column(Integer) + sarachnis = Column(Integer) + scorpia = Column(Integer) + skotizo = Column(Integer) + Tempoross = Column(Integer, nullable=False) + the_gauntlet = Column(Integer) + the_corrupted_gauntlet = Column(Integer) + theatre_of_blood = Column(Integer) + theatre_of_blood_hard = Column(Integer) + thermonuclear_smoke_devil = Column(Integer) + tombs_of_amascut = Column(Integer) + tombs_of_amascut_expert = Column(Integer) + tzkal_zuk = Column(Integer) + tztok_jad = Column(Integer) + venenatis = Column(Integer) + vetion = Column(Integer) + vorkath = Column(Integer) + wintertodt = Column(Integer) + zalcano = Column(Integer) + zulrah = Column(Integer) + + # New columns added + rifts_closed = Column(Integer, default=0) + artio = Column(Integer, default=0) + calvarion = Column(Integer, default=0) + duke_sucellus = Column(Integer, default=0) + spindel = Column(Integer, default=0) + the_leviathan = Column(Integer, default=0) + the_whisperer = Column(Integer, default=0) + vardorvis = Column(Integer, default=0) + + +class PlayerHiscoreDataXPChange(Base): + __tablename__ = "playerHiscoreDataXPChange" + + id = Column(Integer, primary_key=True) + timestamp = Column( + DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP") + ) + ts_date = Column(Date) + Player_id = Column( + ForeignKey("Players.id", ondelete="RESTRICT", onupdate="RESTRICT"), + nullable=False, + index=True, + ) + total = Column(BigInteger) + attack = Column(Integer) + defence = Column(Integer) + strength = Column(Integer) + hitpoints = Column(Integer) + ranged = Column(Integer) + prayer = Column(Integer) + magic = Column(Integer) + cooking = Column(Integer) + woodcutting = Column(Integer) + fletching = Column(Integer) + fishing = Column(Integer) + firemaking = Column(Integer) + crafting = Column(Integer) + smithing = Column(Integer) + mining = Column(Integer) + herblore = Column(Integer) + agility = Column(Integer) + thieving = Column(Integer) + slayer = Column(Integer) + farming = Column(Integer) + runecraft = Column(Integer) + hunter = Column(Integer) + construction = Column(Integer) + league = Column(Integer) + bounty_hunter_hunter = Column(Integer) + bounty_hunter_rogue = Column(Integer) + cs_all = Column(Integer) + cs_beginner = Column(Integer) + cs_easy = Column(Integer) + cs_medium = Column(Integer) + cs_hard = Column(Integer) + cs_elite = Column(Integer) + cs_master = Column(Integer) + lms_rank = Column(Integer) + soul_wars_zeal = Column(Integer) + abyssal_sire = Column(Integer) + alchemical_hydra = Column(Integer) + barrows_chests = Column(Integer) + bryophyta = Column(Integer) + callisto = Column(Integer) + cerberus = Column(Integer) + chambers_of_xeric = Column(Integer) + chambers_of_xeric_challenge_mode = Column(Integer) + chaos_elemental = Column(Integer) + chaos_fanatic = Column(Integer) + commander_zilyana = Column(Integer) + corporeal_beast = Column(Integer) + crazy_archaeologist = Column(Integer) + dagannoth_prime = Column(Integer) + dagannoth_rex = Column(Integer) + dagannoth_supreme = Column(Integer) + deranged_archaeologist = Column(Integer) + general_graardor = Column(Integer) + giant_mole = Column(Integer) + grotesque_guardians = Column(Integer) + hespori = Column(Integer) + kalphite_queen = Column(Integer) + king_black_dragon = Column(Integer) + kraken = Column(Integer) + kreearra = Column(Integer) + kril_tsutsaroth = Column(Integer) + mimic = Column(Integer) + nightmare = Column(Integer) + nex = Column(Integer) + obor = Column(Integer) + phantom_muspah = Column(Integer) + phosanis_nightmare = Column(Integer) + sarachnis = Column(Integer) + scorpia = Column(Integer) + skotizo = Column(Integer) + Tempoross = Column(Integer, nullable=False) + the_gauntlet = Column(Integer) + the_corrupted_gauntlet = Column(Integer) + theatre_of_blood = Column(Integer) + theatre_of_blood_hard = Column(Integer) + thermonuclear_smoke_devil = Column(Integer) + tombs_of_amascut = Column(Integer) + tombs_of_amascut_expert = Column(Integer) + tzkal_zuk = Column(Integer) + tztok_jad = Column(Integer) + venenatis = Column(Integer) + vetion = Column(Integer) + vorkath = Column(Integer) + wintertodt = Column(Integer) + zalcano = Column(Integer) + zulrah = Column(Integer) + # New columns added + rifts_closed = Column(Integer, default=0) + artio = Column(Integer, default=0) + calvarion = Column(Integer, default=0) + duke_sucellus = Column(Integer, default=0) + spindel = Column(Integer, default=0) + the_leviathan = Column(Integer, default=0) + the_whisperer = Column(Integer, default=0) + vardorvis = Column(Integer, default=0) diff --git a/bases/bot_detector/api_private/src/core/database/models/player.py b/bases/bot_detector/api_private/src/core/database/models/player.py new file mode 100644 index 0000000..726f376 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/database/models/player.py @@ -0,0 +1,19 @@ +from sqlalchemy import Boolean, Column, DateTime, Integer, Text +from src.core.database.database import Base + +class Player(Base): + __tablename__ = "Players" + + id = Column(Integer, primary_key=True, autoincrement=True) + name = Column(Text) + created_at = Column(DateTime) + updated_at = Column(DateTime) + possible_ban = Column(Boolean) + confirmed_ban = Column(Boolean) + confirmed_player = Column(Boolean) + label_id = Column(Integer) + label_jagex = Column(Integer) + ironman = Column(Boolean) + hardcore_ironman = Column(Boolean) + ultimate_ironman = Column(Boolean) + normalized_name = Column(Text) diff --git a/bases/bot_detector/api_private/src/core/database/models/prediction.py b/bases/bot_detector/api_private/src/core/database/models/prediction.py new file mode 100644 index 0000000..748f7e8 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/database/models/prediction.py @@ -0,0 +1,39 @@ +from sqlalchemy import Column, Integer, String, TIMESTAMP, DECIMAL +from src.core.database.database import Base + + + +class Prediction(Base): + __tablename__ = "Predictions" + + id = Column(Integer, primary_key=True, autoincrement=True) + name = Column(String(12)) + prediction = Column(String(50)) + created = Column(TIMESTAMP) + predicted_confidence = Column(DECIMAL(5, 2)) + real_player = Column(DECIMAL(5, 2), default=0) + pvm_melee_bot = Column(DECIMAL(5, 2), default=0) + smithing_bot = Column(DECIMAL(5, 2), default=0) + magic_bot = Column(DECIMAL(5, 2), default=0) + fishing_bot = Column(DECIMAL(5, 2), default=0) + mining_bot = Column(DECIMAL(5, 2), default=0) + crafting_bot = Column(DECIMAL(5, 2), default=0) + pvm_ranged_magic_bot = Column(DECIMAL(5, 2), default=0) + pvm_ranged_bot = Column(DECIMAL(5, 2), default=0) + hunter_bot = Column(DECIMAL(5, 2), default=0) + fletching_bot = Column(DECIMAL(5, 2), default=0) + clue_scroll_bot = Column(DECIMAL(5, 2), default=0) + lms_bot = Column(DECIMAL(5, 2), default=0) + agility_bot = Column(DECIMAL(5, 2), default=0) + wintertodt_bot = Column(DECIMAL(5, 2), default=0) + runecrafting_bot = Column(DECIMAL(5, 2), default=0) + zalcano_bot = Column(DECIMAL(5, 2), default=0) + woodcutting_bot = Column(DECIMAL(5, 2), default=0) + thieving_bot = Column(DECIMAL(5, 2), default=0) + soul_wars_bot = Column(DECIMAL(5, 2), default=0) + cooking_bot = Column(DECIMAL(5, 2), default=0) + vorkath_bot = Column(DECIMAL(5, 2), default=0) + barrows_bot = Column(DECIMAL(5, 2), default=0) + herblore_bot = Column(DECIMAL(5, 2), default=0) + unknown_bot = Column(DECIMAL(5, 2), default=0) + diff --git a/bases/bot_detector/api_private/src/core/database/models/report.py b/bases/bot_detector/api_private/src/core/database/models/report.py new file mode 100644 index 0000000..28e65a9 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/database/models/report.py @@ -0,0 +1,29 @@ +from sqlalchemy import BigInteger, Column, Integer, SmallInteger, TIMESTAMP +from src.core.database.database import Base + +class Report(Base): + __tablename__ = "Reports" + + ID = Column(BigInteger, primary_key=True, autoincrement=True) + created_at = Column(TIMESTAMP) + reportedID = Column(Integer) + reportingID = Column(Integer) + region_id = Column(Integer) + x_coord = Column(Integer) + y_coord = Column(Integer) + z_coord = Column(Integer) + timestamp = Column(TIMESTAMP) + manual_detect = Column(SmallInteger) + on_members_world = Column(Integer) + on_pvp_world = Column(SmallInteger) + world_number = Column(Integer) + equip_head_id = Column(Integer) + equip_amulet_id = Column(Integer) + equip_torso_id = Column(Integer) + equip_legs_id = Column(Integer) + equip_boots_id = Column(Integer) + equip_cape_id = Column(Integer) + equip_hands_id = Column(Integer) + equip_weapon_id = Column(Integer) + equip_shield_id = Column(Integer) + equip_ge_value = Column(BigInteger) diff --git a/bases/bot_detector/api_private/src/core/database/models/scraper_data.py b/bases/bot_detector/api_private/src/core/database/models/scraper_data.py new file mode 100644 index 0000000..e14021a --- /dev/null +++ b/bases/bot_detector/api_private/src/core/database/models/scraper_data.py @@ -0,0 +1,22 @@ +from sqlalchemy import Column, Date, DateTime, func +from sqlalchemy.dialects.mysql import BIGINT, SMALLINT + +from src.core.database.database import Base + + +class ScraperData(Base): + __tablename__ = "scraper_data" + + scraper_id = Column(BIGINT, primary_key=True, autoincrement=True) + created_at = Column(DateTime, nullable=False, server_default=func.now()) + player_id = Column(SMALLINT, nullable=False) + record_date = Column(Date, nullable=True) + + +class ScraperDataLatest(Base): + __tablename__ = "scraper_data_latest" + + scraper_id = Column(BIGINT) + created_at = Column(DateTime, nullable=False, server_default=func.now()) + player_id = Column(BIGINT, primary_key=True) + record_date = Column(Date, nullable=True) diff --git a/bases/bot_detector/api_private/src/core/database/models/scraper_data_v3.py b/bases/bot_detector/api_private/src/core/database/models/scraper_data_v3.py new file mode 100644 index 0000000..4c51043 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/database/models/scraper_data_v3.py @@ -0,0 +1,107 @@ +from sqlalchemy import ( + BigInteger, + Column, + Date, + DateTime, + ForeignKey, + Index, + Integer, + SmallInteger, + String, + UniqueConstraint, +) +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() + + +class Skill(Base): + __tablename__ = "skill" + + skill_id = Column(SmallInteger(), primary_key=True, autoincrement=True) + skill_name = Column(String(50), nullable=False, unique=True) + + +class Activity(Base): + __tablename__ = "activity" + + activity_id = Column(SmallInteger(), primary_key=True, autoincrement=True) + activity_name = Column(String(50), nullable=False, unique=True) + + +class PlayerSkill(Base): + __tablename__ = "player_skill" + + player_skill_id = Column(BigInteger(), primary_key=True, autoincrement=True) + skill_id = Column(SmallInteger(), nullable=False) + skill_value = Column(Integer(), nullable=False, default=0) + + __table_args__ = ( + UniqueConstraint("skill_id", "skill_value", name="unique_skill_value"), + ) + + +class PlayerActivity(Base): + __tablename__ = "player_activity" + + player_activity_id = Column(BigInteger(), primary_key=True, autoincrement=True) + activity_id = Column(SmallInteger(), nullable=False) + activity_value = Column(Integer(), nullable=False, default=0) + + __table_args__ = ( + UniqueConstraint("activity_id", "activity_value", name="unique_activity_value"), + ) + + +class ScraperDataV3(Base): + __tablename__ = "scraper_data_v3" + + scrape_id = Column(BigInteger(), primary_key=True, autoincrement=True) + scrape_ts = Column(DateTime, nullable=False) + scrape_date = Column(Date, nullable=False) + player_id = Column(Integer, nullable=False) + + __table_args__ = ( + UniqueConstraint("player_id", "scrape_date", name="unique_player_scrape"), + Index("idx_scrape_ts", "scrape_ts"), + ) + + +class ScraperPlayerSkill(Base): + __tablename__ = "scraper_player_skill" + + scrape_id = Column( + BigInteger(), + ForeignKey("scraper_data_v3.scrape_id"), + primary_key=True, + ) + player_skill_id = Column( + BigInteger(), + ForeignKey("player_skill.player_skill_id"), + primary_key=True, + ) + + __table_args__ = ( + Index("idx_scrape_id", "scrape_id"), + Index("idx_player_skill_id", "player_skill_id"), + ) + + +class ScraperPlayerActivity(Base): + __tablename__ = "scraper_player_activity" + + scrape_id = Column( + BigInteger(), + ForeignKey("scraper_data_v3.scrape_id"), + primary_key=True, + ) + player_activity_id = Column( + BigInteger(), + ForeignKey("player_activity.player_activity_id"), + primary_key=True, + ) + + __table_args__ = ( + Index("idx_scrape_id", "scrape_id"), + Index("idx_player_activity_id", "player_activity_id"), + ) diff --git a/bases/bot_detector/api_private/src/core/database/models/skills.py b/bases/bot_detector/api_private/src/core/database/models/skills.py new file mode 100644 index 0000000..a7e4080 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/database/models/skills.py @@ -0,0 +1,27 @@ +from sqlalchemy import Column, ForeignKey, Integer, String +from sqlalchemy.dialects.mysql import BIGINT, TINYINT + +from src.core.database.database import Base + + +class Skills(Base): + __tablename__ = "skills" + + skill_id = Column(TINYINT, primary_key=True, autoincrement=True) + skill_name = Column(String(50), nullable=False) + + +class PlayerSkills(Base): + __tablename__ = "player_skills" + + scraper_id = Column( + BIGINT, + ForeignKey("scraper_data.scraper_id", ondelete="CASCADE"), + primary_key=True, + ) + skill_id = Column( + TINYINT, + ForeignKey("skills.skill_id", ondelete="CASCADE"), + primary_key=True, + ) + skill_value = Column(Integer, nullable=False, default=0) diff --git a/bases/bot_detector/api_private/src/core/fastapi/dependencies/.gitkeep b/bases/bot_detector/api_private/src/core/fastapi/dependencies/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/bases/bot_detector/api_private/src/core/fastapi/dependencies/session.py b/bases/bot_detector/api_private/src/core/fastapi/dependencies/session.py new file mode 100644 index 0000000..08f5092 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/fastapi/dependencies/session.py @@ -0,0 +1,9 @@ +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.database.database import SessionFactory + + +# Dependency to get an asynchronous session +async def get_session() -> AsyncSession: + async with SessionFactory() as session: + yield session diff --git a/bases/bot_detector/api_private/src/core/fastapi/dependencies/to_jagex_name.py b/bases/bot_detector/api_private/src/core/fastapi/dependencies/to_jagex_name.py new file mode 100644 index 0000000..571229c --- /dev/null +++ b/bases/bot_detector/api_private/src/core/fastapi/dependencies/to_jagex_name.py @@ -0,0 +1,3 @@ +# Define the to_jagex_name dependency +async def to_jagex_name(name: str) -> str: + return name.lower().replace("_", " ").replace("-", " ").strip() \ No newline at end of file diff --git a/bases/bot_detector/api_private/src/core/fastapi/middleware/logging.py b/bases/bot_detector/api_private/src/core/fastapi/middleware/logging.py new file mode 100644 index 0000000..1209826 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/fastapi/middleware/logging.py @@ -0,0 +1,27 @@ +from fastapi import Request +from starlette.middleware.base import BaseHTTPMiddleware +import logging +import time + +logger = logging.getLogger(__name__) + + +class LoggingMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + start_time = time.time() + response = await call_next(request) + process_time = time.time() - start_time + + query_params_list = [ + (key, value if key != "token" else "***") + for key, value in request.query_params.items() + ] + + logger.debug( + { + "url": request.url.path, + "params": query_params_list, + "process_time": f"{process_time:.4f}", + } + ) + return response diff --git a/bases/bot_detector/api_private/src/core/kafka/engine.py b/bases/bot_detector/api_private/src/core/kafka/engine.py new file mode 100644 index 0000000..e1eb122 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/kafka/engine.py @@ -0,0 +1,72 @@ +import asyncio +from aiokafka import AIOKafkaConsumer, AIOKafkaProducer +from asyncio import Queue +import json + +def retry_on_exception(max_retries=3, retry_interval=5): + def decorator(func): + async def wrapper(*args, **kwargs): + retries = 0 + while retries < max_retries: + try: + await func(*args, **kwargs) + except Exception as e: + print(f"Error: {e}") + retries += 1 + await asyncio.sleep(retry_interval) + else: + break + return wrapper + return decorator + +class AioKafkaEngine: + def __init__(self, bootstrap_servers: list[str], topic: str, message_queue: Queue): + self.bootstrap_servers = bootstrap_servers + self.topic = topic + self.consumer = None + self.producer = None + self.message_queue = message_queue + + async def start_consumer(self, group_id: str): + self.consumer = AIOKafkaConsumer( + self.topic, + bootstrap_servers=self.bootstrap_servers, + value_serializer=lambda v: json.dumps(v).encode(), + group_id=group_id, + ) + await self.consumer.start() + + async def start_producer(self): + self.producer = AIOKafkaProducer( + bootstrap_servers=self.bootstrap_servers, + value_serializer=lambda v: json.dumps(v).encode() + ) + await self.producer.start() + + @retry_on_exception(max_retries=3, retry_interval=5) + async def consume_messages(self): + if self.consumer is None: + raise ValueError("Consumer not started. Call start_consumer() first.") + + async for message in self.consumer: + value = message.value + self.message_queue.put_nowait(value) + + @retry_on_exception(max_retries=3, retry_interval=5) + async def produce_messages(self): + if self.producer is None: + raise ValueError("Producer not started. Call start_producer() first.") + while True: + message = await self.message_queue.get() + await self.producer.send(self.topic, value=message) + + async def stop_consumer(self): + if self.consumer: + await self.consumer.stop() + + async def stop_producer(self): + if self.producer: + await self.producer.stop() + + def is_ready(self): + return self.consumer is not None or self.producer is not None \ No newline at end of file diff --git a/bases/bot_detector/api_private/src/core/logging_config.py b/bases/bot_detector/api_private/src/core/logging_config.py new file mode 100644 index 0000000..e35d4f3 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/logging_config.py @@ -0,0 +1,42 @@ +import json +import logging +import sys + +from src.core.config import settings + +# # log formatting +formatter = logging.Formatter( + json.dumps( + { + "ts": "%(asctime)s", + "name": "%(name)s", + "function": "%(funcName)s", + "level": "%(levelname)s", + "msg": json.dumps("%(message)s"), + } + ) +) + +stream_handler = logging.StreamHandler(sys.stdout) + +stream_handler.setFormatter(formatter) + +handlers = [stream_handler] + +logging.basicConfig(level=logging.DEBUG, handlers=handlers) + +# set imported loggers to warning +# logging.getLogger("urllib3").setLevel(logging.DEBUG) +# logging.getLogger("uvicorn").setLevel(logging.DEBUG) +# logging.getLogger("aiomysql").setLevel(logging.ERROR) +# logging.getLogger("aiokafka").setLevel(logging.WARNING) + +if settings.ENV == "PRD": + uvicorn_error = logging.getLogger("uvicorn.error") + uvicorn_error.disabled = True + uvicorn_access = logging.getLogger("uvicorn.access") + uvicorn_access.disabled = True + +# # https://github.com/aio-libs/aiomysql/issues/103 +# # https://github.com/coleifer/peewee/issues/2229 +# warnings.filterwarnings("ignore", ".*Duplicate entry.*") diff --git a/bases/bot_detector/api_private/src/core/server.py b/bases/bot_detector/api_private/src/core/server.py new file mode 100644 index 0000000..dc20696 --- /dev/null +++ b/bases/bot_detector/api_private/src/core/server.py @@ -0,0 +1,51 @@ +import logging + +from fastapi import FastAPI +from fastapi.middleware import Middleware +from fastapi.middleware.cors import CORSMiddleware + +from src import api +from src.core.fastapi.middleware.logging import LoggingMiddleware + +logger = logging.getLogger(__name__) + + +def init_routers(_app: FastAPI) -> None: + _app.include_router(api.router) + + +def make_middleware() -> list[Middleware]: + middleware = [ + Middleware( + CORSMiddleware, + allow_origins=[ + "http://osrsbotdetector.com/", + "https://osrsbotdetector.com/", + "http://localhost", + "http://localhost:8080", + ], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ), + Middleware(LoggingMiddleware), + ] + return middleware + + +def create_app() -> FastAPI: + _app = FastAPI( + title="Bot-Detector-API", + description="Bot-Detector-API", + middleware=make_middleware(), + ) + init_routers(_app=_app) + return _app + + +app = create_app() + + +@app.get("/") +async def root(): + return {"message": "Hello World"} diff --git a/bases/bot_detector/api_private/tests/.gitkeep b/bases/bot_detector/api_private/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/bases/bot_detector/api_private/tests/_test_highscore_benchmark.py b/bases/bot_detector/api_private/tests/_test_highscore_benchmark.py new file mode 100644 index 0000000..a14edb9 --- /dev/null +++ b/bases/bot_detector/api_private/tests/_test_highscore_benchmark.py @@ -0,0 +1,44 @@ +import asyncio + +import httpx +import pytest + + +@pytest.mark.asyncio +async def test_highscore_v2(benchmark, custom_client): + player_ids = list(range(1, 101)) # Or any other player IDs you want to use + async with httpx.AsyncClient() as client: + + async def request(player_id): + endpoint = "http://localhost:5000/v2/highscore/latest" + params = {"player_id": player_id} + response = await client.get(url=endpoint, params=params) + return response + + async def run_requests(): + tasks = [ + asyncio.create_task(request(player_id)) for player_id in player_ids + ] + return await asyncio.gather(*tasks) + + _ = await benchmark(run_requests) + + +@pytest.mark.asyncio +async def test_highscore_v3(benchmark, custom_client): + player_ids = list(range(1, 101)) # Or any other player IDs you want to use + async with httpx.AsyncClient() as client: + + async def request(player_id): + endpoint = "http://localhost:5000/v3/highscore/latest" + params = {"player_id": player_id} + response = await client.get(url=endpoint, params=params) + return response + + async def run_requests(): + tasks = [ + asyncio.create_task(request(player_id)) for player_id in player_ids + ] + return await asyncio.gather(*tasks) + + _ = await benchmark(run_requests) diff --git a/bases/bot_detector/api_private/tests/benchmark.py b/bases/bot_detector/api_private/tests/benchmark.py new file mode 100644 index 0000000..825c40d --- /dev/null +++ b/bases/bot_detector/api_private/tests/benchmark.py @@ -0,0 +1,39 @@ +import logging +from time import perf_counter + + +class Benchmark: + results = [] + + def __init__(self, name, suppress_logging=False): + self.name = name + self.suppress_logging = suppress_logging + if suppress_logging: + self.original_level = self.set_logging_level("httpcore", logging.INFO) + + async def __aenter__(self): + self.time_start = perf_counter() + return self + + async def __aexit__(self, exc_type, exc, tb): + duration = perf_counter() - self.time_start + self.results.append((self.name, duration)) + if self.suppress_logging: + self.set_logging_level("httpcore", self.original_level) + return False + + @classmethod + def output_results(cls): + print("{:<10} {:<10}".format("Name", "Duration")) + durations = [] + for name, duration in cls.results: + print("{:<10} {:<10.3f}".format(name, duration)) + durations.append(duration) + return durations + + @staticmethod + def set_logging_level(logger_name, level): + logger = logging.getLogger(logger_name) + original_level = logger.level + logger.setLevel(level) + return original_level diff --git a/bases/bot_detector/api_private/tests/conftest.py b/bases/bot_detector/api_private/tests/conftest.py new file mode 100644 index 0000000..79823af --- /dev/null +++ b/bases/bot_detector/api_private/tests/conftest.py @@ -0,0 +1,57 @@ +# conftest.py +import os +import sys +from contextlib import asynccontextmanager + +import pytest +from fastapi import FastAPI +from httpx import AsyncClient +from httpx._transports.asgi import ASGITransport +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.orm import sessionmaker + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +from src.core import server # noqa: E402 +from src.core.fastapi.dependencies.session import get_session # noqa: E402 + + +# Create an async SQLAlchemy engine +async def get_session_override(): + engine = create_async_engine( + "mysql+aiomysql://root:root_bot_buster@localhost:3307/playerdata", + pool_timeout=30, + pool_recycle=30, + echo=True, + pool_pre_ping=True, + ) + + # Create a session factory + SessionFactory = sessionmaker( + bind=engine, + expire_on_commit=False, + class_=AsyncSession, # Use AsyncSession for asynchronous operations + ) + async with SessionFactory() as session: + session: AsyncSession + yield session + await engine.dispose() + return + + +server.app.dependency_overrides[get_session] = get_session_override + + +@pytest.fixture +def app() -> FastAPI: + return server.app + + +@pytest.fixture +@asynccontextmanager +async def custom_client(app: FastAPI): + base_url = "http://srv.test/" + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url=base_url) as client: + yield client diff --git a/bases/bot_detector/api_private/tests/pytest.ini b/bases/bot_detector/api_private/tests/pytest.ini new file mode 100644 index 0000000..12681bf --- /dev/null +++ b/bases/bot_detector/api_private/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +asyncio_mode=auto +addopts = -s diff --git a/bases/bot_detector/api_private/tests/test_highscore.py b/bases/bot_detector/api_private/tests/test_highscore.py new file mode 100644 index 0000000..b8bdcb2 --- /dev/null +++ b/bases/bot_detector/api_private/tests/test_highscore.py @@ -0,0 +1,45 @@ +import pytest +from httpx import AsyncClient + + +@pytest.mark.asyncio +async def test_one_hs_id_v2(custom_client): + endpoint = "/v2/highscore/latest" + async with custom_client as client: + client: AsyncClient + params = {"player_id": 1} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) == 1 + assert isinstance( + json_response[0], dict + ), f"expected dict, got {type(json_response[0])}, {json_response=}" + + player = json_response[0] + + assert player.get("Player_id") == 1, f"expected id: 1 got: {player=}" + + +@pytest.mark.asyncio +async def test_one_hs_id_v3(custom_client): + endpoint = "/v3/highscore/latest" + async with custom_client as client: + client: AsyncClient + params = {"player_id": 1} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) == 1 + assert isinstance( + json_response[0], dict + ), f"expected dict, got {type(json_response[0])}, {json_response=}" + + player = json_response[0] + assert player.get("player_id") == 1, f"expected id: 1 got: {player=}" diff --git a/bases/bot_detector/api_private/tests/test_highscore_benchmark_custom.py b/bases/bot_detector/api_private/tests/test_highscore_benchmark_custom.py new file mode 100644 index 0000000..7dafae0 --- /dev/null +++ b/bases/bot_detector/api_private/tests/test_highscore_benchmark_custom.py @@ -0,0 +1,81 @@ +import random +import statistics + +import pytest +from benchmark import Benchmark +from httpx import AsyncClient + +# Global variable to store the results +benchmark_results = {"v2": [], "v3": []} +player_ids = [random.randint(1, 250) for i in range(10)] +ITERATIONS = 1 + + +async def request(client: AsyncClient, endpoint: str, player_id: int): + params = {"player_id": player_id, "many": 1, "limit": 5000} + response = await client.get(url=endpoint, params=params) + return response + + +async def bench(iterations, client, endpoint, player_ids): + for _ in range(iterations): + for player_id in player_ids: + async with Benchmark("requests", suppress_logging=True) as b: + await request(client, endpoint, player_id) + return b + + +@pytest.mark.asyncio +async def test_highscore_custom_benchmark_v2(custom_client): + # Clear the results from the previous tests + Benchmark.results.clear() + endpoint = "/v2/highscore/latest" + async with custom_client as client: + client: AsyncClient + b = await bench(ITERATIONS, client, endpoint, player_ids) + for name, duration in b.results: + benchmark_results["v2"].append(duration) + + +@pytest.mark.asyncio +async def test_highscore_custom_benchmark_v3(custom_client): + # Clear the results from the previous tests + Benchmark.results.clear() + endpoint = "/v3/highscore/latest" + async with custom_client as client: + client: AsyncClient + b = await bench(ITERATIONS, client, endpoint, player_ids) + for name, duration in b.results: + benchmark_results["v3"].append(duration) + + +def test_output_results(): + print("v2 results:") + avg_time = statistics.mean(benchmark_results["v2"]) + median_time = statistics.median(benchmark_results["v2"]) + stdev_time = ( + statistics.stdev(benchmark_results["v2"]) + if len(benchmark_results["v2"]) > 1 + else 0 # if length of results empty return 0 + ) + max_time = max(benchmark_results["v2"]) + min_time = min(benchmark_results["v2"]) + print( + f"average {avg_time:.3f} seconds, median {median_time:.3f} seconds, stdev {stdev_time:.3f} seconds, max {max_time:.3f} seconds, min {min_time:.3f} seconds" + ) + assert avg_time > 0 + + print("v3 results:") + avg_time = statistics.mean(benchmark_results["v3"]) + median_time = statistics.median(benchmark_results["v3"]) + stdev_time = ( + statistics.stdev(benchmark_results["v3"]) + if len(benchmark_results["v3"]) > 1 + else 0 + ) + max_time = max(benchmark_results["v3"]) + min_time = min(benchmark_results["v3"]) + print( + f"average {avg_time:.3f} seconds, median {median_time:.3f} seconds, stdev {stdev_time:.3f} seconds, max {max_time:.3f} seconds, min {min_time:.3f} seconds" + ) + assert avg_time > 0 diff --git a/bases/bot_detector/api_private/tests/test_highscore_v2.py b/bases/bot_detector/api_private/tests/test_highscore_v2.py new file mode 100644 index 0000000..f2c0f84 --- /dev/null +++ b/bases/bot_detector/api_private/tests/test_highscore_v2.py @@ -0,0 +1,263 @@ +import pytest +from httpx import AsyncClient + + +@pytest.mark.asyncio +async def test_one_hs_id_v2(custom_client): + endpoint = "/v2/highscore/latest" + async with custom_client as client: + client: AsyncClient + params = {"player_id": 1} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) == 1 + assert isinstance( + json_response[0], dict + ), f"expected dict, got {type(json_response[0])}, {json_response=}" + + player = json_response[0] + assert player.get("Player_id") == 1, f"expected Player_id: 1 got: {player=}" + + +@pytest.mark.asyncio +async def test_highscore_latest_v2(custom_client): + endpoint = "/v2/highscore/latest" + async with custom_client as client: + client: AsyncClient + params = {"player_id": 1} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) == 1 + assert isinstance( + json_response[0], dict + ), f"expected dict, got {type(json_response[0])}, {json_response=}" + + player = json_response[0] + # assert player.get("Player_id") == 1, f"expected Player_id: 1 got: {player=}" + + # List of keys that should be present in the player dictionary + keys = [ + "id", + "timestamp", + "ts_date", + "Player_id", + "name", + "total", + "attack", + "defence", + "strength", + "hitpoints", + "ranged", + "prayer", + "magic", + "cooking", + "woodcutting", + "fletching", + "fishing", + "firemaking", + "crafting", + "smithing", + "mining", + "herblore", + "agility", + "thieving", + "slayer", + "farming", + "runecraft", + "hunter", + "construction", + "league", + "bounty_hunter_hunter", + "bounty_hunter_rogue", + "cs_all", + "cs_beginner", + "cs_easy", + "cs_medium", + "cs_hard", + "cs_elite", + "cs_master", + "lms_rank", + "soul_wars_zeal", + "abyssal_sire", + "alchemical_hydra", + "barrows_chests", + "bryophyta", + "callisto", + "cerberus", + "chambers_of_xeric", + "chambers_of_xeric_challenge_mode", + "chaos_elemental", + "chaos_fanatic", + "commander_zilyana", + "corporeal_beast", + "crazy_archaeologist", + "dagannoth_prime", + "dagannoth_rex", + "dagannoth_supreme", + "deranged_archaeologist", + "general_graardor", + "giant_mole", + "grotesque_guardians", + "hespori", + "kalphite_queen", + "king_black_dragon", + "kraken", + "kreearra", + "kril_tsutsaroth", + "mimic", + "nex", + "nightmare", + "phosanis_nightmare", + "obor", + "phantom_muspah", + "sarachnis", + "scorpia", + "skotizo", + "tempoross", + "the_gauntlet", + "the_corrupted_gauntlet", + "theatre_of_blood", + "theatre_of_blood_hard", + "thermonuclear_smoke_devil", + "tombs_of_amascut", + "tombs_of_amascut_expert", + "tzkal_zuk", + "tztok_jad", + "venenatis", + "vetion", + "vorkath", + "wintertodt", + "zalcano", + "zulrah", + "rifts_closed", + "artio", + "calvarion", + "duke_sucellus", + "spindel", + "the_leviathan", + "the_whisperer", + "vardorvis", + ] + + # Check if all keys are present in the player dictionary + for key in keys: + assert key in player, f"Key {key} not found in player dictionary" + + # Dictionary with expected types + expected_types = { + "id": int, + "timestamp": str, + "ts_date": str, + "Player_id": int, + "total": int, + "attack": int, + "defence": int, + "strength": int, + "hitpoints": int, + "ranged": int, + "prayer": int, + "magic": int, + "cooking": int, + "woodcutting": int, + "fletching": int, + "fishing": int, + "firemaking": int, + "crafting": int, + "smithing": int, + "mining": int, + "herblore": int, + "agility": int, + "thieving": int, + "slayer": int, + "farming": int, + "runecraft": int, + "hunter": int, + "construction": int, + "league": int, + "bounty_hunter_hunter": int, + "bounty_hunter_rogue": int, + "cs_all": int, + "cs_beginner": int, + "cs_easy": int, + "cs_medium": int, + "cs_hard": int, + "cs_elite": int, + "cs_master": int, + "lms_rank": int, + "soul_wars_zeal": int, + "abyssal_sire": int, + "alchemical_hydra": int, + "barrows_chests": int, + "bryophyta": int, + "callisto": int, + "cerberus": int, + "chambers_of_xeric": int, + "chambers_of_xeric_challenge_mode": int, + "chaos_elemental": int, + "chaos_fanatic": int, + "commander_zilyana": int, + "corporeal_beast": int, + "crazy_archaeologist": int, + "dagannoth_prime": int, + "dagannoth_rex": int, + "dagannoth_supreme": int, + "deranged_archaeologist": int, + "general_graardor": int, + "giant_mole": int, + "grotesque_guardians": int, + "hespori": int, + "kalphite_queen": int, + "king_black_dragon": int, + "kraken": int, + "kreearra": int, + "kril_tsutsaroth": int, + "mimic": int, + "nex": int, + "nightmare": int, + "phosanis_nightmare": int, + "obor": int, + "phantom_muspah": int, + "sarachnis": int, + "scorpia": int, + "skotizo": int, + "tempoross": int, + "the_gauntlet": int, + "the_corrupted_gauntlet": int, + "theatre_of_blood": int, + "theatre_of_blood_hard": int, + "thermonuclear_smoke_devil": int, + "tombs_of_amascut": int, + "tombs_of_amascut_expert": int, + "tzkal_zuk": int, + "tztok_jad": int, + "venenatis": int, + "vetion": int, + "vorkath": int, + "wintertodt": int, + "zalcano": int, + "zulrah": int, + "rifts_closed": int, + "artio": int, + "calvarion": int, + "duke_sucellus": int, + "spindel": int, + "the_leviathan": int, + "the_whisperer": int, + "vardorvis": int, + } + + # Check if the type of each value in the returned player dictionary matches the expected type + for key, expected_type in expected_types.items(): + value = player.get(key) + # if value is not None: + assert isinstance( + value, expected_type + ), f"Key {key} has incorrect type. Expected: {expected_type}, Got: {type(value)}" diff --git a/bases/bot_detector/api_private/tests/test_highscore_v3.py b/bases/bot_detector/api_private/tests/test_highscore_v3.py new file mode 100644 index 0000000..c8e59b7 --- /dev/null +++ b/bases/bot_detector/api_private/tests/test_highscore_v3.py @@ -0,0 +1,65 @@ +import pytest +from httpx import AsyncClient + + +@pytest.mark.asyncio +async def test_one_hs_id_v3(custom_client): + endpoint = "/v3/highscore/latest" + async with custom_client as client: + client: AsyncClient + params = { + "player_id": 1, + } + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) == 1 + assert isinstance( + json_response[0], dict + ), f"expected dict, got {type(json_response[0])}, {json_response=}" + + player = json_response[0] + assert player.get("player_id") == 1, f"expected id: 1 got: {player=}" + + +@pytest.mark.asyncio +async def test_many_hs_id_v3(custom_client): + endpoint = "/v3/highscore/latest" + async with custom_client as client: + client: AsyncClient + params = {"player_id": 1, "many": 1} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) >= 1 + assert isinstance( + json_response[0], dict + ), f"expected dict, got {type(json_response[0])}, {json_response=}" + + player = json_response[0] + assert player.get("player_id") == 1, f"expected id: 1 got: {player=}" + + +@pytest.mark.asyncio +async def test_highscore_latest_v3(custom_client): + # TODO: discuss how we are going to test the "keys" + endpoint = "/v3/highscore/latest" + async with custom_client as client: + client: AsyncClient + params = {"player_id": 1} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) == 1 + assert isinstance( + json_response[0], dict + ), f"expected dict, got {type(json_response[0])}, {json_response=}" diff --git a/bases/bot_detector/api_private/tests/test_player.py b/bases/bot_detector/api_private/tests/test_player.py new file mode 100644 index 0000000..e5e04c5 --- /dev/null +++ b/bases/bot_detector/api_private/tests/test_player.py @@ -0,0 +1,76 @@ +import pytest +from httpx import AsyncClient + + +@pytest.mark.asyncio +async def test_one_player_id(custom_client): + endpoint = "/v2/player" + + async with custom_client as client: + client: AsyncClient + params = {"player_id": 1, "greater_than": 0} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) == 1 + assert isinstance(json_response[0], dict) + player = json_response[0] + assert player.get("id") == 1 + + +@pytest.mark.asyncio +async def test_one_player_name(custom_client): + endpoint = "/v2/player" + + async with custom_client as client: + client: AsyncClient + params = {"player_name": "Player_0", "greater_than": 0} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) == 1 + assert isinstance(json_response[0], dict) + player = json_response[0] + assert player.get("name") == "Player_0" + + +@pytest.mark.asyncio +async def test_many_player(custom_client): + endpoint = "/v2/player" + + async with custom_client as client: + client: AsyncClient + params = {"player_id": 1, "greater_than": 1} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) > 1 + assert isinstance(json_response[0], dict) + player = json_response[0] + assert player.get("id") == 1 + + +@pytest.mark.asyncio +async def test_player_label(custom_client): + endpoint = "/v2/player" + + async with custom_client as client: + client: AsyncClient + params = {"label_id": 0} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) >= 1 + assert isinstance(json_response[0], dict) From c5c962bd871a0269dd6f53f45bc4bb04c13d33d1 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Fri, 6 Jun 2025 22:02:38 -0400 Subject: [PATCH 02/39] removing redundant files --- bases/bot_detector/api_private/Dockerfile | 31 --- bases/bot_detector/api_private/LICENSE | 24 -- bases/bot_detector/api_private/Makefile | 103 ------- .../api_private/docker-compose-standalone.yml | 65 ----- .../api_private/kafka_setup/.dockerignore | 1 - .../api_private/kafka_setup/Dockerfile | 13 - .../kafka_setup/kafka_data/.gitkeep | 0 .../api_private/kafka_setup/requirements.txt | 1 - .../api_private/kafka_setup/setup_kafka.py | 92 ------- .../bot_detector/api_private/mysql/Dockerfile | 3 - .../docker-entrypoint-initdb.d/00_init.sql | 1 - .../docker-entrypoint-initdb.d/01_tables.sql | 254 ------------------ .../docker-entrypoint-initdb.d/02_data.sql | 73 ----- .../api_private/mysql_setup/Dockerfile | 10 - .../api_private/mysql_setup/__init__.py | 0 .../api_private/mysql_setup/models.py | 179 ------------ .../api_private/mysql_setup/requirements.txt | 8 - .../api_private/mysql_setup/setup_mysql.py | 108 -------- bases/bot_detector/api_private/notes.md | 40 --- .../api_private/src/api/v2/highscore.py | 14 +- 20 files changed, 11 insertions(+), 1009 deletions(-) delete mode 100644 bases/bot_detector/api_private/Dockerfile delete mode 100644 bases/bot_detector/api_private/LICENSE delete mode 100644 bases/bot_detector/api_private/Makefile delete mode 100644 bases/bot_detector/api_private/docker-compose-standalone.yml delete mode 100644 bases/bot_detector/api_private/kafka_setup/.dockerignore delete mode 100644 bases/bot_detector/api_private/kafka_setup/Dockerfile delete mode 100644 bases/bot_detector/api_private/kafka_setup/kafka_data/.gitkeep delete mode 100644 bases/bot_detector/api_private/kafka_setup/requirements.txt delete mode 100644 bases/bot_detector/api_private/kafka_setup/setup_kafka.py delete mode 100644 bases/bot_detector/api_private/mysql/Dockerfile delete mode 100644 bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/00_init.sql delete mode 100644 bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/01_tables.sql delete mode 100644 bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/02_data.sql delete mode 100644 bases/bot_detector/api_private/mysql_setup/Dockerfile delete mode 100644 bases/bot_detector/api_private/mysql_setup/__init__.py delete mode 100644 bases/bot_detector/api_private/mysql_setup/models.py delete mode 100644 bases/bot_detector/api_private/mysql_setup/requirements.txt delete mode 100644 bases/bot_detector/api_private/mysql_setup/setup_mysql.py delete mode 100644 bases/bot_detector/api_private/notes.md diff --git a/bases/bot_detector/api_private/Dockerfile b/bases/bot_detector/api_private/Dockerfile deleted file mode 100644 index 516a9ed..0000000 --- a/bases/bot_detector/api_private/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM python:3.11-slim as base - -ARG api_port -ENV UVICORN_PORT ${api_port} - -ARG root_path -ENV UVICORN_ROOT_PATH ${root_path} - -# Keeps Python from generating .pyc files in the container -ENV PYTHONDONTWRITEBYTECODE=1 - -# Turns off buffering for easier container logging -ENV PYTHONUNBUFFERED=1 - -# set the working directory -WORKDIR /project - -# install dependencies -COPY ./requirements.txt /project -RUN pip install --no-cache-dir -r requirements.txt - -# copy the scripts to the folder -COPY ./src /project/src - -# production image -FROM base as production -# Creates a non-root user with an explicit UID and adds permission to access the /project folder -RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /project -USER appuser - -CMD ["uvicorn", "src.core.server:app", "--proxy-headers", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/bases/bot_detector/api_private/LICENSE b/bases/bot_detector/api_private/LICENSE deleted file mode 100644 index f75bfcc..0000000 --- a/bases/bot_detector/api_private/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ -BSD 2-Clause License - -Copyright (c) 2023, OSRS Bot-Detector - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bases/bot_detector/api_private/Makefile b/bases/bot_detector/api_private/Makefile deleted file mode 100644 index 4b473ba..0000000 --- a/bases/bot_detector/api_private/Makefile +++ /dev/null @@ -1,103 +0,0 @@ -.PHONY: clean clean-test clean-pyc clean-build build help -.DEFAULT_GOAL := help - -define PRINT_HELP_PYSCRIPT -import re, sys - -for line in sys.stdin: - match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) - if match: - target, help = match.groups() - print("%-20s %s" % (target, help)) -endef -export PRINT_HELP_PYSCRIPT - -help: - @python3 -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) - -clean-pyc: ## clean python cache files - find . -name '*.pyc' -exec rm -f {} + - find . -name '*.pyo' -exec rm -f {} + - find . -name '*~' -exec rm -f {} + - find . -name '__pycache__' -exec rm -fr {} + - find . -name '.pytest_cache' -exec rm -fr {} + - -clean-test: ## cleanup pytests leftovers - rm -f .coverage - rm -fr htmlcov/ - rm -fr test_results/ - rm -f *report.html - rm -f log.html - rm -f test-results.html - rm -f output.xml - -test: clean ## Run pytest unit tests - python3 -m pytest - -test-debug: ## Run unit tests with debugging enabled - python3 -m pytest --pdb - -test-coverage: clean ## Run unit tests and check code coverage - PYTHONPATH=src python3 -m pytest --cov=src tests/ --disable-warnings - -docker-up: ## Startup docker - docker-compose --verbose up - -docker-build: ## Startup docker - docker-compose --verbose up --build - -setup: requirements pre-commit-setup docker-build test-setup api-setup ## setup & run after downloaded repo - -pre-commit-setup: ## Install pre-commit - python3 -m pip install pre-commit - pre-commit --version - -pre-commit: ## Run pre-commit - pre-commit run --all-files - -test-setup: - python3 -m pip install pytest - python3 -m pip install pytest-benchmark - -list-benchmarks: - pytest-benchmark list - -benchmark: - pytest tests/test_highscore_benchmark.py --benchmark-min-rounds=1000 - -create-venv: - python3 -m venv .venv - source .venv/bin/activate - -requirements: - python3 -m pip install -r requirements.txt - python3 -m pip install pytest-asyncio==0.23.6 - python3 -m pip install httpx==0.27.0 - python3 -m pip install pre-commit==3.6.2 - python3 -m pip install ruff==0.1.15 - pre-commit install - -docker-restart: - docker compose down - docker compose up --build -d - -docker-test: - docker compose down - docker compose up --build -d - pytest - -api-setup: - python3 -m pip install "fastapi[all]" - -env-setup: - touch .env - echo "KAFKA_HOST= 'localhost:9092'" >> .env - echo "DATABASE_URL= 'mysql+aiomysql://root:root_bot_buster@localhost:3307/playerdata'" >> .env - echo "ENV='DEV'" >> .env - echo "POOL_RECYCLE='60'" >> .env - echo "POOL_TIMEOUT='30'" >> .env - -docs: - open http://localhost:5000/docs - xdg-open http://localhost:5000/docs - . http://localhost:5000/docs diff --git a/bases/bot_detector/api_private/docker-compose-standalone.yml b/bases/bot_detector/api_private/docker-compose-standalone.yml deleted file mode 100644 index a5b5303..0000000 --- a/bases/bot_detector/api_private/docker-compose-standalone.yml +++ /dev/null @@ -1,65 +0,0 @@ -version: '3' -services: - mysql: - container_name: database - build: - context: ./mysql - image: bot-detector/mysql:latest - environment: - - MYSQL_ROOT_PASSWORD=root_bot_buster - volumes: - - ./mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d - # - ./mysql/mount:/var/lib/mysql # creates persistence - ports: - - 3307:3306 - networks: - - botdetector-network - healthcheck: - test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] - timeout: 20s - retries: 10 - - mysql_setup: - container_name: mysql_setup - image: bot-detector/mysql_setup - build: - context: ./mysql_setup - command: ["python", "-u","setup_mysql.py"] - networks: - - botdetector-network - depends_on: - mysql: - condition: service_healthy - - private_api: - container_name: private_api - build: - context: . - dockerfile: Dockerfile - target: base - args: - root_path: / - api_port: 5000 - # command: bash -c "apt update && apt install -y curl && sleep infinity" - command: uvicorn src.core.server:app --host 0.0.0.0 --reload --reload-include src/* - ports: - - 5000:5000 - networks: - - botdetector-network - # this overrides the env_file for the specific variable - environment: - - KAFKA_HOST=kafka:9092 - - DATABASE_URL=mysql+aiomysql://root:root_bot_buster@mysql:3306/playerdata - - ENV=DEV - - POOL_RECYCLE=60 - - POOL_TIMEOUT=30 - # env_file: - # - .env - volumes: - - ./src:/project/src - depends_on: - - mysql - - mysql_setup - -networks: - botdetector-network: diff --git a/bases/bot_detector/api_private/kafka_setup/.dockerignore b/bases/bot_detector/api_private/kafka_setup/.dockerignore deleted file mode 100644 index 121473d..0000000 --- a/bases/bot_detector/api_private/kafka_setup/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -kafka_data/*.json \ No newline at end of file diff --git a/bases/bot_detector/api_private/kafka_setup/Dockerfile b/bases/bot_detector/api_private/kafka_setup/Dockerfile deleted file mode 100644 index 9fec847..0000000 --- a/bases/bot_detector/api_private/kafka_setup/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM python:3.11-slim - -WORKDIR /app - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -COPY . . - -# Set the KAFKA_BROKER environment variable during container runtime -ENV KAFKA_BROKER=localhost:9094 - -CMD ["python", "setup_kafka.py"] \ No newline at end of file diff --git a/bases/bot_detector/api_private/kafka_setup/kafka_data/.gitkeep b/bases/bot_detector/api_private/kafka_setup/kafka_data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/bases/bot_detector/api_private/kafka_setup/requirements.txt b/bases/bot_detector/api_private/kafka_setup/requirements.txt deleted file mode 100644 index 34aa892..0000000 --- a/bases/bot_detector/api_private/kafka_setup/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -kafka-python \ No newline at end of file diff --git a/bases/bot_detector/api_private/kafka_setup/setup_kafka.py b/bases/bot_detector/api_private/kafka_setup/setup_kafka.py deleted file mode 100644 index 54ce59d..0000000 --- a/bases/bot_detector/api_private/kafka_setup/setup_kafka.py +++ /dev/null @@ -1,92 +0,0 @@ -# setup_kafka.py -import json -from kafka.admin import NewTopic, KafkaAdminClient -from kafka import KafkaProducer -import os -import zipfile - - -def create_topics(): - # Get the Kafka broker address from the environment variable - kafka_broker = os.environ.get("KAFKA_BROKER", "localhost:9094") - - # Create Kafka topics - admin_client = KafkaAdminClient(bootstrap_servers=kafka_broker) - - topics = admin_client.list_topics() - print("existing topics", topics) - - if not topics == []: - admin_client.delete_topics(topics) - - res = admin_client.create_topics( - [ - NewTopic( - name="player", - num_partitions=3, - replication_factor=1, - ), - NewTopic( - name="scraper", - num_partitions=4, - replication_factor=1, - ), - NewTopic( - name="reports", - num_partitions=4, - replication_factor=1, - ), - ] - ) - - print("created_topic", res) - - topics = admin_client.list_topics() - print("all topics", topics) - return - - -def send_json_to_kafka(file_path, producer, topic): - with open(file_path) as file: - data = json.load(file) - - for record in data: - # record = json.dumps(record).encode("utf-8") - producer.send(topic, value=record) - return - - -def insert_data(): - # Get the Kafka broker address from the environment variable - kafka_broker = os.environ.get("KAFKA_BROKER", "localhost:9094") - - zip_file_path = "kafka_data/kafka_data.zip" - extracted_folder = "kafka_data" - - print("Extracting data from the zip archive...") - # Extract the files from the zip archive - with zipfile.ZipFile(zip_file_path, "r") as zip_ref: - zip_ref.extractall(extracted_folder) - - # Create the Kafka producer - producer = KafkaProducer( - bootstrap_servers=kafka_broker, - value_serializer=lambda x: json.dumps(x).encode(), - ) - - for file_name in os.listdir(extracted_folder): - if file_name.endswith(".json"): - file_path = os.path.join(extracted_folder, file_name) - print(f"Processing file: {file_path}") - send_json_to_kafka(file_path, producer, "player") - - print("Data insertion completed.") - - -def setup_kafka(): - create_topics() - insert_data() - - -if __name__ == "__main__": - setup_kafka() \ No newline at end of file diff --git a/bases/bot_detector/api_private/mysql/Dockerfile b/bases/bot_detector/api_private/mysql/Dockerfile deleted file mode 100644 index 659f748..0000000 --- a/bases/bot_detector/api_private/mysql/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM mysql:latest - -EXPOSE 3306 \ No newline at end of file diff --git a/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/00_init.sql b/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/00_init.sql deleted file mode 100644 index 7d82066..0000000 --- a/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/00_init.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE playerdata; \ No newline at end of file diff --git a/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/01_tables.sql b/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/01_tables.sql deleted file mode 100644 index 1ae2a3f..0000000 --- a/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/01_tables.sql +++ /dev/null @@ -1,254 +0,0 @@ -USE playerdata; - -CREATE TABLE Labels ( - id int NOT NULL AUTO_INCREMENT, - label varchar(50) NOT NULL, - PRIMARY KEY (id), - UNIQUE KEY Unique_label (label) USING BTREE -) -; - -CREATE TABLE Players ( - id INT PRIMARY KEY AUTO_INCREMENT, - name TEXT, - created_at TIMESTAMP, - updated_at TIMESTAMP, - possible_ban BOOLEAN, - confirmed_ban BOOLEAN, - confirmed_player BOOLEAN, - label_id INTEGER, - label_jagex INTEGER, - ironman BOOLEAN, - hardcore_ironman BOOLEAN, - ultimate_ironman BOOLEAN, - normalized_name TEXT -); - -CREATE TABLE Reports ( - ID BIGINT PRIMARY KEY AUTO_INCREMENT, - created_at TIMESTAMP, - reportedID INT, - reportingID INT, - region_id INT, - x_coord INT, - y_coord INT, - z_coord INT, - timestamp TIMESTAMP, - manual_detect SMALLINT, - on_members_world INT, - on_pvp_world SMALLINT, - world_number INT, - equip_head_id INT, - equip_amulet_id INT, - equip_torso_id INT, - equip_legs_id INT, - equip_boots_id INT, - equip_cape_id INT, - equip_hands_id INT, - equip_weapon_id INT, - equip_shield_id INT, - equip_ge_value BIGINT, - CONSTRAINT FK_Reported_Players_id FOREIGN KEY (reportedID) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT, - CONSTRAINT FK_Reporting_Players_id FOREIGN KEY (reportingID) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT -); - - -CREATE TABLE Predictions ( - id INT PRIMARY KEY AUTO_INCREMENT, - name VARCHAR(12), - prediction VARCHAR(50), - created TIMESTAMP, - predicted_confidence DECIMAL(5, 2), - real_player DECIMAL(5, 2) DEFAULT 0, - pvm_melee_bot DECIMAL(5, 2) DEFAULT 0, - smithing_bot DECIMAL(5, 2) DEFAULT 0, - magic_bot DECIMAL(5, 2) DEFAULT 0, - fishing_bot DECIMAL(5, 2) DEFAULT 0, - mining_bot DECIMAL(5, 2) DEFAULT 0, - crafting_bot DECIMAL(5, 2) DEFAULT 0, - pvm_ranged_magic_bot DECIMAL(5, 2) DEFAULT 0, - pvm_ranged_bot DECIMAL(5, 2) DEFAULT 0, - hunter_bot DECIMAL(5, 2) DEFAULT 0, - fletching_bot DECIMAL(5, 2) DEFAULT 0, - clue_scroll_bot DECIMAL(5, 2) DEFAULT 0, - lms_bot DECIMAL(5, 2) DEFAULT 0, - agility_bot DECIMAL(5, 2) DEFAULT 0, - wintertodt_bot DECIMAL(5, 2) DEFAULT 0, - runecrafting_bot DECIMAL(5, 2) DEFAULT 0, - zalcano_bot DECIMAL(5, 2) DEFAULT 0, - woodcutting_bot DECIMAL(5, 2) DEFAULT 0, - thieving_bot DECIMAL(5, 2) DEFAULT 0, - soul_wars_bot DECIMAL(5, 2) DEFAULT 0, - cooking_bot DECIMAL(5, 2) DEFAULT 0, - vorkath_bot DECIMAL(5, 2) DEFAULT 0, - barrows_bot DECIMAL(5, 2) DEFAULT 0, - herblore_bot DECIMAL(5, 2) DEFAULT 0, - unknown_bot DECIMAL(5, 2) DEFAULT 0 -); - -CREATE TABLE playerHiscoreData ( - id bigint NOT NULL AUTO_INCREMENT, - timestamp datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - ts_date date DEFAULT NULL, - Player_id int NOT NULL, - total bigint DEFAULT '0', - attack int DEFAULT '0', - defence int DEFAULT '0', - strength int DEFAULT '0', - hitpoints int DEFAULT '0', - ranged int DEFAULT '0', - prayer int DEFAULT '0', - magic int DEFAULT '0', - cooking int DEFAULT '0', - woodcutting int DEFAULT '0', - fletching int DEFAULT '0', - fishing int DEFAULT '0', - firemaking int DEFAULT '0', - crafting int DEFAULT '0', - smithing int DEFAULT '0', - mining int DEFAULT '0', - herblore int DEFAULT '0', - agility int DEFAULT '0', - thieving int DEFAULT '0', - slayer int DEFAULT '0', - farming int DEFAULT '0', - runecraft int DEFAULT '0', - hunter int DEFAULT '0', - construction int DEFAULT '0', - league int DEFAULT '0', - bounty_hunter_hunter int DEFAULT '0', - bounty_hunter_rogue int DEFAULT '0', - cs_all int DEFAULT '0', - cs_beginner int DEFAULT '0', - cs_easy int DEFAULT '0', - cs_medium int DEFAULT '0', - cs_hard int DEFAULT '0', - cs_elite int DEFAULT '0', - cs_master int DEFAULT '0', - lms_rank int DEFAULT '0', - soul_wars_zeal int DEFAULT '0', - abyssal_sire int DEFAULT '0', - alchemical_hydra int DEFAULT '0', - barrows_chests int DEFAULT '0', - bryophyta int DEFAULT '0', - callisto int DEFAULT '0', - cerberus int DEFAULT '0', - chambers_of_xeric int DEFAULT '0', - chambers_of_xeric_challenge_mode int DEFAULT '0', - chaos_elemental int DEFAULT '0', - chaos_fanatic int DEFAULT '0', - commander_zilyana int DEFAULT '0', - corporeal_beast int DEFAULT '0', - crazy_archaeologist int DEFAULT '0', - dagannoth_prime int DEFAULT '0', - dagannoth_rex int DEFAULT '0', - dagannoth_supreme int DEFAULT '0', - deranged_archaeologist int DEFAULT '0', - general_graardor int DEFAULT '0', - giant_mole int DEFAULT '0', - grotesque_guardians int DEFAULT '0', - hespori int DEFAULT '0', - kalphite_queen int DEFAULT '0', - king_black_dragon int DEFAULT '0', - kraken int DEFAULT '0', - kreearra int DEFAULT '0', - kril_tsutsaroth int DEFAULT '0', - mimic int DEFAULT '0', - nex int DEFAULT '0', - nightmare int DEFAULT '0', - phosanis_nightmare int DEFAULT '0', - obor int DEFAULT '0', - phantom_muspah int DEFAULT '0', - sarachnis int DEFAULT '0', - scorpia int DEFAULT '0', - skotizo int DEFAULT '0', - tempoross int DEFAULT '0', - the_gauntlet int DEFAULT '0', - the_corrupted_gauntlet int DEFAULT '0', - theatre_of_blood int DEFAULT '0', - theatre_of_blood_hard int DEFAULT '0', - thermonuclear_smoke_devil int DEFAULT '0', - tombs_of_amascut int DEFAULT '0', - tombs_of_amascut_expert int DEFAULT '0', - tzkal_zuk int DEFAULT '0', - tztok_jad int DEFAULT '0', - venenatis int DEFAULT '0', - vetion int DEFAULT '0', - vorkath int DEFAULT '0', - wintertodt int DEFAULT '0', - zalcano int DEFAULT '0', - zulrah int DEFAULT '0', - rifts_closed int DEFAULT '0', - artio int DEFAULT '0', - calvarion int DEFAULT '0', - duke_sucellus int DEFAULT '0', - spindel int DEFAULT '0', - the_leviathan int DEFAULT '0', - the_whisperer int DEFAULT '0', - vardorvis int DEFAULT '0', - PRIMARY KEY (id), - UNIQUE KEY idx_playerHiscoreData_Player_id_timestamp (Player_id,timestamp), - UNIQUE KEY Unique_player_date (Player_id,ts_date), - CONSTRAINT FK_Players_id FOREIGN KEY (Player_id) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT -); - -/* --- V3 -*/ -CREATE TABLE skill ( - skill_id tinyint unsigned NOT NULL AUTO_INCREMENT, - skill_name varchar(50) NOT NULL, - PRIMARY KEY (skill_id), - UNIQUE KEY unique_skill_name (skill_name) -); - -CREATE TABLE activity ( - activity_id tinyint unsigned NOT NULL AUTO_INCREMENT, - activity_name varchar(50) NOT NULL, - PRIMARY KEY (activity_id), - UNIQUE KEY unique_activity_name (activity_name) -); - -CREATE TABLE player_skill ( - player_skill_id BIGINT unsigned NOT NULL AUTO_INCREMENT, - skill_id tinyint unsigned NOT NULL, - skill_value int unsigned NOT NULL DEFAULT '0', - PRIMARY KEY (player_skill_id), - UNIQUE KEY unique_skill_value (skill_id, skill_value) -); - -CREATE TABLE player_activity ( - player_activity_id bigint unsigned NOT NULL AUTO_INCREMENT, - activity_id tinyint unsigned NOT NULL, - activity_value int unsigned NOT NULL DEFAULT '0', - PRIMARY KEY (player_activity_id), - UNIQUE KEY unique_activity_value (activity_id, activity_value) -); - -CREATE TABLE scraper_data_v3 ( - scrape_id bigint unsigned NOT NULL AUTO_INCREMENT, - scrape_ts DATETIME NOT NULL, - scrape_date DATE NOT NULL, - player_id INT NOT NULL, - PRIMARY KEY (scrape_id), - UNIQUE KEY unique_player_scrape (player_id, scrape_date), - INDEX idx_scrape_ts (scrape_ts) -); - -CREATE TABLE scraper_player_skill ( - scrape_id BIGINT unsigned NOT NULL, - player_skill_id BIGINT unsigned NOT NULL, - PRIMARY KEY (scrape_id, player_skill_id), - KEY idx_scrape_id (scrape_id), - KEY idx_player_skill_id (player_skill_id) -) -PARTITION BY HASH (scrape_id) PARTITIONS 10; - -CREATE TABLE scraper_player_activity ( - scrape_id BIGINT unsigned NOT NULL, - player_activity_id BIGINT unsigned NOT NULL, - PRIMARY KEY (scrape_id, player_activity_id), - KEY idx_scrape_id (scrape_id), - KEY idx_player_activity_id (player_activity_id) -) -PARTITION BY HASH (scrape_id) PARTITIONS 10; diff --git a/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/02_data.sql b/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/02_data.sql deleted file mode 100644 index 6f088ea..0000000 --- a/bases/bot_detector/api_private/mysql/docker-entrypoint-initdb.d/02_data.sql +++ /dev/null @@ -1,73 +0,0 @@ -USE playerdata; - -INSERT INTO Labels (id, label) -VALUES - (1, 'Real_Player'), - (4, 'Wintertodt_bot'), - (5, 'Mining_bot'), - (7, 'Hunter_bot'), - (8, 'Herblore_bot'), - (9, 'Fletching_bot'), - (10, 'Fishing_bot'), - (11, 'Crafting_bot'), - (12, 'Cooking_bot'), - (13, 'Woodcutting_bot'), - (15, 'Smithing_bot'), - (17, 'Magic_bot'), - (19, 'PVM_Ranged_Magic_bot'), - (21, 'Agility_bot'), - (27, 'Zalcano_bot'), - (38, 'Runecrafting_bot'), - (40, 'PVM_Ranged_bot'), - (41, 'PVM_Melee_bot'), - (42, 'Thieving_bot'), - (52, 'LMS_bot'), - (56, 'Fishing_Cooking_bot'), - (57, 'mort_myre_fungus_bot'), - (59, 'temp_real_player'), - (61, 'Soul_Wars_bot'), - (64, 'Construction_Magic_bot'), - (65, 'Vorkath_bot'), - (66, 'Clue_Scroll_bot'), - (67, 'Barrows_bot'), - (76, 'Woodcutting_Mining_bot'), - ( - 77, 'Woodcutting_Firemaking_bot' - ), - (84, 'Mage_Guild_Store_bot'), - (87, 'Phosani_bot'), - (89, 'Unknown_bot'), - (90, 'Blast_mine_bot'), - (91, 'Zulrah_bot'), - (92, 'test_label'), - (109, 'Nex_bot'), - (110, 'Gauntlet_bot'); - -INSERT INTO Labels (label) VALUES ("Unkown"); -UPDATE Labels set id=0 where label="Unkown"; - - -INSERT INTO skill (skill_name) VALUES -('attack'), ('defence'), ('strength'), ('hitpoints'), ('ranged'), ('prayer'), -('magic'), ('cooking'), ('woodcutting'), ('fletching'), ('fishing'), ('firemaking'), -('crafting'), ('smithing'), ('mining'), ('herblore'), ('agility'), ('thieving'), -('slayer'), ('farming'), ('runecraft'), ('hunter'), ('construction') -; - -INSERT INTO activity (activity_name) VALUES -('league'), ('bounty_hunter_hunter'), ('bounty_hunter_rogue'), ('cs_all'), ('cs_beginner'), -('cs_easy'), ('cs_medium'), ('cs_hard'), ('cs_elite'), ('cs_master'), ('lms_rank'), -('soul_wars_zeal'), ('abyssal_sire'), ('alchemical_hydra'), ('barrows_chests'), ('bryophyta'), -('callisto'), ('cerberus'), ('chambers_of_xeric'), ('chambers_of_xeric_challenge_mode'), -('chaos_elemental'), ('chaos_fanatic'), ('commander_zilyana'), ('corporeal_beast'), -('crazy_archaeologist'), ('dagannoth_prime'), ('dagannoth_rex'), ('dagannoth_supreme'), -('deranged_archaeologist'), ('general_graardor'), ('giant_mole'), ('grotesque_guardians'), -('hespori'), ('kalphite_queen'), ('king_black_dragon'), ('kraken'), ('kreearra'), -('kril_tsutsaroth'), ('mimic'), ('nightmare'), ('nex'), ('phosanis_nightmare'), ('obor'), -('phantom_muspah'), ('sarachnis'), ('scorpia'), ('skotizo'), ('tempoross'), ('the_gauntlet'), -('the_corrupted_gauntlet'), ('theatre_of_blood'), ('theatre_of_blood_hard'), -('thermonuclear_smoke_devil'), ('tombs_of_amascut'), ('tombs_of_amascut_expert'), ('tzkal_zuk'), -('tztok_jad'), ('venenatis'), ('vetion'), ('vorkath'), ('wintertodt'), ('zalcano'), ('zulrah'), -('rifts_closed'), ('artio'), ('calvarion'), ('duke_sucellus'), ('spindel'), ('the_leviathan'), -('the_whisperer'), ('vardorvis') -; diff --git a/bases/bot_detector/api_private/mysql_setup/Dockerfile b/bases/bot_detector/api_private/mysql_setup/Dockerfile deleted file mode 100644 index 014cd75..0000000 --- a/bases/bot_detector/api_private/mysql_setup/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.11-slim - -WORKDIR /app - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -COPY . . - -CMD ["python", "setup_mysql.py"] diff --git a/bases/bot_detector/api_private/mysql_setup/__init__.py b/bases/bot_detector/api_private/mysql_setup/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bases/bot_detector/api_private/mysql_setup/models.py b/bases/bot_detector/api_private/mysql_setup/models.py deleted file mode 100644 index a755e20..0000000 --- a/bases/bot_detector/api_private/mysql_setup/models.py +++ /dev/null @@ -1,179 +0,0 @@ -# script to insert all the data we need -import random -from datetime import datetime - -from sqlalchemy import ( - TIMESTAMP, - BigInteger, - Boolean, - Column, - Date, - DateTime, - ForeignKey, - Index, - Integer, - SmallInteger, - String, - UniqueConstraint, - create_engine, -) -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker - -Base = declarative_base() -random.seed(42) - -# Define other SQLAlchemy models for remaining tables in a similar manner - -# Create an engine and bind the base -engine = create_engine("mysql+pymysql://root:root_bot_buster@mysql:3306/playerdata") -Base.metadata.create_all(engine) - -# Create a session -Session = sessionmaker(bind=engine) -session = Session() - - -class Labels(Base): - __tablename__ = "Labels" - - id = Column(Integer, primary_key=True) - label = Column(String) - - -class Players(Base): - __tablename__ = "Players" - - id = Column(Integer, primary_key=True) - name = Column(String) - created_at = Column(DateTime, default=datetime.utcnow) - updated_at = Column(DateTime, default=datetime.utcnow) - possible_ban = Column(Boolean, default=True) - confirmed_ban = Column(Boolean, default=False) - confirmed_player = Column(Boolean, default=False) - label_id = Column(Integer) - label_jagex = Column(Integer) - # ironman = Column(Boolean) - # hardcore_ironman = Column(Boolean) - # ultimate_ironman = Column(Boolean) - normalized_name = Column(String) - - -class Report(Base): - __tablename__ = "Reports" - - ID = Column(BigInteger, primary_key=True, autoincrement=True) - created_at = Column(TIMESTAMP) - reportedID = Column(Integer) - reportingID = Column(Integer) - region_id = Column(Integer) - x_coord = Column(Integer) - y_coord = Column(Integer) - z_coord = Column(Integer) - timestamp = Column(TIMESTAMP) - manual_detect = Column(SmallInteger) - on_members_world = Column(Integer) - on_pvp_world = Column(SmallInteger) - world_number = Column(Integer) - equip_head_id = Column(Integer) - equip_amulet_id = Column(Integer) - equip_torso_id = Column(Integer) - equip_legs_id = Column(Integer) - equip_boots_id = Column(Integer) - equip_cape_id = Column(Integer) - equip_hands_id = Column(Integer) - equip_weapon_id = Column(Integer) - equip_shield_id = Column(Integer) - equip_ge_value = Column(BigInteger) - - -class Skill(Base): - __tablename__ = "skill" - - skill_id = Column(SmallInteger(), primary_key=True, autoincrement=True) - skill_name = Column(String(50), nullable=False, unique=True) - - -class Activity(Base): - __tablename__ = "activity" - - activity_id = Column(SmallInteger(), primary_key=True, autoincrement=True) - activity_name = Column(String(50), nullable=False, unique=True) - - -class PlayerSkill(Base): - __tablename__ = "player_skill" - - player_skill_id = Column(BigInteger(), primary_key=True, autoincrement=True) - skill_id = Column(SmallInteger(), nullable=False) - skill_value = Column(Integer(), nullable=False, default=0) - - __table_args__ = ( - UniqueConstraint("skill_id", "skill_value", name="unique_skill_value"), - ) - - -class PlayerActivity(Base): - __tablename__ = "player_activity" - - player_activity_id = Column(BigInteger(), primary_key=True, autoincrement=True) - activity_id = Column(SmallInteger(), nullable=False) - activity_value = Column(Integer(), nullable=False, default=0) - - __table_args__ = ( - UniqueConstraint("activity_id", "activity_value", name="unique_activity_value"), - ) - - -class ScraperDataV3(Base): - __tablename__ = "scraper_data_v3" - - scrape_id = Column(BigInteger(), primary_key=True, autoincrement=True) - scrape_ts = Column(DateTime, nullable=False) - scrape_date = Column(Date, nullable=False) - player_id = Column(Integer, nullable=False) - - __table_args__ = ( - UniqueConstraint("player_id", "scrape_date", name="unique_player_scrape"), - Index("idx_scrape_ts", "scrape_ts"), - ) - - -class ScraperPlayerSkill(Base): - __tablename__ = "scraper_player_skill" - - scrape_id = Column( - BigInteger(), - ForeignKey("scraper_data_v3.scrape_id"), - primary_key=True, - ) - player_skill_id = Column( - BigInteger(), - ForeignKey("player_skill.player_skill_id"), - primary_key=True, - ) - - __table_args__ = ( - Index("idx_scrape_id", "scrape_id"), - Index("idx_player_skill_id", "player_skill_id"), - ) - - -class ScraperPlayerActivity(Base): - __tablename__ = "scraper_player_activity" - - scrape_id = Column( - BigInteger(), - ForeignKey("scraper_data_v3.scrape_id"), - primary_key=True, - ) - player_activity_id = Column( - BigInteger(), - ForeignKey("player_activity.player_activity_id"), - primary_key=True, - ) - - __table_args__ = ( - Index("idx_scrape_id", "scrape_id"), - Index("idx_player_activity_id", "player_activity_id"), - ) diff --git a/bases/bot_detector/api_private/mysql_setup/requirements.txt b/bases/bot_detector/api_private/mysql_setup/requirements.txt deleted file mode 100644 index 1ab8080..0000000 --- a/bases/bot_detector/api_private/mysql_setup/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -aiomysql==0.2.0 -cffi==1.16.0 -cryptography==42.0.5 -greenlet==3.0.3 -pycparser==2.22 -PyMySQL==1.1.0 -SQLAlchemy==2.0.28 -typing_extensions==4.10.0 diff --git a/bases/bot_detector/api_private/mysql_setup/setup_mysql.py b/bases/bot_detector/api_private/mysql_setup/setup_mysql.py deleted file mode 100644 index 30e5931..0000000 --- a/bases/bot_detector/api_private/mysql_setup/setup_mysql.py +++ /dev/null @@ -1,108 +0,0 @@ -import random -from datetime import datetime, timedelta - -from models import ( - Labels, - PlayerActivity, - Players, - PlayerSkill, - Report, - ScraperDataV3, - ScraperPlayerActivity, - ScraperPlayerSkill, - session, -) -from sqlalchemy.exc import IntegrityError - - -def random_date(): - return datetime.utcnow() - timedelta(days=random.randint(0, 365)) - - -def get_labels(): - # Query the labels table to get all id values - label_ids = session.query(Labels.id).all() - label_ids = [id[0] for id in label_ids] # Convert list of tuples to list of ids - return label_ids - - -def insert_players(len_players, label_ids: list): - # Insert data into Players table - for i in range(len_players): - print(f"Player_{i}") - # Check if the player already exists - existing_player = session.query(Players).filter_by(name=f"Player_{i}").first() - if not existing_player: - player = Players( - name=f"Player_{i}", - created_at=random_date(), - updated_at=random_date(), - possible_ban=random.choice([True, False]), - confirmed_ban=random.choice([True, False]), - confirmed_player=random.choice([True, False]), - label_id=random.choice(label_ids), # Select a random id from label_ids - label_jagex=random.randint(0, 2), - normalized_name=f"Player_{i}", - ) - session.add(player) - session.commit() - return - - -def insert_reports(len_reports, len_players): - for i in range(1, len_reports + 1): - print(f"Report_{i}") - # pick random player - reporter = random.randint(1, len_players) - reported = random.randint(1, len_players) - - if reporter == reported: - reported = random.randint(1, len_players) - - try: - session.add( - Report( - created_at=random_date(), - reportedID=reporter, - reportingID=reported, - region_id=random.randint(1, 30000), - x_coord=random.randint(1, 30000), - y_coord=random.randint(1, 30000), - z_coord=random.randint(1, 30000), - timestamp=random_date(), - manual_detect=random.choice([0, 1]), - on_members_world=random.choice([0, 1]), - on_pvp_world=random.choice([0, 1]), - world_number=random.randint(1, 300), - equip_head_id=random.randint(1, 30000), - equip_amulet_id=random.randint(1, 30000), - equip_torso_id=random.randint(1, 30000), - equip_legs_id=random.randint(1, 30000), - equip_boots_id=random.randint(1, 30000), - equip_cape_id=random.randint(1, 30000), - equip_hands_id=random.randint(1, 30000), - equip_weapon_id=random.randint(1, 30000), - equip_shield_id=random.randint(1, 30000), - equip_ge_value=random.randint(1, 2000000000), - ) - ) - except IntegrityError: - session.rollback() # Rollback the transaction if a duplicate entry is encountered - finally: - session.commit() - - -def generate_random_scraper_data(len_scrapers, len_players, skill_ids, activity_ids): - # TODO: - ... - - -def main(): - len_players = 250 - label_ids = get_labels() - insert_players(len_players, label_ids) - insert_reports(len_reports=10_000, len_players=len_players) - - -if __name__ == "__main__": - main() diff --git a/bases/bot_detector/api_private/notes.md b/bases/bot_detector/api_private/notes.md deleted file mode 100644 index 9d5da08..0000000 --- a/bases/bot_detector/api_private/notes.md +++ /dev/null @@ -1,40 +0,0 @@ -# kubectl -```sh -kubectl port-forward -n kafka svc/bd-prd-kafka-service 9094:9094 -kubectl port-forward -n database svc/mysql 3306:3306 -kubectl port-forward -n bd-prd svc/private-api-svc 5000:5000 -``` - -```sh -python -m venv .venv -.venv\Scripts\activate -python -m pip install --upgrade pip -pip install -r requirements.txt -``` - -```sh -.venv\Scripts\activate -pip freeze > requirements.txt -``` - -# linux / wsl -to open vscode in wsl just open vs code, type `wsl` in the terminal than type `code .` - -tip: you can trim your command line path with -```sh -nano ~/.bashrc -``` -add at the botom, exit nano with ctrl + x then press y to save -```sh -PROMPT_DIRTRIM=3 -``` -restart your shell - -```sh -sudo apt update -y && sudo apt upgrade -y -sudo apt install python3.10-venv -y -``` -```sh -python3 -m venv .venv -touch .venv\bin\activate -``` diff --git a/bases/bot_detector/api_private/src/api/v2/highscore.py b/bases/bot_detector/api_private/src/api/v2/highscore.py index 8a509ae..ee9dc60 100644 --- a/bases/bot_detector/api_private/src/api/v2/highscore.py +++ b/bases/bot_detector/api_private/src/api/v2/highscore.py @@ -2,11 +2,19 @@ from fastapi import APIRouter, Depends, Query -from src.app.repositories import PlayerActivityRepo, PlayerSkillsRepo, ScraperDataRepo -from src.app.views.response.highscore import PlayerHiscoreData +from bases.bot_detector.api_private.src.app.repositories import ( + PlayerActivityRepo, + PlayerSkillsRepo, + ScraperDataRepo, +) +from bases.bot_detector.api_private.src.app.views.response.highscore import ( + PlayerHiscoreData, +) # from src.app.repositories.highscore import HighscoreRepo -from src.core.fastapi.dependencies.session import get_session +from bases.bot_detector.api_private.src.core.fastapi.dependencies.session import ( + get_session, +) logger = logging.getLogger(__name__) From 8ba0ef87c49274213ff4a7744811c35ea9bfbe5f Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Fri, 6 Jun 2025 22:02:50 -0400 Subject: [PATCH 03/39] referance fixes --- .../bot_detector/api_private/src/__init__.py | 0 .../api_private/src/api/v2/player.py | 6 ++- .../api_private/src/api/v3/highscore.py | 12 +++-- .../src/app/repositories/highscore.py | 8 +-- .../src/app/repositories/player.py | 2 +- .../src/app/repositories/player_activities.py | 9 +++- .../src/app/repositories/player_skills.py | 6 ++- .../src/app/repositories/scraper_data.py | 4 +- .../api_private/src/core/database/database.py | 2 +- .../src/core/database/models/activities.py | 2 +- .../src/core/database/models/highscore.py | 2 +- .../src/core/database/models/player.py | 4 +- .../src/core/database/models/prediction.py | 5 +- .../src/core/database/models/report.py | 6 ++- .../src/core/database/models/scraper_data.py | 2 +- .../src/core/database/models/skills.py | 2 +- .../src/core/fastapi/dependencies/session.py | 2 +- .../api_private/src/core/logging_config.py | 2 +- .../api_private/src/core/server.py | 6 ++- .../api_private/tests/conftest.py | 6 ++- .../tests/test_highscore_benchmark_custom.py | 3 +- pyproject.toml | 2 + uv.lock | 54 +++++++++++++++++++ 23 files changed, 114 insertions(+), 33 deletions(-) create mode 100644 bases/bot_detector/api_private/src/__init__.py diff --git a/bases/bot_detector/api_private/src/__init__.py b/bases/bot_detector/api_private/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bases/bot_detector/api_private/src/api/v2/player.py b/bases/bot_detector/api_private/src/api/v2/player.py index 743a58b..9c5991b 100644 --- a/bases/bot_detector/api_private/src/api/v2/player.py +++ b/bases/bot_detector/api_private/src/api/v2/player.py @@ -1,7 +1,9 @@ from fastapi import APIRouter, Depends, Query -from src.app.repositories.player import PlayerRepo -from src.core.fastapi.dependencies.session import get_session +from bases.bot_detector.api_private.src.app.repositories.player import PlayerRepo +from bases.bot_detector.api_private.src.core.fastapi.dependencies.session import ( + get_session, +) router = APIRouter() diff --git a/bases/bot_detector/api_private/src/api/v3/highscore.py b/bases/bot_detector/api_private/src/api/v3/highscore.py index 466446f..d1c808a 100644 --- a/bases/bot_detector/api_private/src/api/v3/highscore.py +++ b/bases/bot_detector/api_private/src/api/v3/highscore.py @@ -3,9 +3,15 @@ from fastapi import APIRouter, Depends, Query -from src.app.repositories import ScraperDataRepo -from src.app.views.response import ActivityView, ScraperDataView, SkillView -from src.core.fastapi.dependencies.session import get_session +from bases.bot_detector.api_private.src.app.repositories import ScraperDataRepo +from bases.bot_detector.api_private.src.app.views.response import ( + ActivityView, + ScraperDataView, + SkillView, +) +from bases.bot_detector.api_private.src.core.fastapi.dependencies.session import ( + get_session, +) logger = logging.getLogger(__name__) diff --git a/bases/bot_detector/api_private/src/app/repositories/highscore.py b/bases/bot_detector/api_private/src/app/repositories/highscore.py index 9dadedb..a310121 100644 --- a/bases/bot_detector/api_private/src/app/repositories/highscore.py +++ b/bases/bot_detector/api_private/src/app/repositories/highscore.py @@ -5,11 +5,13 @@ from sqlalchemy.orm import aliased from sqlalchemy.sql.expression import Select -from src.app.repositories.abstract_repo import AbstractAPI -from src.core.database.models import ( # playerHiscoreData,; PlayerHiscoreDataXPChange, +from bases.bot_detector.api_private.src.app.repositories.abstract_repo import ( + AbstractAPI, +) +from bases.bot_detector.api_private.src.core.database.models import ( # playerHiscoreData,; PlayerHiscoreDataXPChange, PlayerHiscoreDataLatest, ) -from src.core.database.models.player import Player +from bases.bot_detector.api_private.src.core.database.models.player import Player logger = logging.getLogger(__name__) diff --git a/bases/bot_detector/api_private/src/app/repositories/player.py b/bases/bot_detector/api_private/src/app/repositories/player.py index 32e0823..f6f39af 100644 --- a/bases/bot_detector/api_private/src/app/repositories/player.py +++ b/bases/bot_detector/api_private/src/app/repositories/player.py @@ -2,7 +2,7 @@ from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession from sqlalchemy.sql.expression import Select -from src.core.database.models.player import Player +from bases.bot_detector.api_private.src.core.database.models.player import Player class PlayerRepo: diff --git a/bases/bot_detector/api_private/src/app/repositories/player_activities.py b/bases/bot_detector/api_private/src/app/repositories/player_activities.py index b6c0993..6e91a3c 100644 --- a/bases/bot_detector/api_private/src/app/repositories/player_activities.py +++ b/bases/bot_detector/api_private/src/app/repositories/player_activities.py @@ -3,8 +3,13 @@ from sqlalchemy.orm import aliased from sqlalchemy.sql.expression import Select -from src.app.repositories.abstract_repo import AbstractAPI -from src.core.database.models import Activities, PlayerActivities +from bases.bot_detector.api_private.src.app.repositories.abstract_repo import ( + AbstractAPI, +) +from bases.bot_detector.api_private.src.core.database.models import ( + Activities, + PlayerActivities, +) class PlayerActivityRepo(AbstractAPI): diff --git a/bases/bot_detector/api_private/src/app/repositories/player_skills.py b/bases/bot_detector/api_private/src/app/repositories/player_skills.py index ea9705d..125c2b5 100644 --- a/bases/bot_detector/api_private/src/app/repositories/player_skills.py +++ b/bases/bot_detector/api_private/src/app/repositories/player_skills.py @@ -3,8 +3,10 @@ from sqlalchemy.orm import aliased from sqlalchemy.sql.expression import Select -from src.app.repositories.abstract_repo import AbstractAPI -from src.core.database.models import PlayerSkills, Skills +from bases.bot_detector.api_private.src.app.repositories.abstract_repo import ( + AbstractAPI, +) +from bases.bot_detector.api_private.src.core.database.models import PlayerSkills, Skills class PlayerSkillsRepo(AbstractAPI): diff --git a/bases/bot_detector/api_private/src/app/repositories/scraper_data.py b/bases/bot_detector/api_private/src/app/repositories/scraper_data.py index 56ba093..d868354 100644 --- a/bases/bot_detector/api_private/src/app/repositories/scraper_data.py +++ b/bases/bot_detector/api_private/src/app/repositories/scraper_data.py @@ -2,8 +2,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import aliased -from src.core.database.models.player import Player -from src.core.database.models.scraper_data_v3 import ( +from bases.bot_detector.api_private.src.core.database.models.player import Player +from bases.bot_detector.api_private.src.core.database.models.scraper_data_v3 import ( Activity, PlayerActivity, PlayerSkill, diff --git a/bases/bot_detector/api_private/src/core/database/database.py b/bases/bot_detector/api_private/src/core/database/database.py index b02fb3f..7bd0520 100644 --- a/bases/bot_detector/api_private/src/core/database/database.py +++ b/bases/bot_detector/api_private/src/core/database/database.py @@ -1,7 +1,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.orm import declarative_base, sessionmaker -from src.core.config import settings +from bases.bot_detector.api_private.src.core.config import settings # Create an async SQLAlchemy engine engine = create_async_engine( diff --git a/bases/bot_detector/api_private/src/core/database/models/activities.py b/bases/bot_detector/api_private/src/core/database/models/activities.py index e252c72..cfb3cf2 100644 --- a/bases/bot_detector/api_private/src/core/database/models/activities.py +++ b/bases/bot_detector/api_private/src/core/database/models/activities.py @@ -1,7 +1,7 @@ from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.dialects.mysql import BIGINT, TINYINT -from src.core.database.database import Base +from bases.bot_detector.api_private.src.core.database.database import Base class Activities(Base): diff --git a/bases/bot_detector/api_private/src/core/database/models/highscore.py b/bases/bot_detector/api_private/src/core/database/models/highscore.py index e28bcb4..4bb8051 100644 --- a/bases/bot_detector/api_private/src/core/database/models/highscore.py +++ b/bases/bot_detector/api_private/src/core/database/models/highscore.py @@ -9,7 +9,7 @@ text, ) -from src.core.database.database import Base +from bases.bot_detector.api_private.src.core.database.database import Base class playerHiscoreData(Base): diff --git a/bases/bot_detector/api_private/src/core/database/models/player.py b/bases/bot_detector/api_private/src/core/database/models/player.py index 726f376..768b073 100644 --- a/bases/bot_detector/api_private/src/core/database/models/player.py +++ b/bases/bot_detector/api_private/src/core/database/models/player.py @@ -1,5 +1,7 @@ from sqlalchemy import Boolean, Column, DateTime, Integer, Text -from src.core.database.database import Base + +from bases.bot_detector.api_private.src.core.database.database import Base + class Player(Base): __tablename__ = "Players" diff --git a/bases/bot_detector/api_private/src/core/database/models/prediction.py b/bases/bot_detector/api_private/src/core/database/models/prediction.py index 748f7e8..e664c4f 100644 --- a/bases/bot_detector/api_private/src/core/database/models/prediction.py +++ b/bases/bot_detector/api_private/src/core/database/models/prediction.py @@ -1,6 +1,6 @@ -from sqlalchemy import Column, Integer, String, TIMESTAMP, DECIMAL -from src.core.database.database import Base +from sqlalchemy import DECIMAL, TIMESTAMP, Column, Integer, String +from bases.bot_detector.api_private.src.core.database.database import Base class Prediction(Base): @@ -36,4 +36,3 @@ class Prediction(Base): barrows_bot = Column(DECIMAL(5, 2), default=0) herblore_bot = Column(DECIMAL(5, 2), default=0) unknown_bot = Column(DECIMAL(5, 2), default=0) - diff --git a/bases/bot_detector/api_private/src/core/database/models/report.py b/bases/bot_detector/api_private/src/core/database/models/report.py index 28e65a9..c6e7b06 100644 --- a/bases/bot_detector/api_private/src/core/database/models/report.py +++ b/bases/bot_detector/api_private/src/core/database/models/report.py @@ -1,5 +1,7 @@ -from sqlalchemy import BigInteger, Column, Integer, SmallInteger, TIMESTAMP -from src.core.database.database import Base +from sqlalchemy import TIMESTAMP, BigInteger, Column, Integer, SmallInteger + +from bases.bot_detector.api_private.src.core.database.database import Base + class Report(Base): __tablename__ = "Reports" diff --git a/bases/bot_detector/api_private/src/core/database/models/scraper_data.py b/bases/bot_detector/api_private/src/core/database/models/scraper_data.py index e14021a..3117cad 100644 --- a/bases/bot_detector/api_private/src/core/database/models/scraper_data.py +++ b/bases/bot_detector/api_private/src/core/database/models/scraper_data.py @@ -1,7 +1,7 @@ from sqlalchemy import Column, Date, DateTime, func from sqlalchemy.dialects.mysql import BIGINT, SMALLINT -from src.core.database.database import Base +from bases.bot_detector.api_private.src.core.database.database import Base class ScraperData(Base): diff --git a/bases/bot_detector/api_private/src/core/database/models/skills.py b/bases/bot_detector/api_private/src/core/database/models/skills.py index a7e4080..cca52ca 100644 --- a/bases/bot_detector/api_private/src/core/database/models/skills.py +++ b/bases/bot_detector/api_private/src/core/database/models/skills.py @@ -1,7 +1,7 @@ from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.dialects.mysql import BIGINT, TINYINT -from src.core.database.database import Base +from bases.bot_detector.api_private.src.core.database.database import Base class Skills(Base): diff --git a/bases/bot_detector/api_private/src/core/fastapi/dependencies/session.py b/bases/bot_detector/api_private/src/core/fastapi/dependencies/session.py index 08f5092..7a59fd6 100644 --- a/bases/bot_detector/api_private/src/core/fastapi/dependencies/session.py +++ b/bases/bot_detector/api_private/src/core/fastapi/dependencies/session.py @@ -1,6 +1,6 @@ from sqlalchemy.ext.asyncio import AsyncSession -from src.core.database.database import SessionFactory +from bases.bot_detector.api_private.src.core.database.database import SessionFactory # Dependency to get an asynchronous session diff --git a/bases/bot_detector/api_private/src/core/logging_config.py b/bases/bot_detector/api_private/src/core/logging_config.py index e35d4f3..3db98f4 100644 --- a/bases/bot_detector/api_private/src/core/logging_config.py +++ b/bases/bot_detector/api_private/src/core/logging_config.py @@ -2,7 +2,7 @@ import logging import sys -from src.core.config import settings +from .config import settings # # log formatting formatter = logging.Formatter( diff --git a/bases/bot_detector/api_private/src/core/server.py b/bases/bot_detector/api_private/src/core/server.py index dc20696..a8ac5d7 100644 --- a/bases/bot_detector/api_private/src/core/server.py +++ b/bases/bot_detector/api_private/src/core/server.py @@ -4,8 +4,10 @@ from fastapi.middleware import Middleware from fastapi.middleware.cors import CORSMiddleware -from src import api -from src.core.fastapi.middleware.logging import LoggingMiddleware +from bases.bot_detector.api_private.src import api +from bases.bot_detector.api_private.src.core.fastapi.middleware.logging import ( + LoggingMiddleware, +) logger = logging.getLogger(__name__) diff --git a/bases/bot_detector/api_private/tests/conftest.py b/bases/bot_detector/api_private/tests/conftest.py index 79823af..39d4328 100644 --- a/bases/bot_detector/api_private/tests/conftest.py +++ b/bases/bot_detector/api_private/tests/conftest.py @@ -13,8 +13,10 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from src.core import server # noqa: E402 -from src.core.fastapi.dependencies.session import get_session # noqa: E402 +from bases.bot_detector.api_private.src.core import server # noqa: E402 +from bases.bot_detector.api_private.src.core.fastapi.dependencies.session import ( + get_session, # noqa: E402 +) # Create an async SQLAlchemy engine diff --git a/bases/bot_detector/api_private/tests/test_highscore_benchmark_custom.py b/bases/bot_detector/api_private/tests/test_highscore_benchmark_custom.py index 7dafae0..d8a4a13 100644 --- a/bases/bot_detector/api_private/tests/test_highscore_benchmark_custom.py +++ b/bases/bot_detector/api_private/tests/test_highscore_benchmark_custom.py @@ -2,9 +2,10 @@ import statistics import pytest -from benchmark import Benchmark from httpx import AsyncClient +from .benchmark import Benchmark + # Global variable to store the results benchmark_results = {"v2": [], "v3": []} player_ids = [random.randint(1, 250) for i in range(10)] diff --git a/pyproject.toml b/pyproject.toml index 34ed746..2b0a9b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,8 @@ dev = [ "ruff>=0.8.4", "pytest-asyncio>=0.25.0", "pytest-cov>=6.1.1", + "httpx>=0.27.0", + "pytest-benchmark>=3.0.0", ] diff --git a/uv.lock b/uv.lock index 8157d95..04425ac 100644 --- a/uv.lock +++ b/uv.lock @@ -260,10 +260,12 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "httpx" }, { name = "kafka-python" }, { name = "polylith-cli" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-benchmark" }, { name = "pytest-cov" }, { name = "ruff" }, ] @@ -286,10 +288,12 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "httpx", specifier = ">=0.27.0" }, { name = "kafka-python", specifier = ">=2.0.2" }, { name = "polylith-cli", specifier = ">=1.24.0" }, { name = "pytest", specifier = ">=8.3.4" }, { name = "pytest-asyncio", specifier = ">=0.25.0" }, + { name = "pytest-benchmark", specifier = ">=3.0.0" }, { name = "pytest-cov", specifier = ">=6.1.1" }, { name = "ruff", specifier = ">=0.8.4" }, ] @@ -696,6 +700,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515 }, ] +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784 }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517 }, +] + [[package]] name = "idna" version = "3.10" @@ -1004,6 +1036,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/b6/c5319caea262f4821995dca2107483b94a3345d4607ad797c76cb9c36bcc/propcache-0.2.1-py3-none-any.whl", hash = "sha256:52277518d6aae65536e9cea52d4e7fd2f7a66f4aa2d30ed3f2fcea620ace3c54", size = 11818 }, ] +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335 }, +] + [[package]] name = "pycparser" version = "2.22" @@ -1153,6 +1194,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/56/2ee0cab25c11d4e38738a2a98c645a8f002e2ecf7b5ed774c70d53b92bb1/pytest_asyncio-0.25.0-py3-none-any.whl", hash = "sha256:db5432d18eac6b7e28b46dcd9b69921b55c3b1086e85febfe04e70b18d9e81b3", size = 19245 }, ] +[[package]] +name = "pytest-benchmark" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259 }, +] + [[package]] name = "pytest-cov" version = "6.1.1" From bd48cd596a8b637ac107acac544561e991e99e5b Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Fri, 6 Jun 2025 22:15:20 -0400 Subject: [PATCH 04/39] proj for api private --- projects/api_private/pyproject.toml | 44 +++ projects/api_private/uv.lock | 440 ++++++++++++++++++++++++++++ pyproject.toml | 1 - 3 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 projects/api_private/pyproject.toml create mode 100644 projects/api_private/uv.lock diff --git a/projects/api_private/pyproject.toml b/projects/api_private/pyproject.toml new file mode 100644 index 0000000..159ca9f --- /dev/null +++ b/projects/api_private/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["hatchling", "hatch-polylith-bricks"] +build-backend = "hatchling.build" + +[project] +name = "api_private" +version = "0.1.0" +requires-python = ">=3.10" + +dependencies = [ + # core + "fastapi>=0.115.12", + "uvicorn>=0.34.2", + + # config / validation + "pydantic-settings>=2.9.1", + "python-dotenv>=1.0.1", + + # Kafka (choose one) + "kafka-python>=2.0.2", + + # Database + "sqlalchemy>=2.0.40", + "asyncmy>=0.2.9", +] + +[tool.poetry.group.dev.dependencies] +pytest = "^8.3" +pytest-asyncio = "^0.25" +httpx = "^0.27" +watchfiles = "^0.21" +pytest-benchmark = "^3.0.0" + +[project.scripts] +scrape_task_producer = "bot_detector.api_private.core.server:run" + +[tool.hatch.build.hooks.polylith-bricks] +packages = ["bot_detector"] + +[tool.polylith.bricks] +"../../bases/bot_detector/api_private" = "bot_detector/api_private" +"../../components/bot_detector/kafka" = "bot_detector/kafka" +"../../components/bot_detector/structs" = "bot_detector/structs" +"../../components/bot_detector/logfmt" = "bot_detector/logfmt" \ No newline at end of file diff --git a/projects/api_private/uv.lock b/projects/api_private/uv.lock new file mode 100644 index 0000000..f258160 --- /dev/null +++ b/projects/api_private/uv.lock @@ -0,0 +1,440 @@ +version = 1 +revision = 1 +requires-python = ">=3.10" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643 }, +] + +[[package]] +name = "anyio" +version = "4.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "sniffio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/7d/4c1bd541d4dffa1b52bd83fb8527089e097a106fc90b467a7313b105f840/anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028", size = 190949 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916 }, +] + +[[package]] +name = "api-private" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "asyncmy" }, + { name = "fastapi" }, + { name = "kafka-python" }, + { name = "pydantic-settings" }, + { name = "python-dotenv" }, + { name = "sqlalchemy" }, + { name = "uvicorn" }, +] + +[package.metadata] +requires-dist = [ + { name = "asyncmy", specifier = ">=0.2.9" }, + { name = "fastapi", specifier = ">=0.115.12" }, + { name = "kafka-python", specifier = ">=2.0.2" }, + { name = "pydantic-settings", specifier = ">=2.9.1" }, + { name = "python-dotenv", specifier = ">=1.0.1" }, + { name = "sqlalchemy", specifier = ">=2.0.40" }, + { name = "uvicorn", specifier = ">=0.34.2" }, +] + +[[package]] +name = "asyncmy" +version = "0.2.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/76/55cc0577f9e838c5a5213bf33159b9e484c9d9820a2bafd4d6bfa631bf86/asyncmy-0.2.10.tar.gz", hash = "sha256:f4b67edadf7caa56bdaf1c2e6cf451150c0a86f5353744deabe4426fe27aff4e", size = 63889 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/c9/412b137c52f6c6437faba27412ccb32721571c42e59bc4f799796316866b/asyncmy-0.2.10-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:c2237c8756b8f374099bd320c53b16f7ec0cee8258f00d72eed5a2cd3d251066", size = 1803880 }, + { url = "https://files.pythonhosted.org/packages/74/f3/c9520f489dc42a594c8ad3cbe2088ec511245a3c55c3333e6fa949838420/asyncmy-0.2.10-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:6e98d4fbf7ea0d99dfecb24968c9c350b019397ba1af9f181d51bb0f6f81919b", size = 1736363 }, + { url = "https://files.pythonhosted.org/packages/52/9c/3c531a414290cbde9313cad54bb525caf6b1055ffa56bb271bf70512b533/asyncmy-0.2.10-cp310-cp310-manylinux_2_17_i686.manylinux_2_5_i686.manylinux1_i686.manylinux2014_i686.whl", hash = "sha256:b1b1ee03556c7eda6422afc3aca132982a84706f8abf30f880d642f50670c7ed", size = 4970043 }, + { url = "https://files.pythonhosted.org/packages/03/64/176ed8a79d3a24b2e8ba7a11b429553f29fea20276537651526f3a87660b/asyncmy-0.2.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e2b97672ea3f0b335c0ffd3da1a5727b530f82f5032cd87e86c3aa3ac6df7f3", size = 5168645 }, + { url = "https://files.pythonhosted.org/packages/81/3f/46f126663649784ab6586bc9b482bca432a35588714170621db8d33d76e4/asyncmy-0.2.10-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c6471ce1f9ae1e6f0d55adfb57c49d0bcf5753a253cccbd33799ddb402fe7da2", size = 4988493 }, + { url = "https://files.pythonhosted.org/packages/5f/c6/acce7ea4b74e092582d65744418940b2b8c661102a22a638f58e7b651c6f/asyncmy-0.2.10-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:10e2a10fe44a2b216a1ae58fbdafa3fed661a625ec3c030c560c26f6ab618522", size = 5158496 }, + { url = "https://files.pythonhosted.org/packages/d5/01/d8fa0291083e9a0d899addda1f7608da37d28fff9bb4df1bd6f7f37354db/asyncmy-0.2.10-cp310-cp310-win32.whl", hash = "sha256:a791ab117787eb075bc37ed02caa7f3e30cca10f1b09ec7eeb51d733df1d49fc", size = 1624372 }, + { url = "https://files.pythonhosted.org/packages/cf/a0/ad6669fd2870492749c189a72c881716a3727b7f0bc972fc8cea7a40879c/asyncmy-0.2.10-cp310-cp310-win_amd64.whl", hash = "sha256:bd16fdc0964a4a1a19aec9797ca631c3ff2530013fdcd27225fc2e48af592804", size = 1694174 }, + { url = "https://files.pythonhosted.org/packages/72/1a/21b4af0d19862cc991f1095f006981a4f898599060dfa59f136e292b3e9a/asyncmy-0.2.10-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:7af0f1f31f800a8789620c195e92f36cce4def68ee70d625534544d43044ed2a", size = 1806974 }, + { url = "https://files.pythonhosted.org/packages/1d/ce/3579a88123ead38e60e0b6e744224907e3d7a668518f9a46ed584df4f788/asyncmy-0.2.10-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:800116ab85dc53b24f484fb644fefffac56db7367a31e7d62f4097d495105a2c", size = 1738218 }, + { url = "https://files.pythonhosted.org/packages/e2/39/10646bbafce22025be25aa709e83f0cdd3fb9089304cf9d3169a80540850/asyncmy-0.2.10-cp311-cp311-manylinux_2_17_i686.manylinux_2_5_i686.manylinux1_i686.manylinux2014_i686.whl", hash = "sha256:39525e9d7e557b83db268ed14b149a13530e0d09a536943dba561a8a1c94cc07", size = 5346417 }, + { url = "https://files.pythonhosted.org/packages/8f/f8/3fb0d0481def3a0900778f7d04f50028a4a2d987087a2f1e718e6c236e01/asyncmy-0.2.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76e199d6b57918999efc702d2dbb182cb7ba8c604cdfc912517955219b16eaea", size = 5553197 }, + { url = "https://files.pythonhosted.org/packages/82/a5/8281e8c0999fc6303b5b522ee82d1e338157a74f8bbbaa020e392b69156a/asyncmy-0.2.10-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9ca8fdd7dbbf2d9b4c2d3a5fac42b058707d6a483b71fded29051b8ae198a250", size = 5337915 }, + { url = "https://files.pythonhosted.org/packages/fe/f4/425108f5c6976ceb67b8f95bc73480fe777a95e7a89a29299664f5cb380f/asyncmy-0.2.10-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0df23db54e38602c803dacf1bbc1dcc4237a87223e659681f00d1a319a4f3826", size = 5524662 }, + { url = "https://files.pythonhosted.org/packages/ff/32/17291b12dce380abbbec888ea9d4e863fd2116530bf2c87c1ab40b39f9d1/asyncmy-0.2.10-cp311-cp311-win32.whl", hash = "sha256:a16633032be020b931acfd7cd1862c7dad42a96ea0b9b28786f2ec48e0a86757", size = 1622375 }, + { url = "https://files.pythonhosted.org/packages/e2/a3/76e65877de5e6fc853373908079adb711f80ed09aab4e152a533e0322375/asyncmy-0.2.10-cp311-cp311-win_amd64.whl", hash = "sha256:cca06212575922216b89218abd86a75f8f7375fc9c28159ea469f860785cdbc7", size = 1696693 }, + { url = "https://files.pythonhosted.org/packages/b8/82/5a4b1aedae9b35f7885f10568437d80507d7a6704b51da2fc960a20c4948/asyncmy-0.2.10-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:42295530c5f36784031f7fa42235ef8dd93a75d9b66904de087e68ff704b4f03", size = 1783558 }, + { url = "https://files.pythonhosted.org/packages/39/24/0fce480680531a29b51e1d2680a540c597e1a113aa1dc58cb7483c123a6b/asyncmy-0.2.10-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:641a853ffcec762905cbeceeb623839c9149b854d5c3716eb9a22c2b505802af", size = 1729268 }, + { url = "https://files.pythonhosted.org/packages/c8/96/74dc1aaf1ab0bde88d3c6b3a70bd25f18796adb4e91b77ad580efe232df5/asyncmy-0.2.10-cp312-cp312-manylinux_2_17_i686.manylinux_2_5_i686.manylinux1_i686.manylinux2014_i686.whl", hash = "sha256:c554874223dd36b1cfc15e2cd0090792ea3832798e8fe9e9d167557e9cf31b4d", size = 5343513 }, + { url = "https://files.pythonhosted.org/packages/9a/04/14662ff5b9cfab5cc11dcf91f2316e2f80d88fbd2156e458deef3e72512a/asyncmy-0.2.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd16e84391dde8edb40c57d7db634706cbbafb75e6a01dc8b68a63f8dd9e44ca", size = 5592344 }, + { url = "https://files.pythonhosted.org/packages/7c/ac/3cf0abb3acd4f469bd012a1b4a01968bac07a142fca510da946b6ab1bf4f/asyncmy-0.2.10-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9f6b44c4bf4bb69a2a1d9d26dee302473099105ba95283b479458c448943ed3c", size = 5300819 }, + { url = "https://files.pythonhosted.org/packages/5c/23/6d05254d1c89ad15e7f32eb3df277afc7bbb2220faa83a76bea0b7bc6407/asyncmy-0.2.10-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:16d398b1aad0550c6fe1655b6758455e3554125af8aaf1f5abdc1546078c7257", size = 5548799 }, + { url = "https://files.pythonhosted.org/packages/fe/32/b7ce9782c741b6a821a0d11772f180f431a5c3ba6eaf2e6dfa1c3cbcf4df/asyncmy-0.2.10-cp312-cp312-win32.whl", hash = "sha256:59d2639dcc23939ae82b93b40a683c15a091460a3f77fa6aef1854c0a0af99cc", size = 1597544 }, + { url = "https://files.pythonhosted.org/packages/94/08/7de4f4a17196c355e4706ceba0ab60627541c78011881a7c69f41c6414c5/asyncmy-0.2.10-cp312-cp312-win_amd64.whl", hash = "sha256:4c6674073be97ffb7ac7f909e803008b23e50281131fef4e30b7b2162141a574", size = 1679064 }, + { url = "https://files.pythonhosted.org/packages/83/32/3317d5290737a3c4685343fe37e02567518357c46ed87c51f47139d31ded/asyncmy-0.2.10-pp310-pypy310_pp73-macosx_13_0_x86_64.whl", hash = "sha256:f10c977c60a95bd6ec6b8654e20c8f53bad566911562a7ad7117ca94618f05d3", size = 1627680 }, + { url = "https://files.pythonhosted.org/packages/e9/e1/afeb50deb0554006c48b9f4f7b6b726e0aa42fa96d7cfbd3fdd0800765e2/asyncmy-0.2.10-pp310-pypy310_pp73-macosx_14_0_arm64.whl", hash = "sha256:aab07fbdb9466beaffef136ffabe388f0d295d8d2adb8f62c272f1d4076515b9", size = 1593957 }, + { url = "https://files.pythonhosted.org/packages/be/c1/56d3721e2b2eab84320058c3458da168d143446031eca3799aed481c33d2/asyncmy-0.2.10-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux_2_5_i686.manylinux1_i686.manylinux2014_i686.whl", hash = "sha256:63144322ade68262201baae73ad0c8a06b98a3c6ae39d1f3f21c41cc5287066a", size = 1756531 }, + { url = "https://files.pythonhosted.org/packages/ac/1a/295f06eb8e5926749265e08da9e2dc0dc14e0244bf36843997a1c8e18a50/asyncmy-0.2.10-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9659d95c6f2a611aec15bdd928950df937bf68bc4bbb68b809ee8924b6756067", size = 1752746 }, + { url = "https://files.pythonhosted.org/packages/ab/09/3a5351acc6273c28333cad8193184de0070c617fd8385fd8ba23d789e08d/asyncmy-0.2.10-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8ced4bd938e95ede0fb9fa54755773df47bdb9f29f142512501e613dd95cf4a4", size = 1614903 }, +] + +[[package]] +name = "click" +version = "8.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/6c/8ca2efa64cf75a977a0d7fac081354553ebe483345c734fb6b6515d96bbc/click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202", size = 286342 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215 }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674 }, +] + +[[package]] +name = "fastapi" +version = "0.115.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/55/ae499352d82338331ca1e28c7f4a63bfd09479b16395dce38cf50a39e2c2/fastapi-0.115.12.tar.gz", hash = "sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681", size = 295236 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/b3/b51f09c2ba432a576fe63758bddc81f78f0c6309d9e5c10d194313bf021e/fastapi-0.115.12-py3-none-any.whl", hash = "sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d", size = 95164 }, +] + +[[package]] +name = "greenlet" +version = "3.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/92/bb85bd6e80148a4d2e0c59f7c0c2891029f8fd510183afc7d8d2feeed9b6/greenlet-3.2.3.tar.gz", hash = "sha256:8b0dd8ae4c0d6f5e54ee55ba935eeb3d735a9b58a8a1e5b5cbab64e01a39f365", size = 185752 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/db/b4c12cff13ebac2786f4f217f06588bccd8b53d260453404ef22b121fc3a/greenlet-3.2.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:1afd685acd5597349ee6d7a88a8bec83ce13c106ac78c196ee9dde7c04fe87be", size = 268977 }, + { url = "https://files.pythonhosted.org/packages/52/61/75b4abd8147f13f70986df2801bf93735c1bd87ea780d70e3b3ecda8c165/greenlet-3.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:761917cac215c61e9dc7324b2606107b3b292a8349bdebb31503ab4de3f559ac", size = 627351 }, + { url = "https://files.pythonhosted.org/packages/35/aa/6894ae299d059d26254779a5088632874b80ee8cf89a88bca00b0709d22f/greenlet-3.2.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:a433dbc54e4a37e4fff90ef34f25a8c00aed99b06856f0119dcf09fbafa16392", size = 638599 }, + { url = "https://files.pythonhosted.org/packages/30/64/e01a8261d13c47f3c082519a5e9dbf9e143cc0498ed20c911d04e54d526c/greenlet-3.2.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:72e77ed69312bab0434d7292316d5afd6896192ac4327d44f3d613ecb85b037c", size = 634482 }, + { url = "https://files.pythonhosted.org/packages/47/48/ff9ca8ba9772d083a4f5221f7b4f0ebe8978131a9ae0909cf202f94cd879/greenlet-3.2.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:68671180e3849b963649254a882cd544a3c75bfcd2c527346ad8bb53494444db", size = 633284 }, + { url = "https://files.pythonhosted.org/packages/e9/45/626e974948713bc15775b696adb3eb0bd708bec267d6d2d5c47bb47a6119/greenlet-3.2.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49c8cfb18fb419b3d08e011228ef8a25882397f3a859b9fe1436946140b6756b", size = 582206 }, + { url = "https://files.pythonhosted.org/packages/b1/8e/8b6f42c67d5df7db35b8c55c9a850ea045219741bb14416255616808c690/greenlet-3.2.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:efc6dc8a792243c31f2f5674b670b3a95d46fa1c6a912b8e310d6f542e7b0712", size = 1111412 }, + { url = "https://files.pythonhosted.org/packages/05/46/ab58828217349500a7ebb81159d52ca357da747ff1797c29c6023d79d798/greenlet-3.2.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:731e154aba8e757aedd0781d4b240f1225b075b4409f1bb83b05ff410582cf00", size = 1135054 }, + { url = "https://files.pythonhosted.org/packages/68/7f/d1b537be5080721c0f0089a8447d4ef72839039cdb743bdd8ffd23046e9a/greenlet-3.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:96c20252c2f792defe9a115d3287e14811036d51e78b3aaddbee23b69b216302", size = 296573 }, + { url = "https://files.pythonhosted.org/packages/fc/2e/d4fcb2978f826358b673f779f78fa8a32ee37df11920dc2bb5589cbeecef/greenlet-3.2.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:784ae58bba89fa1fa5733d170d42486580cab9decda3484779f4759345b29822", size = 270219 }, + { url = "https://files.pythonhosted.org/packages/16/24/929f853e0202130e4fe163bc1d05a671ce8dcd604f790e14896adac43a52/greenlet-3.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0921ac4ea42a5315d3446120ad48f90c3a6b9bb93dd9b3cf4e4d84a66e42de83", size = 630383 }, + { url = "https://files.pythonhosted.org/packages/d1/b2/0320715eb61ae70c25ceca2f1d5ae620477d246692d9cc284c13242ec31c/greenlet-3.2.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d2971d93bb99e05f8c2c0c2f4aa9484a18d98c4c3bd3c62b65b7e6ae33dfcfaf", size = 642422 }, + { url = "https://files.pythonhosted.org/packages/bd/49/445fd1a210f4747fedf77615d941444349c6a3a4a1135bba9701337cd966/greenlet-3.2.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c667c0bf9d406b77a15c924ef3285e1e05250948001220368e039b6aa5b5034b", size = 638375 }, + { url = "https://files.pythonhosted.org/packages/7e/c8/ca19760cf6eae75fa8dc32b487e963d863b3ee04a7637da77b616703bc37/greenlet-3.2.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:592c12fb1165be74592f5de0d70f82bc5ba552ac44800d632214b76089945147", size = 637627 }, + { url = "https://files.pythonhosted.org/packages/65/89/77acf9e3da38e9bcfca881e43b02ed467c1dedc387021fc4d9bd9928afb8/greenlet-3.2.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29e184536ba333003540790ba29829ac14bb645514fbd7e32af331e8202a62a5", size = 585502 }, + { url = "https://files.pythonhosted.org/packages/97/c6/ae244d7c95b23b7130136e07a9cc5aadd60d59b5951180dc7dc7e8edaba7/greenlet-3.2.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:93c0bb79844a367782ec4f429d07589417052e621aa39a5ac1fb99c5aa308edc", size = 1114498 }, + { url = "https://files.pythonhosted.org/packages/89/5f/b16dec0cbfd3070658e0d744487919740c6d45eb90946f6787689a7efbce/greenlet-3.2.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:751261fc5ad7b6705f5f76726567375bb2104a059454e0226e1eef6c756748ba", size = 1139977 }, + { url = "https://files.pythonhosted.org/packages/66/77/d48fb441b5a71125bcac042fc5b1494c806ccb9a1432ecaa421e72157f77/greenlet-3.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:83a8761c75312361aa2b5b903b79da97f13f556164a7dd2d5448655425bd4c34", size = 297017 }, + { url = "https://files.pythonhosted.org/packages/f3/94/ad0d435f7c48debe960c53b8f60fb41c2026b1d0fa4a99a1cb17c3461e09/greenlet-3.2.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:25ad29caed5783d4bd7a85c9251c651696164622494c00802a139c00d639242d", size = 271992 }, + { url = "https://files.pythonhosted.org/packages/93/5d/7c27cf4d003d6e77749d299c7c8f5fd50b4f251647b5c2e97e1f20da0ab5/greenlet-3.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88cd97bf37fe24a6710ec6a3a7799f3f81d9cd33317dcf565ff9950c83f55e0b", size = 638820 }, + { url = "https://files.pythonhosted.org/packages/c6/7e/807e1e9be07a125bb4c169144937910bf59b9d2f6d931578e57f0bce0ae2/greenlet-3.2.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:baeedccca94880d2f5666b4fa16fc20ef50ba1ee353ee2d7092b383a243b0b0d", size = 653046 }, + { url = "https://files.pythonhosted.org/packages/9d/ab/158c1a4ea1068bdbc78dba5a3de57e4c7aeb4e7fa034320ea94c688bfb61/greenlet-3.2.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:be52af4b6292baecfa0f397f3edb3c6092ce071b499dd6fe292c9ac9f2c8f264", size = 647701 }, + { url = "https://files.pythonhosted.org/packages/cc/0d/93729068259b550d6a0288da4ff72b86ed05626eaf1eb7c0d3466a2571de/greenlet-3.2.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0cc73378150b8b78b0c9fe2ce56e166695e67478550769536a6742dca3651688", size = 649747 }, + { url = "https://files.pythonhosted.org/packages/f6/f6/c82ac1851c60851302d8581680573245c8fc300253fc1ff741ae74a6c24d/greenlet-3.2.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:706d016a03e78df129f68c4c9b4c4f963f7d73534e48a24f5f5a7101ed13dbbb", size = 605461 }, + { url = "https://files.pythonhosted.org/packages/98/82/d022cf25ca39cf1200650fc58c52af32c90f80479c25d1cbf57980ec3065/greenlet-3.2.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:419e60f80709510c343c57b4bb5a339d8767bf9aef9b8ce43f4f143240f88b7c", size = 1121190 }, + { url = "https://files.pythonhosted.org/packages/f5/e1/25297f70717abe8104c20ecf7af0a5b82d2f5a980eb1ac79f65654799f9f/greenlet-3.2.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:93d48533fade144203816783373f27a97e4193177ebaaf0fc396db19e5d61163", size = 1149055 }, + { url = "https://files.pythonhosted.org/packages/1f/8f/8f9e56c5e82eb2c26e8cde787962e66494312dc8cb261c460e1f3a9c88bc/greenlet-3.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:7454d37c740bb27bdeddfc3f358f26956a07d5220818ceb467a483197d84f849", size = 297817 }, + { url = "https://files.pythonhosted.org/packages/b1/cf/f5c0b23309070ae93de75c90d29300751a5aacefc0a3ed1b1d8edb28f08b/greenlet-3.2.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:500b8689aa9dd1ab26872a34084503aeddefcb438e2e7317b89b11eaea1901ad", size = 270732 }, + { url = "https://files.pythonhosted.org/packages/48/ae/91a957ba60482d3fecf9be49bc3948f341d706b52ddb9d83a70d42abd498/greenlet-3.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a07d3472c2a93117af3b0136f246b2833fdc0b542d4a9799ae5f41c28323faef", size = 639033 }, + { url = "https://files.pythonhosted.org/packages/6f/df/20ffa66dd5a7a7beffa6451bdb7400d66251374ab40b99981478c69a67a8/greenlet-3.2.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:8704b3768d2f51150626962f4b9a9e4a17d2e37c8a8d9867bbd9fa4eb938d3b3", size = 652999 }, + { url = "https://files.pythonhosted.org/packages/51/b4/ebb2c8cb41e521f1d72bf0465f2f9a2fd803f674a88db228887e6847077e/greenlet-3.2.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5035d77a27b7c62db6cf41cf786cfe2242644a7a337a0e155c80960598baab95", size = 647368 }, + { url = "https://files.pythonhosted.org/packages/8e/6a/1e1b5aa10dced4ae876a322155705257748108b7fd2e4fae3f2a091fe81a/greenlet-3.2.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2d8aa5423cd4a396792f6d4580f88bdc6efcb9205891c9d40d20f6e670992efb", size = 650037 }, + { url = "https://files.pythonhosted.org/packages/26/f2/ad51331a157c7015c675702e2d5230c243695c788f8f75feba1af32b3617/greenlet-3.2.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2c724620a101f8170065d7dded3f962a2aea7a7dae133a009cada42847e04a7b", size = 608402 }, + { url = "https://files.pythonhosted.org/packages/26/bc/862bd2083e6b3aff23300900a956f4ea9a4059de337f5c8734346b9b34fc/greenlet-3.2.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:873abe55f134c48e1f2a6f53f7d1419192a3d1a4e873bace00499a4e45ea6af0", size = 1119577 }, + { url = "https://files.pythonhosted.org/packages/86/94/1fc0cc068cfde885170e01de40a619b00eaa8f2916bf3541744730ffb4c3/greenlet-3.2.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:024571bbce5f2c1cfff08bf3fbaa43bbc7444f580ae13b0099e95d0e6e67ed36", size = 1147121 }, + { url = "https://files.pythonhosted.org/packages/27/1a/199f9587e8cb08a0658f9c30f3799244307614148ffe8b1e3aa22f324dea/greenlet-3.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:5195fb1e75e592dd04ce79881c8a22becdfa3e6f500e7feb059b1e6fdd54d3e3", size = 297603 }, + { url = "https://files.pythonhosted.org/packages/d8/ca/accd7aa5280eb92b70ed9e8f7fd79dc50a2c21d8c73b9a0856f5b564e222/greenlet-3.2.3-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:3d04332dddb10b4a211b68111dabaee2e1a073663d117dc10247b5b1642bac86", size = 271479 }, + { url = "https://files.pythonhosted.org/packages/55/71/01ed9895d9eb49223280ecc98a557585edfa56b3d0e965b9fa9f7f06b6d9/greenlet-3.2.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8186162dffde068a465deab08fc72c767196895c39db26ab1c17c0b77a6d8b97", size = 683952 }, + { url = "https://files.pythonhosted.org/packages/ea/61/638c4bdf460c3c678a0a1ef4c200f347dff80719597e53b5edb2fb27ab54/greenlet-3.2.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f4bfbaa6096b1b7a200024784217defedf46a07c2eee1a498e94a1b5f8ec5728", size = 696917 }, + { url = "https://files.pythonhosted.org/packages/22/cc/0bd1a7eb759d1f3e3cc2d1bc0f0b487ad3cc9f34d74da4b80f226fde4ec3/greenlet-3.2.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:ed6cfa9200484d234d8394c70f5492f144b20d4533f69262d530a1a082f6ee9a", size = 692443 }, + { url = "https://files.pythonhosted.org/packages/67/10/b2a4b63d3f08362662e89c103f7fe28894a51ae0bc890fabf37d1d780e52/greenlet-3.2.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:02b0df6f63cd15012bed5401b47829cfd2e97052dc89da3cfaf2c779124eb892", size = 692995 }, + { url = "https://files.pythonhosted.org/packages/5a/c6/ad82f148a4e3ce9564056453a71529732baf5448ad53fc323e37efe34f66/greenlet-3.2.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:86c2d68e87107c1792e2e8d5399acec2487a4e993ab76c792408e59394d52141", size = 655320 }, + { url = "https://files.pythonhosted.org/packages/5c/4f/aab73ecaa6b3086a4c89863d94cf26fa84cbff63f52ce9bc4342b3087a06/greenlet-3.2.3-cp314-cp314-win_amd64.whl", hash = "sha256:8c47aae8fbbfcf82cc13327ae802ba13c9c36753b67e760023fd116bc124a62a", size = 301236 }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515 }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, +] + +[[package]] +name = "kafka-python" +version = "2.2.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/cf/f55532155c7e2c54fdbdceb1412425f894fd0a6897575fa3454e541ce186/kafka_python-2.2.11.tar.gz", hash = "sha256:8ff8bcc158f48b47ba516536a5b1287db75a8ceff13d639da917ec52e171acde", size = 347568 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/e1/89df26df8a93d24c4ad01db6cffb65894ef3e39c1cedf4a281178e7cb669/kafka_python-2.2.11-py2.py3-none-any.whl", hash = "sha256:c285ce322108382ea9fd62273aab175d9a6959866145cf7cf9d4ca447b632372", size = 309650 }, +] + +[[package]] +name = "pydantic" +version = "2.11.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f0/86/8ce9040065e8f924d642c58e4a344e33163a07f6b57f836d0d734e0ad3fb/pydantic-2.11.5.tar.gz", hash = "sha256:7f853db3d0ce78ce8bbb148c401c2cdd6431b3473c0cdff2755c7690952a7b7a", size = 787102 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/69/831ed22b38ff9b4b64b66569f0e5b7b97cf3638346eb95a2147fdb49ad5f/pydantic-2.11.5-py3-none-any.whl", hash = "sha256:f9c26ba06f9747749ca1e5c94d6a85cb84254577553c8785576fd38fa64dc0f7", size = 444229 }, +] + +[[package]] +name = "pydantic-core" +version = "2.33.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/88/5f2260bdfae97aabf98f1778d43f69574390ad787afb646292a638c923d4/pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc", size = 435195 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/92/b31726561b5dae176c2d2c2dc43a9c5bfba5d32f96f8b4c0a600dd492447/pydantic_core-2.33.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8", size = 2028817 }, + { url = "https://files.pythonhosted.org/packages/a3/44/3f0b95fafdaca04a483c4e685fe437c6891001bf3ce8b2fded82b9ea3aa1/pydantic_core-2.33.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d", size = 1861357 }, + { url = "https://files.pythonhosted.org/packages/30/97/e8f13b55766234caae05372826e8e4b3b96e7b248be3157f53237682e43c/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d", size = 1898011 }, + { url = "https://files.pythonhosted.org/packages/9b/a3/99c48cf7bafc991cc3ee66fd544c0aae8dc907b752f1dad2d79b1b5a471f/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572", size = 1982730 }, + { url = "https://files.pythonhosted.org/packages/de/8e/a5b882ec4307010a840fb8b58bd9bf65d1840c92eae7534c7441709bf54b/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02", size = 2136178 }, + { url = "https://files.pythonhosted.org/packages/e4/bb/71e35fc3ed05af6834e890edb75968e2802fe98778971ab5cba20a162315/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b", size = 2736462 }, + { url = "https://files.pythonhosted.org/packages/31/0d/c8f7593e6bc7066289bbc366f2235701dcbebcd1ff0ef8e64f6f239fb47d/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2", size = 2005652 }, + { url = "https://files.pythonhosted.org/packages/d2/7a/996d8bd75f3eda405e3dd219ff5ff0a283cd8e34add39d8ef9157e722867/pydantic_core-2.33.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a", size = 2113306 }, + { url = "https://files.pythonhosted.org/packages/ff/84/daf2a6fb2db40ffda6578a7e8c5a6e9c8affb251a05c233ae37098118788/pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac", size = 2073720 }, + { url = "https://files.pythonhosted.org/packages/77/fb/2258da019f4825128445ae79456a5499c032b55849dbd5bed78c95ccf163/pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a", size = 2244915 }, + { url = "https://files.pythonhosted.org/packages/d8/7a/925ff73756031289468326e355b6fa8316960d0d65f8b5d6b3a3e7866de7/pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b", size = 2241884 }, + { url = "https://files.pythonhosted.org/packages/0b/b0/249ee6d2646f1cdadcb813805fe76265745c4010cf20a8eba7b0e639d9b2/pydantic_core-2.33.2-cp310-cp310-win32.whl", hash = "sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22", size = 1910496 }, + { url = "https://files.pythonhosted.org/packages/66/ff/172ba8f12a42d4b552917aa65d1f2328990d3ccfc01d5b7c943ec084299f/pydantic_core-2.33.2-cp310-cp310-win_amd64.whl", hash = "sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640", size = 1955019 }, + { url = "https://files.pythonhosted.org/packages/3f/8d/71db63483d518cbbf290261a1fc2839d17ff89fce7089e08cad07ccfce67/pydantic_core-2.33.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7", size = 2028584 }, + { url = "https://files.pythonhosted.org/packages/24/2f/3cfa7244ae292dd850989f328722d2aef313f74ffc471184dc509e1e4e5a/pydantic_core-2.33.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246", size = 1855071 }, + { url = "https://files.pythonhosted.org/packages/b3/d3/4ae42d33f5e3f50dd467761304be2fa0a9417fbf09735bc2cce003480f2a/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f", size = 1897823 }, + { url = "https://files.pythonhosted.org/packages/f4/f3/aa5976e8352b7695ff808599794b1fba2a9ae2ee954a3426855935799488/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc", size = 1983792 }, + { url = "https://files.pythonhosted.org/packages/d5/7a/cda9b5a23c552037717f2b2a5257e9b2bfe45e687386df9591eff7b46d28/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de", size = 2136338 }, + { url = "https://files.pythonhosted.org/packages/2b/9f/b8f9ec8dd1417eb9da784e91e1667d58a2a4a7b7b34cf4af765ef663a7e5/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a", size = 2730998 }, + { url = "https://files.pythonhosted.org/packages/47/bc/cd720e078576bdb8255d5032c5d63ee5c0bf4b7173dd955185a1d658c456/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef", size = 2003200 }, + { url = "https://files.pythonhosted.org/packages/ca/22/3602b895ee2cd29d11a2b349372446ae9727c32e78a94b3d588a40fdf187/pydantic_core-2.33.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e", size = 2113890 }, + { url = "https://files.pythonhosted.org/packages/ff/e6/e3c5908c03cf00d629eb38393a98fccc38ee0ce8ecce32f69fc7d7b558a7/pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d", size = 2073359 }, + { url = "https://files.pythonhosted.org/packages/12/e7/6a36a07c59ebefc8777d1ffdaf5ae71b06b21952582e4b07eba88a421c79/pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30", size = 2245883 }, + { url = "https://files.pythonhosted.org/packages/16/3f/59b3187aaa6cc0c1e6616e8045b284de2b6a87b027cce2ffcea073adf1d2/pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf", size = 2241074 }, + { url = "https://files.pythonhosted.org/packages/e0/ed/55532bb88f674d5d8f67ab121a2a13c385df382de2a1677f30ad385f7438/pydantic_core-2.33.2-cp311-cp311-win32.whl", hash = "sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51", size = 1910538 }, + { url = "https://files.pythonhosted.org/packages/fe/1b/25b7cccd4519c0b23c2dd636ad39d381abf113085ce4f7bec2b0dc755eb1/pydantic_core-2.33.2-cp311-cp311-win_amd64.whl", hash = "sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab", size = 1952909 }, + { url = "https://files.pythonhosted.org/packages/49/a9/d809358e49126438055884c4366a1f6227f0f84f635a9014e2deb9b9de54/pydantic_core-2.33.2-cp311-cp311-win_arm64.whl", hash = "sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65", size = 1897786 }, + { url = "https://files.pythonhosted.org/packages/18/8a/2b41c97f554ec8c71f2a8a5f85cb56a8b0956addfe8b0efb5b3d77e8bdc3/pydantic_core-2.33.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc", size = 2009000 }, + { url = "https://files.pythonhosted.org/packages/a1/02/6224312aacb3c8ecbaa959897af57181fb6cf3a3d7917fd44d0f2917e6f2/pydantic_core-2.33.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7", size = 1847996 }, + { url = "https://files.pythonhosted.org/packages/d6/46/6dcdf084a523dbe0a0be59d054734b86a981726f221f4562aed313dbcb49/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025", size = 1880957 }, + { url = "https://files.pythonhosted.org/packages/ec/6b/1ec2c03837ac00886ba8160ce041ce4e325b41d06a034adbef11339ae422/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011", size = 1964199 }, + { url = "https://files.pythonhosted.org/packages/2d/1d/6bf34d6adb9debd9136bd197ca72642203ce9aaaa85cfcbfcf20f9696e83/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f", size = 2120296 }, + { url = "https://files.pythonhosted.org/packages/e0/94/2bd0aaf5a591e974b32a9f7123f16637776c304471a0ab33cf263cf5591a/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88", size = 2676109 }, + { url = "https://files.pythonhosted.org/packages/f9/41/4b043778cf9c4285d59742281a769eac371b9e47e35f98ad321349cc5d61/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1", size = 2002028 }, + { url = "https://files.pythonhosted.org/packages/cb/d5/7bb781bf2748ce3d03af04d5c969fa1308880e1dca35a9bd94e1a96a922e/pydantic_core-2.33.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b", size = 2100044 }, + { url = "https://files.pythonhosted.org/packages/fe/36/def5e53e1eb0ad896785702a5bbfd25eed546cdcf4087ad285021a90ed53/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1", size = 2058881 }, + { url = "https://files.pythonhosted.org/packages/01/6c/57f8d70b2ee57fc3dc8b9610315949837fa8c11d86927b9bb044f8705419/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6", size = 2227034 }, + { url = "https://files.pythonhosted.org/packages/27/b9/9c17f0396a82b3d5cbea4c24d742083422639e7bb1d5bf600e12cb176a13/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea", size = 2234187 }, + { url = "https://files.pythonhosted.org/packages/b0/6a/adf5734ffd52bf86d865093ad70b2ce543415e0e356f6cacabbc0d9ad910/pydantic_core-2.33.2-cp312-cp312-win32.whl", hash = "sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290", size = 1892628 }, + { url = "https://files.pythonhosted.org/packages/43/e4/5479fecb3606c1368d496a825d8411e126133c41224c1e7238be58b87d7e/pydantic_core-2.33.2-cp312-cp312-win_amd64.whl", hash = "sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2", size = 1955866 }, + { url = "https://files.pythonhosted.org/packages/0d/24/8b11e8b3e2be9dd82df4b11408a67c61bb4dc4f8e11b5b0fc888b38118b5/pydantic_core-2.33.2-cp312-cp312-win_arm64.whl", hash = "sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab", size = 1888894 }, + { url = "https://files.pythonhosted.org/packages/46/8c/99040727b41f56616573a28771b1bfa08a3d3fe74d3d513f01251f79f172/pydantic_core-2.33.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f", size = 2015688 }, + { url = "https://files.pythonhosted.org/packages/3a/cc/5999d1eb705a6cefc31f0b4a90e9f7fc400539b1a1030529700cc1b51838/pydantic_core-2.33.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6", size = 1844808 }, + { url = "https://files.pythonhosted.org/packages/6f/5e/a0a7b8885c98889a18b6e376f344da1ef323d270b44edf8174d6bce4d622/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef", size = 1885580 }, + { url = "https://files.pythonhosted.org/packages/3b/2a/953581f343c7d11a304581156618c3f592435523dd9d79865903272c256a/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a", size = 1973859 }, + { url = "https://files.pythonhosted.org/packages/e6/55/f1a813904771c03a3f97f676c62cca0c0a4138654107c1b61f19c644868b/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916", size = 2120810 }, + { url = "https://files.pythonhosted.org/packages/aa/c3/053389835a996e18853ba107a63caae0b9deb4a276c6b472931ea9ae6e48/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a", size = 2676498 }, + { url = "https://files.pythonhosted.org/packages/eb/3c/f4abd740877a35abade05e437245b192f9d0ffb48bbbbd708df33d3cda37/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d", size = 2000611 }, + { url = "https://files.pythonhosted.org/packages/59/a7/63ef2fed1837d1121a894d0ce88439fe3e3b3e48c7543b2a4479eb99c2bd/pydantic_core-2.33.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56", size = 2107924 }, + { url = "https://files.pythonhosted.org/packages/04/8f/2551964ef045669801675f1cfc3b0d74147f4901c3ffa42be2ddb1f0efc4/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5", size = 2063196 }, + { url = "https://files.pythonhosted.org/packages/26/bd/d9602777e77fc6dbb0c7db9ad356e9a985825547dce5ad1d30ee04903918/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e", size = 2236389 }, + { url = "https://files.pythonhosted.org/packages/42/db/0e950daa7e2230423ab342ae918a794964b053bec24ba8af013fc7c94846/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162", size = 2239223 }, + { url = "https://files.pythonhosted.org/packages/58/4d/4f937099c545a8a17eb52cb67fe0447fd9a373b348ccfa9a87f141eeb00f/pydantic_core-2.33.2-cp313-cp313-win32.whl", hash = "sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849", size = 1900473 }, + { url = "https://files.pythonhosted.org/packages/a0/75/4a0a9bac998d78d889def5e4ef2b065acba8cae8c93696906c3a91f310ca/pydantic_core-2.33.2-cp313-cp313-win_amd64.whl", hash = "sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9", size = 1955269 }, + { url = "https://files.pythonhosted.org/packages/f9/86/1beda0576969592f1497b4ce8e7bc8cbdf614c352426271b1b10d5f0aa64/pydantic_core-2.33.2-cp313-cp313-win_arm64.whl", hash = "sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9", size = 1893921 }, + { url = "https://files.pythonhosted.org/packages/a4/7d/e09391c2eebeab681df2b74bfe6c43422fffede8dc74187b2b0bf6fd7571/pydantic_core-2.33.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac", size = 1806162 }, + { url = "https://files.pythonhosted.org/packages/f1/3d/847b6b1fed9f8ed3bb95a9ad04fbd0b212e832d4f0f50ff4d9ee5a9f15cf/pydantic_core-2.33.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5", size = 1981560 }, + { url = "https://files.pythonhosted.org/packages/6f/9a/e73262f6c6656262b5fdd723ad90f518f579b7bc8622e43a942eec53c938/pydantic_core-2.33.2-cp313-cp313t-win_amd64.whl", hash = "sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9", size = 1935777 }, + { url = "https://files.pythonhosted.org/packages/30/68/373d55e58b7e83ce371691f6eaa7175e3a24b956c44628eb25d7da007917/pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa", size = 2023982 }, + { url = "https://files.pythonhosted.org/packages/a4/16/145f54ac08c96a63d8ed6442f9dec17b2773d19920b627b18d4f10a061ea/pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29", size = 1858412 }, + { url = "https://files.pythonhosted.org/packages/41/b1/c6dc6c3e2de4516c0bb2c46f6a373b91b5660312342a0cf5826e38ad82fa/pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d", size = 1892749 }, + { url = "https://files.pythonhosted.org/packages/12/73/8cd57e20afba760b21b742106f9dbdfa6697f1570b189c7457a1af4cd8a0/pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e", size = 2067527 }, + { url = "https://files.pythonhosted.org/packages/e3/d5/0bb5d988cc019b3cba4a78f2d4b3854427fc47ee8ec8e9eaabf787da239c/pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c", size = 2108225 }, + { url = "https://files.pythonhosted.org/packages/f1/c5/00c02d1571913d496aabf146106ad8239dc132485ee22efe08085084ff7c/pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec", size = 2069490 }, + { url = "https://files.pythonhosted.org/packages/22/a8/dccc38768274d3ed3a59b5d06f59ccb845778687652daa71df0cab4040d7/pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052", size = 2237525 }, + { url = "https://files.pythonhosted.org/packages/d4/e7/4f98c0b125dda7cf7ccd14ba936218397b44f50a56dd8c16a3091df116c3/pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c", size = 2238446 }, + { url = "https://files.pythonhosted.org/packages/ce/91/2ec36480fdb0b783cd9ef6795753c1dea13882f2e68e73bce76ae8c21e6a/pydantic_core-2.33.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808", size = 2066678 }, + { url = "https://files.pythonhosted.org/packages/7b/27/d4ae6487d73948d6f20dddcd94be4ea43e74349b56eba82e9bdee2d7494c/pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8", size = 2025200 }, + { url = "https://files.pythonhosted.org/packages/f1/b8/b3cb95375f05d33801024079b9392a5ab45267a63400bf1866e7ce0f0de4/pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593", size = 1859123 }, + { url = "https://files.pythonhosted.org/packages/05/bc/0d0b5adeda59a261cd30a1235a445bf55c7e46ae44aea28f7bd6ed46e091/pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612", size = 1892852 }, + { url = "https://files.pythonhosted.org/packages/3e/11/d37bdebbda2e449cb3f519f6ce950927b56d62f0b84fd9cb9e372a26a3d5/pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7", size = 2067484 }, + { url = "https://files.pythonhosted.org/packages/8c/55/1f95f0a05ce72ecb02a8a8a1c3be0579bbc29b1d5ab68f1378b7bebc5057/pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e", size = 2108896 }, + { url = "https://files.pythonhosted.org/packages/53/89/2b2de6c81fa131f423246a9109d7b2a375e83968ad0800d6e57d0574629b/pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8", size = 2069475 }, + { url = "https://files.pythonhosted.org/packages/b8/e9/1f7efbe20d0b2b10f6718944b5d8ece9152390904f29a78e68d4e7961159/pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf", size = 2239013 }, + { url = "https://files.pythonhosted.org/packages/3c/b2/5309c905a93811524a49b4e031e9851a6b00ff0fb668794472ea7746b448/pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb", size = 2238715 }, + { url = "https://files.pythonhosted.org/packages/32/56/8a7ca5d2cd2cda1d245d34b1c9a942920a718082ae8e54e5f3e5a58b7add/pydantic_core-2.33.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1", size = 2066757 }, +] + +[[package]] +name = "pydantic-settings" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/67/1d/42628a2c33e93f8e9acbde0d5d735fa0850f3e6a2f8cb1eb6c40b9a732ac/pydantic_settings-2.9.1.tar.gz", hash = "sha256:c509bf79d27563add44e8446233359004ed85066cd096d8b510f715e6ef5d268", size = 163234 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/5f/d6d641b490fd3ec2c4c13b4244d68deea3a1b970a97be64f34fb5504ff72/pydantic_settings-2.9.1-py3-none-any.whl", hash = "sha256:59b4f431b1defb26fe620c71a7d3968a710d719f5f4cdbbdb7926edeb770f6ef", size = 44356 }, +] + +[[package]] +name = "python-dotenv" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/88/2c/7bb1416c5620485aa793f2de31d3df393d3686aa8a8506d11e10e13c5baf/python_dotenv-1.1.0.tar.gz", hash = "sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5", size = 39920 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/18/98a99ad95133c6a6e2005fe89faedf294a748bd5dc803008059409ac9b1e/python_dotenv-1.1.0-py3-none-any.whl", hash = "sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d", size = 20256 }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.41" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "(python_full_version < '3.14' and platform_machine == 'AMD64') or (python_full_version < '3.14' and platform_machine == 'WIN32') or (python_full_version < '3.14' and platform_machine == 'aarch64') or (python_full_version < '3.14' and platform_machine == 'amd64') or (python_full_version < '3.14' and platform_machine == 'ppc64le') or (python_full_version < '3.14' and platform_machine == 'win32') or (python_full_version < '3.14' and platform_machine == 'x86_64')" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/66/45b165c595ec89aa7dcc2c1cd222ab269bc753f1fc7a1e68f8481bd957bf/sqlalchemy-2.0.41.tar.gz", hash = "sha256:edba70118c4be3c2b1f90754d308d0b79c6fe2c0fdc52d8ddf603916f83f4db9", size = 9689424 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/12/d7c445b1940276a828efce7331cb0cb09d6e5f049651db22f4ebb0922b77/sqlalchemy-2.0.41-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b1f09b6821406ea1f94053f346f28f8215e293344209129a9c0fcc3578598d7b", size = 2117967 }, + { url = "https://files.pythonhosted.org/packages/6f/b8/cb90f23157e28946b27eb01ef401af80a1fab7553762e87df51507eaed61/sqlalchemy-2.0.41-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1936af879e3db023601196a1684d28e12f19ccf93af01bf3280a3262c4b6b4e5", size = 2107583 }, + { url = "https://files.pythonhosted.org/packages/9e/c2/eef84283a1c8164a207d898e063edf193d36a24fb6a5bb3ce0634b92a1e8/sqlalchemy-2.0.41-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2ac41acfc8d965fb0c464eb8f44995770239668956dc4cdf502d1b1ffe0d747", size = 3186025 }, + { url = "https://files.pythonhosted.org/packages/bd/72/49d52bd3c5e63a1d458fd6d289a1523a8015adedbddf2c07408ff556e772/sqlalchemy-2.0.41-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81c24e0c0fde47a9723c81d5806569cddef103aebbf79dbc9fcbb617153dea30", size = 3186259 }, + { url = "https://files.pythonhosted.org/packages/4f/9e/e3ffc37d29a3679a50b6bbbba94b115f90e565a2b4545abb17924b94c52d/sqlalchemy-2.0.41-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:23a8825495d8b195c4aa9ff1c430c28f2c821e8c5e2d98089228af887e5d7e29", size = 3126803 }, + { url = "https://files.pythonhosted.org/packages/8a/76/56b21e363f6039978ae0b72690237b38383e4657281285a09456f313dd77/sqlalchemy-2.0.41-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:60c578c45c949f909a4026b7807044e7e564adf793537fc762b2489d522f3d11", size = 3148566 }, + { url = "https://files.pythonhosted.org/packages/3b/92/11b8e1b69bf191bc69e300a99badbbb5f2f1102f2b08b39d9eee2e21f565/sqlalchemy-2.0.41-cp310-cp310-win32.whl", hash = "sha256:118c16cd3f1b00c76d69343e38602006c9cfb9998fa4f798606d28d63f23beda", size = 2086696 }, + { url = "https://files.pythonhosted.org/packages/5c/88/2d706c9cc4502654860f4576cd54f7db70487b66c3b619ba98e0be1a4642/sqlalchemy-2.0.41-cp310-cp310-win_amd64.whl", hash = "sha256:7492967c3386df69f80cf67efd665c0f667cee67032090fe01d7d74b0e19bb08", size = 2110200 }, + { url = "https://files.pythonhosted.org/packages/37/4e/b00e3ffae32b74b5180e15d2ab4040531ee1bef4c19755fe7926622dc958/sqlalchemy-2.0.41-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6375cd674fe82d7aa9816d1cb96ec592bac1726c11e0cafbf40eeee9a4516b5f", size = 2121232 }, + { url = "https://files.pythonhosted.org/packages/ef/30/6547ebb10875302074a37e1970a5dce7985240665778cfdee2323709f749/sqlalchemy-2.0.41-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f8c9fdd15a55d9465e590a402f42082705d66b05afc3ffd2d2eb3c6ba919560", size = 2110897 }, + { url = "https://files.pythonhosted.org/packages/9e/21/59df2b41b0f6c62da55cd64798232d7349a9378befa7f1bb18cf1dfd510a/sqlalchemy-2.0.41-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32f9dc8c44acdee06c8fc6440db9eae8b4af8b01e4b1aee7bdd7241c22edff4f", size = 3273313 }, + { url = "https://files.pythonhosted.org/packages/62/e4/b9a7a0e5c6f79d49bcd6efb6e90d7536dc604dab64582a9dec220dab54b6/sqlalchemy-2.0.41-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90c11ceb9a1f482c752a71f203a81858625d8df5746d787a4786bca4ffdf71c6", size = 3273807 }, + { url = "https://files.pythonhosted.org/packages/39/d8/79f2427251b44ddee18676c04eab038d043cff0e764d2d8bb08261d6135d/sqlalchemy-2.0.41-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:911cc493ebd60de5f285bcae0491a60b4f2a9f0f5c270edd1c4dbaef7a38fc04", size = 3209632 }, + { url = "https://files.pythonhosted.org/packages/d4/16/730a82dda30765f63e0454918c982fb7193f6b398b31d63c7c3bd3652ae5/sqlalchemy-2.0.41-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03968a349db483936c249f4d9cd14ff2c296adfa1290b660ba6516f973139582", size = 3233642 }, + { url = "https://files.pythonhosted.org/packages/04/61/c0d4607f7799efa8b8ea3c49b4621e861c8f5c41fd4b5b636c534fcb7d73/sqlalchemy-2.0.41-cp311-cp311-win32.whl", hash = "sha256:293cd444d82b18da48c9f71cd7005844dbbd06ca19be1ccf6779154439eec0b8", size = 2086475 }, + { url = "https://files.pythonhosted.org/packages/9d/8e/8344f8ae1cb6a479d0741c02cd4f666925b2bf02e2468ddaf5ce44111f30/sqlalchemy-2.0.41-cp311-cp311-win_amd64.whl", hash = "sha256:3d3549fc3e40667ec7199033a4e40a2f669898a00a7b18a931d3efb4c7900504", size = 2110903 }, + { url = "https://files.pythonhosted.org/packages/3e/2a/f1f4e068b371154740dd10fb81afb5240d5af4aa0087b88d8b308b5429c2/sqlalchemy-2.0.41-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:81f413674d85cfd0dfcd6512e10e0f33c19c21860342a4890c3a2b59479929f9", size = 2119645 }, + { url = "https://files.pythonhosted.org/packages/9b/e8/c664a7e73d36fbfc4730f8cf2bf930444ea87270f2825efbe17bf808b998/sqlalchemy-2.0.41-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:598d9ebc1e796431bbd068e41e4de4dc34312b7aa3292571bb3674a0cb415dd1", size = 2107399 }, + { url = "https://files.pythonhosted.org/packages/5c/78/8a9cf6c5e7135540cb682128d091d6afa1b9e48bd049b0d691bf54114f70/sqlalchemy-2.0.41-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a104c5694dfd2d864a6f91b0956eb5d5883234119cb40010115fd45a16da5e70", size = 3293269 }, + { url = "https://files.pythonhosted.org/packages/3c/35/f74add3978c20de6323fb11cb5162702670cc7a9420033befb43d8d5b7a4/sqlalchemy-2.0.41-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6145afea51ff0af7f2564a05fa95eb46f542919e6523729663a5d285ecb3cf5e", size = 3303364 }, + { url = "https://files.pythonhosted.org/packages/6a/d4/c990f37f52c3f7748ebe98883e2a0f7d038108c2c5a82468d1ff3eec50b7/sqlalchemy-2.0.41-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b46fa6eae1cd1c20e6e6f44e19984d438b6b2d8616d21d783d150df714f44078", size = 3229072 }, + { url = "https://files.pythonhosted.org/packages/15/69/cab11fecc7eb64bc561011be2bd03d065b762d87add52a4ca0aca2e12904/sqlalchemy-2.0.41-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41836fe661cc98abfae476e14ba1906220f92c4e528771a8a3ae6a151242d2ae", size = 3268074 }, + { url = "https://files.pythonhosted.org/packages/5c/ca/0c19ec16858585d37767b167fc9602593f98998a68a798450558239fb04a/sqlalchemy-2.0.41-cp312-cp312-win32.whl", hash = "sha256:a8808d5cf866c781150d36a3c8eb3adccfa41a8105d031bf27e92c251e3969d6", size = 2084514 }, + { url = "https://files.pythonhosted.org/packages/7f/23/4c2833d78ff3010a4e17f984c734f52b531a8c9060a50429c9d4b0211be6/sqlalchemy-2.0.41-cp312-cp312-win_amd64.whl", hash = "sha256:5b14e97886199c1f52c14629c11d90c11fbb09e9334fa7bb5f6d068d9ced0ce0", size = 2111557 }, + { url = "https://files.pythonhosted.org/packages/d3/ad/2e1c6d4f235a97eeef52d0200d8ddda16f6c4dd70ae5ad88c46963440480/sqlalchemy-2.0.41-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4eeb195cdedaf17aab6b247894ff2734dcead6c08f748e617bfe05bd5a218443", size = 2115491 }, + { url = "https://files.pythonhosted.org/packages/cf/8d/be490e5db8400dacc89056f78a52d44b04fbf75e8439569d5b879623a53b/sqlalchemy-2.0.41-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d4ae769b9c1c7757e4ccce94b0641bc203bbdf43ba7a2413ab2523d8d047d8dc", size = 2102827 }, + { url = "https://files.pythonhosted.org/packages/a0/72/c97ad430f0b0e78efaf2791342e13ffeafcbb3c06242f01a3bb8fe44f65d/sqlalchemy-2.0.41-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a62448526dd9ed3e3beedc93df9bb6b55a436ed1474db31a2af13b313a70a7e1", size = 3225224 }, + { url = "https://files.pythonhosted.org/packages/5e/51/5ba9ea3246ea068630acf35a6ba0d181e99f1af1afd17e159eac7e8bc2b8/sqlalchemy-2.0.41-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc56c9788617b8964ad02e8fcfeed4001c1f8ba91a9e1f31483c0dffb207002a", size = 3230045 }, + { url = "https://files.pythonhosted.org/packages/78/2f/8c14443b2acea700c62f9b4a8bad9e49fc1b65cfb260edead71fd38e9f19/sqlalchemy-2.0.41-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c153265408d18de4cc5ded1941dcd8315894572cddd3c58df5d5b5705b3fa28d", size = 3159357 }, + { url = "https://files.pythonhosted.org/packages/fc/b2/43eacbf6ccc5276d76cea18cb7c3d73e294d6fb21f9ff8b4eef9b42bbfd5/sqlalchemy-2.0.41-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f67766965996e63bb46cfbf2ce5355fc32d9dd3b8ad7e536a920ff9ee422e23", size = 3197511 }, + { url = "https://files.pythonhosted.org/packages/fa/2e/677c17c5d6a004c3c45334ab1dbe7b7deb834430b282b8a0f75ae220c8eb/sqlalchemy-2.0.41-cp313-cp313-win32.whl", hash = "sha256:bfc9064f6658a3d1cadeaa0ba07570b83ce6801a1314985bf98ec9b95d74e15f", size = 2082420 }, + { url = "https://files.pythonhosted.org/packages/e9/61/e8c1b9b6307c57157d328dd8b8348ddc4c47ffdf1279365a13b2b98b8049/sqlalchemy-2.0.41-cp313-cp313-win_amd64.whl", hash = "sha256:82ca366a844eb551daff9d2e6e7a9e5e76d2612c8564f58db6c19a726869c1df", size = 2108329 }, + { url = "https://files.pythonhosted.org/packages/1c/fc/9ba22f01b5cdacc8f5ed0d22304718d2c758fce3fd49a5372b886a86f37c/sqlalchemy-2.0.41-py3-none-any.whl", hash = "sha256:57df5dc6fdb5ed1a88a1ed2195fd31927e705cad62dedd86b46972752a80f576", size = 1911224 }, +] + +[[package]] +name = "starlette" +version = "0.46.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/20/08dfcd9c983f6a6f4a1000d934b9e6d626cff8d2eeb77a89a68eef20a2b7/starlette-0.46.2.tar.gz", hash = "sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5", size = 2580846 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/0c/9d30a4ebeb6db2b25a841afbb80f6ef9a854fc3b41be131d249a977b4959/starlette-0.46.2-py3-none-any.whl", hash = "sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35", size = 72037 }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/bc/51647cd02527e87d05cb083ccc402f93e441606ff1f01739a62c8ad09ba5/typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4", size = 107423 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/e0/552843e0d356fbb5256d21449fa957fa4eff3bbc135a74a691ee70c7c5da/typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af", size = 43839 }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/b1/0c11f5058406b3af7609f121aaa6b609744687f1d158b3c3a5bf4cc94238/typing_inspection-0.4.1.tar.gz", hash = "sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28", size = 75726 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/69/cd203477f944c353c31bade965f880aa1061fd6bf05ded0726ca845b6ff7/typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51", size = 14552 }, +] + +[[package]] +name = "uvicorn" +version = "0.34.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/ad/713be230bcda622eaa35c28f0d328c3675c371238470abdea52417f17a8e/uvicorn-0.34.3.tar.gz", hash = "sha256:35919a9a979d7a59334b6b10e05d77c1d0d574c50e0fc98b8b1a0f165708b55a", size = 76631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/0d/8adfeaa62945f90d19ddc461c55f4a50c258af7662d34b6a3d5d1f8646f6/uvicorn-0.34.3-py3-none-any.whl", hash = "sha256:16246631db62bdfbf069b0645177d6e8a77ba950cfedbfd093acef9444e4d885", size = 62431 }, +] diff --git a/pyproject.toml b/pyproject.toml index 2b0a9b6..f840eaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,6 @@ dev = [ "pytest-asyncio>=0.25.0", "pytest-cov>=6.1.1", "httpx>=0.27.0", - "pytest-benchmark>=3.0.0", ] From 57ece58461b89a6b5223b651f526160475802758 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Thu, 5 Jun 2025 23:26:19 -0400 Subject: [PATCH 05/39] fixing pytest for local run --- .env.test | 3 ++ .../api_public/src/core/config.py | 6 +++- .../api_public/src/core/database/database.py | 29 +++++++++++++------ pyproject.toml | 1 + .../bot_detector/hiscore_scraper/__init__.py | 3 -- .../bot_detector/hiscore_scraper/test_core.py | 5 ++-- uv.lock | 14 +++++++++ 7 files changed, 46 insertions(+), 15 deletions(-) create mode 100644 .env.test diff --git a/.env.test b/.env.test new file mode 100644 index 0000000..26835f9 --- /dev/null +++ b/.env.test @@ -0,0 +1,3 @@ +DATABASE_URL=sqlite+aiosqlite:///:memory: +DEBUG=true +KAFKA_BOOTSTRAP_SERVERS=kafka:9092 \ No newline at end of file diff --git a/bases/bot_detector/api_public/src/core/config.py b/bases/bot_detector/api_public/src/core/config.py index 1e08eb8..f4354e8 100644 --- a/bases/bot_detector/api_public/src/core/config.py +++ b/bases/bot_detector/api_public/src/core/config.py @@ -1,9 +1,13 @@ import asyncio +import sys from dotenv import find_dotenv, load_dotenv from pydantic_settings import BaseSettings -load_dotenv(find_dotenv()) +if "pytest" in sys.modules: + load_dotenv(find_dotenv(".env.test")) +else: + load_dotenv(find_dotenv()) # fallback to normal .env class Settings(BaseSettings): diff --git a/bases/bot_detector/api_public/src/core/database/database.py b/bases/bot_detector/api_public/src/core/database/database.py index 089e29b..83cbec6 100644 --- a/bases/bot_detector/api_public/src/core/database/database.py +++ b/bases/bot_detector/api_public/src/core/database/database.py @@ -3,15 +3,26 @@ from sqlalchemy.orm import declarative_base, sessionmaker # Create an async SQLAlchemy engine -engine = create_async_engine( - settings.DATABASE_URL, - pool_pre_ping=True, - pool_size=10, - max_overflow=90, - pool_timeout=settings.POOL_TIMEOUT, - pool_recycle=settings.POOL_RECYCLE, - # echo=(settings.ENV != "PRD"), -) +from sqlalchemy.pool import StaticPool # add at the top with your imports + +# Create an async SQLAlchemy engine +if settings.DATABASE_URL.startswith("sqlite"): + engine = create_async_engine( + settings.DATABASE_URL, + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + echo=(settings.ENV != "PRD"), + ) +else: + engine = create_async_engine( + settings.DATABASE_URL, + pool_pre_ping=True, + pool_size=10, + max_overflow=90, + pool_timeout=settings.POOL_TIMEOUT, + pool_recycle=settings.POOL_RECYCLE, + echo=(settings.ENV != "PRD"), + ) # Create a session factory SessionFactory = sessionmaker( diff --git a/pyproject.toml b/pyproject.toml index f840eaf..7ee434a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "fastapi>=0.115.12", "uvicorn>=0.34.2", "prometheus-client>=0.21.1", + "aiosqlite>=0.19.0" ] [dependency-groups] diff --git a/test/bases/bot_detector/hiscore_scraper/__init__.py b/test/bases/bot_detector/hiscore_scraper/__init__.py index 3d5905b..e69de29 100644 --- a/test/bases/bot_detector/hiscore_scraper/__init__.py +++ b/test/bases/bot_detector/hiscore_scraper/__init__.py @@ -1,3 +0,0 @@ -from dotenv import find_dotenv, load_dotenv - -load_dotenv(find_dotenv()) diff --git a/test/bases/bot_detector/hiscore_scraper/test_core.py b/test/bases/bot_detector/hiscore_scraper/test_core.py index 71b7501..676f7f9 100644 --- a/test/bases/bot_detector/hiscore_scraper/test_core.py +++ b/test/bases/bot_detector/hiscore_scraper/test_core.py @@ -1,8 +1,9 @@ import os -from bot_detector.hiscore_scraper import core - os.environ["ENVIRONMENT"] = "test" +os.environ["DEBUG"] = "true" + +from bot_detector.hiscore_scraper import core def test_sample(): diff --git a/uv.lock b/uv.lock index 04425ac..2762f95 100644 --- a/uv.lock +++ b/uv.lock @@ -151,6 +151,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/6a/bc7e17a3e87a2985d3e8f4da4cd0f481060eb78fb08596c42be62c90a4d9/aiosignal-1.3.2-py2.py3-none-any.whl", hash = "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5", size = 7597 }, ] +[[package]] +name = "aiosqlite" +version = "0.21.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/7d/8bca2bf9a247c2c5dfeec1d7a5f40db6518f88d314b8bca9da29670d2671/aiosqlite-0.21.0.tar.gz", hash = "sha256:131bb8056daa3bc875608c631c678cda73922a2d4ba8aec373b19f18c17e7aa3", size = 13454 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/10/6c25ed6de94c49f88a91fa5018cb4c0f3625f31d5be9f771ebe5cc7cd506/aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0", size = 15792 }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -246,6 +258,7 @@ source = { editable = "." } dependencies = [ { name = "aiohttp" }, { name = "aiokafkaengine" }, + { name = "aiosqlite" }, { name = "asyncmy" }, { name = "cryptography" }, { name = "fastapi" }, @@ -274,6 +287,7 @@ dev = [ requires-dist = [ { name = "aiohttp", specifier = ">=3.11.11" }, { name = "aiokafkaengine", specifier = ">=0.1.0" }, + { name = "aiosqlite", specifier = ">=0.19.0" }, { name = "asyncmy", specifier = ">=0.2.10" }, { name = "cryptography", specifier = ">=44.0.0" }, { name = "fastapi", specifier = ">=0.115.12" }, From 156782abd087fe62c6821bd90cd9a14ed5f4e42f Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Fri, 6 Jun 2025 17:31:56 -0400 Subject: [PATCH 06/39] remove aiosqlite --- pyproject.toml | 1 - uv.lock | 14 -------------- 2 files changed, 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7ee434a..f840eaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,6 @@ dependencies = [ "fastapi>=0.115.12", "uvicorn>=0.34.2", "prometheus-client>=0.21.1", - "aiosqlite>=0.19.0" ] [dependency-groups] diff --git a/uv.lock b/uv.lock index 2762f95..04425ac 100644 --- a/uv.lock +++ b/uv.lock @@ -151,18 +151,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/6a/bc7e17a3e87a2985d3e8f4da4cd0f481060eb78fb08596c42be62c90a4d9/aiosignal-1.3.2-py2.py3-none-any.whl", hash = "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5", size = 7597 }, ] -[[package]] -name = "aiosqlite" -version = "0.21.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/13/7d/8bca2bf9a247c2c5dfeec1d7a5f40db6518f88d314b8bca9da29670d2671/aiosqlite-0.21.0.tar.gz", hash = "sha256:131bb8056daa3bc875608c631c678cda73922a2d4ba8aec373b19f18c17e7aa3", size = 13454 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/10/6c25ed6de94c49f88a91fa5018cb4c0f3625f31d5be9f771ebe5cc7cd506/aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0", size = 15792 }, -] - [[package]] name = "annotated-types" version = "0.7.0" @@ -258,7 +246,6 @@ source = { editable = "." } dependencies = [ { name = "aiohttp" }, { name = "aiokafkaengine" }, - { name = "aiosqlite" }, { name = "asyncmy" }, { name = "cryptography" }, { name = "fastapi" }, @@ -287,7 +274,6 @@ dev = [ requires-dist = [ { name = "aiohttp", specifier = ">=3.11.11" }, { name = "aiokafkaengine", specifier = ">=0.1.0" }, - { name = "aiosqlite", specifier = ">=0.19.0" }, { name = "asyncmy", specifier = ">=0.2.10" }, { name = "cryptography", specifier = ">=44.0.0" }, { name = "fastapi", specifier = ">=0.115.12" }, From c6da2d0599367c35202bdd60aa8afd24cb3ac3ae Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Fri, 6 Jun 2025 17:41:43 -0400 Subject: [PATCH 07/39] removing, uneeded --- .env.test | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .env.test diff --git a/.env.test b/.env.test deleted file mode 100644 index 26835f9..0000000 --- a/.env.test +++ /dev/null @@ -1,3 +0,0 @@ -DATABASE_URL=sqlite+aiosqlite:///:memory: -DEBUG=true -KAFKA_BOOTSTRAP_SERVERS=kafka:9092 \ No newline at end of file From 7966f8e2067f2634b77e1a134e927ce392c0790b Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Fri, 6 Jun 2025 17:42:55 -0400 Subject: [PATCH 08/39] reverting back --- .../api_public/src/core/config.py | 5 +--- .../api_public/src/core/database/database.py | 29 ++++++------------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/bases/bot_detector/api_public/src/core/config.py b/bases/bot_detector/api_public/src/core/config.py index f4354e8..109d56c 100644 --- a/bases/bot_detector/api_public/src/core/config.py +++ b/bases/bot_detector/api_public/src/core/config.py @@ -4,10 +4,7 @@ from dotenv import find_dotenv, load_dotenv from pydantic_settings import BaseSettings -if "pytest" in sys.modules: - load_dotenv(find_dotenv(".env.test")) -else: - load_dotenv(find_dotenv()) # fallback to normal .env +load_dotenv(find_dotenv()) class Settings(BaseSettings): diff --git a/bases/bot_detector/api_public/src/core/database/database.py b/bases/bot_detector/api_public/src/core/database/database.py index 83cbec6..4374884 100644 --- a/bases/bot_detector/api_public/src/core/database/database.py +++ b/bases/bot_detector/api_public/src/core/database/database.py @@ -3,26 +3,15 @@ from sqlalchemy.orm import declarative_base, sessionmaker # Create an async SQLAlchemy engine -from sqlalchemy.pool import StaticPool # add at the top with your imports - -# Create an async SQLAlchemy engine -if settings.DATABASE_URL.startswith("sqlite"): - engine = create_async_engine( - settings.DATABASE_URL, - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - echo=(settings.ENV != "PRD"), - ) -else: - engine = create_async_engine( - settings.DATABASE_URL, - pool_pre_ping=True, - pool_size=10, - max_overflow=90, - pool_timeout=settings.POOL_TIMEOUT, - pool_recycle=settings.POOL_RECYCLE, - echo=(settings.ENV != "PRD"), - ) +engine = create_async_engine( + settings.DATABASE_URL, + pool_pre_ping=True, + pool_size=10, + max_overflow=90, + pool_timeout=settings.POOL_TIMEOUT, + pool_recycle=settings.POOL_RECYCLE, + echo=(settings.ENV != "PRD"), +) # Create a session factory SessionFactory = sessionmaker( From 94f0d33f9bf42e034b40180db7c1e74f95771be1 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Fri, 6 Jun 2025 17:43:46 -0400 Subject: [PATCH 09/39] remove dependency not used --- bases/bot_detector/api_public/src/core/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bases/bot_detector/api_public/src/core/config.py b/bases/bot_detector/api_public/src/core/config.py index 109d56c..1e08eb8 100644 --- a/bases/bot_detector/api_public/src/core/config.py +++ b/bases/bot_detector/api_public/src/core/config.py @@ -1,5 +1,4 @@ import asyncio -import sys from dotenv import find_dotenv, load_dotenv from pydantic_settings import BaseSettings From 0eccc2bb5c703a10fa51995b4e5acd157023a127 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Fri, 6 Jun 2025 17:53:03 -0400 Subject: [PATCH 10/39] move env loads to init --- test/bases/bot_detector/hiscore_scraper/__init__.py | 8 ++++++++ test/bases/bot_detector/hiscore_scraper/test_core.py | 5 ----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/test/bases/bot_detector/hiscore_scraper/__init__.py b/test/bases/bot_detector/hiscore_scraper/__init__.py index e69de29..7776175 100644 --- a/test/bases/bot_detector/hiscore_scraper/__init__.py +++ b/test/bases/bot_detector/hiscore_scraper/__init__.py @@ -0,0 +1,8 @@ +import os + +from dotenv import find_dotenv, load_dotenv + +load_dotenv(find_dotenv()) + +os.environ["ENVIRONMENT"] = "test" +os.environ["DEBUG"] = "true" diff --git a/test/bases/bot_detector/hiscore_scraper/test_core.py b/test/bases/bot_detector/hiscore_scraper/test_core.py index 676f7f9..7432678 100644 --- a/test/bases/bot_detector/hiscore_scraper/test_core.py +++ b/test/bases/bot_detector/hiscore_scraper/test_core.py @@ -1,8 +1,3 @@ -import os - -os.environ["ENVIRONMENT"] = "test" -os.environ["DEBUG"] = "true" - from bot_detector.hiscore_scraper import core From 1b7b98bcc62430ee4cc4dc89c18afeec84b052e3 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 07:43:07 -0400 Subject: [PATCH 11/39] init docker file. pivioting back to public api --- projects/api_private/Dockerfile | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 projects/api_private/Dockerfile diff --git a/projects/api_private/Dockerfile b/projects/api_private/Dockerfile new file mode 100644 index 0000000..92dd620 --- /dev/null +++ b/projects/api_private/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.11-slim AS base + +ARG api_port +ENV UVICORN_PORT=${api_port} + +ARG root_path +ENV UVICORN_ROOT_PATH=${root_path} + +# Keeps Python from generating .pyc files in the container +ENV PYTHONDONTWRITEBYTECODE=1 + +# Turns off buffering for easier container logging +ENV PYTHONUNBUFFERED=1 + +# set the working directory +WORKDIR /project + +# install dependencies +COPY ./requirements.txt /project +RUN pip install --no-cache-dir -r requirements.txt + +# copy the scripts to the folder +COPY ./src /project/src + +# production image +FROM base AS production +# Creates a non-root user with an explicit UID and adds permission to access the /project folder +RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /project +USER appuser + +CMD ["uvicorn", "src.core.server:app", "--proxy-headers", "--host", "0.0.0.0"] \ No newline at end of file From 9e5207dd021ab5ee7d1ae175fb770544156ec19f Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 12:07:10 -0400 Subject: [PATCH 12/39] removing wait adding private --- docker-compose.yml | 60 +++++++++++++++++++-------------- projects/api_private/Dockerfile | 41 +++++++++++----------- 2 files changed, 55 insertions(+), 46 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1d06266..c66938d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,20 @@ +x-api-common: &api_common + env_file: + - .env + environment: + - KAFKA_HOST=kafka:9092 + - DATABASE_URL=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata + - ENV=DEV + - POOL_RECYCLE=60 + - POOL_TIMEOUT=30 + networks: + - botdetector-network + depends_on: + kafka_setup: + condition: service_completed_successfully + mysql_setup: + condition: service_completed_successfully + services: kafka: image: apache/kafka:3.7.2 @@ -161,30 +178,15 @@ services: condition: service_completed_successfully api_public: + <<: *api_common container_name: api_public image: bd/api_public build: context: . dockerfile: ./projects/api_public/Dockerfile target: production - # command: ["sleep", "infinity"] - env_file: - - .env - environment: - - KAFKA_HOST=kafka:9092 - - DATABASE_URL=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata - - ENV=DEV - - POOL_RECYCLE=60 - - POOL_TIMEOUT=30 ports: - "5000:5000" - networks: - - botdetector-network - depends_on: - kafka_setup: - condition: service_completed_successfully - mysql_setup: - condition: service_completed_successfully healthcheck: test: ["CMD-SHELL", "python -c 'import urllib.request,sys; sys.exit(0 if urllib.request.urlopen(\"http://localhost:5000/\").getcode()==200 else 1)'"] interval: 10s @@ -192,15 +194,23 @@ services: retries: 3 start_period: 30s - wait_for_api_public: - image: alpine:latest - container_name: wait_for_api_public - command: ["sh", "-c", "echo 'api_public healthy'"] - depends_on: - api_public: - condition: service_healthy - networks: - - botdetector-network + api_private: + <<: *api_common + container_name: api_private + image: bd/api_private + build: + context: . + dockerfile: ./projects/api_private/Dockerfile + target: production + ports: + - "6000:6000" + healthcheck: + test: ["CMD-SHELL", "python -c 'import urllib.request,sys; sys.exit(0 if urllib.request.urlopen(\"http://localhost:6000/\").getcode()==200 else 1)'"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + networks: botdetector-network: name: bd-network \ No newline at end of file diff --git a/projects/api_private/Dockerfile b/projects/api_private/Dockerfile index 92dd620..213a988 100644 --- a/projects/api_private/Dockerfile +++ b/projects/api_private/Dockerfile @@ -1,31 +1,30 @@ -FROM python:3.11-slim AS base +FROM python:3.12-slim-bookworm AS base -ARG api_port -ENV UVICORN_PORT=${api_port} +# Copy uv from external repository +COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv /uvx /bin/ -ARG root_path -ENV UVICORN_ROOT_PATH=${root_path} - -# Keeps Python from generating .pyc files in the container -ENV PYTHONDONTWRITEBYTECODE=1 +# set the working directory +WORKDIR /app -# Turns off buffering for easier container logging -ENV PYTHONUNBUFFERED=1 +# Copy only necessary files to run the projects +COPY ./bases ./bases +COPY ./components ./components +COPY ./projects ./projects -# set the working directory -WORKDIR /project +WORKDIR /app/projects/api_private -# install dependencies -COPY ./requirements.txt /project -RUN pip install --no-cache-dir -r requirements.txt +# install dependencies via RUN uv build +RUN uv sync --frozen --no-editable -# copy the scripts to the folder -COPY ./src /project/src +# Production stage: Prepare the final production environment +FROM python:3.12-slim-bookworm AS production -# production image -FROM base AS production # Creates a non-root user with an explicit UID and adds permission to access the /project folder -RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /project +RUN adduser -u 5678 --disabled-password --gecos "" appuser + +WORKDIR /app/projects/api_private +COPY --from=base --chown=appuser /app/projects/api_private/.venv /app/projects/api_private/.venv + USER appuser -CMD ["uvicorn", "src.core.server:app", "--proxy-headers", "--host", "0.0.0.0"] \ No newline at end of file +CMD [".venv/bin/uvicorn", "bot_detector.api_private.src.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] \ No newline at end of file From d407d1adde73c47256ec7a8ad6c2fc17d43baec1 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 12:13:25 -0400 Subject: [PATCH 13/39] correcting paths, bases is not right for context --- .../api_private/src/api/v2/highscore.py | 9 ++++----- .../bot_detector/api_private/src/api/v2/player.py | 7 +++---- .../api_private/src/api/v3/highscore.py | 9 ++++----- .../api_private/src/app/repositories/highscore.py | 15 +++++++-------- .../api_private/src/app/repositories/player.py | 3 +-- .../src/app/repositories/player_activities.py | 13 ++++++------- .../src/app/repositories/player_skills.py | 9 ++++----- .../src/app/repositories/scraper_data.py | 11 +++++------ .../api_private/src/core/database/database.py | 3 +-- .../src/core/database/models/activities.py | 3 +-- .../src/core/database/models/highscore.py | 3 +-- .../src/core/database/models/player.py | 3 +-- .../src/core/database/models/prediction.py | 3 +-- .../src/core/database/models/report.py | 3 +-- .../src/core/database/models/scraper_data.py | 3 +-- .../src/core/database/models/skills.py | 3 +-- .../src/core/fastapi/dependencies/session.py | 3 +-- bases/bot_detector/api_private/src/core/server.py | 9 ++++----- 18 files changed, 47 insertions(+), 65 deletions(-) diff --git a/bases/bot_detector/api_private/src/api/v2/highscore.py b/bases/bot_detector/api_private/src/api/v2/highscore.py index ee9dc60..8e09fb3 100644 --- a/bases/bot_detector/api_private/src/api/v2/highscore.py +++ b/bases/bot_detector/api_private/src/api/v2/highscore.py @@ -1,20 +1,19 @@ import logging -from fastapi import APIRouter, Depends, Query - -from bases.bot_detector.api_private.src.app.repositories import ( +from bot_detector.api_private.src.app.repositories import ( PlayerActivityRepo, PlayerSkillsRepo, ScraperDataRepo, ) -from bases.bot_detector.api_private.src.app.views.response.highscore import ( +from bot_detector.api_private.src.app.views.response.highscore import ( PlayerHiscoreData, ) # from src.app.repositories.highscore import HighscoreRepo -from bases.bot_detector.api_private.src.core.fastapi.dependencies.session import ( +from bot_detector.api_private.src.core.fastapi.dependencies.session import ( get_session, ) +from fastapi import APIRouter, Depends, Query logger = logging.getLogger(__name__) diff --git a/bases/bot_detector/api_private/src/api/v2/player.py b/bases/bot_detector/api_private/src/api/v2/player.py index 9c5991b..07c61bf 100644 --- a/bases/bot_detector/api_private/src/api/v2/player.py +++ b/bases/bot_detector/api_private/src/api/v2/player.py @@ -1,9 +1,8 @@ -from fastapi import APIRouter, Depends, Query - -from bases.bot_detector.api_private.src.app.repositories.player import PlayerRepo -from bases.bot_detector.api_private.src.core.fastapi.dependencies.session import ( +from bot_detector.api_private.src.app.repositories.player import PlayerRepo +from bot_detector.api_private.src.core.fastapi.dependencies.session import ( get_session, ) +from fastapi import APIRouter, Depends, Query router = APIRouter() diff --git a/bases/bot_detector/api_private/src/api/v3/highscore.py b/bases/bot_detector/api_private/src/api/v3/highscore.py index d1c808a..c9e6306 100644 --- a/bases/bot_detector/api_private/src/api/v3/highscore.py +++ b/bases/bot_detector/api_private/src/api/v3/highscore.py @@ -1,17 +1,16 @@ import logging from collections import defaultdict -from fastapi import APIRouter, Depends, Query - -from bases.bot_detector.api_private.src.app.repositories import ScraperDataRepo -from bases.bot_detector.api_private.src.app.views.response import ( +from bot_detector.api_private.src.app.repositories import ScraperDataRepo +from bot_detector.api_private.src.app.views.response import ( ActivityView, ScraperDataView, SkillView, ) -from bases.bot_detector.api_private.src.core.fastapi.dependencies.session import ( +from bot_detector.api_private.src.core.fastapi.dependencies.session import ( get_session, ) +from fastapi import APIRouter, Depends, Query logger = logging.getLogger(__name__) diff --git a/bases/bot_detector/api_private/src/app/repositories/highscore.py b/bases/bot_detector/api_private/src/app/repositories/highscore.py index a310121..db55932 100644 --- a/bases/bot_detector/api_private/src/app/repositories/highscore.py +++ b/bases/bot_detector/api_private/src/app/repositories/highscore.py @@ -1,17 +1,16 @@ import logging -from fastapi.encoders import jsonable_encoder -from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession -from sqlalchemy.orm import aliased -from sqlalchemy.sql.expression import Select - -from bases.bot_detector.api_private.src.app.repositories.abstract_repo import ( +from bot_detector.api_private.src.app.repositories.abstract_repo import ( AbstractAPI, ) -from bases.bot_detector.api_private.src.core.database.models import ( # playerHiscoreData,; PlayerHiscoreDataXPChange, +from bot_detector.api_private.src.core.database.models import ( # playerHiscoreData,; PlayerHiscoreDataXPChange, PlayerHiscoreDataLatest, ) -from bases.bot_detector.api_private.src.core.database.models.player import Player +from bot_detector.api_private.src.core.database.models.player import Player +from fastapi.encoders import jsonable_encoder +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import Select logger = logging.getLogger(__name__) diff --git a/bases/bot_detector/api_private/src/app/repositories/player.py b/bases/bot_detector/api_private/src/app/repositories/player.py index f6f39af..7619490 100644 --- a/bases/bot_detector/api_private/src/app/repositories/player.py +++ b/bases/bot_detector/api_private/src/app/repositories/player.py @@ -1,9 +1,8 @@ +from bot_detector.api_private.src.core.database.models.player import Player from fastapi.encoders import jsonable_encoder from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession from sqlalchemy.sql.expression import Select -from bases.bot_detector.api_private.src.core.database.models.player import Player - class PlayerRepo: def __init__(self, session: AsyncSession) -> None: diff --git a/bases/bot_detector/api_private/src/app/repositories/player_activities.py b/bases/bot_detector/api_private/src/app/repositories/player_activities.py index 6e91a3c..3f32dd7 100644 --- a/bases/bot_detector/api_private/src/app/repositories/player_activities.py +++ b/bases/bot_detector/api_private/src/app/repositories/player_activities.py @@ -1,15 +1,14 @@ -from fastapi.encoders import jsonable_encoder -from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession -from sqlalchemy.orm import aliased -from sqlalchemy.sql.expression import Select - -from bases.bot_detector.api_private.src.app.repositories.abstract_repo import ( +from bot_detector.api_private.src.app.repositories.abstract_repo import ( AbstractAPI, ) -from bases.bot_detector.api_private.src.core.database.models import ( +from bot_detector.api_private.src.core.database.models import ( Activities, PlayerActivities, ) +from fastapi.encoders import jsonable_encoder +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import Select class PlayerActivityRepo(AbstractAPI): diff --git a/bases/bot_detector/api_private/src/app/repositories/player_skills.py b/bases/bot_detector/api_private/src/app/repositories/player_skills.py index 125c2b5..c5e669a 100644 --- a/bases/bot_detector/api_private/src/app/repositories/player_skills.py +++ b/bases/bot_detector/api_private/src/app/repositories/player_skills.py @@ -1,13 +1,12 @@ +from bot_detector.api_private.src.app.repositories.abstract_repo import ( + AbstractAPI, +) +from bot_detector.api_private.src.core.database.models import PlayerSkills, Skills from fastapi.encoders import jsonable_encoder from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession from sqlalchemy.orm import aliased from sqlalchemy.sql.expression import Select -from bases.bot_detector.api_private.src.app.repositories.abstract_repo import ( - AbstractAPI, -) -from bases.bot_detector.api_private.src.core.database.models import PlayerSkills, Skills - class PlayerSkillsRepo(AbstractAPI): def __init__(self, session: AsyncSession) -> None: diff --git a/bases/bot_detector/api_private/src/app/repositories/scraper_data.py b/bases/bot_detector/api_private/src/app/repositories/scraper_data.py index d868354..49d3662 100644 --- a/bases/bot_detector/api_private/src/app/repositories/scraper_data.py +++ b/bases/bot_detector/api_private/src/app/repositories/scraper_data.py @@ -1,9 +1,5 @@ -from sqlalchemy import func, literal, select, union_all -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import aliased - -from bases.bot_detector.api_private.src.core.database.models.player import Player -from bases.bot_detector.api_private.src.core.database.models.scraper_data_v3 import ( +from bot_detector.api_private.src.core.database.models.player import Player +from bot_detector.api_private.src.core.database.models.scraper_data_v3 import ( Activity, PlayerActivity, PlayerSkill, @@ -12,6 +8,9 @@ ScraperPlayerSkill, Skill, ) +from sqlalchemy import func, literal, select, union_all +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import aliased class ScraperDataRepo: diff --git a/bases/bot_detector/api_private/src/core/database/database.py b/bases/bot_detector/api_private/src/core/database/database.py index 7bd0520..7e64082 100644 --- a/bases/bot_detector/api_private/src/core/database/database.py +++ b/bases/bot_detector/api_private/src/core/database/database.py @@ -1,8 +1,7 @@ +from bot_detector.api_private.src.core.config import settings from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.orm import declarative_base, sessionmaker -from bases.bot_detector.api_private.src.core.config import settings - # Create an async SQLAlchemy engine engine = create_async_engine( settings.DATABASE_URL, diff --git a/bases/bot_detector/api_private/src/core/database/models/activities.py b/bases/bot_detector/api_private/src/core/database/models/activities.py index cfb3cf2..6772ad4 100644 --- a/bases/bot_detector/api_private/src/core/database/models/activities.py +++ b/bases/bot_detector/api_private/src/core/database/models/activities.py @@ -1,8 +1,7 @@ +from bot_detector.api_private.src.core.database.database import Base from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.dialects.mysql import BIGINT, TINYINT -from bases.bot_detector.api_private.src.core.database.database import Base - class Activities(Base): __tablename__ = "activities" diff --git a/bases/bot_detector/api_private/src/core/database/models/highscore.py b/bases/bot_detector/api_private/src/core/database/models/highscore.py index 4bb8051..d81fde1 100644 --- a/bases/bot_detector/api_private/src/core/database/models/highscore.py +++ b/bases/bot_detector/api_private/src/core/database/models/highscore.py @@ -1,3 +1,4 @@ +from bot_detector.api_private.src.core.database.database import Base from sqlalchemy import ( BigInteger, Column, @@ -9,8 +10,6 @@ text, ) -from bases.bot_detector.api_private.src.core.database.database import Base - class playerHiscoreData(Base): __tablename__ = "playerHiscoreData" diff --git a/bases/bot_detector/api_private/src/core/database/models/player.py b/bases/bot_detector/api_private/src/core/database/models/player.py index 768b073..4c314e3 100644 --- a/bases/bot_detector/api_private/src/core/database/models/player.py +++ b/bases/bot_detector/api_private/src/core/database/models/player.py @@ -1,7 +1,6 @@ +from bot_detector.api_private.src.core.database.database import Base from sqlalchemy import Boolean, Column, DateTime, Integer, Text -from bases.bot_detector.api_private.src.core.database.database import Base - class Player(Base): __tablename__ = "Players" diff --git a/bases/bot_detector/api_private/src/core/database/models/prediction.py b/bases/bot_detector/api_private/src/core/database/models/prediction.py index e664c4f..4571355 100644 --- a/bases/bot_detector/api_private/src/core/database/models/prediction.py +++ b/bases/bot_detector/api_private/src/core/database/models/prediction.py @@ -1,7 +1,6 @@ +from bot_detector.api_private.src.core.database.database import Base from sqlalchemy import DECIMAL, TIMESTAMP, Column, Integer, String -from bases.bot_detector.api_private.src.core.database.database import Base - class Prediction(Base): __tablename__ = "Predictions" diff --git a/bases/bot_detector/api_private/src/core/database/models/report.py b/bases/bot_detector/api_private/src/core/database/models/report.py index c6e7b06..a01fd4d 100644 --- a/bases/bot_detector/api_private/src/core/database/models/report.py +++ b/bases/bot_detector/api_private/src/core/database/models/report.py @@ -1,7 +1,6 @@ +from bot_detector.api_private.src.core.database.database import Base from sqlalchemy import TIMESTAMP, BigInteger, Column, Integer, SmallInteger -from bases.bot_detector.api_private.src.core.database.database import Base - class Report(Base): __tablename__ = "Reports" diff --git a/bases/bot_detector/api_private/src/core/database/models/scraper_data.py b/bases/bot_detector/api_private/src/core/database/models/scraper_data.py index 3117cad..1693695 100644 --- a/bases/bot_detector/api_private/src/core/database/models/scraper_data.py +++ b/bases/bot_detector/api_private/src/core/database/models/scraper_data.py @@ -1,8 +1,7 @@ +from bot_detector.api_private.src.core.database.database import Base from sqlalchemy import Column, Date, DateTime, func from sqlalchemy.dialects.mysql import BIGINT, SMALLINT -from bases.bot_detector.api_private.src.core.database.database import Base - class ScraperData(Base): __tablename__ = "scraper_data" diff --git a/bases/bot_detector/api_private/src/core/database/models/skills.py b/bases/bot_detector/api_private/src/core/database/models/skills.py index cca52ca..f3ead46 100644 --- a/bases/bot_detector/api_private/src/core/database/models/skills.py +++ b/bases/bot_detector/api_private/src/core/database/models/skills.py @@ -1,8 +1,7 @@ +from bot_detector.api_private.src.core.database.database import Base from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.dialects.mysql import BIGINT, TINYINT -from bases.bot_detector.api_private.src.core.database.database import Base - class Skills(Base): __tablename__ = "skills" diff --git a/bases/bot_detector/api_private/src/core/fastapi/dependencies/session.py b/bases/bot_detector/api_private/src/core/fastapi/dependencies/session.py index 7a59fd6..7ca3c85 100644 --- a/bases/bot_detector/api_private/src/core/fastapi/dependencies/session.py +++ b/bases/bot_detector/api_private/src/core/fastapi/dependencies/session.py @@ -1,7 +1,6 @@ +from bot_detector.api_private.src.core.database.database import SessionFactory from sqlalchemy.ext.asyncio import AsyncSession -from bases.bot_detector.api_private.src.core.database.database import SessionFactory - # Dependency to get an asynchronous session async def get_session() -> AsyncSession: diff --git a/bases/bot_detector/api_private/src/core/server.py b/bases/bot_detector/api_private/src/core/server.py index a8ac5d7..001307d 100644 --- a/bases/bot_detector/api_private/src/core/server.py +++ b/bases/bot_detector/api_private/src/core/server.py @@ -1,14 +1,13 @@ import logging +from bot_detector.api_private.src import api +from bot_detector.api_private.src.core.fastapi.middleware.logging import ( + LoggingMiddleware, +) from fastapi import FastAPI from fastapi.middleware import Middleware from fastapi.middleware.cors import CORSMiddleware -from bases.bot_detector.api_private.src import api -from bases.bot_detector.api_private.src.core.fastapi.middleware.logging import ( - LoggingMiddleware, -) - logger = logging.getLogger(__name__) From 49a18fbf15f27e0de926bc37cc388aa3e9eb4531 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 12:15:11 -0400 Subject: [PATCH 14/39] use 6000 --- projects/api_private/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/api_private/Dockerfile b/projects/api_private/Dockerfile index 213a988..eb01d40 100644 --- a/projects/api_private/Dockerfile +++ b/projects/api_private/Dockerfile @@ -27,4 +27,4 @@ COPY --from=base --chown=appuser /app/projects/api_private/.venv /app/projects/a USER appuser -CMD [".venv/bin/uvicorn", "bot_detector.api_private.src.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] \ No newline at end of file +CMD [".venv/bin/uvicorn", "bot_detector.api_private.src.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "6000", "--log-level", "warning"] \ No newline at end of file From 6aa9c958cbf42ba8e3a0c0ea41269146a19326c6 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 12:16:27 -0400 Subject: [PATCH 15/39] cleanup --- bases/bot_detector/api_private/README.md | 44 ------------ .../api_private/docker-compose.yml | 67 ------------------- .../bot_detector/api_private/requirements.txt | 33 --------- 3 files changed, 144 deletions(-) delete mode 100644 bases/bot_detector/api_private/README.md delete mode 100644 bases/bot_detector/api_private/docker-compose.yml delete mode 100644 bases/bot_detector/api_private/requirements.txt diff --git a/bases/bot_detector/api_private/README.md b/bases/bot_detector/api_private/README.md deleted file mode 100644 index 3fc29b2..0000000 --- a/bases/bot_detector/api_private/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# public-api -structure: -- src: contains all program files -- src.api: contains the api -- src.app: contains the database logic -- src.core: contains the core componetns -- src.core.database: contains all the database functionality - -# setup -## windows -creating a python venv to work in and install the project requirements -```sh -python -m venv .venv -.venv\Scripts\activate -python -m pip install --upgrade pip -pip install -r requirements.txt -``` -## linux -make sure to have pip -```sh -sudo apt update -y && sudo apt upgrade -y -sudo apt install python3.10-venv -y -sudo apt install python3-pip -y -``` -```sh -python3 -m venv .venv -source .venv/bin/activate -python3 -m pip install --upgrade pip -pip install -r requirements.txt -``` -# for admin purposes saving & upgrading -when you added some dependancies update the requirements -```sh -.venv\Scripts\activate -call pip freeze > requirements.txt -``` -when you want to upgrade the dependancies -```sh -.venv\Scripts\activate -powershell "(Get-Content requirements.txt) | ForEach-Object { $_ -replace '==', '>=' } | Set-Content requirements.txt" -call pip install -r requirements.txt --upgrade -call pip freeze > requirements.txt -powershell "(Get-Content requirements.txt) | ForEach-Object { $_ -replace '>=', '==' } | Set-Content requirements.txt" -``` \ No newline at end of file diff --git a/bases/bot_detector/api_private/docker-compose.yml b/bases/bot_detector/api_private/docker-compose.yml deleted file mode 100644 index 0451218..0000000 --- a/bases/bot_detector/api_private/docker-compose.yml +++ /dev/null @@ -1,67 +0,0 @@ -services: - mysql: - container_name: database - build: - context: ./mysql - image: bot-detector/mysql:latest - environment: - - MYSQL_ROOT_PASSWORD=root_bot_buster - volumes: - - ./mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d - # - ./mysql/mount:/var/lib/mysql # creates persistence - ports: - - 3307:3306 - networks: - - botdetector-network - healthcheck: - test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] - timeout: 20s - retries: 10 - - mysql_setup: - container_name: mysql_setup - image: bot-detector/mysql_setup - build: - context: ./mysql_setup - command: ["python", "-u","setup_mysql.py"] - networks: - - botdetector-network - depends_on: - mysql: - condition: service_healthy - - private_api: - container_name: private_api - build: - context: . - dockerfile: Dockerfile - target: base - args: - root_path: / - api_port: 5000 - # command: bash -c "apt update && apt install -y curl && sleep infinity" - command: uvicorn src.core.server:app --host 0.0.0.0 --reload --reload-include src/* - ports: - - 5000:5000 - networks: - - botdetector-network - # this overrides the env_file for the specific variable - environment: - - KAFKA_HOST=kafka:9092 - - DATABASE_URL=mysql+aiomysql://root:root_bot_buster@mysql:3306/playerdata - - ENV=DEV - - POOL_RECYCLE=60 - - POOL_TIMEOUT=30 - # env_file: - # - .env - volumes: - - ./src:/project/src - depends_on: - mysql: - condition: service_healthy - mysql_setup: - condition: service_completed_successfully - - -networks: - botdetector-network: diff --git a/bases/bot_detector/api_private/requirements.txt b/bases/bot_detector/api_private/requirements.txt deleted file mode 100644 index 669de9a..0000000 --- a/bases/bot_detector/api_private/requirements.txt +++ /dev/null @@ -1,33 +0,0 @@ -aiokafka==0.10.0 -aiomysql==0.2.0 -annotated-types==0.6.0 -anyio==4.3.0 -async-timeout==4.0.3 -asyncmy==0.2.9 -cffi==1.16.0 -click==8.1.7 -colorama==0.4.6 -cryptography==42.0.5 -databases==0.9.0 -exceptiongroup==1.2.0 -fastapi==0.110.0 -greenlet==3.0.3 -h11==0.14.0 -httptools==0.6.1 -idna==3.7 -kafka-python==2.0.2 -packaging==24.0 -pycparser==2.21 -pydantic==2.6.4 -pydantic-settings==2.2.1 -pydantic_core==2.16.3 -PyMySQL==1.1.0 -python-dotenv==1.0.1 -PyYAML==6.0.1 -sniffio==1.3.1 -SQLAlchemy==2.0.28 -starlette==0.36.3 -typing_extensions==4.10.0 -uvicorn==0.29.0 -watchfiles==0.21.0 -websockets==12.0 From a9d3d1a2b9072f7ec144f1ab472b882ed3eb45e2 Mon Sep 17 00:00:00 2001 From: extreme4all <> Date: Sat, 7 Jun 2025 22:47:59 +0200 Subject: [PATCH 16/39] refactor: update docker-compose and Dockerfiles to use common healthcheck and environment variables --- docker-compose.yml | 78 +++++++++++++++------------------ projects/api_private/Dockerfile | 3 +- projects/api_public/Dockerfile | 3 +- 3 files changed, 40 insertions(+), 44 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c66938d..3e18fc1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,20 +1,16 @@ -x-api-common: &api_common - env_file: - - .env - environment: - - KAFKA_HOST=kafka:9092 - - DATABASE_URL=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata - - ENV=DEV - - POOL_RECYCLE=60 - - POOL_TIMEOUT=30 - networks: - - botdetector-network - depends_on: +x-common-depends: &common_depends kafka_setup: condition: service_completed_successfully mysql_setup: condition: service_completed_successfully +x-common-api-healthcheck: &api-healthcheck + test: ["CMD-SHELL", "python -c 'import urllib.request,sys; sys.exit(0 if urllib.request.urlopen(\"http://localhost:5000/\").getcode()==200 else 1)'"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + services: kafka: image: apache/kafka:3.7.2 @@ -111,7 +107,6 @@ services: depends_on: mysql: condition: service_healthy - hiscore_scraper: container_name: hiscore_scraper image: bd/hiscore_scraper # tags the image if build @@ -154,11 +149,7 @@ services: - botdetector-network env_file: - .env - depends_on: - kafka_setup: - condition: service_completed_successfully - mysql_setup: - condition: service_completed_successfully + depends_on: *common_depends scrape_task_producer: container_name: scrape_task_producer image: bd/scrape_task_producer # tags the image if build @@ -171,14 +162,8 @@ services: - botdetector-network env_file: - .env - depends_on: - kafka_setup: - condition: service_completed_successfully - mysql_setup: - condition: service_completed_successfully - + depends_on: *common_depends api_public: - <<: *api_common container_name: api_public image: bd/api_public build: @@ -186,31 +171,40 @@ services: dockerfile: ./projects/api_public/Dockerfile target: production ports: - - "5000:5000" - healthcheck: - test: ["CMD-SHELL", "python -c 'import urllib.request,sys; sys.exit(0 if urllib.request.urlopen(\"http://localhost:5000/\").getcode()==200 else 1)'"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 30s - + - "5001:5000" + environment: + - UVICORN_PORT=5000 + - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 + - DATABASE_URL=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata + - ENV=DEV + - POOL_RECYCLE=60 + - POOL_TIMEOUT=30 + networks: + - botdetector-network + depends_on: *common_depends + healthcheck: *api-healthcheck api_private: - <<: *api_common container_name: api_private image: bd/api_private build: context: . dockerfile: ./projects/api_private/Dockerfile target: production + env_file: + - .env + environment: + - UVICORN_PORT=5000 + - KAFKA_HOST=kafka:9092 + - DATABASE_URL=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata + - ENV=DEV + - POOL_RECYCLE=60 + - POOL_TIMEOUT=30 + networks: + - botdetector-network ports: - - "6000:6000" - healthcheck: - test: ["CMD-SHELL", "python -c 'import urllib.request,sys; sys.exit(0 if urllib.request.urlopen(\"http://localhost:6000/\").getcode()==200 else 1)'"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 30s - + - "5002:5000" + depends_on: *common_depends + healthcheck: *api-healthcheck networks: botdetector-network: name: bd-network \ No newline at end of file diff --git a/projects/api_private/Dockerfile b/projects/api_private/Dockerfile index eb01d40..a5096b4 100644 --- a/projects/api_private/Dockerfile +++ b/projects/api_private/Dockerfile @@ -27,4 +27,5 @@ COPY --from=base --chown=appuser /app/projects/api_private/.venv /app/projects/a USER appuser -CMD [".venv/bin/uvicorn", "bot_detector.api_private.src.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "6000", "--log-level", "warning"] \ No newline at end of file +# we let the port be set by the environment variable UVICORN_PORT (default 8000) +CMD [".venv/bin/uvicorn", "bot_detector.api_private.src.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--log-level", "warning"] \ No newline at end of file diff --git a/projects/api_public/Dockerfile b/projects/api_public/Dockerfile index db64bac..dbdcb08 100644 --- a/projects/api_public/Dockerfile +++ b/projects/api_public/Dockerfile @@ -27,4 +27,5 @@ COPY --from=base --chown=appuser /app/projects/api_public/.venv /app/projects/ap USER appuser -CMD [".venv/bin/uvicorn", "bot_detector.api_public.src.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] \ No newline at end of file +# we let the port be set by the environment variable UVICORN_PORT (default 8000) +CMD [".venv/bin/uvicorn", "bot_detector.api_public.src.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--log-level", "warning"] \ No newline at end of file From 9feec185cc8cc2c0ab9aa9fac45e533ecb22cca7 Mon Sep 17 00:00:00 2001 From: extreme4all <> Date: Sat, 7 Jun 2025 22:50:16 +0200 Subject: [PATCH 17/39] refactor: remove deprecated highscore endpoint implementation --- .../api_private/src/api/v2/highscore.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/bases/bot_detector/api_private/src/api/v2/highscore.py b/bases/bot_detector/api_private/src/api/v2/highscore.py index 8e09fb3..e37326a 100644 --- a/bases/bot_detector/api_private/src/api/v2/highscore.py +++ b/bases/bot_detector/api_private/src/api/v2/highscore.py @@ -21,23 +21,6 @@ router = APIRouter() -# @router.get("/highscore/latest", response_model=list[PlayerHiscoreData]) -# async def get_highscore_latest( -# player_id: int, -# label_id: int = None, -# many: bool = False, -# limit: int = Query(default=10, ge=0, le=10_000), -# session=Depends(get_session), -# ): -# repo = HighscoreRepo(session=session) -# data: list[dict] = await repo.select( -# player_id=player_id, label_id=label_id, many=many, limit=limit -# ) - -# data = [{k: v for k, v in d.items() if v} for d in data] -# return [PlayerHiscoreData(**d).model_dump(mode="json") for d in data] - - @router.get("/highscore/latest") async def get_highscore_latest_v2( player_id: int, From 432300e13f21d9460487e6200ccc3dfbda52464f Mon Sep 17 00:00:00 2001 From: extreme4all <> Date: Sat, 7 Jun 2025 22:55:51 +0200 Subject: [PATCH 18/39] refactor: remove unused AioKafkaEngine implementation --- .../api_private/src/core/kafka/engine.py | 72 ------------------- 1 file changed, 72 deletions(-) delete mode 100644 bases/bot_detector/api_private/src/core/kafka/engine.py diff --git a/bases/bot_detector/api_private/src/core/kafka/engine.py b/bases/bot_detector/api_private/src/core/kafka/engine.py deleted file mode 100644 index e1eb122..0000000 --- a/bases/bot_detector/api_private/src/core/kafka/engine.py +++ /dev/null @@ -1,72 +0,0 @@ -import asyncio -from aiokafka import AIOKafkaConsumer, AIOKafkaProducer -from asyncio import Queue -import json - -def retry_on_exception(max_retries=3, retry_interval=5): - def decorator(func): - async def wrapper(*args, **kwargs): - retries = 0 - while retries < max_retries: - try: - await func(*args, **kwargs) - except Exception as e: - print(f"Error: {e}") - retries += 1 - await asyncio.sleep(retry_interval) - else: - break - return wrapper - return decorator - -class AioKafkaEngine: - def __init__(self, bootstrap_servers: list[str], topic: str, message_queue: Queue): - self.bootstrap_servers = bootstrap_servers - self.topic = topic - self.consumer = None - self.producer = None - self.message_queue = message_queue - - async def start_consumer(self, group_id: str): - self.consumer = AIOKafkaConsumer( - self.topic, - bootstrap_servers=self.bootstrap_servers, - value_serializer=lambda v: json.dumps(v).encode(), - group_id=group_id, - ) - await self.consumer.start() - - async def start_producer(self): - self.producer = AIOKafkaProducer( - bootstrap_servers=self.bootstrap_servers, - value_serializer=lambda v: json.dumps(v).encode() - ) - await self.producer.start() - - @retry_on_exception(max_retries=3, retry_interval=5) - async def consume_messages(self): - if self.consumer is None: - raise ValueError("Consumer not started. Call start_consumer() first.") - - async for message in self.consumer: - value = message.value - self.message_queue.put_nowait(value) - - @retry_on_exception(max_retries=3, retry_interval=5) - async def produce_messages(self): - if self.producer is None: - raise ValueError("Producer not started. Call start_producer() first.") - while True: - message = await self.message_queue.get() - await self.producer.send(self.topic, value=message) - - async def stop_consumer(self): - if self.consumer: - await self.consumer.stop() - - async def stop_producer(self): - if self.producer: - await self.producer.stop() - - def is_ready(self): - return self.consumer is not None or self.producer is not None \ No newline at end of file From 307e7baa3747d32951e1a8eea73ea0470a7b36da Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 17:41:21 -0400 Subject: [PATCH 19/39] still runs after removal, cleanup unused dependencies --- projects/api_private/pyproject.toml | 4 ---- projects/api_private/uv.lock | 13 ------------- 2 files changed, 17 deletions(-) diff --git a/projects/api_private/pyproject.toml b/projects/api_private/pyproject.toml index 159ca9f..1dfc581 100644 --- a/projects/api_private/pyproject.toml +++ b/projects/api_private/pyproject.toml @@ -14,10 +14,6 @@ dependencies = [ # config / validation "pydantic-settings>=2.9.1", - "python-dotenv>=1.0.1", - - # Kafka (choose one) - "kafka-python>=2.0.2", # Database "sqlalchemy>=2.0.40", diff --git a/projects/api_private/uv.lock b/projects/api_private/uv.lock index f258160..a45cdaa 100644 --- a/projects/api_private/uv.lock +++ b/projects/api_private/uv.lock @@ -33,9 +33,7 @@ source = { editable = "." } dependencies = [ { name = "asyncmy" }, { name = "fastapi" }, - { name = "kafka-python" }, { name = "pydantic-settings" }, - { name = "python-dotenv" }, { name = "sqlalchemy" }, { name = "uvicorn" }, ] @@ -44,9 +42,7 @@ dependencies = [ requires-dist = [ { name = "asyncmy", specifier = ">=0.2.9" }, { name = "fastapi", specifier = ">=0.115.12" }, - { name = "kafka-python", specifier = ">=2.0.2" }, { name = "pydantic-settings", specifier = ">=2.9.1" }, - { name = "python-dotenv", specifier = ">=1.0.1" }, { name = "sqlalchemy", specifier = ">=2.0.40" }, { name = "uvicorn", specifier = ">=0.34.2" }, ] @@ -204,15 +200,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, ] -[[package]] -name = "kafka-python" -version = "2.2.11" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/cf/f55532155c7e2c54fdbdceb1412425f894fd0a6897575fa3454e541ce186/kafka_python-2.2.11.tar.gz", hash = "sha256:8ff8bcc158f48b47ba516536a5b1287db75a8ceff13d639da917ec52e171acde", size = 347568 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/e1/89df26df8a93d24c4ad01db6cffb65894ef3e39c1cedf4a281178e7cb669/kafka_python-2.2.11-py2.py3-none-any.whl", hash = "sha256:c285ce322108382ea9fd62273aab175d9a6959866145cf7cf9d4ca447b632372", size = 309650 }, -] - [[package]] name = "pydantic" version = "2.11.5" From 84f604f1948e3967ab7709335aa0c2ae19a29c81 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 18:16:43 -0400 Subject: [PATCH 20/39] move tests --- bases/bot_detector/api_private/tests/pytest.ini | 3 --- pytest.ini | 4 +++- .../bases/bot_detector/api_private/__init__.py | 0 .../bot_detector/api_private}/_test_highscore_benchmark.py | 0 .../bases/bot_detector/api_private}/benchmark.py | 0 .../tests => test/bases/bot_detector/api_private}/conftest.py | 0 .../bases/bot_detector/api_private}/test_highscore.py | 0 .../api_private}/test_highscore_benchmark_custom.py | 0 .../bases/bot_detector/api_private}/test_highscore_v2.py | 0 .../bases/bot_detector/api_private}/test_highscore_v3.py | 0 .../bases/bot_detector/api_private}/test_player.py | 0 11 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 bases/bot_detector/api_private/tests/pytest.ini rename bases/bot_detector/api_private/tests/.gitkeep => test/bases/bot_detector/api_private/__init__.py (100%) rename {bases/bot_detector/api_private/tests => test/bases/bot_detector/api_private}/_test_highscore_benchmark.py (100%) rename {bases/bot_detector/api_private/tests => test/bases/bot_detector/api_private}/benchmark.py (100%) rename {bases/bot_detector/api_private/tests => test/bases/bot_detector/api_private}/conftest.py (100%) rename {bases/bot_detector/api_private/tests => test/bases/bot_detector/api_private}/test_highscore.py (100%) rename {bases/bot_detector/api_private/tests => test/bases/bot_detector/api_private}/test_highscore_benchmark_custom.py (100%) rename {bases/bot_detector/api_private/tests => test/bases/bot_detector/api_private}/test_highscore_v2.py (100%) rename {bases/bot_detector/api_private/tests => test/bases/bot_detector/api_private}/test_highscore_v3.py (100%) rename {bases/bot_detector/api_private/tests => test/bases/bot_detector/api_private}/test_player.py (100%) diff --git a/bases/bot_detector/api_private/tests/pytest.ini b/bases/bot_detector/api_private/tests/pytest.ini deleted file mode 100644 index 12681bf..0000000 --- a/bases/bot_detector/api_private/tests/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -asyncio_mode=auto -addopts = -s diff --git a/pytest.ini b/pytest.ini index c385873..2919b90 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,5 @@ [pytest] testpaths = test/bases/bot_detector tests/components/bot_detector -pythonpath = bases components projects \ No newline at end of file +pythonpath = bases components projects +asyncio_mode=auto +addopts = -s \ No newline at end of file diff --git a/bases/bot_detector/api_private/tests/.gitkeep b/test/bases/bot_detector/api_private/__init__.py similarity index 100% rename from bases/bot_detector/api_private/tests/.gitkeep rename to test/bases/bot_detector/api_private/__init__.py diff --git a/bases/bot_detector/api_private/tests/_test_highscore_benchmark.py b/test/bases/bot_detector/api_private/_test_highscore_benchmark.py similarity index 100% rename from bases/bot_detector/api_private/tests/_test_highscore_benchmark.py rename to test/bases/bot_detector/api_private/_test_highscore_benchmark.py diff --git a/bases/bot_detector/api_private/tests/benchmark.py b/test/bases/bot_detector/api_private/benchmark.py similarity index 100% rename from bases/bot_detector/api_private/tests/benchmark.py rename to test/bases/bot_detector/api_private/benchmark.py diff --git a/bases/bot_detector/api_private/tests/conftest.py b/test/bases/bot_detector/api_private/conftest.py similarity index 100% rename from bases/bot_detector/api_private/tests/conftest.py rename to test/bases/bot_detector/api_private/conftest.py diff --git a/bases/bot_detector/api_private/tests/test_highscore.py b/test/bases/bot_detector/api_private/test_highscore.py similarity index 100% rename from bases/bot_detector/api_private/tests/test_highscore.py rename to test/bases/bot_detector/api_private/test_highscore.py diff --git a/bases/bot_detector/api_private/tests/test_highscore_benchmark_custom.py b/test/bases/bot_detector/api_private/test_highscore_benchmark_custom.py similarity index 100% rename from bases/bot_detector/api_private/tests/test_highscore_benchmark_custom.py rename to test/bases/bot_detector/api_private/test_highscore_benchmark_custom.py diff --git a/bases/bot_detector/api_private/tests/test_highscore_v2.py b/test/bases/bot_detector/api_private/test_highscore_v2.py similarity index 100% rename from bases/bot_detector/api_private/tests/test_highscore_v2.py rename to test/bases/bot_detector/api_private/test_highscore_v2.py diff --git a/bases/bot_detector/api_private/tests/test_highscore_v3.py b/test/bases/bot_detector/api_private/test_highscore_v3.py similarity index 100% rename from bases/bot_detector/api_private/tests/test_highscore_v3.py rename to test/bases/bot_detector/api_private/test_highscore_v3.py diff --git a/bases/bot_detector/api_private/tests/test_player.py b/test/bases/bot_detector/api_private/test_player.py similarity index 100% rename from bases/bot_detector/api_private/tests/test_player.py rename to test/bases/bot_detector/api_private/test_player.py From dfc6582b03dcd41be5817e2a124a68992ebda2f7 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 18:20:58 -0400 Subject: [PATCH 21/39] kafka setting is not being used anywhere?, removing --- bases/bot_detector/api_private/src/core/config.py | 4 ++-- docker-compose.yml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bases/bot_detector/api_private/src/core/config.py b/bases/bot_detector/api_private/src/core/config.py index d3dc92c..f05be1f 100644 --- a/bases/bot_detector/api_private/src/core/config.py +++ b/bases/bot_detector/api_private/src/core/config.py @@ -4,8 +4,8 @@ class Settings(BaseSettings): ENV: str DATABASE_URL: str - KAFKA_HOST: str POOL_RECYCLE: int = 25 POOL_TIMEOUT: int = 25 -settings = Settings() \ No newline at end of file + +settings = Settings() diff --git a/docker-compose.yml b/docker-compose.yml index 3e18fc1..44c4420 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -194,7 +194,6 @@ services: - .env environment: - UVICORN_PORT=5000 - - KAFKA_HOST=kafka:9092 - DATABASE_URL=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata - ENV=DEV - POOL_RECYCLE=60 From 3b86991e25e91873589a9b6249fa980a1bc94ccf Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 18:26:21 -0400 Subject: [PATCH 22/39] load env for tests --- test/bases/bot_detector/api_private/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/bases/bot_detector/api_private/__init__.py b/test/bases/bot_detector/api_private/__init__.py index e69de29..3d5905b 100644 --- a/test/bases/bot_detector/api_private/__init__.py +++ b/test/bases/bot_detector/api_private/__init__.py @@ -0,0 +1,3 @@ +from dotenv import find_dotenv, load_dotenv + +load_dotenv(find_dotenv()) From 5d280a3acd0413d6bc96fec8c2bf6dfe6ee0f252 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 18:30:40 -0400 Subject: [PATCH 23/39] spaces, and private api needs dev --- .env.example | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 3d64daa..abe82fb 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,9 @@ # To get an api key please use our referral code (they also have free tier) # https://www.webshare.io/?referral_code=qvpjdwxqsblt PROXY_API_KEY = "" -KAFKA_BOOTSTRAP_SERVERS = "kafka:9092" +KAFKA_BOOTSTRAP_SERVERS = "kafka:9092" # public api, scraper and worker SESSION_TIMEOUT = "60" DATABASE_URL = "mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata" DEBUG = "False" -PYTHONDONTWRITEBYTECODE=1 \ No newline at end of file +PYTHONDONTWRITEBYTECODE = 1 +ENV = "DEV" # private api \ No newline at end of file From 4678fa82bc8f900958e0a3f58d8fb488b988d25f Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 18:32:26 -0400 Subject: [PATCH 24/39] update uvlock --- uv.lock | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/uv.lock b/uv.lock index 04425ac..64fff2f 100644 --- a/uv.lock +++ b/uv.lock @@ -265,7 +265,6 @@ dev = [ { name = "polylith-cli" }, { name = "pytest" }, { name = "pytest-asyncio" }, - { name = "pytest-benchmark" }, { name = "pytest-cov" }, { name = "ruff" }, ] @@ -293,7 +292,6 @@ dev = [ { name = "polylith-cli", specifier = ">=1.24.0" }, { name = "pytest", specifier = ">=8.3.4" }, { name = "pytest-asyncio", specifier = ">=0.25.0" }, - { name = "pytest-benchmark", specifier = ">=3.0.0" }, { name = "pytest-cov", specifier = ">=6.1.1" }, { name = "ruff", specifier = ">=0.8.4" }, ] @@ -1036,15 +1034,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/b6/c5319caea262f4821995dca2107483b94a3345d4607ad797c76cb9c36bcc/propcache-0.2.1-py3-none-any.whl", hash = "sha256:52277518d6aae65536e9cea52d4e7fd2f7a66f4aa2d30ed3f2fcea620ace3c54", size = 11818 }, ] -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335 }, -] - [[package]] name = "pycparser" version = "2.22" @@ -1194,19 +1183,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/56/2ee0cab25c11d4e38738a2a98c645a8f002e2ecf7b5ed774c70d53b92bb1/pytest_asyncio-0.25.0-py3-none-any.whl", hash = "sha256:db5432d18eac6b7e28b46dcd9b69921b55c3b1086e85febfe04e70b18d9e81b3", size = 19245 }, ] -[[package]] -name = "pytest-benchmark" -version = "5.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "py-cpuinfo" }, - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259 }, -] - [[package]] name = "pytest-cov" version = "6.1.1" From 0ee47597585642ded02e6908b1954d0c3418542f Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 20:37:11 -0400 Subject: [PATCH 25/39] poly upgrade for tests --- pyproject.toml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f840eaf..6617bcd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,12 +26,13 @@ dependencies = [ [dependency-groups] dev = [ "kafka-python>=2.0.2", - "polylith-cli>=1.24.0", + "polylith-cli>=1.30.1", "pytest>=8.3.4", "ruff>=0.8.4", "pytest-asyncio>=0.25.0", "pytest-cov>=6.1.1", "httpx>=0.27.0", + "greenlet>=3.0.3", ] @@ -42,3 +43,19 @@ dev-mode-dirs = ["components", "bases", "development", "."] [tool.hatch.metadata] allow-direct-references = true + +[tool.polylith.bricks] +"bases/bot_detector/api_public" = "bot_detector/api_public" +"bases/bot_detector/runemetrics_scraper" = "bot_detector/runemetrics_scraper" +"bases/bot_detector/hiscore_worker" = "bot_detector/hiscore_worker" +"bases/bot_detector/api_private" = "bot_detector/api_private" +"bases/bot_detector/scrape_task_producer" = "bot_detector/scrape_task_producer" +"bases/bot_detector/hiscore_scraper" = "bot_detector/hiscore_scraper" +"components/bot_detector/feedback" = "bot_detector/feedback" +"components/bot_detector/proxy_manager" = "bot_detector/proxy_manager" +"components/bot_detector/structs" = "bot_detector/structs" +"components/bot_detector/kafka" = "bot_detector/kafka" +"components/bot_detector/kafka_client" = "bot_detector/kafka_client" +"components/bot_detector/runemetrics_api" = "bot_detector/runemetrics_api" +"components/bot_detector/logfmt" = "bot_detector/logfmt" +"components/bot_detector/database" = "bot_detector/database" From 0e3534d0be42b57396e02a553ad957e727b973f5 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 21:03:32 -0400 Subject: [PATCH 26/39] poly complaining, removing unused --- projects/api_private/pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/api_private/pyproject.toml b/projects/api_private/pyproject.toml index 1dfc581..52502c6 100644 --- a/projects/api_private/pyproject.toml +++ b/projects/api_private/pyproject.toml @@ -35,6 +35,5 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/api_private" = "bot_detector/api_private" -"../../components/bot_detector/kafka" = "bot_detector/kafka" "../../components/bot_detector/structs" = "bot_detector/structs" "../../components/bot_detector/logfmt" = "bot_detector/logfmt" \ No newline at end of file From 3ffecc31488a254f57a9b585ca4c65c7bfbcec85 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 21:09:22 -0400 Subject: [PATCH 27/39] useful poly commands, run make checks --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Makefile b/Makefile index c7c0788..aa08a93 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,15 @@ restart-%: ## Restart a docker service by name, eg make restart-api_public docker compose build $* docker compose up -d $* +info: + uv run poly info + +libs: + uv run poly libs + +checks: info libs + uv run poly check + setup: uv sync From 6d115bb17bffd771c92011e24fa3ae45a116ab84 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 21:10:25 -0400 Subject: [PATCH 28/39] update cli to support tests --- pyproject.toml | 1 - uv.lock | 53 ++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6617bcd..394e483 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,6 @@ dev = [ "pytest-asyncio>=0.25.0", "pytest-cov>=6.1.1", "httpx>=0.27.0", - "greenlet>=3.0.3", ] diff --git a/uv.lock b/uv.lock index 64fff2f..a8c4956 100644 --- a/uv.lock +++ b/uv.lock @@ -289,7 +289,7 @@ requires-dist = [ dev = [ { name = "httpx", specifier = ">=0.27.0" }, { name = "kafka-python", specifier = ">=2.0.2" }, - { name = "polylith-cli", specifier = ">=1.24.0" }, + { name = "polylith-cli", specifier = ">=1.30.1" }, { name = "pytest", specifier = ">=8.3.4" }, { name = "pytest-asyncio", specifier = ">=0.25.0" }, { name = "pytest-cov", specifier = ">=6.1.1" }, @@ -940,16 +940,17 @@ wheels = [ [[package]] name = "polylith-cli" -version = "1.24.0" +version = "1.30.1" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "pyyaml" }, { name = "rich" }, { name = "tomlkit" }, { name = "typer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ed/7c/709fc263b30253cd7d92bcea460551f81230ab3098dd3bf5c8100256b17b/polylith_cli-1.24.0.tar.gz", hash = "sha256:c167f6174f7e922c8d3bc159e0c1e71fa43425d2db21151e239251aea1f2efaa", size = 32670 } +sdist = { url = "https://files.pythonhosted.org/packages/08/dd/bca06856139a41e195c4657516f4c1a4d6626a46add3136ce33c699e5adf/polylith_cli-1.30.1.tar.gz", hash = "sha256:d9e2eced5ccf3fe3d3c342c2920b0e1ca4d2e95c1117da16c7b165d7c3bcb0db", size = 31571 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/02/62e3f10aed27edf1c5292dcec8afa7014d32daef1265dcd834ea4cea332f/polylith_cli-1.24.0-py3-none-any.whl", hash = "sha256:2fa9df58eeba0fcf63c9ba7ec4c73d8aee639863b76986d06bd5ed74191b10c5", size = 54713 }, + { url = "https://files.pythonhosted.org/packages/98/96/1520d08fa7f7f893399c22ccb3da446561ca647206befb91419b34d9756c/polylith_cli-1.30.1-py3-none-any.whl", hash = "sha256:73fd9e967428fc6c1403750ab7e4bbbace2c0bc44cac2f7efb8ae6ecaf8ba9bc", size = 59151 }, ] [[package]] @@ -1205,6 +1206,50 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/3e/b68c118422ec867fa7ab88444e1274aa40681c606d59ac27de5a5588f082/python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a", size = 19863 }, ] +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199 }, + { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758 }, + { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463 }, + { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280 }, + { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239 }, + { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802 }, + { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527 }, + { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052 }, + { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774 }, + { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612 }, + { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040 }, + { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829 }, + { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167 }, + { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952 }, + { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301 }, + { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638 }, + { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850 }, + { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980 }, + { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873 }, + { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302 }, + { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154 }, + { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223 }, + { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542 }, + { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164 }, + { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611 }, + { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591 }, + { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338 }, + { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309 }, + { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679 }, + { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428 }, + { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361 }, + { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523 }, + { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660 }, + { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597 }, + { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527 }, + { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446 }, +] + [[package]] name = "requests" version = "2.32.3" From cfe4b4756d2485e331bc7d0f5761e99acab8a075 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 21:10:45 -0400 Subject: [PATCH 29/39] move old tests to projects for now as they are mostly integration tests --- .../api_private => projects/api_private/tests}/__init__.py | 0 .../api_private/tests}/_test_highscore_benchmark.py | 2 ++ .../api_private => projects/api_private/tests}/benchmark.py | 0 .../api_private => projects/api_private/tests}/conftest.py | 0 .../api_private/tests}/test_highscore.py | 0 .../api_private/tests}/test_highscore_benchmark_custom.py | 0 .../api_private/tests}/test_highscore_v2.py | 0 .../api_private/tests}/test_highscore_v3.py | 0 .../api_private => projects/api_private/tests}/test_player.py | 0 9 files changed, 2 insertions(+) rename {test/bases/bot_detector/api_private => projects/api_private/tests}/__init__.py (100%) rename {test/bases/bot_detector/api_private => projects/api_private/tests}/_test_highscore_benchmark.py (93%) rename {test/bases/bot_detector/api_private => projects/api_private/tests}/benchmark.py (100%) rename {test/bases/bot_detector/api_private => projects/api_private/tests}/conftest.py (100%) rename {test/bases/bot_detector/api_private => projects/api_private/tests}/test_highscore.py (100%) rename {test/bases/bot_detector/api_private => projects/api_private/tests}/test_highscore_benchmark_custom.py (100%) rename {test/bases/bot_detector/api_private => projects/api_private/tests}/test_highscore_v2.py (100%) rename {test/bases/bot_detector/api_private => projects/api_private/tests}/test_highscore_v3.py (100%) rename {test/bases/bot_detector/api_private => projects/api_private/tests}/test_player.py (100%) diff --git a/test/bases/bot_detector/api_private/__init__.py b/projects/api_private/tests/__init__.py similarity index 100% rename from test/bases/bot_detector/api_private/__init__.py rename to projects/api_private/tests/__init__.py diff --git a/test/bases/bot_detector/api_private/_test_highscore_benchmark.py b/projects/api_private/tests/_test_highscore_benchmark.py similarity index 93% rename from test/bases/bot_detector/api_private/_test_highscore_benchmark.py rename to projects/api_private/tests/_test_highscore_benchmark.py index a14edb9..984a42c 100644 --- a/test/bases/bot_detector/api_private/_test_highscore_benchmark.py +++ b/projects/api_private/tests/_test_highscore_benchmark.py @@ -5,6 +5,7 @@ @pytest.mark.asyncio +@pytest.mark.integration async def test_highscore_v2(benchmark, custom_client): player_ids = list(range(1, 101)) # Or any other player IDs you want to use async with httpx.AsyncClient() as client: @@ -25,6 +26,7 @@ async def run_requests(): @pytest.mark.asyncio +@pytest.mark.integration async def test_highscore_v3(benchmark, custom_client): player_ids = list(range(1, 101)) # Or any other player IDs you want to use async with httpx.AsyncClient() as client: diff --git a/test/bases/bot_detector/api_private/benchmark.py b/projects/api_private/tests/benchmark.py similarity index 100% rename from test/bases/bot_detector/api_private/benchmark.py rename to projects/api_private/tests/benchmark.py diff --git a/test/bases/bot_detector/api_private/conftest.py b/projects/api_private/tests/conftest.py similarity index 100% rename from test/bases/bot_detector/api_private/conftest.py rename to projects/api_private/tests/conftest.py diff --git a/test/bases/bot_detector/api_private/test_highscore.py b/projects/api_private/tests/test_highscore.py similarity index 100% rename from test/bases/bot_detector/api_private/test_highscore.py rename to projects/api_private/tests/test_highscore.py diff --git a/test/bases/bot_detector/api_private/test_highscore_benchmark_custom.py b/projects/api_private/tests/test_highscore_benchmark_custom.py similarity index 100% rename from test/bases/bot_detector/api_private/test_highscore_benchmark_custom.py rename to projects/api_private/tests/test_highscore_benchmark_custom.py diff --git a/test/bases/bot_detector/api_private/test_highscore_v2.py b/projects/api_private/tests/test_highscore_v2.py similarity index 100% rename from test/bases/bot_detector/api_private/test_highscore_v2.py rename to projects/api_private/tests/test_highscore_v2.py diff --git a/test/bases/bot_detector/api_private/test_highscore_v3.py b/projects/api_private/tests/test_highscore_v3.py similarity index 100% rename from test/bases/bot_detector/api_private/test_highscore_v3.py rename to projects/api_private/tests/test_highscore_v3.py diff --git a/test/bases/bot_detector/api_private/test_player.py b/projects/api_private/tests/test_player.py similarity index 100% rename from test/bases/bot_detector/api_private/test_player.py rename to projects/api_private/tests/test_player.py From 76c4bfa045934aba2dcf45927919f7019f49b08b Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 7 Jun 2025 21:26:41 -0400 Subject: [PATCH 30/39] init api private --- test/bases/bot_detector/api_private/__init__.py | 0 test/bases/bot_detector/api_private/test_core.py | 5 +++++ 2 files changed, 5 insertions(+) create mode 100644 test/bases/bot_detector/api_private/__init__.py create mode 100644 test/bases/bot_detector/api_private/test_core.py diff --git a/test/bases/bot_detector/api_private/__init__.py b/test/bases/bot_detector/api_private/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/bases/bot_detector/api_private/test_core.py b/test/bases/bot_detector/api_private/test_core.py new file mode 100644 index 0000000..a0c1906 --- /dev/null +++ b/test/bases/bot_detector/api_private/test_core.py @@ -0,0 +1,5 @@ +from bot_detector.api_private.src import core + + +def test_sample(): + assert core is not None From e741a50e3f67c3aa82c43f704612836e40433058 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sun, 8 Jun 2025 08:57:28 -0400 Subject: [PATCH 31/39] uv uses hatch, removing poetry, adding dev group --- projects/api_private/Dockerfile | 4 ++-- projects/api_private/pyproject.toml | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/projects/api_private/Dockerfile b/projects/api_private/Dockerfile index a5096b4..6929cf2 100644 --- a/projects/api_private/Dockerfile +++ b/projects/api_private/Dockerfile @@ -13,8 +13,8 @@ COPY ./projects ./projects WORKDIR /app/projects/api_private -# install dependencies via RUN uv build -RUN uv sync --frozen --no-editable +# install dependencies via RUN uv build, exclude dev packages group +RUN uv sync --frozen --no-editable --no-dev # Production stage: Prepare the final production environment FROM python:3.12-slim-bookworm AS production diff --git a/projects/api_private/pyproject.toml b/projects/api_private/pyproject.toml index 52502c6..77302b4 100644 --- a/projects/api_private/pyproject.toml +++ b/projects/api_private/pyproject.toml @@ -20,12 +20,14 @@ dependencies = [ "asyncmy>=0.2.9", ] -[tool.poetry.group.dev.dependencies] -pytest = "^8.3" -pytest-asyncio = "^0.25" -httpx = "^0.27" -watchfiles = "^0.21" -pytest-benchmark = "^3.0.0" +[dependency-groups] +dev = [ + "pytest>=8.3", + "pytest-asyncio>=0.25", + "httpx>=0.27", + "watchfiles>=0.21", + "pytest-benchmark>=3.0.0" +] [project.scripts] scrape_task_producer = "bot_detector.api_private.core.server:run" From 139af32a4a65795700210748ef445d7469d4628c Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sun, 8 Jun 2025 08:57:38 -0400 Subject: [PATCH 32/39] sync --- projects/api_private/uv.lock | 247 +++++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) diff --git a/projects/api_private/uv.lock b/projects/api_private/uv.lock index a45cdaa..3453bf9 100644 --- a/projects/api_private/uv.lock +++ b/projects/api_private/uv.lock @@ -38,6 +38,15 @@ dependencies = [ { name = "uvicorn" }, ] +[package.dev-dependencies] +dev = [ + { name = "httpx" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-benchmark" }, + { name = "watchfiles" }, +] + [package.metadata] requires-dist = [ { name = "asyncmy", specifier = ">=0.2.9" }, @@ -47,6 +56,15 @@ requires-dist = [ { name = "uvicorn", specifier = ">=0.34.2" }, ] +[package.metadata.requires-dev] +dev = [ + { name = "httpx", specifier = ">=0.27" }, + { name = "pytest", specifier = ">=8.3" }, + { name = "pytest-asyncio", specifier = ">=0.25" }, + { name = "pytest-benchmark", specifier = ">=3.0.0" }, + { name = "watchfiles", specifier = ">=0.21" }, +] + [[package]] name = "asyncmy" version = "0.2.10" @@ -84,6 +102,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/09/3a5351acc6273c28333cad8193184de0070c617fd8385fd8ba23d789e08d/asyncmy-0.2.10-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8ced4bd938e95ede0fb9fa54755773df47bdb9f29f142512501e613dd95cf4a4", size = 1614903 }, ] +[[package]] +name = "certifi" +version = "2025.4.26" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/9e/c05b3920a3b7d20d3d3310465f50348e5b3694f4f88c6daf736eef3024c4/certifi-2025.4.26.tar.gz", hash = "sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6", size = 160705 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/7e/3db2bd1b1f9e95f7cddca6d6e75e2f2bd9f51b1246e546d88addca0106bd/certifi-2025.4.26-py3-none-any.whl", hash = "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3", size = 159618 }, +] + [[package]] name = "click" version = "8.2.1" @@ -191,6 +218,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515 }, ] +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784 }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517 }, +] + [[package]] name = "idna" version = "3.10" @@ -200,6 +255,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, ] +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050 }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469 }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 }, +] + +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335 }, +] + [[package]] name = "pydantic" version = "2.11.5" @@ -316,6 +407,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b6/5f/d6d641b490fd3ec2c4c13b4244d68deea3a1b970a97be64f34fb5504ff72/pydantic_settings-2.9.1-py3-none-any.whl", hash = "sha256:59b4f431b1defb26fe620c71a7d3968a710d719f5f4cdbbdb7926edeb770f6ef", size = 44356 }, ] +[[package]] +name = "pygments" +version = "2.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/2d/c3338d48ea6cc0feb8446d8e6937e1408088a72a39937982cc6111d17f84/pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f", size = 4968581 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293 }, +] + +[[package]] +name = "pytest" +version = "8.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/aa/405082ce2749be5398045152251ac69c0f3578c7077efc53431303af97ce/pytest-8.4.0.tar.gz", hash = "sha256:14d920b48472ea0dbf68e45b96cd1ffda4705f33307dcc86c676c1b5104838a6", size = 1515232 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/de/afa024cbe022b1b318a3d224125aa24939e99b4ff6f22e0ba639a2eaee47/pytest-8.4.0-py3-none-any.whl", hash = "sha256:f40f825768ad76c0977cbacdf1fd37c6f7a468e460ea6a0636078f8972d4517e", size = 363797 }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/d4/14f53324cb1a6381bef29d698987625d80052bb33932d8e7cbf9b337b17c/pytest_asyncio-1.0.0.tar.gz", hash = "sha256:d15463d13f4456e1ead2594520216b225a16f781e144f8fdf6c5bb4667c48b3f", size = 46960 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/05/ce271016e351fddc8399e546f6e23761967ee09c8c568bbfbecb0c150171/pytest_asyncio-1.0.0-py3-none-any.whl", hash = "sha256:4f024da9f1ef945e680dc68610b52550e36590a67fd31bb3b4943979a1f90ef3", size = 15976 }, +] + +[[package]] +name = "pytest-benchmark" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259 }, +] + [[package]] name = "python-dotenv" version = "1.1.0" @@ -391,6 +534,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/0c/9d30a4ebeb6db2b25a841afbb80f6ef9a854fc3b41be131d249a977b4959/starlette-0.46.2-py3-none-any.whl", hash = "sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35", size = 72037 }, ] +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077 }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429 }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067 }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030 }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898 }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894 }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319 }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273 }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310 }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309 }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762 }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453 }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486 }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349 }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159 }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243 }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645 }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584 }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875 }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418 }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708 }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582 }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543 }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691 }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170 }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530 }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666 }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954 }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724 }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383 }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257 }, +] + [[package]] name = "typing-extensions" version = "4.14.0" @@ -425,3 +607,68 @@ sdist = { url = "https://files.pythonhosted.org/packages/de/ad/713be230bcda622ea wheels = [ { url = "https://files.pythonhosted.org/packages/6d/0d/8adfeaa62945f90d19ddc461c55f4a50c258af7662d34b6a3d5d1f8646f6/uvicorn-0.34.3-py3-none-any.whl", hash = "sha256:16246631db62bdfbf069b0645177d6e8a77ba950cfedbfd093acef9444e4d885", size = 62431 }, ] + +[[package]] +name = "watchfiles" +version = "1.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/03/e2/8ed598c42057de7aa5d97c472254af4906ff0a59a66699d426fc9ef795d7/watchfiles-1.0.5.tar.gz", hash = "sha256:b7529b5dcc114679d43827d8c35a07c493ad6f083633d573d81c660abc5979e9", size = 94537 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/4d/d02e6ea147bb7fff5fd109c694a95109612f419abed46548a930e7f7afa3/watchfiles-1.0.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:5c40fe7dd9e5f81e0847b1ea64e1f5dd79dd61afbedb57759df06767ac719b40", size = 405632 }, + { url = "https://files.pythonhosted.org/packages/60/31/9ee50e29129d53a9a92ccf1d3992751dc56fc3c8f6ee721be1c7b9c81763/watchfiles-1.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c0db396e6003d99bb2d7232c957b5f0b5634bbd1b24e381a5afcc880f7373fb", size = 395734 }, + { url = "https://files.pythonhosted.org/packages/ad/8c/759176c97195306f028024f878e7f1c776bda66ccc5c68fa51e699cf8f1d/watchfiles-1.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b551d4fb482fc57d852b4541f911ba28957d051c8776e79c3b4a51eb5e2a1b11", size = 455008 }, + { url = "https://files.pythonhosted.org/packages/55/1a/5e977250c795ee79a0229e3b7f5e3a1b664e4e450756a22da84d2f4979fe/watchfiles-1.0.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:830aa432ba5c491d52a15b51526c29e4a4b92bf4f92253787f9726fe01519487", size = 459029 }, + { url = "https://files.pythonhosted.org/packages/e6/17/884cf039333605c1d6e296cf5be35fad0836953c3dfd2adb71b72f9dbcd0/watchfiles-1.0.5-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a16512051a822a416b0d477d5f8c0e67b67c1a20d9acecb0aafa3aa4d6e7d256", size = 488916 }, + { url = "https://files.pythonhosted.org/packages/ef/e0/bcb6e64b45837056c0a40f3a2db3ef51c2ced19fda38484fa7508e00632c/watchfiles-1.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe0cbc787770e52a96c6fda6726ace75be7f840cb327e1b08d7d54eadc3bc85", size = 523763 }, + { url = "https://files.pythonhosted.org/packages/24/e9/f67e9199f3bb35c1837447ecf07e9830ec00ff5d35a61e08c2cd67217949/watchfiles-1.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d363152c5e16b29d66cbde8fa614f9e313e6f94a8204eaab268db52231fe5358", size = 502891 }, + { url = "https://files.pythonhosted.org/packages/23/ed/a6cf815f215632f5c8065e9c41fe872025ffea35aa1f80499f86eae922db/watchfiles-1.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ee32c9a9bee4d0b7bd7cbeb53cb185cf0b622ac761efaa2eba84006c3b3a614", size = 454921 }, + { url = "https://files.pythonhosted.org/packages/92/4c/e14978599b80cde8486ab5a77a821e8a982ae8e2fcb22af7b0886a033ec8/watchfiles-1.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29c7fd632ccaf5517c16a5188e36f6612d6472ccf55382db6c7fe3fcccb7f59f", size = 631422 }, + { url = "https://files.pythonhosted.org/packages/b2/1a/9263e34c3458f7614b657f974f4ee61fd72f58adce8b436e16450e054efd/watchfiles-1.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8e637810586e6fe380c8bc1b3910accd7f1d3a9a7262c8a78d4c8fb3ba6a2b3d", size = 625675 }, + { url = "https://files.pythonhosted.org/packages/96/1f/1803a18bd6ab04a0766386a19bcfe64641381a04939efdaa95f0e3b0eb58/watchfiles-1.0.5-cp310-cp310-win32.whl", hash = "sha256:cd47d063fbeabd4c6cae1d4bcaa38f0902f8dc5ed168072874ea11d0c7afc1ff", size = 277921 }, + { url = "https://files.pythonhosted.org/packages/c2/3b/29a89de074a7d6e8b4dc67c26e03d73313e4ecf0d6e97e942a65fa7c195e/watchfiles-1.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:86c0df05b47a79d80351cd179893f2f9c1b1cae49d96e8b3290c7f4bd0ca0a92", size = 291526 }, + { url = "https://files.pythonhosted.org/packages/39/f4/41b591f59021786ef517e1cdc3b510383551846703e03f204827854a96f8/watchfiles-1.0.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:237f9be419e977a0f8f6b2e7b0475ababe78ff1ab06822df95d914a945eac827", size = 405336 }, + { url = "https://files.pythonhosted.org/packages/ae/06/93789c135be4d6d0e4f63e96eea56dc54050b243eacc28439a26482b5235/watchfiles-1.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0da39ff917af8b27a4bdc5a97ac577552a38aac0d260a859c1517ea3dc1a7c4", size = 395977 }, + { url = "https://files.pythonhosted.org/packages/d2/db/1cd89bd83728ca37054512d4d35ab69b5f12b8aa2ac9be3b0276b3bf06cc/watchfiles-1.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cfcb3952350e95603f232a7a15f6c5f86c5375e46f0bd4ae70d43e3e063c13d", size = 455232 }, + { url = "https://files.pythonhosted.org/packages/40/90/d8a4d44ffe960517e487c9c04f77b06b8abf05eb680bed71c82b5f2cad62/watchfiles-1.0.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:68b2dddba7a4e6151384e252a5632efcaa9bc5d1c4b567f3cb621306b2ca9f63", size = 459151 }, + { url = "https://files.pythonhosted.org/packages/6c/da/267a1546f26465dead1719caaba3ce660657f83c9d9c052ba98fb8856e13/watchfiles-1.0.5-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95cf944fcfc394c5f9de794ce581914900f82ff1f855326f25ebcf24d5397418", size = 489054 }, + { url = "https://files.pythonhosted.org/packages/b1/31/33850dfd5c6efb6f27d2465cc4c6b27c5a6f5ed53c6fa63b7263cf5f60f6/watchfiles-1.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ecf6cd9f83d7c023b1aba15d13f705ca7b7d38675c121f3cc4a6e25bd0857ee9", size = 523955 }, + { url = "https://files.pythonhosted.org/packages/09/84/b7d7b67856efb183a421f1416b44ca975cb2ea6c4544827955dfb01f7dc2/watchfiles-1.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:852de68acd6212cd6d33edf21e6f9e56e5d98c6add46f48244bd479d97c967c6", size = 502234 }, + { url = "https://files.pythonhosted.org/packages/71/87/6dc5ec6882a2254cfdd8b0718b684504e737273903b65d7338efaba08b52/watchfiles-1.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5730f3aa35e646103b53389d5bc77edfbf578ab6dab2e005142b5b80a35ef25", size = 454750 }, + { url = "https://files.pythonhosted.org/packages/3d/6c/3786c50213451a0ad15170d091570d4a6554976cf0df19878002fc96075a/watchfiles-1.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:18b3bd29954bc4abeeb4e9d9cf0b30227f0f206c86657674f544cb032296acd5", size = 631591 }, + { url = "https://files.pythonhosted.org/packages/1b/b3/1427425ade4e359a0deacce01a47a26024b2ccdb53098f9d64d497f6684c/watchfiles-1.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ba5552a1b07c8edbf197055bc9d518b8f0d98a1c6a73a293bc0726dce068ed01", size = 625370 }, + { url = "https://files.pythonhosted.org/packages/15/ba/f60e053b0b5b8145d682672024aa91370a29c5c921a88977eb565de34086/watchfiles-1.0.5-cp311-cp311-win32.whl", hash = "sha256:2f1fefb2e90e89959447bc0420fddd1e76f625784340d64a2f7d5983ef9ad246", size = 277791 }, + { url = "https://files.pythonhosted.org/packages/50/ed/7603c4e164225c12c0d4e8700b64bb00e01a6c4eeea372292a3856be33a4/watchfiles-1.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:b6e76ceb1dd18c8e29c73f47d41866972e891fc4cc7ba014f487def72c1cf096", size = 291622 }, + { url = "https://files.pythonhosted.org/packages/a2/c2/99bb7c96b4450e36877fde33690ded286ff555b5a5c1d925855d556968a1/watchfiles-1.0.5-cp311-cp311-win_arm64.whl", hash = "sha256:266710eb6fddc1f5e51843c70e3bebfb0f5e77cf4f27129278c70554104d19ed", size = 283699 }, + { url = "https://files.pythonhosted.org/packages/2a/8c/4f0b9bdb75a1bfbd9c78fad7d8854369283f74fe7cf03eb16be77054536d/watchfiles-1.0.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b5eb568c2aa6018e26da9e6c86f3ec3fd958cee7f0311b35c2630fa4217d17f2", size = 401511 }, + { url = "https://files.pythonhosted.org/packages/dc/4e/7e15825def77f8bd359b6d3f379f0c9dac4eb09dd4ddd58fd7d14127179c/watchfiles-1.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0a04059f4923ce4e856b4b4e5e783a70f49d9663d22a4c3b3298165996d1377f", size = 392715 }, + { url = "https://files.pythonhosted.org/packages/58/65/b72fb817518728e08de5840d5d38571466c1b4a3f724d190cec909ee6f3f/watchfiles-1.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e380c89983ce6e6fe2dd1e1921b9952fb4e6da882931abd1824c092ed495dec", size = 454138 }, + { url = "https://files.pythonhosted.org/packages/3e/a4/86833fd2ea2e50ae28989f5950b5c3f91022d67092bfec08f8300d8b347b/watchfiles-1.0.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fe43139b2c0fdc4a14d4f8d5b5d967f7a2777fd3d38ecf5b1ec669b0d7e43c21", size = 458592 }, + { url = "https://files.pythonhosted.org/packages/38/7e/42cb8df8be9a37e50dd3a818816501cf7a20d635d76d6bd65aae3dbbff68/watchfiles-1.0.5-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee0822ce1b8a14fe5a066f93edd20aada932acfe348bede8aa2149f1a4489512", size = 487532 }, + { url = "https://files.pythonhosted.org/packages/fc/fd/13d26721c85d7f3df6169d8b495fcac8ab0dc8f0945ebea8845de4681dab/watchfiles-1.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a0dbcb1c2d8f2ab6e0a81c6699b236932bd264d4cef1ac475858d16c403de74d", size = 522865 }, + { url = "https://files.pythonhosted.org/packages/a1/0d/7f9ae243c04e96c5455d111e21b09087d0eeaf9a1369e13a01c7d3d82478/watchfiles-1.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a2014a2b18ad3ca53b1f6c23f8cd94a18ce930c1837bd891262c182640eb40a6", size = 499887 }, + { url = "https://files.pythonhosted.org/packages/8e/0f/a257766998e26aca4b3acf2ae97dff04b57071e991a510857d3799247c67/watchfiles-1.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10f6ae86d5cb647bf58f9f655fcf577f713915a5d69057a0371bc257e2553234", size = 454498 }, + { url = "https://files.pythonhosted.org/packages/81/79/8bf142575a03e0af9c3d5f8bcae911ee6683ae93a625d349d4ecf4c8f7df/watchfiles-1.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1a7bac2bde1d661fb31f4d4e8e539e178774b76db3c2c17c4bb3e960a5de07a2", size = 630663 }, + { url = "https://files.pythonhosted.org/packages/f1/80/abe2e79f610e45c63a70d271caea90c49bbf93eb00fa947fa9b803a1d51f/watchfiles-1.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ab626da2fc1ac277bbf752446470b367f84b50295264d2d313e28dc4405d663", size = 625410 }, + { url = "https://files.pythonhosted.org/packages/91/6f/bc7fbecb84a41a9069c2c6eb6319f7f7df113adf113e358c57fc1aff7ff5/watchfiles-1.0.5-cp312-cp312-win32.whl", hash = "sha256:9f4571a783914feda92018ef3901dab8caf5b029325b5fe4558c074582815249", size = 277965 }, + { url = "https://files.pythonhosted.org/packages/99/a5/bf1c297ea6649ec59e935ab311f63d8af5faa8f0b86993e3282b984263e3/watchfiles-1.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:360a398c3a19672cf93527f7e8d8b60d8275119c5d900f2e184d32483117a705", size = 291693 }, + { url = "https://files.pythonhosted.org/packages/7f/7b/fd01087cc21db5c47e5beae507b87965db341cce8a86f9eb12bf5219d4e0/watchfiles-1.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:1a2902ede862969077b97523987c38db28abbe09fb19866e711485d9fbf0d417", size = 283287 }, + { url = "https://files.pythonhosted.org/packages/c7/62/435766874b704f39b2fecd8395a29042db2b5ec4005bd34523415e9bd2e0/watchfiles-1.0.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:0b289572c33a0deae62daa57e44a25b99b783e5f7aed81b314232b3d3c81a11d", size = 401531 }, + { url = "https://files.pythonhosted.org/packages/6e/a6/e52a02c05411b9cb02823e6797ef9bbba0bfaf1bb627da1634d44d8af833/watchfiles-1.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a056c2f692d65bf1e99c41045e3bdcaea3cb9e6b5a53dcaf60a5f3bd95fc9763", size = 392417 }, + { url = "https://files.pythonhosted.org/packages/3f/53/c4af6819770455932144e0109d4854437769672d7ad897e76e8e1673435d/watchfiles-1.0.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9dca99744991fc9850d18015c4f0438865414e50069670f5f7eee08340d8b40", size = 453423 }, + { url = "https://files.pythonhosted.org/packages/cb/d1/8e88df58bbbf819b8bc5cfbacd3c79e01b40261cad0fc84d1e1ebd778a07/watchfiles-1.0.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:894342d61d355446d02cd3988a7326af344143eb33a2fd5d38482a92072d9563", size = 458185 }, + { url = "https://files.pythonhosted.org/packages/ff/70/fffaa11962dd5429e47e478a18736d4e42bec42404f5ee3b92ef1b87ad60/watchfiles-1.0.5-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab44e1580924d1ffd7b3938e02716d5ad190441965138b4aa1d1f31ea0877f04", size = 486696 }, + { url = "https://files.pythonhosted.org/packages/39/db/723c0328e8b3692d53eb273797d9a08be6ffb1d16f1c0ba2bdbdc2a3852c/watchfiles-1.0.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d6f9367b132078b2ceb8d066ff6c93a970a18c3029cea37bfd7b2d3dd2e5db8f", size = 522327 }, + { url = "https://files.pythonhosted.org/packages/cd/05/9fccc43c50c39a76b68343484b9da7b12d42d0859c37c61aec018c967a32/watchfiles-1.0.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2e55a9b162e06e3f862fb61e399fe9f05d908d019d87bf5b496a04ef18a970a", size = 499741 }, + { url = "https://files.pythonhosted.org/packages/23/14/499e90c37fa518976782b10a18b18db9f55ea73ca14641615056f8194bb3/watchfiles-1.0.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0125f91f70e0732a9f8ee01e49515c35d38ba48db507a50c5bdcad9503af5827", size = 453995 }, + { url = "https://files.pythonhosted.org/packages/61/d9/f75d6840059320df5adecd2c687fbc18960a7f97b55c300d20f207d48aef/watchfiles-1.0.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:13bb21f8ba3248386337c9fa51c528868e6c34a707f729ab041c846d52a0c69a", size = 629693 }, + { url = "https://files.pythonhosted.org/packages/fc/17/180ca383f5061b61406477218c55d66ec118e6c0c51f02d8142895fcf0a9/watchfiles-1.0.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:839ebd0df4a18c5b3c1b890145b5a3f5f64063c2a0d02b13c76d78fe5de34936", size = 624677 }, + { url = "https://files.pythonhosted.org/packages/bf/15/714d6ef307f803f236d69ee9d421763707899d6298d9f3183e55e366d9af/watchfiles-1.0.5-cp313-cp313-win32.whl", hash = "sha256:4a8ec1e4e16e2d5bafc9ba82f7aaecfeec990ca7cd27e84fb6f191804ed2fcfc", size = 277804 }, + { url = "https://files.pythonhosted.org/packages/a8/b4/c57b99518fadf431f3ef47a610839e46e5f8abf9814f969859d1c65c02c7/watchfiles-1.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:f436601594f15bf406518af922a89dcaab416568edb6f65c4e5bbbad1ea45c11", size = 291087 }, + { url = "https://files.pythonhosted.org/packages/1a/03/81f9fcc3963b3fc415cd4b0b2b39ee8cc136c42fb10a36acf38745e9d283/watchfiles-1.0.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f59b870db1f1ae5a9ac28245707d955c8721dd6565e7f411024fa374b5362d1d", size = 405947 }, + { url = "https://files.pythonhosted.org/packages/54/97/8c4213a852feb64807ec1d380f42d4fc8bfaef896bdbd94318f8fd7f3e4e/watchfiles-1.0.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9475b0093767e1475095f2aeb1d219fb9664081d403d1dff81342df8cd707034", size = 397276 }, + { url = "https://files.pythonhosted.org/packages/78/12/d4464d19860cb9672efa45eec1b08f8472c478ed67dcd30647c51ada7aef/watchfiles-1.0.5-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc533aa50664ebd6c628b2f30591956519462f5d27f951ed03d6c82b2dfd9965", size = 455550 }, + { url = "https://files.pythonhosted.org/packages/90/fb/b07bcdf1034d8edeaef4c22f3e9e3157d37c5071b5f9492ffdfa4ad4bed7/watchfiles-1.0.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed1cd825158dcaae36acce7b2db33dcbfd12b30c34317a88b8ed80f0541cc57", size = 455542 }, +] From 09aeb59d62485eb3e76bb0395334520668fe4d61 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sun, 8 Jun 2025 08:57:55 -0400 Subject: [PATCH 33/39] not sure what changed --- .../api_private/test_highscore_v3.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 test/bases/bot_detector/api_private/test_highscore_v3.py diff --git a/test/bases/bot_detector/api_private/test_highscore_v3.py b/test/bases/bot_detector/api_private/test_highscore_v3.py new file mode 100644 index 0000000..c4f0d7a --- /dev/null +++ b/test/bases/bot_detector/api_private/test_highscore_v3.py @@ -0,0 +1,47 @@ +# bases/bot_detector/api_private/test/api/v3/test_highscore_unit.py +from datetime import datetime + +from bot_detector.api_private.src.api.v3.highscore import ( + ActivityView, + SkillView, + convert_to_scraper_data_view, +) + + +def test_convert_groups_rows_by_scraper_id(): + # -- Arrange ----------------------------------------------------------------- + sample_rows = [ + { + "scrape_id": 1, + "scrape_ts": datetime(2025, 6, 7, 12, 0), + "scrape_date": datetime(2025, 6, 7).date(), + "player_id": 99, + "player_name": "Bob", + "hs_type": "skill", + "hs_name": "strength", + "hs_value": 42, + }, + { + "scrape_id": 1, + "scrape_ts": datetime(2025, 6, 7, 12, 0), + "scrape_date": datetime(2025, 6, 7).date(), + "player_id": 99, + "player_name": "Bob", + "hs_type": "activity", + "hs_name": "clue-scroll", + "hs_value": 100, + }, + ] + + # -- Act --------------------------------------------------------------------- + result = convert_to_scraper_data_view(sample_rows) + + # -- Assert ------------------------------------------------------------------ + assert len(result) == 1 # rows collapsed into one ScraperDataView + view = result[0] + assert view.scraper_id == 1 + assert view.player_name == "Bob" + assert view.skills == [SkillView(skill_name="strength", skill_value=42)] + assert view.activities == [ + ActivityView(activity_name="clue-scroll", activity_value=100) + ] From a62fc7494601106d7d33ccc5693af973e9ac1848 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sun, 8 Jun 2025 08:58:50 -0400 Subject: [PATCH 34/39] prob should consider removing this line --- bases/bot_detector/api_private/src/core/database/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bases/bot_detector/api_private/src/core/database/database.py b/bases/bot_detector/api_private/src/core/database/database.py index 7e64082..7af6ef9 100644 --- a/bases/bot_detector/api_private/src/core/database/database.py +++ b/bases/bot_detector/api_private/src/core/database/database.py @@ -7,7 +7,7 @@ settings.DATABASE_URL, pool_timeout=settings.POOL_TIMEOUT, pool_recycle=settings.POOL_RECYCLE, - echo=(settings.ENV != "PRD"), + # echo=(settings.ENV != "PRD"), pool_pre_ping=True, ) From 2d1004f517504611d2ef98aaffaf2f9ad9058130 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sun, 8 Jun 2025 12:06:59 -0400 Subject: [PATCH 35/39] remove tests --- projects/api_private/tests/__init__.py | 3 - .../tests/_test_highscore_benchmark.py | 46 --- projects/api_private/tests/benchmark.py | 39 --- projects/api_private/tests/conftest.py | 59 ---- projects/api_private/tests/test_highscore.py | 45 --- .../tests/test_highscore_benchmark_custom.py | 82 ------ .../api_private/tests/test_highscore_v2.py | 263 ------------------ .../api_private/tests/test_highscore_v3.py | 65 ----- projects/api_private/tests/test_player.py | 76 ----- 9 files changed, 678 deletions(-) delete mode 100644 projects/api_private/tests/__init__.py delete mode 100644 projects/api_private/tests/_test_highscore_benchmark.py delete mode 100644 projects/api_private/tests/benchmark.py delete mode 100644 projects/api_private/tests/conftest.py delete mode 100644 projects/api_private/tests/test_highscore.py delete mode 100644 projects/api_private/tests/test_highscore_benchmark_custom.py delete mode 100644 projects/api_private/tests/test_highscore_v2.py delete mode 100644 projects/api_private/tests/test_highscore_v3.py delete mode 100644 projects/api_private/tests/test_player.py diff --git a/projects/api_private/tests/__init__.py b/projects/api_private/tests/__init__.py deleted file mode 100644 index 3d5905b..0000000 --- a/projects/api_private/tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from dotenv import find_dotenv, load_dotenv - -load_dotenv(find_dotenv()) diff --git a/projects/api_private/tests/_test_highscore_benchmark.py b/projects/api_private/tests/_test_highscore_benchmark.py deleted file mode 100644 index 984a42c..0000000 --- a/projects/api_private/tests/_test_highscore_benchmark.py +++ /dev/null @@ -1,46 +0,0 @@ -import asyncio - -import httpx -import pytest - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_highscore_v2(benchmark, custom_client): - player_ids = list(range(1, 101)) # Or any other player IDs you want to use - async with httpx.AsyncClient() as client: - - async def request(player_id): - endpoint = "http://localhost:5000/v2/highscore/latest" - params = {"player_id": player_id} - response = await client.get(url=endpoint, params=params) - return response - - async def run_requests(): - tasks = [ - asyncio.create_task(request(player_id)) for player_id in player_ids - ] - return await asyncio.gather(*tasks) - - _ = await benchmark(run_requests) - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_highscore_v3(benchmark, custom_client): - player_ids = list(range(1, 101)) # Or any other player IDs you want to use - async with httpx.AsyncClient() as client: - - async def request(player_id): - endpoint = "http://localhost:5000/v3/highscore/latest" - params = {"player_id": player_id} - response = await client.get(url=endpoint, params=params) - return response - - async def run_requests(): - tasks = [ - asyncio.create_task(request(player_id)) for player_id in player_ids - ] - return await asyncio.gather(*tasks) - - _ = await benchmark(run_requests) diff --git a/projects/api_private/tests/benchmark.py b/projects/api_private/tests/benchmark.py deleted file mode 100644 index 825c40d..0000000 --- a/projects/api_private/tests/benchmark.py +++ /dev/null @@ -1,39 +0,0 @@ -import logging -from time import perf_counter - - -class Benchmark: - results = [] - - def __init__(self, name, suppress_logging=False): - self.name = name - self.suppress_logging = suppress_logging - if suppress_logging: - self.original_level = self.set_logging_level("httpcore", logging.INFO) - - async def __aenter__(self): - self.time_start = perf_counter() - return self - - async def __aexit__(self, exc_type, exc, tb): - duration = perf_counter() - self.time_start - self.results.append((self.name, duration)) - if self.suppress_logging: - self.set_logging_level("httpcore", self.original_level) - return False - - @classmethod - def output_results(cls): - print("{:<10} {:<10}".format("Name", "Duration")) - durations = [] - for name, duration in cls.results: - print("{:<10} {:<10.3f}".format(name, duration)) - durations.append(duration) - return durations - - @staticmethod - def set_logging_level(logger_name, level): - logger = logging.getLogger(logger_name) - original_level = logger.level - logger.setLevel(level) - return original_level diff --git a/projects/api_private/tests/conftest.py b/projects/api_private/tests/conftest.py deleted file mode 100644 index 39d4328..0000000 --- a/projects/api_private/tests/conftest.py +++ /dev/null @@ -1,59 +0,0 @@ -# conftest.py -import os -import sys -from contextlib import asynccontextmanager - -import pytest -from fastapi import FastAPI -from httpx import AsyncClient -from httpx._transports.asgi import ASGITransport -from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine -from sqlalchemy.orm import sessionmaker - -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - - -from bases.bot_detector.api_private.src.core import server # noqa: E402 -from bases.bot_detector.api_private.src.core.fastapi.dependencies.session import ( - get_session, # noqa: E402 -) - - -# Create an async SQLAlchemy engine -async def get_session_override(): - engine = create_async_engine( - "mysql+aiomysql://root:root_bot_buster@localhost:3307/playerdata", - pool_timeout=30, - pool_recycle=30, - echo=True, - pool_pre_ping=True, - ) - - # Create a session factory - SessionFactory = sessionmaker( - bind=engine, - expire_on_commit=False, - class_=AsyncSession, # Use AsyncSession for asynchronous operations - ) - async with SessionFactory() as session: - session: AsyncSession - yield session - await engine.dispose() - return - - -server.app.dependency_overrides[get_session] = get_session_override - - -@pytest.fixture -def app() -> FastAPI: - return server.app - - -@pytest.fixture -@asynccontextmanager -async def custom_client(app: FastAPI): - base_url = "http://srv.test/" - transport = ASGITransport(app=app) - async with AsyncClient(transport=transport, base_url=base_url) as client: - yield client diff --git a/projects/api_private/tests/test_highscore.py b/projects/api_private/tests/test_highscore.py deleted file mode 100644 index b8bdcb2..0000000 --- a/projects/api_private/tests/test_highscore.py +++ /dev/null @@ -1,45 +0,0 @@ -import pytest -from httpx import AsyncClient - - -@pytest.mark.asyncio -async def test_one_hs_id_v2(custom_client): - endpoint = "/v2/highscore/latest" - async with custom_client as client: - client: AsyncClient - params = {"player_id": 1} - response = await client.get(url=endpoint, params=params) - - assert response.status_code == 200 - assert isinstance(response.json(), list) - - json_response: list[dict] = response.json() - assert len(json_response) == 1 - assert isinstance( - json_response[0], dict - ), f"expected dict, got {type(json_response[0])}, {json_response=}" - - player = json_response[0] - - assert player.get("Player_id") == 1, f"expected id: 1 got: {player=}" - - -@pytest.mark.asyncio -async def test_one_hs_id_v3(custom_client): - endpoint = "/v3/highscore/latest" - async with custom_client as client: - client: AsyncClient - params = {"player_id": 1} - response = await client.get(url=endpoint, params=params) - - assert response.status_code == 200 - assert isinstance(response.json(), list) - - json_response: list[dict] = response.json() - assert len(json_response) == 1 - assert isinstance( - json_response[0], dict - ), f"expected dict, got {type(json_response[0])}, {json_response=}" - - player = json_response[0] - assert player.get("player_id") == 1, f"expected id: 1 got: {player=}" diff --git a/projects/api_private/tests/test_highscore_benchmark_custom.py b/projects/api_private/tests/test_highscore_benchmark_custom.py deleted file mode 100644 index d8a4a13..0000000 --- a/projects/api_private/tests/test_highscore_benchmark_custom.py +++ /dev/null @@ -1,82 +0,0 @@ -import random -import statistics - -import pytest -from httpx import AsyncClient - -from .benchmark import Benchmark - -# Global variable to store the results -benchmark_results = {"v2": [], "v3": []} -player_ids = [random.randint(1, 250) for i in range(10)] -ITERATIONS = 1 - - -async def request(client: AsyncClient, endpoint: str, player_id: int): - params = {"player_id": player_id, "many": 1, "limit": 5000} - response = await client.get(url=endpoint, params=params) - return response - - -async def bench(iterations, client, endpoint, player_ids): - for _ in range(iterations): - for player_id in player_ids: - async with Benchmark("requests", suppress_logging=True) as b: - await request(client, endpoint, player_id) - return b - - -@pytest.mark.asyncio -async def test_highscore_custom_benchmark_v2(custom_client): - # Clear the results from the previous tests - Benchmark.results.clear() - endpoint = "/v2/highscore/latest" - async with custom_client as client: - client: AsyncClient - b = await bench(ITERATIONS, client, endpoint, player_ids) - for name, duration in b.results: - benchmark_results["v2"].append(duration) - - -@pytest.mark.asyncio -async def test_highscore_custom_benchmark_v3(custom_client): - # Clear the results from the previous tests - Benchmark.results.clear() - endpoint = "/v3/highscore/latest" - async with custom_client as client: - client: AsyncClient - b = await bench(ITERATIONS, client, endpoint, player_ids) - for name, duration in b.results: - benchmark_results["v3"].append(duration) - - -def test_output_results(): - print("v2 results:") - avg_time = statistics.mean(benchmark_results["v2"]) - median_time = statistics.median(benchmark_results["v2"]) - stdev_time = ( - statistics.stdev(benchmark_results["v2"]) - if len(benchmark_results["v2"]) > 1 - else 0 # if length of results empty return 0 - ) - max_time = max(benchmark_results["v2"]) - min_time = min(benchmark_results["v2"]) - print( - f"average {avg_time:.3f} seconds, median {median_time:.3f} seconds, stdev {stdev_time:.3f} seconds, max {max_time:.3f} seconds, min {min_time:.3f} seconds" - ) - assert avg_time > 0 - - print("v3 results:") - avg_time = statistics.mean(benchmark_results["v3"]) - median_time = statistics.median(benchmark_results["v3"]) - stdev_time = ( - statistics.stdev(benchmark_results["v3"]) - if len(benchmark_results["v3"]) > 1 - else 0 - ) - max_time = max(benchmark_results["v3"]) - min_time = min(benchmark_results["v3"]) - print( - f"average {avg_time:.3f} seconds, median {median_time:.3f} seconds, stdev {stdev_time:.3f} seconds, max {max_time:.3f} seconds, min {min_time:.3f} seconds" - ) - assert avg_time > 0 diff --git a/projects/api_private/tests/test_highscore_v2.py b/projects/api_private/tests/test_highscore_v2.py deleted file mode 100644 index f2c0f84..0000000 --- a/projects/api_private/tests/test_highscore_v2.py +++ /dev/null @@ -1,263 +0,0 @@ -import pytest -from httpx import AsyncClient - - -@pytest.mark.asyncio -async def test_one_hs_id_v2(custom_client): - endpoint = "/v2/highscore/latest" - async with custom_client as client: - client: AsyncClient - params = {"player_id": 1} - response = await client.get(url=endpoint, params=params) - - assert response.status_code == 200 - assert isinstance(response.json(), list) - - json_response: list[dict] = response.json() - assert len(json_response) == 1 - assert isinstance( - json_response[0], dict - ), f"expected dict, got {type(json_response[0])}, {json_response=}" - - player = json_response[0] - assert player.get("Player_id") == 1, f"expected Player_id: 1 got: {player=}" - - -@pytest.mark.asyncio -async def test_highscore_latest_v2(custom_client): - endpoint = "/v2/highscore/latest" - async with custom_client as client: - client: AsyncClient - params = {"player_id": 1} - response = await client.get(url=endpoint, params=params) - - assert response.status_code == 200 - assert isinstance(response.json(), list) - - json_response: list[dict] = response.json() - assert len(json_response) == 1 - assert isinstance( - json_response[0], dict - ), f"expected dict, got {type(json_response[0])}, {json_response=}" - - player = json_response[0] - # assert player.get("Player_id") == 1, f"expected Player_id: 1 got: {player=}" - - # List of keys that should be present in the player dictionary - keys = [ - "id", - "timestamp", - "ts_date", - "Player_id", - "name", - "total", - "attack", - "defence", - "strength", - "hitpoints", - "ranged", - "prayer", - "magic", - "cooking", - "woodcutting", - "fletching", - "fishing", - "firemaking", - "crafting", - "smithing", - "mining", - "herblore", - "agility", - "thieving", - "slayer", - "farming", - "runecraft", - "hunter", - "construction", - "league", - "bounty_hunter_hunter", - "bounty_hunter_rogue", - "cs_all", - "cs_beginner", - "cs_easy", - "cs_medium", - "cs_hard", - "cs_elite", - "cs_master", - "lms_rank", - "soul_wars_zeal", - "abyssal_sire", - "alchemical_hydra", - "barrows_chests", - "bryophyta", - "callisto", - "cerberus", - "chambers_of_xeric", - "chambers_of_xeric_challenge_mode", - "chaos_elemental", - "chaos_fanatic", - "commander_zilyana", - "corporeal_beast", - "crazy_archaeologist", - "dagannoth_prime", - "dagannoth_rex", - "dagannoth_supreme", - "deranged_archaeologist", - "general_graardor", - "giant_mole", - "grotesque_guardians", - "hespori", - "kalphite_queen", - "king_black_dragon", - "kraken", - "kreearra", - "kril_tsutsaroth", - "mimic", - "nex", - "nightmare", - "phosanis_nightmare", - "obor", - "phantom_muspah", - "sarachnis", - "scorpia", - "skotizo", - "tempoross", - "the_gauntlet", - "the_corrupted_gauntlet", - "theatre_of_blood", - "theatre_of_blood_hard", - "thermonuclear_smoke_devil", - "tombs_of_amascut", - "tombs_of_amascut_expert", - "tzkal_zuk", - "tztok_jad", - "venenatis", - "vetion", - "vorkath", - "wintertodt", - "zalcano", - "zulrah", - "rifts_closed", - "artio", - "calvarion", - "duke_sucellus", - "spindel", - "the_leviathan", - "the_whisperer", - "vardorvis", - ] - - # Check if all keys are present in the player dictionary - for key in keys: - assert key in player, f"Key {key} not found in player dictionary" - - # Dictionary with expected types - expected_types = { - "id": int, - "timestamp": str, - "ts_date": str, - "Player_id": int, - "total": int, - "attack": int, - "defence": int, - "strength": int, - "hitpoints": int, - "ranged": int, - "prayer": int, - "magic": int, - "cooking": int, - "woodcutting": int, - "fletching": int, - "fishing": int, - "firemaking": int, - "crafting": int, - "smithing": int, - "mining": int, - "herblore": int, - "agility": int, - "thieving": int, - "slayer": int, - "farming": int, - "runecraft": int, - "hunter": int, - "construction": int, - "league": int, - "bounty_hunter_hunter": int, - "bounty_hunter_rogue": int, - "cs_all": int, - "cs_beginner": int, - "cs_easy": int, - "cs_medium": int, - "cs_hard": int, - "cs_elite": int, - "cs_master": int, - "lms_rank": int, - "soul_wars_zeal": int, - "abyssal_sire": int, - "alchemical_hydra": int, - "barrows_chests": int, - "bryophyta": int, - "callisto": int, - "cerberus": int, - "chambers_of_xeric": int, - "chambers_of_xeric_challenge_mode": int, - "chaos_elemental": int, - "chaos_fanatic": int, - "commander_zilyana": int, - "corporeal_beast": int, - "crazy_archaeologist": int, - "dagannoth_prime": int, - "dagannoth_rex": int, - "dagannoth_supreme": int, - "deranged_archaeologist": int, - "general_graardor": int, - "giant_mole": int, - "grotesque_guardians": int, - "hespori": int, - "kalphite_queen": int, - "king_black_dragon": int, - "kraken": int, - "kreearra": int, - "kril_tsutsaroth": int, - "mimic": int, - "nex": int, - "nightmare": int, - "phosanis_nightmare": int, - "obor": int, - "phantom_muspah": int, - "sarachnis": int, - "scorpia": int, - "skotizo": int, - "tempoross": int, - "the_gauntlet": int, - "the_corrupted_gauntlet": int, - "theatre_of_blood": int, - "theatre_of_blood_hard": int, - "thermonuclear_smoke_devil": int, - "tombs_of_amascut": int, - "tombs_of_amascut_expert": int, - "tzkal_zuk": int, - "tztok_jad": int, - "venenatis": int, - "vetion": int, - "vorkath": int, - "wintertodt": int, - "zalcano": int, - "zulrah": int, - "rifts_closed": int, - "artio": int, - "calvarion": int, - "duke_sucellus": int, - "spindel": int, - "the_leviathan": int, - "the_whisperer": int, - "vardorvis": int, - } - - # Check if the type of each value in the returned player dictionary matches the expected type - for key, expected_type in expected_types.items(): - value = player.get(key) - # if value is not None: - assert isinstance( - value, expected_type - ), f"Key {key} has incorrect type. Expected: {expected_type}, Got: {type(value)}" diff --git a/projects/api_private/tests/test_highscore_v3.py b/projects/api_private/tests/test_highscore_v3.py deleted file mode 100644 index c8e59b7..0000000 --- a/projects/api_private/tests/test_highscore_v3.py +++ /dev/null @@ -1,65 +0,0 @@ -import pytest -from httpx import AsyncClient - - -@pytest.mark.asyncio -async def test_one_hs_id_v3(custom_client): - endpoint = "/v3/highscore/latest" - async with custom_client as client: - client: AsyncClient - params = { - "player_id": 1, - } - response = await client.get(url=endpoint, params=params) - - assert response.status_code == 200 - assert isinstance(response.json(), list) - - json_response: list[dict] = response.json() - assert len(json_response) == 1 - assert isinstance( - json_response[0], dict - ), f"expected dict, got {type(json_response[0])}, {json_response=}" - - player = json_response[0] - assert player.get("player_id") == 1, f"expected id: 1 got: {player=}" - - -@pytest.mark.asyncio -async def test_many_hs_id_v3(custom_client): - endpoint = "/v3/highscore/latest" - async with custom_client as client: - client: AsyncClient - params = {"player_id": 1, "many": 1} - response = await client.get(url=endpoint, params=params) - - assert response.status_code == 200 - assert isinstance(response.json(), list) - - json_response: list[dict] = response.json() - assert len(json_response) >= 1 - assert isinstance( - json_response[0], dict - ), f"expected dict, got {type(json_response[0])}, {json_response=}" - - player = json_response[0] - assert player.get("player_id") == 1, f"expected id: 1 got: {player=}" - - -@pytest.mark.asyncio -async def test_highscore_latest_v3(custom_client): - # TODO: discuss how we are going to test the "keys" - endpoint = "/v3/highscore/latest" - async with custom_client as client: - client: AsyncClient - params = {"player_id": 1} - response = await client.get(url=endpoint, params=params) - - assert response.status_code == 200 - assert isinstance(response.json(), list) - - json_response: list[dict] = response.json() - assert len(json_response) == 1 - assert isinstance( - json_response[0], dict - ), f"expected dict, got {type(json_response[0])}, {json_response=}" diff --git a/projects/api_private/tests/test_player.py b/projects/api_private/tests/test_player.py deleted file mode 100644 index e5e04c5..0000000 --- a/projects/api_private/tests/test_player.py +++ /dev/null @@ -1,76 +0,0 @@ -import pytest -from httpx import AsyncClient - - -@pytest.mark.asyncio -async def test_one_player_id(custom_client): - endpoint = "/v2/player" - - async with custom_client as client: - client: AsyncClient - params = {"player_id": 1, "greater_than": 0} - response = await client.get(url=endpoint, params=params) - - assert response.status_code == 200 - assert isinstance(response.json(), list) - - json_response: list[dict] = response.json() - assert len(json_response) == 1 - assert isinstance(json_response[0], dict) - player = json_response[0] - assert player.get("id") == 1 - - -@pytest.mark.asyncio -async def test_one_player_name(custom_client): - endpoint = "/v2/player" - - async with custom_client as client: - client: AsyncClient - params = {"player_name": "Player_0", "greater_than": 0} - response = await client.get(url=endpoint, params=params) - - assert response.status_code == 200 - assert isinstance(response.json(), list) - - json_response: list[dict] = response.json() - assert len(json_response) == 1 - assert isinstance(json_response[0], dict) - player = json_response[0] - assert player.get("name") == "Player_0" - - -@pytest.mark.asyncio -async def test_many_player(custom_client): - endpoint = "/v2/player" - - async with custom_client as client: - client: AsyncClient - params = {"player_id": 1, "greater_than": 1} - response = await client.get(url=endpoint, params=params) - - assert response.status_code == 200 - assert isinstance(response.json(), list) - - json_response: list[dict] = response.json() - assert len(json_response) > 1 - assert isinstance(json_response[0], dict) - player = json_response[0] - assert player.get("id") == 1 - - -@pytest.mark.asyncio -async def test_player_label(custom_client): - endpoint = "/v2/player" - - async with custom_client as client: - client: AsyncClient - params = {"label_id": 0} - response = await client.get(url=endpoint, params=params) - - assert response.status_code == 200 - assert isinstance(response.json(), list) - - json_response: list[dict] = response.json() - assert len(json_response) >= 1 - assert isinstance(json_response[0], dict) From 7af47f18eb8553b2fe264f00b1022e5b6a732d19 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sat, 14 Jun 2025 20:38:32 +0200 Subject: [PATCH 36/39] Add abstract methods for selecting highscores in HighscoreDataLatestInterface --- .../bot_detector/database/interfaces/hiscore.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/components/bot_detector/database/interfaces/hiscore.py b/components/bot_detector/database/interfaces/hiscore.py index eacb281..b49149f 100644 --- a/components/bot_detector/database/interfaces/hiscore.py +++ b/components/bot_detector/database/interfaces/hiscore.py @@ -10,6 +10,18 @@ class HighscoreDataLatestInterface(ABC): + @abstractmethod + async def select_highscore( + self, async_session: AsyncSession, player_id: int, label_id: int + ) -> HighscoreDataLatestStruct: + raise NotImplementedError() + + @abstractmethod + async def select_highscore_list( + self, async_session: AsyncSession, player_id: int, label_id: int, limit: int + ) -> HighscoreDataLatestStruct: + raise NotImplementedError() + @abstractmethod async def insert_highscore( self, From 051b598187d13e435ca93bb047614af9a4ce9700 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sat, 14 Jun 2025 20:40:08 +0200 Subject: [PATCH 37/39] bf/select list must return list --- components/bot_detector/database/interfaces/hiscore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/bot_detector/database/interfaces/hiscore.py b/components/bot_detector/database/interfaces/hiscore.py index b49149f..d25a22e 100644 --- a/components/bot_detector/database/interfaces/hiscore.py +++ b/components/bot_detector/database/interfaces/hiscore.py @@ -19,7 +19,7 @@ async def select_highscore( @abstractmethod async def select_highscore_list( self, async_session: AsyncSession, player_id: int, label_id: int, limit: int - ) -> HighscoreDataLatestStruct: + ) -> list[HighscoreDataLatestStruct]: raise NotImplementedError() @abstractmethod From 24fc1ff013f128b9e83390f4c8f732d91f285226 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 14 Jun 2025 22:39:29 -0400 Subject: [PATCH 38/39] hiscore select and list --- .../database/repositories/hiscore.py | 29 +++++++++++++++++-- components/bot_detector/structs/hiscore.py | 5 ++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/components/bot_detector/database/repositories/hiscore.py b/components/bot_detector/database/repositories/hiscore.py index c76c540..cd39815 100644 --- a/components/bot_detector/database/repositories/hiscore.py +++ b/components/bot_detector/database/repositories/hiscore.py @@ -13,8 +13,8 @@ HighscoreDataMonthlyTableStruct, HighscoreDataWeeklyTableStruct, ) -from bot_detector.structs import HighscoreBaseStruct -from sqlalchemy import func +from bot_detector.structs import HighscoreBaseStruct, HighscoreDataLatestStruct +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession logger = logging.getLogger(__name__) @@ -54,6 +54,31 @@ async def insert_highscore( sql = sql.prefix_with("IGNORE") await async_session.execute(sql) + async def select_highscore( + self, async_session: AsyncSession, player_id: int, label_id: int + ) -> HighscoreDataLatestStruct: + stmt = select(HighscoreDataLatestTableStruct).where( + HighscoreDataLatestTableStruct.player_id == player_id + ) + result = await async_session.execute(stmt) + row = result.scalar_one_or_none() + if row is None: + return None # or raise NotFoundError + return HighscoreDataLatestStruct.model_validate(row) + + async def select_highscore_list( + self, async_session: AsyncSession, player_id: int, label_id: int, limit: int + ) -> list[HighscoreDataLatestStruct]: + stmt = ( + select(HighscoreDataLatestTableStruct) + .where(HighscoreDataLatestTableStruct.player_id == player_id) + .order_by(HighscoreDataLatestTableStruct.scrape_date.desc()) + .limit(limit) + ) + result = await async_session.execute(stmt) + rows = result.scalars().all() + return [HighscoreDataLatestStruct.model_validate(row) for row in rows] + class HighscoreDataDailyRepo(HighscoreDataDailyInterface): async def insert_highscore( diff --git a/components/bot_detector/structs/hiscore.py b/components/bot_detector/structs/hiscore.py index b6ffd3c..f1ba485 100644 --- a/components/bot_detector/structs/hiscore.py +++ b/components/bot_detector/structs/hiscore.py @@ -1,4 +1,4 @@ -from datetime import date +from datetime import date, datetime from typing import Optional from pydantic import BaseModel @@ -19,7 +19,8 @@ class HighscoreDataBaseStruct(HighscoreBaseStruct): class HighscoreDataLatestStruct(HighscoreDataBaseStruct): - pass + player_name: Optional[str] = None + created_at: Optional[datetime] = None class HighscoreDataDailyStruct(HighscoreDataBaseStruct): From 7c388d8ea62977283bd7196c92973d2c9840a157 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Sat, 14 Jun 2025 22:39:54 -0400 Subject: [PATCH 39/39] v4? --- .../api_private/src/api/__init__.py | 3 +- .../api_private/src/api/v4/__init__.py | 6 ++ .../api_private/src/api/v4/highscore.py | 61 ++++++++++++++ .../api_private/test_highscore_v4.py | 82 +++++++++++++++++++ 4 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 bases/bot_detector/api_private/src/api/v4/__init__.py create mode 100644 bases/bot_detector/api_private/src/api/v4/highscore.py create mode 100644 test/bases/bot_detector/api_private/test_highscore_v4.py diff --git a/bases/bot_detector/api_private/src/api/__init__.py b/bases/bot_detector/api_private/src/api/__init__.py index 3e8e7d0..760f14d 100644 --- a/bases/bot_detector/api_private/src/api/__init__.py +++ b/bases/bot_detector/api_private/src/api/__init__.py @@ -1,7 +1,8 @@ from fastapi import APIRouter -from . import v2, v3 +from . import v2, v3, v4 router = APIRouter() router.include_router(v2.router, prefix="/v2") router.include_router(v3.router, prefix="/v3") +router.include_router(v4.router, prefix="/v4") diff --git a/bases/bot_detector/api_private/src/api/v4/__init__.py b/bases/bot_detector/api_private/src/api/v4/__init__.py new file mode 100644 index 0000000..8d6dbdc --- /dev/null +++ b/bases/bot_detector/api_private/src/api/v4/__init__.py @@ -0,0 +1,6 @@ +from fastapi import APIRouter + +from . import highscore + +router = APIRouter() +router.include_router(highscore.router) diff --git a/bases/bot_detector/api_private/src/api/v4/highscore.py b/bases/bot_detector/api_private/src/api/v4/highscore.py new file mode 100644 index 0000000..3e1bc2b --- /dev/null +++ b/bases/bot_detector/api_private/src/api/v4/highscore.py @@ -0,0 +1,61 @@ +import logging +from collections import defaultdict +from datetime import datetime + +from bot_detector.api_private.src.app.repositories import ScraperDataRepo +from bot_detector.api_private.src.app.views.response import ( + ActivityView, + ScraperDataView, + SkillView, +) +from bot_detector.api_private.src.core.fastapi.dependencies.session import ( + get_session, +) +from bot_detector.database.repositories.hiscore import HighscoreDataLatestRepo +from bot_detector.structs import HighscoreDataLatestStruct +from fastapi import APIRouter, Depends, Query + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +def convert_latest_struct_to_scraper_data_view( + records: list[HighscoreDataLatestStruct], +) -> list[ScraperDataView]: + return [ + ScraperDataView( + created_at=record.created_at + or datetime.combine(record.scrape_date, datetime.min.time()), + record_date=record.scrape_date, + scraper_id=record.player_id, + player_id=record.player_id, + player_name=record.player_name or "Unknown", + skills=[ + SkillView(skill_name=k, skill_value=v) + for k, v in (record.skills or {}).items() + ], + activities=[ + ActivityView(activity_name=k, activity_value=v) + for k, v in (record.activities or {}).items() + ], + ) + for record in records + ] + + +@router.get("/highscore/latest", response_model=list[ScraperDataView]) +async def get_highscore_latest( + player_id: int, + label_id: int = None, + many: bool = False, + limit: int = Query(default=10, ge=0, le=10_000), + session=Depends(get_session), +): + repo = HighscoreDataLatestRepo() + if many: + rows = await repo.select_highscore_list(session, player_id, label_id, limit) + else: + row = await repo.select_highscore(session, player_id, label_id) + rows = [row] if row else [] + return convert_latest_struct_to_scraper_data_view(rows) diff --git a/test/bases/bot_detector/api_private/test_highscore_v4.py b/test/bases/bot_detector/api_private/test_highscore_v4.py new file mode 100644 index 0000000..3ecba14 --- /dev/null +++ b/test/bases/bot_detector/api_private/test_highscore_v4.py @@ -0,0 +1,82 @@ +from datetime import date + +from bot_detector.api_private.src.api.v4.highscore import ( + ActivityView, + SkillView, + convert_latest_struct_to_scraper_data_view, +) +from bot_detector.structs import HighscoreDataLatestStruct + + +def test_convert_single_struct_to_scraper_data_view(): + # -- Arrange ----------------------------------------------------------------- + sample_rows = [ + HighscoreDataLatestStruct( + player_id=99, + player_name="Bob", + scrape_date=date(2025, 6, 7), + skills={"strength": 42}, + activities={"clue-scroll": 100}, + time_to_live=date(2025, 6, 30), + scrape_year=2025, + scrape_month=6, + scrape_week=23, + ) + ] + + # -- Act --------------------------------------------------------------------- + result = convert_latest_struct_to_scraper_data_view(sample_rows) + + # -- Assert ------------------------------------------------------------------ + assert len(result) == 1 + view = result[0] + assert view.scraper_id == 99 + assert view.player_name == "Bob" + assert view.skills == [SkillView(skill_name="strength", skill_value=42)] + assert view.activities == [ActivityView(activity_name="clue-scroll", activity_value=100)] + + +def test_convert_multiple_structs_to_scraper_data_view(): + # -- Arrange ----------------------------------------------------------------- + sample_rows = [ + HighscoreDataLatestStruct( + player_id=99, + player_name="Bob", + scrape_date=date(2025, 6, 7), + skills={"strength": 42}, + activities={"clue-scroll": 100}, + time_to_live=date(2025, 6, 30), + scrape_year=2025, + scrape_month=6, + scrape_week=23, + ), + HighscoreDataLatestStruct( + player_id=100, + player_name="Alice", + scrape_date=date(2025, 6, 8), + skills={"attack": 35}, + activities={"barrows": 200}, + time_to_live=date(2025, 6, 30), + scrape_year=2025, + scrape_month=6, + scrape_week=23, + ), + ] + + # -- Act --------------------------------------------------------------------- + result = convert_latest_struct_to_scraper_data_view(sample_rows) + + # -- Assert ------------------------------------------------------------------ + assert len(result) == 2 + + view0 = result[0] + assert view0.scraper_id == 99 + assert view0.player_name == "Bob" + assert view0.skills == [SkillView(skill_name="strength", skill_value=42)] + assert view0.activities == [ActivityView(activity_name="clue-scroll", activity_value=100)] + + view1 = result[1] + assert view1.scraper_id == 100 + assert view1.player_name == "Alice" + assert view1.skills == [SkillView(skill_name="attack", skill_value=35)] + assert view1.activities == [ActivityView(activity_name="barrows", activity_value=200)]