Skip to content

Conversation

RusticPotatoes
Copy link
Contributor

No description provided.

Comment on lines +1 to +42
import json
import logging
import sys

from .config import settings

# # log formatting
formatter = logging.Formatter(
json.dumps(
{
"ts": "%(asctime)s",
"name": "%(name)s",
"function": "%(funcName)s",
"level": "%(levelname)s",
"msg": json.dumps("%(message)s"),
}
)
)

stream_handler = logging.StreamHandler(sys.stdout)

stream_handler.setFormatter(formatter)

handlers = [stream_handler]

logging.basicConfig(level=logging.DEBUG, handlers=handlers)

# set imported loggers to warning
# logging.getLogger("urllib3").setLevel(logging.DEBUG)
# logging.getLogger("uvicorn").setLevel(logging.DEBUG)
# logging.getLogger("aiomysql").setLevel(logging.ERROR)
# logging.getLogger("aiokafka").setLevel(logging.WARNING)

if settings.ENV == "PRD":
uvicorn_error = logging.getLogger("uvicorn.error")
uvicorn_error.disabled = True
uvicorn_access = logging.getLogger("uvicorn.access")
uvicorn_access.disabled = True

# # https://github.com/aio-libs/aiomysql/issues/103
# # https://github.com/coleifer/peewee/issues/2229
# warnings.filterwarnings("ignore", ".*Duplicate entry.*")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can use the logfmt component

Comment on lines +18 to +107
class Skill(Base):
__tablename__ = "skill"

skill_id = Column(SmallInteger(), primary_key=True, autoincrement=True)
skill_name = Column(String(50), nullable=False, unique=True)


class Activity(Base):
__tablename__ = "activity"

activity_id = Column(SmallInteger(), primary_key=True, autoincrement=True)
activity_name = Column(String(50), nullable=False, unique=True)


class PlayerSkill(Base):
__tablename__ = "player_skill"

player_skill_id = Column(BigInteger(), primary_key=True, autoincrement=True)
skill_id = Column(SmallInteger(), nullable=False)
skill_value = Column(Integer(), nullable=False, default=0)

__table_args__ = (
UniqueConstraint("skill_id", "skill_value", name="unique_skill_value"),
)


class PlayerActivity(Base):
__tablename__ = "player_activity"

player_activity_id = Column(BigInteger(), primary_key=True, autoincrement=True)
activity_id = Column(SmallInteger(), nullable=False)
activity_value = Column(Integer(), nullable=False, default=0)

__table_args__ = (
UniqueConstraint("activity_id", "activity_value", name="unique_activity_value"),
)


class ScraperDataV3(Base):
__tablename__ = "scraper_data_v3"

scrape_id = Column(BigInteger(), primary_key=True, autoincrement=True)
scrape_ts = Column(DateTime, nullable=False)
scrape_date = Column(Date, nullable=False)
player_id = Column(Integer, nullable=False)

__table_args__ = (
UniqueConstraint("player_id", "scrape_date", name="unique_player_scrape"),
Index("idx_scrape_ts", "scrape_ts"),
)


class ScraperPlayerSkill(Base):
__tablename__ = "scraper_player_skill"

scrape_id = Column(
BigInteger(),
ForeignKey("scraper_data_v3.scrape_id"),
primary_key=True,
)
player_skill_id = Column(
BigInteger(),
ForeignKey("player_skill.player_skill_id"),
primary_key=True,
)

__table_args__ = (
Index("idx_scrape_id", "scrape_id"),
Index("idx_player_skill_id", "player_skill_id"),
)


class ScraperPlayerActivity(Base):
__tablename__ = "scraper_player_activity"

scrape_id = Column(
BigInteger(),
ForeignKey("scraper_data_v3.scrape_id"),
primary_key=True,
)
player_activity_id = Column(
BigInteger(),
ForeignKey("player_activity.player_activity_id"),
primary_key=True,
)

__table_args__ = (
Index("idx_scrape_id", "scrape_id"),
Index("idx_player_activity_id", "player_activity_id"),
)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these tables are no more

Comment on lines +1 to +26
from bot_detector.api_private.src.core.database.database import Base
from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy.dialects.mysql import BIGINT, TINYINT


class Skills(Base):
__tablename__ = "skills"

skill_id = Column(TINYINT, primary_key=True, autoincrement=True)
skill_name = Column(String(50), nullable=False)


class PlayerSkills(Base):
__tablename__ = "player_skills"

scraper_id = Column(
BIGINT,
ForeignKey("scraper_data.scraper_id", ondelete="CASCADE"),
primary_key=True,
)
skill_id = Column(
TINYINT,
ForeignKey("skills.skill_id", ondelete="CASCADE"),
primary_key=True,
)
skill_value = Column(Integer, nullable=False, default=0)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these tables are no more

Comment on lines +1 to +21
from bot_detector.api_private.src.core.database.database import Base
from sqlalchemy import Column, Date, DateTime, func
from sqlalchemy.dialects.mysql import BIGINT, SMALLINT


class ScraperData(Base):
__tablename__ = "scraper_data"

scraper_id = Column(BIGINT, primary_key=True, autoincrement=True)
created_at = Column(DateTime, nullable=False, server_default=func.now())
player_id = Column(SMALLINT, nullable=False)
record_date = Column(Date, nullable=True)


class ScraperDataLatest(Base):
__tablename__ = "scraper_data_latest"

scraper_id = Column(BIGINT)
created_at = Column(DateTime, nullable=False, server_default=func.now())
player_id = Column(BIGINT, primary_key=True)
record_date = Column(Date, nullable=True)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these tables are no more

Comment on lines +1 to +349
from bot_detector.api_private.src.core.database.database import Base
from sqlalchemy import (
BigInteger,
Column,
Date,
DateTime,
ForeignKey,
Index,
Integer,
text,
)


class playerHiscoreData(Base):
__tablename__ = "playerHiscoreData"
__table_args__ = (
Index("Unique_player_time", "Player_id", "timestamp", unique=True),
Index("Unique_player_date", "Player_id", "ts_date", unique=True),
)

id = Column(Integer, primary_key=True, autoincrement=True)
timestamp = Column(
DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP")
)
ts_date = Column(Date)
Player_id = Column(
ForeignKey("Players.id", ondelete="RESTRICT", onupdate="RESTRICT"),
nullable=False,
)
total = Column(BigInteger)
attack = Column(Integer)
defence = Column(Integer)
strength = Column(Integer)
hitpoints = Column(Integer)
ranged = Column(Integer)
prayer = Column(Integer)
magic = Column(Integer)
cooking = Column(Integer)
woodcutting = Column(Integer)
fletching = Column(Integer)
fishing = Column(Integer)
firemaking = Column(Integer)
crafting = Column(Integer)
smithing = Column(Integer)
mining = Column(Integer)
herblore = Column(Integer)
agility = Column(Integer)
thieving = Column(Integer)
slayer = Column(Integer)
farming = Column(Integer)
runecraft = Column(Integer)
hunter = Column(Integer)
construction = Column(Integer)
league = Column(Integer)
bounty_hunter_hunter = Column(Integer)
bounty_hunter_rogue = Column(Integer)
cs_all = Column(Integer)
cs_beginner = Column(Integer)
cs_easy = Column(Integer)
cs_medium = Column(Integer)
cs_hard = Column(Integer)
cs_elite = Column(Integer)
cs_master = Column(Integer)
lms_rank = Column(Integer)
soul_wars_zeal = Column(Integer)
abyssal_sire = Column(Integer)
alchemical_hydra = Column(Integer)
barrows_chests = Column(Integer)
bryophyta = Column(Integer)
callisto = Column(Integer)
cerberus = Column(Integer)
chambers_of_xeric = Column(Integer)
chambers_of_xeric_challenge_mode = Column(Integer)
chaos_elemental = Column(Integer)
chaos_fanatic = Column(Integer)
commander_zilyana = Column(Integer)
corporeal_beast = Column(Integer)
crazy_archaeologist = Column(Integer)
dagannoth_prime = Column(Integer)
dagannoth_rex = Column(Integer)
dagannoth_supreme = Column(Integer)
deranged_archaeologist = Column(Integer)
general_graardor = Column(Integer)
giant_mole = Column(Integer)
grotesque_guardians = Column(Integer)
hespori = Column(Integer)
kalphite_queen = Column(Integer)
king_black_dragon = Column(Integer)
kraken = Column(Integer)
kreearra = Column(Integer)
kril_tsutsaroth = Column(Integer)
mimic = Column(Integer)
nightmare = Column(Integer)
nex = Column(Integer)
phosanis_nightmare = Column(Integer)
obor = Column(Integer)
phantom_muspah = Column(Integer)
sarachnis = Column(Integer)
scorpia = Column(Integer)
skotizo = Column(Integer)
tempoross = Column(Integer)
the_gauntlet = Column(Integer)
the_corrupted_gauntlet = Column(Integer)
theatre_of_blood = Column(Integer)
theatre_of_blood_hard = Column(Integer)
thermonuclear_smoke_devil = Column(Integer)
tombs_of_amascut = Column(Integer)
tombs_of_amascut_expert = Column(Integer)
tzkal_zuk = Column(Integer)
tztok_jad = Column(Integer)
venenatis = Column(Integer)
vetion = Column(Integer)
vorkath = Column(Integer)
wintertodt = Column(Integer)
zalcano = Column(Integer)
zulrah = Column(Integer)

# New columns added
rifts_closed = Column(Integer, default=0)
artio = Column(Integer, default=0)
calvarion = Column(Integer, default=0)
duke_sucellus = Column(Integer, default=0)
spindel = Column(Integer, default=0)
the_leviathan = Column(Integer, default=0)
the_whisperer = Column(Integer, default=0)
vardorvis = Column(Integer, default=0)


class PlayerHiscoreDataLatest(Base):
__tablename__ = "playerHiscoreDataLatest"

id = Column(Integer, primary_key=True)
timestamp = Column(
DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP")
)
ts_date = Column(Date)
Player_id = Column(
ForeignKey("Players.id", ondelete="RESTRICT", onupdate="RESTRICT"),
nullable=False,
unique=True,
)
total = Column(BigInteger)
attack = Column(Integer)
defence = Column(Integer)
strength = Column(Integer)
hitpoints = Column(Integer)
ranged = Column(Integer)
prayer = Column(Integer)
magic = Column(Integer)
cooking = Column(Integer)
woodcutting = Column(Integer)
fletching = Column(Integer)
fishing = Column(Integer)
firemaking = Column(Integer)
crafting = Column(Integer)
smithing = Column(Integer)
mining = Column(Integer)
herblore = Column(Integer)
agility = Column(Integer)
thieving = Column(Integer)
slayer = Column(Integer)
farming = Column(Integer)
runecraft = Column(Integer)
hunter = Column(Integer)
construction = Column(Integer)
league = Column(Integer)
bounty_hunter_hunter = Column(Integer)
bounty_hunter_rogue = Column(Integer)
cs_all = Column(Integer)
cs_beginner = Column(Integer)
cs_easy = Column(Integer)
cs_medium = Column(Integer)
cs_hard = Column(Integer)
cs_elite = Column(Integer)
cs_master = Column(Integer)
lms_rank = Column(Integer)
soul_wars_zeal = Column(Integer)
abyssal_sire = Column(Integer)
alchemical_hydra = Column(Integer)
barrows_chests = Column(Integer)
bryophyta = Column(Integer)
callisto = Column(Integer)
cerberus = Column(Integer)
chambers_of_xeric = Column(Integer)
chambers_of_xeric_challenge_mode = Column(Integer)
chaos_elemental = Column(Integer)
chaos_fanatic = Column(Integer)
commander_zilyana = Column(Integer)
corporeal_beast = Column(Integer)
crazy_archaeologist = Column(Integer)
dagannoth_prime = Column(Integer)
dagannoth_rex = Column(Integer)
dagannoth_supreme = Column(Integer)
deranged_archaeologist = Column(Integer)
general_graardor = Column(Integer)
giant_mole = Column(Integer)
grotesque_guardians = Column(Integer)
hespori = Column(Integer)
kalphite_queen = Column(Integer)
king_black_dragon = Column(Integer)
kraken = Column(Integer)
kreearra = Column(Integer)
kril_tsutsaroth = Column(Integer)
mimic = Column(Integer)
nightmare = Column(Integer)
nex = Column(Integer)
phosanis_nightmare = Column(Integer)
obor = Column(Integer)
phantom_muspah = Column(Integer)
sarachnis = Column(Integer)
scorpia = Column(Integer)
skotizo = Column(Integer)
Tempoross = Column(Integer, nullable=False)
the_gauntlet = Column(Integer)
the_corrupted_gauntlet = Column(Integer)
theatre_of_blood = Column(Integer)
theatre_of_blood_hard = Column(Integer)
thermonuclear_smoke_devil = Column(Integer)
tombs_of_amascut = Column(Integer)
tombs_of_amascut_expert = Column(Integer)
tzkal_zuk = Column(Integer)
tztok_jad = Column(Integer)
venenatis = Column(Integer)
vetion = Column(Integer)
vorkath = Column(Integer)
wintertodt = Column(Integer)
zalcano = Column(Integer)
zulrah = Column(Integer)

# New columns added
rifts_closed = Column(Integer, default=0)
artio = Column(Integer, default=0)
calvarion = Column(Integer, default=0)
duke_sucellus = Column(Integer, default=0)
spindel = Column(Integer, default=0)
the_leviathan = Column(Integer, default=0)
the_whisperer = Column(Integer, default=0)
vardorvis = Column(Integer, default=0)


class PlayerHiscoreDataXPChange(Base):
__tablename__ = "playerHiscoreDataXPChange"

id = Column(Integer, primary_key=True)
timestamp = Column(
DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP")
)
ts_date = Column(Date)
Player_id = Column(
ForeignKey("Players.id", ondelete="RESTRICT", onupdate="RESTRICT"),
nullable=False,
index=True,
)
total = Column(BigInteger)
attack = Column(Integer)
defence = Column(Integer)
strength = Column(Integer)
hitpoints = Column(Integer)
ranged = Column(Integer)
prayer = Column(Integer)
magic = Column(Integer)
cooking = Column(Integer)
woodcutting = Column(Integer)
fletching = Column(Integer)
fishing = Column(Integer)
firemaking = Column(Integer)
crafting = Column(Integer)
smithing = Column(Integer)
mining = Column(Integer)
herblore = Column(Integer)
agility = Column(Integer)
thieving = Column(Integer)
slayer = Column(Integer)
farming = Column(Integer)
runecraft = Column(Integer)
hunter = Column(Integer)
construction = Column(Integer)
league = Column(Integer)
bounty_hunter_hunter = Column(Integer)
bounty_hunter_rogue = Column(Integer)
cs_all = Column(Integer)
cs_beginner = Column(Integer)
cs_easy = Column(Integer)
cs_medium = Column(Integer)
cs_hard = Column(Integer)
cs_elite = Column(Integer)
cs_master = Column(Integer)
lms_rank = Column(Integer)
soul_wars_zeal = Column(Integer)
abyssal_sire = Column(Integer)
alchemical_hydra = Column(Integer)
barrows_chests = Column(Integer)
bryophyta = Column(Integer)
callisto = Column(Integer)
cerberus = Column(Integer)
chambers_of_xeric = Column(Integer)
chambers_of_xeric_challenge_mode = Column(Integer)
chaos_elemental = Column(Integer)
chaos_fanatic = Column(Integer)
commander_zilyana = Column(Integer)
corporeal_beast = Column(Integer)
crazy_archaeologist = Column(Integer)
dagannoth_prime = Column(Integer)
dagannoth_rex = Column(Integer)
dagannoth_supreme = Column(Integer)
deranged_archaeologist = Column(Integer)
general_graardor = Column(Integer)
giant_mole = Column(Integer)
grotesque_guardians = Column(Integer)
hespori = Column(Integer)
kalphite_queen = Column(Integer)
king_black_dragon = Column(Integer)
kraken = Column(Integer)
kreearra = Column(Integer)
kril_tsutsaroth = Column(Integer)
mimic = Column(Integer)
nightmare = Column(Integer)
nex = Column(Integer)
obor = Column(Integer)
phantom_muspah = Column(Integer)
phosanis_nightmare = Column(Integer)
sarachnis = Column(Integer)
scorpia = Column(Integer)
skotizo = Column(Integer)
Tempoross = Column(Integer, nullable=False)
the_gauntlet = Column(Integer)
the_corrupted_gauntlet = Column(Integer)
theatre_of_blood = Column(Integer)
theatre_of_blood_hard = Column(Integer)
thermonuclear_smoke_devil = Column(Integer)
tombs_of_amascut = Column(Integer)
tombs_of_amascut_expert = Column(Integer)
tzkal_zuk = Column(Integer)
tztok_jad = Column(Integer)
venenatis = Column(Integer)
vetion = Column(Integer)
vorkath = Column(Integer)
wintertodt = Column(Integer)
zalcano = Column(Integer)
zulrah = Column(Integer)
# New columns added
rifts_closed = Column(Integer, default=0)
artio = Column(Integer, default=0)
calvarion = Column(Integer, default=0)
duke_sucellus = Column(Integer, default=0)
spindel = Column(Integer, default=0)
the_leviathan = Column(Integer, default=0)
the_whisperer = Column(Integer, default=0)
vardorvis = Column(Integer, default=0)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these tables are no more

Comment on lines +1 to +26
from bot_detector.api_private.src.core.database.database import Base
from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy.dialects.mysql import BIGINT, TINYINT


class Activities(Base):
__tablename__ = "activities"

activity_id = Column(TINYINT, primary_key=True, autoincrement=True)
activity_name = Column(String(50), nullable=False)


class PlayerActivities(Base):
__tablename__ = "player_activities"

scraper_id = Column(
BIGINT,
ForeignKey("scraper_data.scraper_id", ondelete="CASCADE"),
primary_key=True,
)
activity_id = Column(
TINYINT,
ForeignKey("activities.activity_id", ondelete="CASCADE"),
primary_key=True,
)
activity_value = Column(Integer, nullable=False, default=0)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these tables are no more

Comment on lines +1 to +30
from bot_detector.api_private.src.core.database.database import Base
from sqlalchemy import TIMESTAMP, BigInteger, Column, Integer, SmallInteger


class Report(Base):
__tablename__ = "Reports"

ID = Column(BigInteger, primary_key=True, autoincrement=True)
created_at = Column(TIMESTAMP)
reportedID = Column(Integer)
reportingID = Column(Integer)
region_id = Column(Integer)
x_coord = Column(Integer)
y_coord = Column(Integer)
z_coord = Column(Integer)
timestamp = Column(TIMESTAMP)
manual_detect = Column(SmallInteger)
on_members_world = Column(Integer)
on_pvp_world = Column(SmallInteger)
world_number = Column(Integer)
equip_head_id = Column(Integer)
equip_amulet_id = Column(Integer)
equip_torso_id = Column(Integer)
equip_legs_id = Column(Integer)
equip_boots_id = Column(Integer)
equip_cape_id = Column(Integer)
equip_hands_id = Column(Integer)
equip_weapon_id = Column(Integer)
equip_shield_id = Column(Integer)
equip_ge_value = Column(BigInteger)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these tables are no more

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so is this unused? or needs refactoring to a new schema?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lets not migrate the tests like this

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lets not migrate the tests like this

router = APIRouter()


def convert_latest_struct_to_scraper_data_view(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for the v4 i'd not convert, and keep it the HighscoreDataLatestStruct we only convert the v3 model to keep backwards compatibility, the ML component uses the /highscore/latest route

stmt = (
select(HighscoreDataLatestTableStruct)
.where(HighscoreDataLatestTableStruct.player_id == player_id)
.order_by(HighscoreDataLatestTableStruct.scrape_date.desc())
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i would not order by scrape_date,

) -> list[HighscoreDataLatestStruct]:
stmt = (
select(HighscoreDataLatestTableStruct)
.where(HighscoreDataLatestTableStruct.player_id == player_id)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i may need to reconsider the interface, the intended way is that we use player_id with a greater than statement if player_id is provided.

would it be more clear if the interface had player_id_gt: int | None & player_id_lt: int | None, label_id: int | None, limit: Field(default=100, gt=0, lt=10000)

await async_session.execute(sql)

async def select_highscore(
self, async_session: AsyncSession, player_id: int, label_id: int
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm seeing this i think it would make sense to leave out the label_id as i don't see a scenario where you'd use it to select a single highscore

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

Successfully merging this pull request may close these issues.

2 participants