From 4350e09b327aef87d2a9c296769d71b77748ee36 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sun, 28 Jul 2024 13:57:01 +0200 Subject: [PATCH 1/4] cleanup unused functions --- src/api/v1/report.py | 268 ++----------------------------------------- 1 file changed, 10 insertions(+), 258 deletions(-) diff --git a/src/api/v1/report.py b/src/api/v1/report.py index 0786189..1ff9f41 100644 --- a/src/api/v1/report.py +++ b/src/api/v1/report.py @@ -29,77 +29,9 @@ logger = logging.getLogger(__name__) router = APIRouter() -# TODO: put these in the correct place -report_maximum = 5000 -front_time_buffer = 3600 -back_time_buffer = 25200 upper_gear_cost = 1_000_000_000_000 # TODO: cleanup thse functions - - -async def sql_select_players(names: List[str]) -> List: - _names = await functions.jagexify_names_list(names=names) - sql = select(Player) - sql = sql.where(Player.name.in_(_names)) - async with PLAYERDATA_ENGINE.get_session() as session: - session: AsyncSession = session - async with session.begin(): - data = await session.execute(sql) - data = functions.sqlalchemy_result(data) - return [] if not data else data.rows2dict() - - -async def sql_insert_player(new_names: List[dict]) -> None: - sql: Insert = insert(Player) - sql = sql.prefix_with("ignore") - async with PLAYERDATA_ENGINE.get_session() as session: - session: AsyncSession = session - async with session.begin(): - await session.execute(sql, new_names) - - -async def sql_insert_report(param: dict) -> None: - sql: Insert = insert(stgReport) - sql = sql.prefix_with("ignore") - async with PLAYERDATA_ENGINE.get_session() as session: - session: AsyncSession = session - async with session.begin(): - await session.execute(sql, param) - - -async def parse_detection(data: dict) -> dict: - gmt = time.gmtime(data["ts"]) - human_time = time.strftime("%Y-%m-%d %H:%M:%S", gmt) - - equipment = data.get("equipment", {}) - - param = { - "reportedID": data.get("id"), - "reportingID": data.get("reporter_id"), - "region_id": data.get("region_id"), - "x_coord": data.get("x_coord"), - "y_coord": data.get("y_coord"), - "z_coord": data.get("z_coord"), - "timestamp": human_time, - "manual_detect": data.get("manual_detect"), - "on_members_world": data.get("on_members_world"), - "on_pvp_world": data.get("on_pvp_world"), - "world_number": data.get("world_number"), - "equip_head_id": equipment.get("equip_head_id"), - "equip_amulet_id": equipment.get("equip_amulet_id"), - "equip_torso_id": equipment.get("equip_torso_id"), - "equip_legs_id": equipment.get("equip_legs_id"), - "equip_boots_id": equipment.get("equip_boots_id"), - "equip_cape_id": equipment.get("equip_cape_id"), - "equip_hands_id": equipment.get("equip_hands_id"), - "equip_weapon_id": equipment.get("equip_weapon_id"), - "equip_shield_id": equipment.get("equip_shield_id"), - "equip_ge_value": data.get("equip_ge_value", 0), - } - return param - - class equipment(BaseModel): equip_head_id: int = Field(0, ge=0) equip_amulet_id: int = Field(0, ge=0) @@ -174,7 +106,6 @@ async def get_reports( data = functions.sqlalchemy_result(data) return data.rows2dict() - @router.put("/report", tags=["Report"]) async def update_reports( old_user_id: int, new_user_id: int, token: str, request: Request @@ -201,23 +132,6 @@ async def update_reports( return {"detail": f"{data.rowcount} rows updated to reportingID = {new_user_id}."} - -async def insert_active_reporter(reporter: str): - if "anonymoususer" in reporter: - return - try: - sql: Text = text("INSERT INTO activeReporters (name) VALUES (:reporter)") - - async with PLAYERDATA_ENGINE.get_session() as session: - session: AsyncSession = session - async with session.begin(): - await session.execute(sql, {"reporter": reporter}) - except Exception as e: - e = str(e) - if "Duplicate entry" in e: - return - logger.error(str(e)) - async def insert_report_v2(detections: list[detection]): url = 'http://public-api-svc.bd-prd.svc:5000/v2/report' try: @@ -247,141 +161,7 @@ async def insert_report( asyncio.create_task(insert_report_v2(detections)) return {"detail": "ok"} - # if random.randint(1, 10) == 1: - # asyncio.create_task(insert_report_v2(detections)) - - # # remove duplicates - # df = pd.DataFrame([d.dict() for d in detections]) - # df.drop_duplicates(subset=["reporter", "reported", "region_id"], inplace=True) - - # # data validation, there can only be one reporter, and it is unrealistic to send more then 5k reports. - # if len(df) > 5000 or df["reporter"].nunique() > 1: - # logger.warning({"message": "Too many reports."}) - # return - - # # data validation, checks for correct timing - # now = int(time.time()) - # now_upper = int(now + 3600) - # now_lower = int(now - 25200) - - # df_time = df.ts - # mask = (df_time > now_upper) | (df_time < now_lower) - # if len(df_time[mask].values) > 0: - # logger.warning( - # { - # "message": "Data contains out of bounds time", - # "reporter": df["reporter"].unique(), - # "time": df_time[mask].values[0], - # } - # ) - # return - - # logger.info({"message": f"Received: {len(df)} from: {df['reporter'].unique()}"}) - - # # Normalize names - # df["reporter"] = df["reporter"].apply( - # lambda name: name.lower().replace("_", " ").replace("-", " ").strip() - # ) - # df["reported"] = df["reported"].apply( - # lambda name: name.lower().replace("_", " ").replace("-", " ").strip() - # ) - - # # Get a list of unqiue reported names and reporter name - # names = list(df["reported"].unique()) - # names.extend(df["reporter"].unique()) - - # # validate all names - # valid_names = [ - # await functions.to_jagex_name(name) - # for name in names - # if await functions.is_valid_rsn(name) - # ] - # # logger.debug(f"Valid names: {len(valid_names)}") - # # logger.debug(f"{valid_names=}") - - # # Get IDs for all unique valid names - # data = await sql_select_players(valid_names) - # # logger.debug(f"Found players before insert: {len(data)}") - # # logger.debug(f"{data=}") - - # # Create entries for players that do not yet exist in Players table - # existing_names = [d["name"] for d in data] - # # logger.debug(f"{existing_names=}") - - # new_names = set([name for name in valid_names]).difference(existing_names) - # # logger.debug(f"{new_names=}") - - # # Get new player id's - # if new_names: - # param = [ - # {"name": name, "normalized_name": await functions.to_jagex_name(name)} - # for name in new_names - # ] - # await functions.batch_function(sql_insert_player, param) - # players = await sql_select_players(new_names) - # # logger.debug(f"Found players after insert: {len(players)=}, {len(new_names)=}") - # data.extend(players) - - # # Insert detections into Reports table with user ids - # # add reported & reporter id - # df_names = pd.DataFrame(data) - - # if len(df) == 0: - # logger.warning( - # {"message": "empty dataframe, before merge", "detections": detections} - # ) - # return - - # if len(df_names) == 0: - # logger.warning( - # {"message": "empty dataframe names, before merge", "detections": detections} - # ) - # return - - # df = df.merge(df_names, left_on="reported", right_on="name") - - # if len(df) == 0: - # logger.warning( - # {"message": "empty dataframe, after merge", "detections": detections} - # ) - # return - - # reporter = df["reporter"].unique() - - # if len(reporter) != 1: - # logger.warning({"message": "No reporter", "detections": detections}) - # return - - # reporter_id = df_names.query(f"name == {reporter}")["id"].to_list() - - # if len(reporter_id) == 0: - # logger.warning({"message": "No reporter in df_names", "detections": detections}) - # return - - # asyncio.create_task(insert_active_reporter(reporter[0])) - - # # if reporter_id[0] == 657248: - # # logger.debug({"message": "Temporary ignoring anonymous reporter"}) - # # return - - # df["reporter_id"] = reporter_id[0] - - # if manual_detect: - # df["manual_detect"] = manual_detect - - # # Parse data to param - # data = df.to_dict("records") - # param = [await parse_detection(d) for d in data] - - # # Parse query - # await functions.batch_function(sql_insert_report, param) - # return {"detail": "ok"} - - -@router.get("/report/count", tags=["Report"]) -async def get_report_count_v1(name: str): - """ - """ +async def select_report_count_v1(name:str, manual_detect:int): name = await functions.to_jagex_name(name) voter: Player = aliased(Player, name="voter") @@ -390,7 +170,7 @@ async def get_report_count_v1(name: str): sub_query: Select = select(Report.reportedID.distinct().label("reportedID")) sub_query = sub_query.join(voter, Report.reportingID == voter.id) sub_query = sub_query.where(voter.name == name) - sub_query = sub_query.where(Report.manual_detect == 0) + sub_query = sub_query.where(Report.manual_detect == manual_detect) # Create an alias for the subquery sub_query_alias = sub_query.alias("DistinctReports") @@ -418,45 +198,17 @@ async def get_report_count_v1(name: str): data = [{k: v for k, v in zip(keys, d)} for d in data] return data +@router.get("/report/count", tags=["Report"]) +async def get_report_count_v1(name: str): + """ + """ + data = await select_report_count_v1(name=name, manual_detect=0) + return data @router.get("/report/manual/count", tags=["Report"]) async def get_report_manual_count_v1(name: str): """ Get the calculated player report count """ - name = await functions.to_jagex_name(name) - - voter: Player = aliased(Player, name="voter") - subject: Player = aliased(Player, name="subject") - - sub_query: Select = select(Report.reportedID.distinct().label("reportedID")) - sub_query = sub_query.join(voter, Report.reportingID == voter.id) - sub_query = sub_query.where(voter.name == name) - sub_query = sub_query.where(Report.manual_detect == 1) - - # Create an alias for the subquery - sub_query_alias = sub_query.alias("DistinctReports") - - sql: Select = select( - func.count(subject.id), - subject.confirmed_ban, - subject.possible_ban, - subject.confirmed_player, - ) - sql = sql.select_from(sub_query_alias) - sql = sql.join( - subject, sub_query_alias.c.reportedID == subject.id - ) # Use c to access columns - sql = sql.group_by( - subject.confirmed_ban, subject.possible_ban, subject.confirmed_player - ) - - keys = ["count", "confirmed_ban", "possible_ban", "confirmed_player"] - # execute query - async with PLAYERDATA_ENGINE.get_session() as session: - session: AsyncSession = session - async with session.begin(): - data = await session.execute(sql) - data = [{k: v for k, v in zip(keys, d)} for d in data] - - return data + data = await select_report_count_v1(name=name, manual_detect=1) + return data \ No newline at end of file From 11fc55e88a7297fb0410d97d0d9bdf82cac9162b Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sun, 28 Jul 2024 14:48:28 +0200 Subject: [PATCH 2/4] ruff check --fix --- src/api/legacy/legacy.py | 16 ++++++++-------- src/api/v1/hiscore.py | 13 ++++++------- src/api/v1/label.py | 1 - src/api/v1/player.py | 13 ++++++------- src/api/v1/prediction.py | 10 +++++----- src/api/v1/report.py | 9 +-------- src/api/v1/scraper.py | 7 +++---- src/api/v2/highscore.py | 1 - src/app/repositories/highscore.py | 1 - src/app/repositories/player.py | 4 ++-- src/core/server.py | 3 --- src/database/functions.py | 8 ++++---- src/kafka/highscore.py | 3 --- src/kafka/modules/abc.py | 1 - tests/test_feedback.py | 3 +-- tests/test_hiscore.py | 6 +++--- tests/test_label.py | 3 +-- tests/test_player.py | 2 +- tests/test_report.py | 3 +-- tests/test_scraper.py | 2 +- tests/utils/test_logging_helpers.py | 2 +- 21 files changed, 44 insertions(+), 67 deletions(-) diff --git a/src/api/legacy/legacy.py b/src/api/legacy/legacy.py index 8060b64..3f5ce9d 100644 --- a/src/api/legacy/legacy.py +++ b/src/api/legacy/legacy.py @@ -9,7 +9,7 @@ import pandas as pd from src.core import config -from src.database.database import DISCORD_ENGINE, EngineType +from src.database.database import DISCORD_ENGINE from src.database import functions from src.database.functions import execute_sql, list_to_string, verify_token from src.utils import logging_helpers @@ -800,7 +800,7 @@ async def receive_plugin_feedback(feedback: Feedback, version: str = None): voter_data = await sql_insert_player(player_name) if not len(voter_data) > 0: - raise HTTPException(status_code=405, detail=f"Voter does not exist") + raise HTTPException(status_code=405, detail="Voter does not exist") feedback_params["voter_id"] = voter_data.get("id") exclude = ["player_name"] @@ -914,7 +914,7 @@ async def verify_bot(token: str, bots: bots, request: Request): label = bots["label"] if len(playerNames) == 0: - raise HTTPException(status_code=405, detail=f"Invalid Parameters") + raise HTTPException(status_code=405, detail="Invalid Parameters") data = [] for name in playerNames: @@ -992,17 +992,17 @@ async def verify_discord_user( code = verify_data.get("code", "") if len(code) != 4: - raise HTTPException(status_code=400, detail=f"Please provide a 4 digit code.") + raise HTTPException(status_code=400, detail="Please provide a 4 digit code.") try: provided_code = int(code) except ValueError: - raise HTTPException(status_code=400, detail=f"Please provide a 4 digit code.") + raise HTTPException(status_code=400, detail="Please provide a 4 digit code.") player = await sql_get_player(verify_data["player_name"]) if player == None: - raise HTTPException(status_code=400, detail=f"Could not find player") + raise HTTPException(status_code=400, detail="Could not find player") pending_discord = await sql_get_unverified_discord_user(player["id"]) @@ -1011,7 +1011,7 @@ async def verify_discord_user( token_id = token_info.get("id") if not pending_discord: - raise HTTPException(status_code=400, detail=f"No pending links for this user.") + raise HTTPException(status_code=400, detail="No pending links for this user.") found_code = False for record in pending_discord: @@ -1021,7 +1021,7 @@ async def verify_discord_user( break if not (found_code): - raise HTTPException(status_code=400, detail=f"Linking code is incorrect.") + raise HTTPException(status_code=400, detail="Linking code is incorrect.") return {"ok": "ok"} diff --git a/src/api/v1/hiscore.py b/src/api/v1/hiscore.py index 3e44665..244fae6 100644 --- a/src/api/v1/hiscore.py +++ b/src/api/v1/hiscore.py @@ -2,7 +2,6 @@ from src.database.functions import PLAYERDATA_ENGINE from sqlalchemy.ext.asyncio import AsyncSession -from src.database.database import EngineType from src.database.functions import sqlalchemy_result, verify_token from src.database.models import ( Player, @@ -220,22 +219,22 @@ async def get_latest_hiscore_data_by_player_features( sql = select(PlayerHiscoreDataLatest) # filters - if not possible_ban is None: + if possible_ban is not None: sql = sql.where(Player.possible_ban == possible_ban) - if not confirmed_ban is None: + if confirmed_ban is not None: sql = sql.where(Player.confirmed_ban == confirmed_ban) - if not confirmed_player is None: + if confirmed_player is not None: sql = sql.where(Player.confirmed_player == confirmed_player) - if not label_id is None: + if label_id is not None: sql = sql.where(Player.label_id == label_id) - if not label_jagex is None: + if label_jagex is not None: sql = sql.where(Player.label_jagex == label_jagex) - if not greater_than is None: + if greater_than is not None: sql = sql.where(Player.id >= greater_than) # paging diff --git a/src/api/v1/label.py b/src/api/v1/label.py index 8d96836..ef2fdaf 100644 --- a/src/api/v1/label.py +++ b/src/api/v1/label.py @@ -1,4 +1,3 @@ -from src.database.database import EngineType from src.database.functions import sqlalchemy_result, verify_token from src.database.models import Label as dbLabel from fastapi import APIRouter diff --git a/src/api/v1/player.py b/src/api/v1/player.py index 5a0a1f2..1f100a9 100644 --- a/src/api/v1/player.py +++ b/src/api/v1/player.py @@ -1,10 +1,9 @@ import time -from typing import List, Optional +from typing import Optional from src.database import functions from src.database.functions import PLAYERDATA_ENGINE from sqlalchemy.ext.asyncio import AsyncSession -from src.database.database import Engine, EngineType from src.database.functions import sqlalchemy_result, verify_token from src.database.models import Player as dbPlayer from src.utils import logging_helpers @@ -107,19 +106,19 @@ async def get_bulk_player_data_from_the_plugin_database( # filters # filters - if not possible_ban is None: + if possible_ban is not None: sql = sql.where(dbPlayer.possible_ban == possible_ban) - if not confirmed_ban is None: + if confirmed_ban is not None: sql = sql.where(dbPlayer.confirmed_ban == confirmed_ban) - if not confirmed_player is None: + if confirmed_player is not None: sql = sql.where(dbPlayer.confirmed_player == confirmed_player) - if not label_id is None: + if label_id is not None: sql = sql.where(dbPlayer.label_id == label_id) - if not label_jagex is None: + if label_jagex is not None: sql = sql.where(dbPlayer.label_jagex == label_jagex) # query pagination diff --git a/src/api/v1/prediction.py b/src/api/v1/prediction.py index 38ac8e6..8ea7ab4 100644 --- a/src/api/v1/prediction.py +++ b/src/api/v1/prediction.py @@ -213,19 +213,19 @@ async def gets_predictions_by_player_features( sql: Select = select(dbPrediction) # filters - if not possible_ban is None: + if possible_ban is not None: sql = sql.where(Player.possible_ban == possible_ban) - if not confirmed_ban is None: + if confirmed_ban is not None: sql = sql.where(Player.confirmed_ban == confirmed_ban) - if not confirmed_player is None: + if confirmed_player is not None: sql = sql.where(Player.confirmed_player == confirmed_player) - if not label_id is None: + if label_id is not None: sql = sql.where(Player.label_id == label_id) - if not label_jagex is None: + if label_jagex is not None: sql = sql.where(Player.label_jagex == label_jagex) # paging diff --git a/src/api/v1/report.py b/src/api/v1/report.py index 1ff9f41..11fe9d0 100644 --- a/src/api/v1/report.py +++ b/src/api/v1/report.py @@ -1,17 +1,12 @@ import logging -import time from datetime import date from typing import List, Optional import asyncio -import pandas as pd from src.database import functions from src.database.functions import PLAYERDATA_ENGINE from src.database.models import ( Player, Report, - playerReports, - playerReportsManual, - stgReport, ) from src.utils import logging_helpers from fastapi import APIRouter, HTTPException, Query, Request, status @@ -20,10 +15,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import aliased from sqlalchemy.sql import func -from sqlalchemy.sql.expression import Insert, Select, insert, select, update -from sqlalchemy import Text, text +from sqlalchemy.sql.expression import Select, select, update import aiohttp -import random import traceback logger = logging.getLogger(__name__) diff --git a/src/api/v1/scraper.py b/src/api/v1/scraper.py index f006d91..069d9d1 100644 --- a/src/api/v1/scraper.py +++ b/src/api/v1/scraper.py @@ -6,12 +6,11 @@ from src.database.functions import PLAYERDATA_ENGINE from sqlalchemy.ext.asyncio import AsyncSession -from src.database.database import EngineType from src.database.functions import batch_function, execute_sql, verify_token from src.database.models import Player as dbPlayer from src.database.models import playerHiscoreData from src.utils import logging_helpers -from fastapi import APIRouter, BackgroundTasks, Request +from fastapi import APIRouter, Request from pydantic import BaseModel from sqlalchemy.exc import InternalError, OperationalError from sqlalchemy.sql.expression import insert, update @@ -168,7 +167,7 @@ async def sqla_update_player(players: List): ) await session.execute(sql, player) dbplayer.remove(player) - except (OperationalError, InternalError) as e: + except (OperationalError, InternalError): await handle_lock(sqla_update_player, dbplayer) return @@ -185,7 +184,7 @@ async def sqla_insert_hiscore(hiscores: List): async with session.begin(): await session.execute(sql, hiscore) dbhiscores.remove(hiscore) - except (OperationalError, InternalError) as e: + except (OperationalError, InternalError): await handle_lock(sqla_insert_hiscore, dbhiscores) return diff --git a/src/api/v2/highscore.py b/src/api/v2/highscore.py index ba34ae8..300aff3 100644 --- a/src/api/v2/highscore.py +++ b/src/api/v2/highscore.py @@ -8,7 +8,6 @@ PlayerHiscoreDataLatest as RepositoryPlayerHiscoreDataLatest, ) from src.app.schemas.highscore import PlayerHiscoreData as SchemaPlayerHiscoreData -from pydantic import BaseModel router = APIRouter(tags=["Hiscore"]) diff --git a/src/app/repositories/highscore.py b/src/app/repositories/highscore.py index 937bf6a..14048c4 100644 --- a/src/app/repositories/highscore.py +++ b/src/app/repositories/highscore.py @@ -3,7 +3,6 @@ from pydantic import BaseModel, ValidationError from sqlalchemy import select, union_all from sqlalchemy.dialects.mysql import insert -from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession from sqlalchemy.sql.expression import Insert, Select, and_ diff --git a/src/app/repositories/player.py b/src/app/repositories/player.py index a35ded0..f7cb5a9 100644 --- a/src/app/repositories/player.py +++ b/src/app/repositories/player.py @@ -1,9 +1,9 @@ import logging from pydantic import ValidationError -from sqlalchemy import delete, insert, select, update +from sqlalchemy import insert, select, update from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession -from sqlalchemy.sql.expression import Delete, Insert, Select, Update, and_ +from sqlalchemy.sql.expression import Insert, Select, Update from src.app.schemas.player import Player as SchemaPlayer from src.database.database import PLAYERDATA_ENGINE diff --git a/src/core/server.py b/src/core/server.py index 16236f5..e95feb2 100644 --- a/src/core/server.py +++ b/src/core/server.py @@ -1,4 +1,3 @@ -import asyncio import json import logging import time @@ -10,8 +9,6 @@ from fastapi.responses import JSONResponse from src import api -from src.core import config -from src.kafka.highscore import HighscoreProcessor logger = logging.getLogger(__name__) diff --git a/src/database/functions.py b/src/database/functions.py index 7841bda..8f6e1da 100644 --- a/src/database/functions.py +++ b/src/database/functions.py @@ -17,7 +17,7 @@ # Although never directly used, the engines are imported to add a permanent reference # to these entities to prevent the # garbage collector from trying to dispose of our engines. -from src.database.database import PLAYERDATA_ENGINE, Engine, EngineType +from src.database.database import PLAYERDATA_ENGINE, Engine from src.database.models import ApiPermission, ApiUsage, ApiUser, ApiUserPerm logger = logging.getLogger(__name__) @@ -186,15 +186,15 @@ async def verify_token(token: str, verification: str, route: str = None) -> bool # If len api_user == 0; user does not have necessary permissions if len(api_user) == 0: - raise HTTPException(status_code=401, detail=f"Insufficent Permissions") + raise HTTPException(status_code=401, detail="Insufficent Permissions") api_user = api_user[0] if api_user["is_active"] != 1: - raise HTTPException(status_code=403, detail=f"Insufficent Permissions") + raise HTTPException(status_code=403, detail="Insufficent Permissions") if (len(usage_data) > api_user["ratelimit"]) and (api_user["ratelimit"] != -1): - raise HTTPException(status_code=429, detail=f"Your Ratelimit has been reached.") + raise HTTPException(status_code=429, detail="Your Ratelimit has been reached.") return True diff --git a/src/kafka/highscore.py b/src/kafka/highscore.py index c62708e..806295c 100644 --- a/src/kafka/highscore.py +++ b/src/kafka/highscore.py @@ -1,7 +1,5 @@ -import json import logging -from aiokafka import AIOKafkaConsumer, ConsumerRecord, TopicPartition from src.app.repositories.highscore import PlayerHiscoreData as RepoHiscore from src.app.repositories.player import Player as RepositoryPlayer @@ -9,7 +7,6 @@ from src.app.schemas.player import Player as SchemaPlayer from src.database.models import playerHiscoreData as dbHiscore from src.kafka.modules.kafka_consumer import KafkaMessageConsumer -from src.kafka.modules.kafka_producer import KafkaMessageProducer from asyncio import Queue import asyncio import time diff --git a/src/kafka/modules/abc.py b/src/kafka/modules/abc.py index 81dc075..e664761 100644 --- a/src/kafka/modules/abc.py +++ b/src/kafka/modules/abc.py @@ -1,5 +1,4 @@ from abc import ABC, abstractmethod -from aiokafka import ConsumerRecord class AbstractConsumer(ABC): diff --git a/tests/test_feedback.py b/tests/test_feedback.py index 9f5d04c..a8a542b 100644 --- a/tests/test_feedback.py +++ b/tests/test_feedback.py @@ -3,7 +3,6 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import pytest def test_get_feedback(test_client): @@ -24,5 +23,5 @@ def test_get_feedback(test_client): # type check if response.ok: - error = f"Invalid response return type, expected list[dict]" + error = "Invalid response return type, expected list[dict]" assert isinstance(response.json(), list), error diff --git a/tests/test_hiscore.py b/tests/test_hiscore.py index 9312b55..007276b 100644 --- a/tests/test_hiscore.py +++ b/tests/test_hiscore.py @@ -30,7 +30,7 @@ def test_get_player_hiscore_data(test_client): # type check if response.ok: - error = f"Invalid response return type, expected list[dict]" + error = "Invalid response return type, expected list[dict]" assert isinstance(response.json(), list), error @@ -58,7 +58,7 @@ def test_get_latest_hiscore_data_for_an_account(test_client): # type check if response.ok: - error = f"Invalid response return type, expected list[dict]" + error = "Invalid response return type, expected list[dict]" assert isinstance(response.json(), list), error @@ -114,7 +114,7 @@ def test_get_account_hiscore_xp_change(test_client): # type check if response.ok: - error = f"Invalid response return type, expected list[dict]" + error = "Invalid response return type, expected list[dict]" assert isinstance(response.json(), list), error diff --git a/tests/test_label.py b/tests/test_label.py index 24fb364..9dcc432 100644 --- a/tests/test_label.py +++ b/tests/test_label.py @@ -3,7 +3,6 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import pytest from src.core.config import token @@ -29,5 +28,5 @@ def test_get_labels(test_client): # type check if response.ok: - error = f"Invalid response return type, expected list[dict]" + error = "Invalid response return type, expected list[dict]" assert isinstance(response.json(), list), error diff --git a/tests/test_player.py b/tests/test_player.py index e342f4b..10f5267 100644 --- a/tests/test_player.py +++ b/tests/test_player.py @@ -30,5 +30,5 @@ def test_get_player_information(test_client): # type check if response.ok: - error = f"Invalid response return type, expected list[dict]" + error = "Invalid response return type, expected list[dict]" assert isinstance(response.json(), list), error diff --git a/tests/test_report.py b/tests/test_report.py index aff0b48..ffe3673 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -3,7 +3,6 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import pytest from src.core.config import token @@ -29,5 +28,5 @@ def test_report_count(test_client): # type check if response.ok: - error = f"Invalid response return type, expected list[dict]" + error = "Invalid response return type, expected list[dict]" assert isinstance(response.json(), list), error diff --git a/tests/test_scraper.py b/tests/test_scraper.py index 03e3f26..f6916d3 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -22,5 +22,5 @@ def test_scraper_players(test_client): # type check if response.ok: - error = f"Invalid response return type, expected list[dict]" + error = "Invalid response return type, expected list[dict]" assert isinstance(response.json(), list), error diff --git a/tests/utils/test_logging_helpers.py b/tests/utils/test_logging_helpers.py index 0ad894b..b20ebad 100644 --- a/tests/utils/test_logging_helpers.py +++ b/tests/utils/test_logging_helpers.py @@ -8,7 +8,7 @@ from api.routers import hiscore from src.utils import logging_helpers -request_path = f"/v1/hiscore/Latest" +request_path = "/v1/hiscore/Latest" request_params = {"player_id": 1, "token": token} From 42e3755867371b90f42d7932444e391ff421ebcc Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sun, 28 Jul 2024 19:15:59 +0200 Subject: [PATCH 3/4] query report_sighting --- docker-compose.yml | 47 ++- mysql/Dockerfile | 3 + mysql/docker-entrypoint-initdb.d/00_init.sql | 1 + .../docker-entrypoint-initdb.d/01_tables.sql | 128 ++++++++ mysql/docker-entrypoint-initdb.d/02_data.sql | 275 ++++++++++++++++++ src/api/v1/report.py | 100 ++++++- 6 files changed, 516 insertions(+), 38 deletions(-) create mode 100644 mysql/Dockerfile create mode 100644 mysql/docker-entrypoint-initdb.d/00_init.sql create mode 100644 mysql/docker-entrypoint-initdb.d/01_tables.sql create mode 100644 mysql/docker-entrypoint-initdb.d/02_data.sql diff --git a/docker-compose.yml b/docker-compose.yml index 06ebfcf..adb2feb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,34 +1,28 @@ version: '3' services: mysql: + container_name: database build: - context: ../bot-detector-mysql - dockerfile: Dockerfile - image: bot-detector/bd-mysql:latest + context: ./mysql + image: mysql environment: - MYSQL_ROOT_PASSWORD=root_bot_buster - - MYSQL_USER=botssuck - - MYSQL_PASSWORD=botdetector volumes: - - ../bot-detector-mysql/mount:/var/lib/mysql - - '../bot-detector-mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d' + - ./mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + - ./mysql/conf.d:/etc/mysql/conf.d + # - ./mysql/mount:/var/lib/mysql # creates persistence ports: - - "3306:3306" + - 3307:3306 networks: - botdetector-network - - kafka: - image: bitnami/kafka:latest - environment: - - ALLOW_PLAINTEXT_LISTENER=yes - - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094 - - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094 - - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true - ports: - - 9094:9094 - networks: - - botdetector-network + healthcheck: + test: "mysqladmin ping -h localhost -u root -proot_bot_buster" + # during this period fails are not considered + start_period: 30s + # time between cmd + interval: 30s + # time given to the cmd + timeout: 5s api: build: @@ -41,18 +35,17 @@ services: command: uvicorn src.core.server:app --host 0.0.0.0 --reload --reload-include src/* container_name: bd-dev-api environment: - - sql_uri=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata - - discord_sql_uri=mysql+asyncmy://root:root_bot_buster@mysql:3306/discord + - sql_uri=mysql+asyncmy://root:root_bot_buster@mysql/playerdata + - discord_sql_uri=mysql+asyncmy://root:root_bot_buster@mysql/discord - token=verify_ban volumes: - - ../Bot-Detector-Core-Files/src:/project/src:rw + - ./src:/project/src:rw ports: - 5000:5000 networks: - botdetector-network depends_on: - - mysql - - kafka - + mysql: + condition: service_healthy networks: botdetector-network: diff --git a/mysql/Dockerfile b/mysql/Dockerfile new file mode 100644 index 0000000..4244cbe --- /dev/null +++ b/mysql/Dockerfile @@ -0,0 +1,3 @@ +FROM mysql:8.0.32 + +EXPOSE 3306 \ No newline at end of file diff --git a/mysql/docker-entrypoint-initdb.d/00_init.sql b/mysql/docker-entrypoint-initdb.d/00_init.sql new file mode 100644 index 0000000..7d82066 --- /dev/null +++ b/mysql/docker-entrypoint-initdb.d/00_init.sql @@ -0,0 +1 @@ +CREATE DATABASE playerdata; \ No newline at end of file diff --git a/mysql/docker-entrypoint-initdb.d/01_tables.sql b/mysql/docker-entrypoint-initdb.d/01_tables.sql new file mode 100644 index 0000000..9d59bbb --- /dev/null +++ b/mysql/docker-entrypoint-initdb.d/01_tables.sql @@ -0,0 +1,128 @@ +USE playerdata; +-- Create a table for Players +CREATE TABLE Players ( + id INT PRIMARY KEY AUTO_INCREMENT, + name TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP, + possible_ban BOOLEAN, + confirmed_ban BOOLEAN, + confirmed_player BOOLEAN, + label_id INTEGER, + label_jagex INTEGER, + ironman BOOLEAN, + hardcore_ironman BOOLEAN, + ultimate_ironman BOOLEAN, + normalized_name TEXT +); + +CREATE TABLE `Labels` ( + `id` int NOT NULL AUTO_INCREMENT, + `label` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `Unique_label` (`label`) USING BTREE +) +; + +-- Create a table for Reports +CREATE TABLE Reports ( + ID BIGINT PRIMARY KEY AUTO_INCREMENT, + created_at TIMESTAMP, + reportedID INT, + reportingID INT, + region_id INT, + x_coord INT, + y_coord INT, + z_coord INT, + timestamp TIMESTAMP, + manual_detect SMALLINT, + on_members_world INT, + on_pvp_world SMALLINT, + world_number INT, + equip_head_id INT, + equip_amulet_id INT, + equip_torso_id INT, + equip_legs_id INT, + equip_boots_id INT, + equip_cape_id INT, + equip_hands_id INT, + equip_weapon_id INT, + equip_shield_id INT, + equip_ge_value BIGINT, + CONSTRAINT `FK_Reported_Players_id` FOREIGN KEY (`reportedID`) REFERENCES `Players` (`id`) ON DELETE RESTRICT ON UPDATE RESTRICT, + CONSTRAINT `FK_Reporting_Players_id` FOREIGN KEY (`reportingID`) REFERENCES `Players` (`id`) ON DELETE RESTRICT ON UPDATE RESTRICT +); +-- Create a table for Predictions +CREATE TABLE Predictions ( + id INT PRIMARY KEY AUTO_INCREMENT, + name VARCHAR(12), + prediction VARCHAR(50), + created TIMESTAMP, + predicted_confidence DECIMAL(5, 2), + real_player DECIMAL(5, 2) DEFAULT 0, + pvm_melee_bot DECIMAL(5, 2) DEFAULT 0, + smithing_bot DECIMAL(5, 2) DEFAULT 0, + magic_bot DECIMAL(5, 2) DEFAULT 0, + fishing_bot DECIMAL(5, 2) DEFAULT 0, + mining_bot DECIMAL(5, 2) DEFAULT 0, + crafting_bot DECIMAL(5, 2) DEFAULT 0, + pvm_ranged_magic_bot DECIMAL(5, 2) DEFAULT 0, + pvm_ranged_bot DECIMAL(5, 2) DEFAULT 0, + hunter_bot DECIMAL(5, 2) DEFAULT 0, + fletching_bot DECIMAL(5, 2) DEFAULT 0, + clue_scroll_bot DECIMAL(5, 2) DEFAULT 0, + lms_bot DECIMAL(5, 2) DEFAULT 0, + agility_bot DECIMAL(5, 2) DEFAULT 0, + wintertodt_bot DECIMAL(5, 2) DEFAULT 0, + runecrafting_bot DECIMAL(5, 2) DEFAULT 0, + zalcano_bot DECIMAL(5, 2) DEFAULT 0, + woodcutting_bot DECIMAL(5, 2) DEFAULT 0, + thieving_bot DECIMAL(5, 2) DEFAULT 0, + soul_wars_bot DECIMAL(5, 2) DEFAULT 0, + cooking_bot DECIMAL(5, 2) DEFAULT 0, + vorkath_bot DECIMAL(5, 2) DEFAULT 0, + barrows_bot DECIMAL(5, 2) DEFAULT 0, + herblore_bot DECIMAL(5, 2) DEFAULT 0, + zulrah_bot DECIMAL(5, 2) DEFAULT 0, + gauntlet_bot DECIMAL(5, 2) DEFAULT 0, + nex_bot DECIMAL(5, 2) DEFAULT 0, + unknown_bot DECIMAL(5, 2) DEFAULT 0 +); +-- Create a table for Feedback +CREATE TABLE PredictionsFeedback ( + id INT PRIMARY KEY AUTO_INCREMENT, + ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + voter_id INT NOT NULL, + subject_id INT NOT NULL, + prediction VARCHAR(50) NOT NULL, + confidence FLOAT NOT NULL, + vote INT NOT NULL DEFAULT '0', + feedback_text TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci, + reviewed TINYINT NOT NULL DEFAULT '0', + reviewer_id INT DEFAULT NULL, + user_notified TINYINT NOT NULL DEFAULT '0', + proposed_label VARCHAR(50) DEFAULT NULL, + UNIQUE KEY Unique_Vote ( + prediction, + subject_id, + voter_id + ) USING BTREE, + CONSTRAINT `FK_Subject_ID` FOREIGN KEY (`subject_id`) REFERENCES `Players` (`id`) ON DELETE RESTRICT ON UPDATE RESTRICT, + CONSTRAINT `FK_Voter_ID` FOREIGN KEY (`voter_id`) REFERENCES `Players` (`id`) ON DELETE RESTRICT ON UPDATE RESTRICT +); + +CREATE TABLE report_sighting ( + `report_sighting_id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `reporting_id` INT UNSIGNED NOT NULL, + `reported_id` INT UNSIGNED NOT NULL, + `manual_detect` TINYINT(1) DEFAULT 0, + PRIMARY key (`report_sighting_id`), + UNIQUE KEY unique_sighting (`reporting_id`, `reported_id`, `manual_detect`), + KEY idx_reported_id (`reported_id`) +); + +CREATE TABLE report_migrated ( + `reporting_id` INT UNSIGNED NOT NULL, + `migrated` TINYINT UNSIGNED, + PRIMARY KEY (`reporting_id`) +); \ No newline at end of file diff --git a/mysql/docker-entrypoint-initdb.d/02_data.sql b/mysql/docker-entrypoint-initdb.d/02_data.sql new file mode 100644 index 0000000..0b4e593 --- /dev/null +++ b/mysql/docker-entrypoint-initdb.d/02_data.sql @@ -0,0 +1,275 @@ +USE playerdata; + +-- Insert data into the Players table +DELIMITER $$ + +CREATE PROCEDURE InsertRandomPlayers(IN NUM INT, IN possible_ban BOOL, IN confirmed_ban BOOL, IN confirmed_player BOOL) +BEGIN + DECLARE i INT DEFAULT 1; + + WHILE i <= NUM DO + INSERT INTO Players ( + name, + created_at, + updated_at, + possible_ban, + confirmed_ban, + confirmed_player, + label_id, + label_jagex, + ironman, + hardcore_ironman, + ultimate_ironman, + normalized_name + ) + SELECT + UUID() AS name, -- updated later + NOW() AS created_at, -- updated later + NOW() AS updated_at, -- updated later + possible_ban, + confirmed_ban, + confirmed_player, + 0 AS label_id, + ROUND(RAND() * 1) AS label_jagex, -- doesn't matter? + null AS ironman, + null AS hardcore_ironman, + null AS ultimate_ironman, + UUID() AS normalized_name -- updated later + FROM dual; + + SET i = i + 1; + END WHILE; +END $$ + +DELIMITER ; + + +call InsertRandomPlayers(100, 1,0,0); +call InsertRandomPlayers(100, 1,1,0); +call InsertRandomPlayers(100, 0,0,1); + +UPDATE Players +SET + name = CONCAT('player', id), + normalized_name = CONCAT('player', id) +; + + +-- Insert data into the Reports table +INSERT INTO + Reports ( + created_at, + reportedID, + reportingID, + region_id, + x_coord, + y_coord, + z_coord, + timestamp, + manual_detect, + on_members_world, + on_pvp_world, + world_number, + equip_head_id, + equip_amulet_id, + equip_torso_id, + equip_legs_id, + equip_boots_id, + equip_cape_id, + equip_hands_id, + equip_weapon_id, + equip_shield_id, + equip_ge_value + ) +SELECT + NOW() - INTERVAL FLOOR(RAND(42) * 365) DAY AS created_at, + p1.id AS reportedID, + p2.id AS reportingID, + ROUND(RAND(42) * 1000) AS region_id, + -- Random region_id + ROUND(RAND(42) * 1000) AS x_coord, + -- Random x_coord + ROUND(RAND(42) * 1000) AS y_coord, + -- Random y_coord + ROUND(RAND(42) * 1000) AS z_coord, + -- Random z_coord + NOW() - INTERVAL FLOOR(RAND(42) * 365) DAY AS timestamp, + ROUND(RAND(42)) AS manual_detect, + -- Random manual_detect (0 or 1) + ROUND(RAND(42) * 1000) AS on_members_world, + -- Random on_members_world + ROUND(RAND(42)) AS on_pvp_world, + -- Random on_pvp_world (0 or 1) + ROUND(RAND(42) * 100) AS world_number, + -- Random world_number + ROUND(RAND(42) * 1000) AS equip_head_id, + -- Random equip_head_id + ROUND(RAND(42) * 1000) AS equip_amulet_id, + -- Random equip_amulet_id + ROUND(RAND(42) * 1000) AS equip_torso_id, + -- Random equip_torso_id + ROUND(RAND(42) * 1000) AS equip_legs_id, + -- Random equip_legs_id + ROUND(RAND(42) * 1000) AS equip_boots_id, + -- Random equip_boots_id + ROUND(RAND(42) * 1000) AS equip_cape_id, + -- Random equip_cape_id + ROUND(RAND(42) * 1000) AS equip_hands_id, + -- Random equip_hands_id + ROUND(RAND(42) * 1000) AS equip_weapon_id, + -- Random equip_weapon_id + ROUND(RAND(42) * 1000) AS equip_shield_id, + -- Random equip_shield_id + ROUND(RAND(42) * 10000) AS equip_ge_value -- Random equip_ge_value +FROM Players p1 + CROSS JOIN Players p2 +WHERE + p1.id <> p2.id -- Ensure reportedID and reportingID are different +ORDER BY + RAND(42) -- Randomize the order of the combinations +LIMIT + 10000 -- Limit the number of combinations to insert +; + +INSERT INTO Predictions (name, predicted_confidence, prediction, created) +SELECT + name, + FLOOR(RAND(6)*(100-1)+1), + CASE FLOOR(RAND(6)*(25-1)+1) + WHEN 1 THEN 'real_player' + WHEN 2 THEN 'pvm_melee_bot' + WHEN 3 THEN 'smithing_bot' + WHEN 4 THEN 'magic_bot' + WHEN 5 THEN 'fishing_bot' + WHEN 6 THEN 'mining_bot' + WHEN 7 THEN 'crafting_bot' + WHEN 8 THEN 'pvm_ranged_magic_bot' + WHEN 9 THEN 'pvm_ranged_bot' + WHEN 10 THEN 'hunter_bot' + WHEN 11 THEN 'fletching_bot' + WHEN 12 THEN 'clue_scroll_bot' + WHEN 13 THEN 'lms_bot' + WHEN 14 THEN 'agility_bot' + WHEN 15 THEN 'wintertodt_bot' + WHEN 16 THEN 'runecrafting_bot' + WHEN 17 THEN 'zalcano_bot' + WHEN 18 THEN 'woodcutting_bot' + WHEN 19 THEN 'thieving_bot' + WHEN 20 THEN 'soul_wars_bot' + WHEN 21 THEN 'cooking_bot' + WHEN 22 THEN 'vorkath_bot' + WHEN 23 THEN 'barrows_bot' + WHEN 24 THEN 'herblore_bot' + ELSE 'unknown_bot' + END , + FROM_UNIXTIME( + TIMESTAMPDIFF(SECOND, '2020-01-01 00:00:00', '2022-12-31 23:59:59') * RAND(42) + + UNIX_TIMESTAMP('2020-01-01 00:00:00') + ) +FROM `Players` +where 1=1 + AND name not LIKE 'anonymoususer%' +ORDER BY RAND(42) +LIMIT 250 +; + +INSERT INTO playerdata.Labels (label) VALUES + ('Agility_bot'), + ('Barrows_bot'), + ('Blast_mine_bot'), + ('Clue_Scroll_bot'), + ('Construction_Magic_bot'), + ('Cooking_bot'), + ('Crafting_bot'), + ('Fishing_bot'), + ('Fishing_Cooking_bot'), + ('Fletching_bot'), + ('Herblore_bot'), + ('Hunter_bot'), + ('LMS_bot'), + ('Mage_Guild_Store_bot'), + ('Magic_bot'), + ('Mining_bot'), + ('mort_myre_fungus_bot'), + ('Phosani_bot'), + ('PVM_Melee_bot'), + ('PVM_Ranged_bot'), + ('PVM_Ranged_Magic_bot'), + ('Real_Player'), + ('Runecrafting_bot'), + ('Smithing_bot'), + ('Soul_Wars_bot'), + ('temp_real_player'), + ('test_label'), + ('Thieving_bot'), + ('Unknown'), + ('Unknown_bot'), + ('Vorkath_bot'), + ('Wintertodt_bot'), + ('Woodcutting_bot'), + ('Woodcutting_Firemaking_bot'), + ('Woodcutting_Mining_bot'), + ('Zalcano_bot'), + ('Zulrah_bot') +; + +INSERT INTO PredictionsFeedback (voter_id, subject_id, prediction, confidence, feedback_text, vote, proposed_label) +SELECT + pl1.id AS voter_id, + pl2.id AS subject_id, + pr.prediction, + FLOOR(RAND(6)*(100-1)+1)/100 AS confidence, -- Generate a random confidence value between 0 and 1 + "" AS feedback_text, + CASE + WHEN FLOOR(RAND(6)*(100-1)+1) < 33 THEN -1 + WHEN FLOOR(RAND(6)*(100-1)+1) < 66 THEN 0 + ELSE 1 + END AS vote, + (SELECT label FROM Labels ORDER BY RAND(42) LIMIT 1) AS proposed_label +FROM (SELECT * FROM Players ORDER BY RAND(42) LIMIT 1000) pl1 +JOIN (SELECT * FROM Players ORDER BY RAND(42) LIMIT 1000) pl2 ON pl1.id <> pl2.id +JOIN Predictions pr ON pr.id = pl2.id +ORDER BY RAND(42) +LIMIT 100; + +UPDATE PredictionsFeedback + SET proposed_label = prediction +WHERE 1=1 + AND vote = 1 +; + +DELIMITER $$ + +INSERT INTO Players ( + name, + created_at, + updated_at, + possible_ban, + confirmed_ban, + confirmed_player, + label_id, + label_jagex +) VALUES + ("anonymoususer 382e728f 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0), + ("anonymoususer 382e7259 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0), + ("anonymoususer 382e7221 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0), + ("anonymoususer 382e71ee 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0), + ("anonymoususer 382e71bb 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0), + ("anonymoususer 382e7179 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0), + ("anonymoususer 382e7133 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0), + ("anonymoususer 382e70ef 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0), + ("anonymoususer 382e7089 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0), + ("anonymoususer 382e6def 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0) +; + +UPDATE `Players` +SET + created_at = NOW() - INTERVAL FLOOR(RAND(42) * 365) DAY, + updated_at = NOW() - INTERVAL FLOOR(RAND(41) * 365) DAY +; +UPDATE `Players` +SET + name=replace(name,'-',' '), + normalized_name=replace(name,'-',' ') +WHERE name LIKE 'anonymoususer%' +; \ No newline at end of file diff --git a/src/api/v1/report.py b/src/api/v1/report.py index 11fe9d0..dded7b0 100644 --- a/src/api/v1/report.py +++ b/src/api/v1/report.py @@ -14,7 +14,7 @@ from pydantic.fields import Field from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import aliased -from sqlalchemy.sql import func +from sqlalchemy.sql import func, text from sqlalchemy.sql.expression import Select, select, update import aiohttp import traceback @@ -24,6 +24,7 @@ upper_gear_cost = 1_000_000_000_000 + # TODO: cleanup thse functions class equipment(BaseModel): equip_head_id: int = Field(0, ge=0) @@ -99,6 +100,7 @@ async def get_reports( data = functions.sqlalchemy_result(data) return data.rows2dict() + @router.put("/report", tags=["Report"]) async def update_reports( old_user_id: int, new_user_id: int, token: str, request: Request @@ -125,24 +127,32 @@ async def update_reports( return {"detail": f"{data.rowcount} rows updated to reportingID = {new_user_id}."} + async def insert_report_v2(detections: list[detection]): - url = 'http://public-api-svc.bd-prd.svc:5000/v2/report' + url = "http://public-api-svc.bd-prd.svc:5000/v2/report" try: data = [d.dict() for d in detections] async with aiohttp.ClientSession() as session: async with session.post(url=url, json=data) as response: if not response.ok: response_text = await response.text() - logger.warning(f"Request to {url} failed with status {response.status} and response: {response_text}") + logger.warning( + f"Request to {url} failed with status {response.status} and response: {response_text}" + ) except aiohttp.ClientError as e: # Log client-specific errors with request details - logger.error(f"Client error during request to {url} with payload {data}: {str(e)}") + logger.error( + f"Client error during request to {url} with payload {data}: {str(e)}" + ) logger.debug(f"Traceback: {traceback.format_exc()}") except Exception as e: # Log general exceptions with traceback - logger.error(f"Unexpected error during request to {url} with payload {data}: {str(e)}") + logger.error( + f"Unexpected error during request to {url} with payload {data}: {str(e)}" + ) logger.debug(f"Traceback: {traceback.format_exc()}") + @router.post("/report", status_code=status.HTTP_201_CREATED, tags=["Report"]) async def insert_report( detections: List[detection], @@ -154,7 +164,8 @@ async def insert_report( asyncio.create_task(insert_report_v2(detections)) return {"detail": "ok"} -async def select_report_count_v1(name:str, manual_detect:int): + +async def select_report_count_v1(name: str, manual_detect: int): name = await functions.to_jagex_name(name) voter: Player = aliased(Player, name="voter") @@ -191,17 +202,84 @@ async def select_report_count_v1(name:str, manual_detect:int): data = [{k: v for k, v in zip(keys, d)} for d in data] return data -@router.get("/report/count", tags=["Report"]) -async def get_report_count_v1(name: str): + +async def select_or_insert_migration(name: str): + sql_select = """ + SELECT + migrated + FROM report_migrated mg + JOIN Players pl ON mg.reporting_id = pl.id + WHERE pl.name = :name """ + sql_insert = """ + INSERT report_migrated (reporting_id, migrated) + SELECT id, 0 as migrated FROM Players pl where pl.name = :name + """ + + params = {"name": name} + async with PLAYERDATA_ENGINE.get_session() as session: + session: AsyncSession = session + async with session.begin(): + data = await session.execute(text(sql_select), params=params) + result = data.mappings().first() + if not result: + logger.debug(f"start migration: {name}") + await session.execute(text(sql_insert), params=params) + data = await session.execute(text(sql_select), params=params) + result = data.mappings().first() + return result + + +async def select_report_count_v2(name: str, manual_detect: int): + sql_select = """ + select + count(sr.reporting_id) as count, + subject.confirmed_ban, + subject.possible_ban, + subject.confirmed_player + from report_sighting sr + join Players voter ON sr.reporting_id = voter.id + join Players subject ON sr.reported_id = subject.id + WHERE voter.name = :name and sr.manual_detect = :manual_detect + GROUP BY + subject.confirmed_ban, + subject.possible_ban, + subject.confirmed_player """ - data = await select_report_count_v1(name=name, manual_detect=0) + params = {"name": name, "manual_detect": manual_detect} + + async with PLAYERDATA_ENGINE.get_session() as session: + session: AsyncSession = session + async with session.begin(): + data = await session.execute(text(sql_select), params=params) + result = data.mappings().all() + return result + +@router.get("/report/count", tags=["Report"]) +async def get_report_count_v1(name: str): + """ """ + migrated_record = await select_or_insert_migration(name=name) + is_migrated = migrated_record.get("migrated") + if is_migrated: + logger.debug("v2") + data = await select_report_count_v2(name=name, manual_detect=0) + else: + logger.debug("v1") + data = await select_report_count_v1(name=name, manual_detect=0) return data + @router.get("/report/manual/count", tags=["Report"]) async def get_report_manual_count_v1(name: str): """ Get the calculated player report count """ - data = await select_report_count_v1(name=name, manual_detect=1) - return data \ No newline at end of file + migrated_record = await select_or_insert_migration(name=name) + is_migrated = migrated_record.get("migrated") + if is_migrated: + logger.debug("v2") + data = await select_report_count_v2(name=name, manual_detect=1) + else: + logger.debug("v1") + data = await select_report_count_v1(name=name, manual_detect=1) + return data From 449acd9480f1beda65375d5eb5ea6ff54e9665ab Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sun, 28 Jul 2024 19:29:33 +0200 Subject: [PATCH 4/4] extra logging --- src/api/v1/report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/v1/report.py b/src/api/v1/report.py index dded7b0..38fff08 100644 --- a/src/api/v1/report.py +++ b/src/api/v1/report.py @@ -277,9 +277,9 @@ async def get_report_manual_count_v1(name: str): migrated_record = await select_or_insert_migration(name=name) is_migrated = migrated_record.get("migrated") if is_migrated: - logger.debug("v2") + logger.debug(f"v2 - {name=}") data = await select_report_count_v2(name=name, manual_detect=1) else: - logger.debug("v1") + logger.debug(f"v1 - {name=}") data = await select_report_count_v1(name=name, manual_detect=1) return data