diff --git a/src/lambda_functions/otto-bot-player-load/.gitignore b/src/lambda_functions/otto-bot-player-load/.gitignore new file mode 100644 index 0000000..66d4f40 --- /dev/null +++ b/src/lambda_functions/otto-bot-player-load/.gitignore @@ -0,0 +1,2 @@ +*/ +deployment.zip \ No newline at end of file diff --git a/src/lambda_functions/otto-bot-player-load/lambda_function.py b/src/lambda_functions/otto-bot-player-load/lambda_function.py new file mode 100644 index 0000000..54771b9 --- /dev/null +++ b/src/lambda_functions/otto-bot-player-load/lambda_function.py @@ -0,0 +1,88 @@ +from urllib import parse as urlparse +import base64 +import json +import os +import requests +import boto3 + +client = boto3.client('lambda') + +def lambda_handler(event, context): + + print(event) + + msg_map = dict(urlparse.parse_qsl(base64.b64decode(str(event['body'])).decode('ascii'))) + + print(event['requestContext']) + + msg_map['stage'] = event['requestContext']['stage'] + + print(msg_map) + + payload_json = msg_map.get('payload', None) + if payload_json: + payload = json.loads(msg_map['payload']) + search_value = payload.get('value', None) + if search_value: + search_parameters = { + "search_name" : search_value, + "stage" : msg_map['stage'] + } + + search_version = os.environ[f'{msg_map["stage"]}_search_version'] + + response = client.invoke( + FunctionName = os.environ['player_search_lambda_arn'], + InvocationType = 'RequestResponse', + Payload = json.dumps(search_parameters), + Qualifier = search_version + ) + + lambda_response = json.load(response['Payload']) + + if 'body' in lambda_response: + player_list = json.loads(lambda_response['body']) + options = list() + for player_dict in player_list: + name = f"{', '.join([player_dict['name'], player_dict['positions'], player_dict.get('org', 'FA')])}" + value = int(player_dict['_id']) + + options.append( + { + 'text': { + 'type': 'plain_text', + 'text': name + }, + 'value': str(value) + } + ) + + response = { + 'options': options + } + + print(json.dumps(response)) + return { + 'statusCode': 200, + 'body': json.dumps(response), + "headers": { + 'Content-Type': 'application/json', + } + } + + + return { + 'statusCode': 200 + } + + + + return { + 'statusCode': 404, + 'body': json.dumps('No search value provided') + } + + return { + 'statusCode': 404, + 'body': json.dumps('No payload provided') + } \ No newline at end of file diff --git a/src/lambda_functions/otto-bot-queue-processor/lambda_function.py b/src/lambda_functions/otto-bot-queue-processor/lambda_function.py index c993855..d88931d 100644 --- a/src/lambda_functions/otto-bot-queue-processor/lambda_function.py +++ b/src/lambda_functions/otto-bot-queue-processor/lambda_function.py @@ -6,6 +6,187 @@ client = boto3.client('lambda') +TRADE_TEMPLATE = """ + "blocks": [ + { + "type": "section", + "block_id": "team_1", + "text": { + "type": "mrkdwn", + "text": "Team 1 Players" + }, + "accessory": { + "type": "multi_external_select", + "placeholder": { + "type": "plain_text", + "text": "Select options", + "emoji": true + }, + "action_id": "player-search-action-1" + } + }, + { + "type": "section", + "block_id": "team_2", + "text": { + "type": "mrkdwn", + "text": "Team 2 Players" + }, + "accessory": { + "type": "multi_external_select", + "placeholder": { + "type": "plain_text", + "text": "Select options", + "emoji": true + }, + "action_id": "player-search-action-2" + } + }, + { + "type": "input", + "block_id": "league_number", + "element": { + "type": "plain_text_input", + "action_id": "plain_text_input-action" + }, + "label": { + "type": "plain_text", + "text": "League Number", + "emoji": false + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Format:" + }, + "block_id": "format_block", + "accessory": { + "type": "static_select", + "placeholder": { + "type": "plain_text", + "text": "Select scoring format" + }, + "initial_option": { + "text": { + "type": "plain_text", + "text": "FanGraphs Points" + }, + "value": "3" + }, + "options": [ + { + "text": { + "type": "plain_text", + "text": "FanGraphs Points" + }, + "value": "3" + }, + { + "text": { + "type": "plain_text", + "text": "SABR Points" + }, + "value": "4" + }, + { + "text": { + "type": "plain_text", + "text": "Classic 4x4" + }, + "value": "1" + }, + { + "text": { + "type": "plain_text", + "text": "Old School 5x5" + }, + "value": "2" + }, + { + "text": { + "type": "plain_text", + "text": "H2H FanGraphs Points" + }, + "value": "5" + }, + { + "text": { + "type": "plain_text", + "text": "H2H SABR Points" + }, + "value": "6" + } + ], + "action_id": "format_select_action" + } + }, + { + "type": "input", + "block_id": "loan_type", + "element": { + "type": "radio_buttons", + "initial_option": { + "text": { + "type": "plain_text", + "text": "Full Loan", + "emoji": true + }, + "value": "full-loan", + }, + "options": [ + { + "text": { + "type": "plain_text", + "text": "Full Loan", + "emoji": true + }, + "value": "full-loan", + + }, + { + "text": { + "type": "plain_text", + "text": "No Loan", + "emoji": true + }, + "value": "no-loan" + }, + { + "text": { + "type": "plain_text", + "text": "Other", + "emoji": true + }, + "value": "partial-loan" + } + ], + "action_id": "checkboxes-action" + }, + "label": { + "type": "plain_text", + "text": "Loan amount", + "emoji": true + } + }, + { + "type": "input", + "block_id": "partial_loan", + "element": { + "type": "plain_text_input", + "action_id": "plain_text_input-action" + }, + "label": { + "type": "plain_text", + "text": "Partial loan amount", + "emoji": true + }, + "optional": true + } + ] +""" + VIEW_TEMPLATE = """ { "type": "modal", @@ -13,7 +194,7 @@ "private_metadata": , "title": { "type": "plain_text", - "text": "Otto-bot Wizard", + "text": "", "emoji": true }, "submit": { @@ -135,6 +316,13 @@ "text": "*FanGraphs Player Page*" }, "value": "fg" + }, + { + "text": { + "type": "mrkdwn", + "text": "*StatCast Player Page*" + }, + "value": "sc" } ], "initial_options": [{ @@ -152,6 +340,8 @@ def lambda_handler(event, context): + print(event) + if 'command' in event: msg_map = event else: @@ -162,15 +352,45 @@ def lambda_handler(event, context): if msg_map['command'].startswith('/link-player'): return show_player(msg_map) + if msg_map['command'].startswith('/trade-review'): + return show_trade_window(msg_map) + return { 'statusCode': 404, 'body': json.dumps('Not a valid command') } +def mongo_client_warm(msg_map): + search_parameters = { + "league_id" : '160' + } + + search_version = os.environ[f'{msg_map["stage"]}_search_version'] + + _ = client.invoke( + FunctionName = os.environ['player_search_lambda_arn'], + InvocationType = 'RequestResponse', + Payload = json.dumps(search_parameters), + Qualifier = search_version + ) + +def show_trade_window(msg_map): + print('in show trade window') + #Warm MongoClient + mongo_client_warm(msg_map) + view = create_view(msg_map, TRADE_TEMPLATE, 'Trade Review Wizard') + update_res = update_view(msg_map, view) + print(update_res) + + return { + 'statusCode': 200 + } + def show_player(msg_map): search_parameters = { - "search_name" : msg_map['text'] + "search_name" : msg_map['text'], + "stage" : msg_map['stage'] } search_version = os.environ[f'{msg_map["stage"]}_search_version'] @@ -184,16 +404,19 @@ def show_player(msg_map): lambda_response = json.load(response['Payload']) - player_list = json.loads(lambda_response['body']) + if 'body' in lambda_response: + player_list = json.loads(lambda_response['body']) + else: + player_list = None if player_list: blocks = get_modal_response_block_show_players(player_list) else: blocks = get_empty_player_list_blocks(msg_map['text']) - view = create_view(msg_map, blocks) + view = create_view(msg_map, blocks, 'Player Search Wizard') update_res = update_view(msg_map, view) - + print('sending 200') return { 'statusCode': 200 } @@ -211,11 +434,12 @@ def update_view(msg_map, view): res = urllib.request.urlopen(request).read().decode('utf-8') return res -def create_view(msg_map, blocks): +def create_view(msg_map, blocks, title): view = VIEW_TEMPLATE view = view.replace('<callbackid>', '"' + msg_map['trigger_id'] + '"') - view = view.replace('<metadata>', '"' + msg_map['command'] +"," + msg_map['response_url'] + '"') + view = view.replace('<metadata>', '"' + msg_map['command'] +"," + msg_map['response_url'] + ',' + msg_map['channel_id']+ ',' + msg_map['team_id']+'"') view = view.replace('<blocks>', blocks) + view = view.replace('<title>', title) return view def get_empty_player_list_blocks(name): @@ -237,10 +461,12 @@ def get_modal_response_block_show_players(player_list): text_player_list = [] for player_dict in player_list: name = f"{player_dict['name']}, {player_dict['positions']}, {player_dict['org']}" - if player_dict['fg_majorleagueid']: + if 'fg_majorleagueid' in player_dict: id = f'{player_dict['ottoneu_id']},{player_dict['fg_majorleagueid']}' else: id = f'{player_dict['ottoneu_id']},{player_dict['fg_minorleagueid']}' + if 'mlbam_id' in player_dict and player_dict['mlbam_id'] != '0': + id += f',{player_dict['mlbam_id']}' text_player_list.append(f'{{ \ "text": {{ \ "type": "plain_text", \ diff --git a/src/lambda_functions/otto-bot/.gitignore b/src/lambda_functions/otto-bot/.gitignore new file mode 100644 index 0000000..53b66ce --- /dev/null +++ b/src/lambda_functions/otto-bot/.gitignore @@ -0,0 +1,3 @@ +*/ +mongo.conf +deployment.zip \ No newline at end of file diff --git a/src/lambda_functions/otto-bot/lambda_function.py b/src/lambda_functions/otto-bot/lambda_function.py index 512143c..068ab91 100644 --- a/src/lambda_functions/otto-bot/lambda_function.py +++ b/src/lambda_functions/otto-bot/lambda_function.py @@ -5,17 +5,18 @@ import boto3 import requests import os +import time -client = boto3.client('lambda') +lambda_client = boto3.client('lambda') sqs = boto3.client('sqs') -valid_commands = ['/link-player'] -loading_commands = ['/link-player'] + +valid_commands = ['/link-player', '/trade-review'] +loading_commands = ['/link-player', '/trade-review'] def lambda_handler(event, context): msg_map = dict(urlparse.parse_qsl(base64.b64decode(str(event['body'])).decode('ascii'))) - print(event['requestContext']) msg_map['stage'] = event['requestContext']['stage'] @@ -31,61 +32,19 @@ def lambda_handler(event, context): elif payload['type'] == 'view_submission': metadata = payload['view']['private_metadata'].split(',') command = metadata[0] - if command == '/link-player': - vals = payload['view']['state']['values'] - selected_player = vals['player_block']['player_selection_action']['selected_option'] - player_text = selected_player['text']['text'] - name_split = player_text.split(',') - ids = selected_player['value'].split(',') - - link_types = [sel['value'] for sel in vals['link_block']['checkboxes-action']['selected_options']] - - if not link_types: - print(msg_map['payload']) - return { - "response_action": "errors", - "errors": { - "link_block": "You may not select a due date in the past" - } - } - - if 'ottoneu' in link_types: - - selected_format = vals['format_block']['format_select_action']['selected_option']['value'] - - otto_player_link = f'https://ottoneu.fangraphs.com/playercard/{ids[0]}/{selected_format}' - - text_response = f'<{otto_player_link}|{name_split[0]}> {", ".join(name_split[1:])}' - - if 'fg' in link_types: - fg_link = f'http://www.fangraphs.com/statss.aspx?playerid={ids[1]}' - text_response += f' (<{fg_link}|FG>)' - - elif 'fg' in link_types: - fg_link = f'http://www.fangraphs.com/statss.aspx?playerid={ids[1]}' - text_response = f'<{fg_link}|{name_split[0]}> {", ".join(name_split[1:])}' - - else: - return { - 'statusCode': 400, - 'body': json.dumps(f'Invalid link type(s) "{link_types}" selected".') - } - - response_dict = {} - response_dict['response_type'] = 'in_channel' - response_dict['text'] = text_response - - header = {'Content-Type': 'application/json'} - response = requests.post(metadata[1], headers=header, data=json.dumps(response_dict)) - - return { - 'statusCode': 200 - } + if command.startswith('/link-player'): + return link_player_result(payload, msg_map, metadata) + if command.startswith('/trade-review'): + return trade_review_result(payload, msg_map, metadata) + return { + 'statusCode': 400, + 'body': json.dumps(f'Not a valid slash command: {msg_map.get('command', None)}') + } valid_command = False input_command = msg_map.get('command', None) for command in valid_commands: - if input_command and command.startswith(command): + if input_command and input_command.startswith(command): valid_command = True break @@ -97,8 +56,14 @@ def lambda_handler(event, context): 'body': json.dumps(f'Not a valid slash command: {msg_map.get('command', None)}') } + #if not msg_map.get('text', None): + # return { + # 'statusCode': 400, + # 'body': json.dumps(f'No arguments given for slash command: {msg_map['command']}') + # } + for command in loading_commands: - if input_command and command.startswith(command): + if input_command and input_command.startswith(command): modal_res = initiate_loading_modal(msg_map) print(modal_res) if modal_res['ok']: @@ -107,7 +72,7 @@ def lambda_handler(event, context): print(modal_res) return { 'statusCode': 400, - 'body': json.dumps(f'Error creating interactive dialog') + 'body': json.dumps('Error creating interactive dialog') } queueurl = sqs.get_queue_url(QueueName='Otto-bot-queue')['QueueUrl'] @@ -116,13 +81,271 @@ def lambda_handler(event, context): except: return { 'statusCode': 400, - 'body': json.dumps(f'Error when submitting to the queue.') + 'body': json.dumps('Error when submitting to the queue.') } return { 'statusCode': 202 } +def trade_review_result(payload, msg_map, metadata): + vals = payload['view']['state']['values'] + + selected_format = vals['format_block']['format_select_action']['selected_option']['value'] + + league_id = vals['league_number']['plain_text_input-action']['value'] + + search_parameters = { + "league_id" : league_id + } + + search_version = os.environ[f'{msg_map["stage"]}_league_load_version'] + + response = lambda_client.invoke( + FunctionName = os.environ['league_load_lambda_arn'], + InvocationType = 'RequestResponse', + Payload = json.dumps(search_parameters), + Qualifier = search_version + ) + + lambda_response = json.load(response['Payload']) + + if 'body' in lambda_response: + player_dict = json.loads(lambda_response['body']) + else: + player_dict = None + + if not player_dict: + return { + 'statusCode': 200, + "headers": { + 'Content-Type': 'application/json', + }, + 'body': json.dumps({ + 'response_action': 'errors', + 'errors': { + 'league_number': f'League Id {league_id} not available in database.' + } + }) + } + + loan_type = vals['loan_type']['checkboxes-action']['selected_option']['value'] + if loan_type == 'partial-loan': + try: + partial_loan_amount = vals['partial_loan']['plain_text_input-action']['value'] + int_check = int(partial_loan_amount) + except ValueError: + partial_loan_amount = None + else: + partial_loan_amount = None + + if selected_format == '1': + text_response = 'Scoring: 4x4' + elif selected_format == '2': + text_response = 'Scoring: 5x5' + elif selected_format == '3': + text_response = 'Scoring: FGP' + elif selected_format == '4': + text_response = 'Scoring: SABR' + elif selected_format == '5': + text_response = 'Scoring: FGP H2H' + elif selected_format == '6': + text_response = 'Scoring: SABR H2H' + else: + return { + 'statusCode': 400, + 'body': json.dumps(f'Invalid format {selected_format}.') + } + + team_1_players_options = vals['team_1']['player-search-action-1']['selected_options'] + + team_1_salaries = 0 + team_1_names = list() + for option in team_1_players_options: + rostered = player_dict.get(option['value'], None) + if not rostered: + print('not_rostered 1') + return { + 'statusCode': 200, + "headers": { + 'Content-Type': 'application/json', + }, + 'body': json.dumps({ + 'response_action': 'errors', + 'errors': { + 'league_number': f'{option["text"]["text"].split(", ")[0]} is not rostered in league {league_id}' + } + }) + } + salary = rostered.get('Salary') + team_1_salaries += int(salary.split('$')[1]) + team_1_names.append((option['value'], salary, option['text']['text'].split(', '))) + + team_2_players_options = vals['team_2']['player-search-action-2']['selected_options'] + + team_2_salaries = 0 + team_2_names = list() + for option in team_2_players_options: + rostered = player_dict.get(option['value']) + if not rostered: + print('not_rostered 2') + return { + 'statusCode': 200, + "headers": { + 'Content-Type': 'application/json', + }, + 'body': json.dumps({ + 'response_action': 'errors', + 'errors': { + 'league_number': f'{option["text"]["text"].split(", ")[0]} is not rostered in league {league_id}' + } + }) + } + salary = rostered.get('Salary') + team_2_salaries += int(salary.split('$')[1]) + team_2_names.append((option['value'], salary, option['text']['text'].split(', '))) + + team_1_more_salary = team_1_salaries > team_2_salaries + salary_diff = abs(team_1_salaries - team_2_salaries) + + if partial_loan_amount and int(partial_loan_amount) < 0: + team_1_more_salary = not team_1_more_salary + + text_response += '\n:one: ' + + text_response += '\n\t\t'.join(f'{option[1]} <https://ottoneu.fangraphs.com/playercard/{option[0]}/{selected_format}|{option[2][0]}> {", ".join(option[2][1:])}' for option in team_1_names) + + if team_1_more_salary: + if loan_type == 'full-loan': + text_response += f'\n\t\tFull Loan (${salary_diff})' + elif partial_loan_amount: + net_int = (salary_diff - abs(int(partial_loan_amount))) + if net_int < 0: + net = f'-${abs(net_int)}' + else: + net = f'${net_int}' + text_response += f'\n\t\t${abs(int(partial_loan_amount))} Loan (Net {net})' + else: + text_response += f'\n\t\tNo Loan (Net -${salary_diff})' + + text_response += '\n:two: ' + + text_response += '\n\t\t'.join(f'{option[1]} <https://ottoneu.fangraphs.com/playercard/{option[0]}/{selected_format}|{option[2][0]}> {", ".join(option[2][1:])}' for option in team_2_names) + + if not team_1_more_salary: + if loan_type == 'full-loan': + text_response += f'\n\t\tFull Loan (${salary_diff})' + elif partial_loan_amount: + net_int = (salary_diff - abs(int(partial_loan_amount))) + if net_int < 0: + net = f'-${abs(net_int)}' + else: + net = f'${net_int}' + text_response += f'\n\t\t${abs(int(partial_loan_amount))} Loan (Net {net})' + else: + text_response += f'\n\t\tNo Loan (Net -${salary_diff})' + + response_dict = {} + response_dict['response_type'] = 'in_channel' + response_dict['text'] = text_response + response_dict['channel'] = metadata[2] + response_dict['token'] = os.environ[f'{msg_map["stage"]}_{metadata[3]}_token'] + response_dict['unfurl_links'] = False + + header = {'Content-Type': 'application/x-www-form-urlencoded'} + + response = requests.post('https://slack.com/api/chat.postMessage', headers=header, data=urllib.parse.urlencode(response_dict)) + + print(response.content) + + ts = json.loads(response.content.decode('utf-8'))['ts'] + + react_dict = dict() + react_dict['channel'] = metadata[2] + react_dict['token'] = os.environ[f'{msg_map["stage"]}_{metadata[3]}_token'] + react_dict['timestamp'] = ts + + react_dict['name'] = 'one' + response = requests.post('https://slack.com/api/reactions.add', headers=header, data=urllib.parse.urlencode(react_dict)) + + time.sleep(0.2) + + react_dict['name'] = 'two' + response = requests.post('https://slack.com/api/reactions.add', headers=header, data=urllib.parse.urlencode(react_dict)) + + time.sleep(0.2) + + react_dict['name'] = 'scales' + response = requests.post('https://slack.com/api/reactions.add', headers=header, data=urllib.parse.urlencode(react_dict)) + + print(response.content) + + return { + 'statusCode': 200 + } + +def link_player_result(payload, msg_map, metadata): + vals = payload['view']['state']['values'] + selected_player = vals['player_block']['player_selection_action']['selected_option'] + player_text = selected_player['text']['text'] + name_split = player_text.split(',') + ids = selected_player['value'].split(',') + link_types = [sel['value'] for sel in vals['link_block']['checkboxes-action']['selected_options']] + + if not link_types: + print(msg_map['payload']) + return { + "response_action": "errors", + "errors": { + "link_block": "Must select at least one linkage" + } + } + + if 'ottoneu' in link_types: + + selected_format = vals['format_block']['format_select_action']['selected_option']['value'] + + otto_player_link = f'https://ottoneu.fangraphs.com/playercard/{ids[0]}/{selected_format}' + + text_response = f'<{otto_player_link}|{name_split[0]}> {", ".join(name_split[1:])}' + + if 'fg' in link_types: + fg_link = f'http://www.fangraphs.com/statss.aspx?playerid={ids[1]}' + text_response += f' (<{fg_link}|FG>)' + + if 'sc' in link_types: + sc_link = f'https://baseballsavant.mlb.com/savant-player/{ids[2]}' + text_response += f' (<{sc_link}|SC>)' + + elif 'fg' in link_types: + fg_link = f'http://www.fangraphs.com/statss.aspx?playerid={ids[1]}' + text_response = f'<{fg_link}|{name_split[0]}> {", ".join(name_split[1:])}' + + if 'sc' in link_types: + sc_link = f'https://baseballsavant.mlb.com/savant-player/{ids[2]}' + text_response += f' (<{sc_link}|SC>)' + + elif 'sc' in link_types: + sc_link = f'https://baseballsavant.mlb.com/savant-player/{ids[2]}' + text_response = f'<{sc_link}|{name_split[0]}> {", ".join(name_split[1:])}' + + else: + return { + 'statusCode': 400, + 'body': json.dumps(f'Invalid link type(s) "{link_types}" selected".') + } + + response_dict = {} + response_dict['response_type'] = 'in_channel' + response_dict['text'] = text_response + + header = {'Content-Type': 'application/json'} + response = requests.post(metadata[1], headers=header, data=json.dumps(response_dict)) + + return { + 'statusCode': 200 + } + def initiate_loading_modal(msg_map): post_url = 'https://slack.com/api/views.open' view = get_modal() @@ -145,7 +368,7 @@ def get_modal(): "callback_id": <callbackid>, "title": { "type": "plain_text", - "text": "Otto-bot Wizard", + "text": "Otto-bot", "emoji": true }, "submit": { @@ -163,7 +386,7 @@ def get_modal(): "type": "header", "text": { "type": "plain_text", - "text": "Loading players..." + "text": "Loading..." } } ] diff --git a/src/lambda_functions/ottoneu-db-update/lambda_function.py b/src/lambda_functions/ottoneu-db-update/lambda_function.py new file mode 100644 index 0000000..435bf4d --- /dev/null +++ b/src/lambda_functions/ottoneu-db-update/lambda_function.py @@ -0,0 +1,55 @@ +import json +import boto3 +import os +import math + +sqs = boto3.client('sqs') +lambda_client = boto3.client('lambda') +queueurl = sqs.get_queue_url(QueueName='Ottoneu-db-update-queue')['QueueUrl'] + +def lambda_handler(event, context): + + league_id_str = os.environ['league_ids_list'] + + print(league_id_str) + + league_ids = json.loads(league_id_str) + + print(f'Total Number of Leagues: {len(league_ids)}') + + chunk_size = math.ceil(float(len(league_ids) / float(os.environ['number_of_chunks']))) + + league_id_chunks = divide_lists(league_ids, chunk_size) + + for id_chunk in league_id_chunks: + print(f'length = {len(id_chunk)}') + print(id_chunk) + msg_map = dict() + msg_map['league_ids'] = id_chunk + + try: + sqs.send_message(QueueUrl=queueurl, MessageBody=json.dumps(msg_map)) + except Exception as e: + print(e) + return { + 'statusCode': 400, + 'body': json.dumps('Error when submitting to the queue.') + } + + response = lambda_client.invoke( + FunctionName = os.environ['player_put_arn'], + InvocationType = 'RequestResponse' + ) + + print(response) + + return { + 'statusCode': 200, + 'body': json.dumps('Update success') + } + +def divide_lists(l, n): + + # looping till length l + for i in range(0, len(l), n): + yield l[i:i + n] \ No newline at end of file diff --git a/src/lambda_functions/ottoneu-league-load/.gitignore b/src/lambda_functions/ottoneu-league-load/.gitignore new file mode 100644 index 0000000..53b66ce --- /dev/null +++ b/src/lambda_functions/ottoneu-league-load/.gitignore @@ -0,0 +1,3 @@ +*/ +mongo.conf +deployment.zip \ No newline at end of file diff --git a/src/lambda_functions/ottoneu-league-load/lambda_function.py b/src/lambda_functions/ottoneu-league-load/lambda_function.py new file mode 100644 index 0000000..fe49bf6 --- /dev/null +++ b/src/lambda_functions/ottoneu-league-load/lambda_function.py @@ -0,0 +1,35 @@ +from pymongo import MongoClient +import os + +import json + +mongo_client = MongoClient(host=os.environ.get("ATLAS_URI")) +ottoneu_db = mongo_client.ottoneu + +def lambda_handler(event, context): + + try: + if "league_id" in event: + league_id = event['league_id'] + else: + league_id = event["queryStringParameters"]["league_id"] + except KeyError: + return { + 'statusCode': 400, + 'body': json.dumps('Body does not have league_id parameter.') + } + if not league_id: + return { + 'statusCode': 400, + 'body': json.dumps('Empty string passed as league_id') + } + + roster_cursor = ottoneu_db.leagues.find({'_id': league_id}) + + player_dict = next(roster_cursor, None)['rosters'] + print(player_dict) + + return { + 'statusCode': 200, + 'body': json.dumps(player_dict) + } diff --git a/src/lambda_functions/ottoneu-league-roster-load/.gitignore b/src/lambda_functions/ottoneu-league-roster-load/.gitignore new file mode 100644 index 0000000..53b66ce --- /dev/null +++ b/src/lambda_functions/ottoneu-league-roster-load/.gitignore @@ -0,0 +1,3 @@ +*/ +mongo.conf +deployment.zip \ No newline at end of file diff --git a/src/lambda_functions/ottoneu-league-roster-load/lambda_function.py b/src/lambda_functions/ottoneu-league-roster-load/lambda_function.py new file mode 100644 index 0000000..37d67ed --- /dev/null +++ b/src/lambda_functions/ottoneu-league-roster-load/lambda_function.py @@ -0,0 +1,41 @@ +import json +import pandas as pd +import requests +from bs4 import BeautifulSoup as Soup +from io import StringIO + +def lambda_handler(event, context): + ''''Scrapes the /rosterexport page for a league (in csv format) and returns a DataFrame of the information. Index is Ottoneu Id''' + try: + if "league_id" in event: + lg_id = event['league_id'] + else: + lg_id = event["queryStringParameters"].get("league_id", None) + except KeyError: + return { + 'statusCode': 400, + 'body': json.dumps('Body does not have search_name parameter.') + } + if not lg_id: + return { + 'statusCode': 400, + 'body': json.dumps('Empty string passed as search name') + } + + roster_export_url = f'https://ottoneu.fangraphs.com/{lg_id}/rosterexport' + response = requests.get(roster_export_url) + + rost_soup = Soup(response.text, 'html.parser') + df = pd.read_csv(StringIO(rost_soup.contents[0])) + df = df[df['Salary'].notna()] + df = df[['ottoneu ID', 'TeamID', 'Team Name', 'Salary']] + df.set_index("ottoneu ID", inplace=True) + df.index = df.index.astype(str, copy = False) + print(df.head()) + + roster_dict = df.to_dict('index') + + return { + 'statusCode': 200, + 'body': json.dumps(roster_dict) + } \ No newline at end of file diff --git a/src/lambda_functions/ottoneu-player-search/.gitignore b/src/lambda_functions/ottoneu-player-search/.gitignore new file mode 100644 index 0000000..53b66ce --- /dev/null +++ b/src/lambda_functions/ottoneu-player-search/.gitignore @@ -0,0 +1,3 @@ +*/ +mongo.conf +deployment.zip \ No newline at end of file diff --git a/src/lambda_functions/ottoneu-player-search/lambda_function.py b/src/lambda_functions/ottoneu-player-search/lambda_function.py index e006dc7..d47acf8 100644 --- a/src/lambda_functions/ottoneu-player-search/lambda_function.py +++ b/src/lambda_functions/ottoneu-player-search/lambda_function.py @@ -1,57 +1,68 @@ import json -import boto3 -from boto3.dynamodb.conditions import Key from urllib.parse import unquote +import os +from pymongo import MongoClient + +client = MongoClient(host=os.environ.get("ATLAS_URI")) +ottoneu_db = client.ottoneu def lambda_handler(event, context): - try: - if "search_name" in event: - search_name = event['search_name'] - else: - search_name = event["queryStringParameters"]["search_name"] - except KeyError: - return { - 'statusCode': 400, - 'body': json.dumps('Body does not have search_name parameter.') - } - if not search_name: - return { - 'statusCode': 400, - 'body': json.dumps('Empty string passed as search name') - } - search_name = unquote(search_name) - search_name = normalize(search_name) - search_name = clean_full_name(search_name) - - dynamodb = boto3.resource('dynamodb') - table = dynamodb.Table('ottoneu-player-db') + if "search_name" in event: + search_name = event['search_name'] + elif "queryStringParameters" in event: + search_name = event["queryStringParameters"].get("search_name", None) + else: + search_name = None + + if search_name: + return player_search(search_name) - split = search_name.split() - if len(split) == 1: - index = 'search_last_name' - elif split[0] in ['DE', 'DEL', 'DI', 'VAN', 'LA', 'ST']: - index = 'search_last_name' + if "league_id" in event: + league_id = event['league_id'] + elif "queryStringParameters" in event: + league_id = event["queryStringParameters"].get("league_id", None) else: - index = 'search_name' + league_id = None + + if league_id: + return league_search(league_id) + + return { + 'statusCode': 400, + 'body': json.dumps('Invalid search paramters') + } + +def league_search(league_id: str): + roster_cursor = ottoneu_db.leagues.find({'_id': league_id}) + + player_dict = next(roster_cursor, None)['rosters'] - items = table.query( - IndexName=f"{index}-index", - KeyConditionExpression=Key(f"{index}").eq(search_name), - ) + return { + 'statusCode': 200, + 'body': json.dumps(player_dict) + } - results = [] +def player_search(search_name:str): + search_name = unquote(search_name) + search_name = normalize(search_name) + search_name = f'.*{search_name}.*' - for item in items['Items']: + players_col = ottoneu_db.players + + results_cursor = players_col.find({'search_name': {'$regex': search_name, '$options': 'i'}}) + + results = list() + for item in results_cursor: result = {} for key, val in item.items(): result[key] = val - result['ottoneu_id'] = int(str(result['ottoneu_id'])) + result['ottoneu_id'] = int(str(result['_id'])) results.append(result) return { 'statusCode': 200, 'body': json.dumps(results) - } + } normalMap = {'À': 'A', 'Á': 'A', 'Â': 'A', 'Ã': 'A', 'Ä': 'A', 'à': 'a', 'á': 'a', 'â': 'a', 'ã': 'a', 'ä': 'a', 'ª': 'A', @@ -67,22 +78,6 @@ def lambda_handler(event, context): 'Ç': 'C', 'ç': 'c', '§': 'S', '³': '3', '²': '2', '¹': '1'} -def clean_full_name(value:str) -> str: - cleaned = normalize(value) - cleaned = cleaned.replace('.', '') - cleaned = clear_if_ends_with(cleaned, ' JR') - cleaned = clear_if_ends_with(cleaned, ' SR') - cleaned = clear_if_ends_with(cleaned, ' II') - cleaned = clear_if_ends_with(cleaned, ' III') - cleaned = clear_if_ends_with(cleaned, ' IV') - cleaned = clear_if_ends_with(cleaned, ' V') - return cleaned - -def clear_if_ends_with(val:str, check:str) -> str: - if val.endswith(check): - return val[:-len(check)].strip() - return val - def normalize(value:str) -> str: """Function that removes most diacritics from strings and returns value in all caps""" normalize = str.maketrans(normalMap) diff --git a/src/lambda_functions/ottoneu-player-search/requirements.txt b/src/lambda_functions/ottoneu-player-search/requirements.txt index 6a69d95..e66f8a4 100644 --- a/src/lambda_functions/ottoneu-player-search/requirements.txt +++ b/src/lambda_functions/ottoneu-player-search/requirements.txt @@ -1 +1 @@ -boto3=1.34.43 \ No newline at end of file +pymongo==4.8.0 \ No newline at end of file diff --git a/src/lambda_functions/ottoneu-players-put/.gitignore b/src/lambda_functions/ottoneu-players-put/.gitignore index 0a00d70..53b66ce 100644 --- a/src/lambda_functions/ottoneu-players-put/.gitignore +++ b/src/lambda_functions/ottoneu-players-put/.gitignore @@ -1 +1,3 @@ -*/ \ No newline at end of file +*/ +mongo.conf +deployment.zip \ No newline at end of file diff --git a/src/lambda_functions/ottoneu-players-put/lambda_function.py b/src/lambda_functions/ottoneu-players-put/lambda_function.py index 87b135c..b256543 100644 --- a/src/lambda_functions/ottoneu-players-put/lambda_function.py +++ b/src/lambda_functions/ottoneu-players-put/lambda_function.py @@ -3,32 +3,68 @@ from pandas import DataFrame from typing import List import requests -import boto3 +import os + +from pymongo import MongoClient, UpdateOne + +import io +import re +import zipfile + +from typing import List, Iterable + +import pandas as pd +import requests + +client = MongoClient(host=os.environ.get("ATLAS_URI")) +ottoneu_db = client.ottoneu def lambda_handler(event, context): try: df = get_avg_salary_df() - except: + except Exception as e: + print(e) return { 'statusCode': 500, 'body': json.dumps('Error getting player universe') } - - dynamodb = boto3.resource('dynamodb') - table = dynamodb.Table('ottoneu-player-db') - + + players_col = ottoneu_db.players + try: - with table.batch_writer() as batch: - for idx, row in df.iterrows(): - player_dict = row.to_dict() - player_dict['ottoneu_id'] = int(idx) - batch.put_item(Item=player_dict) - except: + players = [] + for idx, row in df.iterrows(): + player_dict = row.to_dict() + player_dict = {k:v for k,v in player_dict.items() if v} + #player_dict['ottoneu_id'] = int(idx) + player_dict['_id'] = int(idx) + if player_dict.get('ottoneu_id'): + player_dict.pop('ottoneu_id') + + players.append(UpdateOne({'_id': player_dict['_id']}, {'$set': player_dict}, upsert=True)) + players_col.bulk_write(players, ordered=False) + except Exception as e: + print(e) return { 'statusCode': 500, 'body': json.dumps('Error writing player universe') } + mlbam_update_players = players_col.find({"$and": [ + {'mlbam_id': {"$exists": False}}, + {'fg_majorleagueid': {'$exists': True}} + ] + }) + + mlbam_updates = [] + for player in mlbam_update_players: + p_df = playerid_reverse_lookup([int(player['fg_majorleagueid'])], key_type='fangraphs') + if len(p_df) > 0: + mlbam_id = p_df.loc[0,'key_mlbam'].item() + mlbam_updates.append(UpdateOne({'_id': player['_id']}, {'$set': {'mlbam_id': mlbam_id}}, upsert=False)) + if mlbam_updates: + players_col.bulk_write(mlbam_updates, ordered=False) + return { 'statusCode': 200, 'body': json.dumps('Player universe updated') @@ -45,7 +81,7 @@ def get_avg_salary_df(game_type : int = None) -> DataFrame: rows = salary_soup.find_all('player') parsed_rows = [__parse_avg_salary_row(row) for row in rows] df = DataFrame(parsed_rows) - df.columns = ['ottoneu_id','name','search_name','search_last_name','fg_majorleagueid','fg_minorleagueid','positions','org'] + df.columns = ['ottoneu_id','name','search_name','fg_majorleagueid','fg_minorleagueid','positions','org'] df.set_index('ottoneu_id', inplace=True) df.index = df.index.astype(int, copy=False) @@ -57,11 +93,10 @@ def __parse_avg_salary_row(row) -> List[str]: parsed_row = [] parsed_row.append(row.get('ottoneu_id')) parsed_row.append(row.get('name')) - full_search_name = clean_full_name(row.get('name')) + full_search_name = normalize(row.get('name')) parsed_row.append(full_search_name) - search_last_name = get_search_last_name(full_search_name) - parsed_row.append(search_last_name) - parsed_row.append(str(row.get('fg_majorleague_id'))) + fg_major_id = str(row.get('fg_majorleague_id')) + parsed_row.append(fg_major_id) parsed_row.append(row.get('fg_minorleague_id')) parsed_row.append(row.find('positions').text) parsed_row.append(row.find('mlb_org').text) @@ -81,35 +116,112 @@ def __parse_avg_salary_row(row) -> List[str]: 'Ç': 'C', 'ç': 'c', '§': 'S', '³': '3', '²': '2', '¹': '1'} -def get_search_last_name(name:str) -> str: - name_split = name.split() - multi_word_ln = ['DE', 'DEL', 'DI', 'VAN', 'LA', 'ST'] - for mwl in multi_word_ln: - if mwl in name_split: - index = name_split.index(mwl) - return ' '.join(name_split[index:]) - return name_split[-1] - -def clean_full_name(value:str) -> str: - cleaned = normalize(value) - cleaned = cleaned.replace('.', '') - cleaned = clear_if_ends_with(cleaned, ' JR') - cleaned = clear_if_ends_with(cleaned, ' SR') - cleaned = clear_if_ends_with(cleaned, ' II') - cleaned = clear_if_ends_with(cleaned, ' III') - cleaned = clear_if_ends_with(cleaned, ' IV') - cleaned = clear_if_ends_with(cleaned, ' V') - cleaned = ' '.join(cleaned.split) - return cleaned - -def clear_if_ends_with(val:str, check:str) -> str: - if val.endswith(check): - return val[:-len(check)].strip() - return val - - def normalize(value:str) -> str: """Function that removes most diacritics from strings and returns value in all caps""" normalize = str.maketrans(normalMap) val = value.translate(normalize) - return val.upper() \ No newline at end of file + return val.upper() + +# This is adapted from pybaseball to elimiate the need for a heavy-weight library w/ dependencies + +url = "https://github.com/chadwickbureau/register/archive/refs/heads/master.zip" +PEOPLE_FILE_PATTERN = re.compile("/people.+csv$") + +_client = None + +def _extract_people_files(zip_archive: zipfile.ZipFile) -> Iterable[zipfile.ZipInfo]: + return filter( + lambda zip_info: re.search(PEOPLE_FILE_PATTERN, zip_info.filename), + zip_archive.infolist(), + ) + + +def _extract_people_table(zip_archive: zipfile.ZipFile) -> pd.DataFrame: + dfs = map( + lambda zip_info: pd.read_csv( + io.BytesIO(zip_archive.read(zip_info.filename)), + low_memory=False + ), + _extract_people_files(zip_archive), + ) + return pd.concat(dfs, axis=0) + + +def chadwick_register(save: bool = False) -> pd.DataFrame: + ''' Get the Chadwick register Database ''' + print('Gathering player lookup table. This may take a moment.') + s = requests.get(url).content + mlb_only_cols = ['key_retro', 'key_bbref', 'key_fangraphs', 'mlb_played_first', 'mlb_played_last'] + cols_to_keep = ['name_last', 'name_first', 'key_mlbam'] + mlb_only_cols + table = _extract_people_table( + zipfile.ZipFile(io.BytesIO(s)) + ).loc[:, cols_to_keep] + + table.dropna(how='all', subset=mlb_only_cols, inplace=True) # Keep only the major league rows + table.reset_index(inplace=True, drop=True) + + table[['key_mlbam', 'key_fangraphs']] = table[['key_mlbam', 'key_fangraphs']].fillna(-1) + # originally returned as floats which is wrong + table[['key_mlbam', 'key_fangraphs']] = table[['key_mlbam', 'key_fangraphs']].astype(int) + + # Reorder the columns to the right order + table = table[cols_to_keep] + + return table + +def get_lookup_table(save=False): + table = chadwick_register(save) + # make these lowercase to avoid capitalization mistakes when searching + table['name_last'] = table['name_last'].str.lower() + table['name_first'] = table['name_first'].str.lower() + return table + +class _PlayerSearchClient: + def __init__(self) -> None: + self.table = get_lookup_table() + + def reverse_lookup(self, player_ids: List[str], key_type: str = 'mlbam') -> pd.DataFrame: + """Retrieve a table of player information given a list of player ids + + :param player_ids: list of player ids + :type player_ids: list + :param key_type: name of the key type being looked up (one of "mlbam", "retro", "bbref", or "fangraphs") + :type key_type: str + + :rtype: :class:`pandas.core.frame.DataFrame` + """ + key_types = ( + 'mlbam', + 'retro', + 'bbref', + 'fangraphs', + ) + + if key_type not in key_types: + raise ValueError(f'[Key Type: {key_type}] Invalid; Key Type must be one of {key_types}') + + key = f'key_{key_type}' + + results = self.table[self.table[key].isin(player_ids)] + results = results.reset_index(drop=True) + + return results + +def _get_client() -> _PlayerSearchClient: + global _client + if _client is None: + _client = _PlayerSearchClient() + return _client + +def playerid_reverse_lookup(player_ids: List[str], key_type: str = 'mlbam') -> pd.DataFrame: + """Retrieve a table of player information given a list of player ids + + :param player_ids: list of player ids + :type player_ids: list + :param key_type: name of the key type being looked up (one of "mlbam", "retro", "bbref", or "fangraphs") + :type key_type: str + + :rtype: :class:`pandas.core.frame.DataFrame` + """ + client = _get_client() + return client.reverse_lookup(player_ids, key_type) \ No newline at end of file diff --git a/src/lambda_functions/ottoneu-players-put/requirements.txt b/src/lambda_functions/ottoneu-players-put/requirements.txt index 8664ba4..3aa9d7f 100644 --- a/src/lambda_functions/ottoneu-players-put/requirements.txt +++ b/src/lambda_functions/ottoneu-players-put/requirements.txt @@ -1,3 +1,3 @@ beautifulsoup4==4.12.0 requests==2.28.0 -boto3=1.34.43 \ No newline at end of file +pymong==4.8.0 \ No newline at end of file diff --git a/src/lambda_functions/ottoneu-save-rosters/.gitignore b/src/lambda_functions/ottoneu-save-rosters/.gitignore new file mode 100644 index 0000000..53b66ce --- /dev/null +++ b/src/lambda_functions/ottoneu-save-rosters/.gitignore @@ -0,0 +1,3 @@ +*/ +mongo.conf +deployment.zip \ No newline at end of file diff --git a/src/lambda_functions/ottoneu-save-rosters/lambda_function.py b/src/lambda_functions/ottoneu-save-rosters/lambda_function.py new file mode 100644 index 0000000..fde1ea1 --- /dev/null +++ b/src/lambda_functions/ottoneu-save-rosters/lambda_function.py @@ -0,0 +1,89 @@ +import json +from pymongo import MongoClient, UpdateOne +import os +import boto3 +import requests +import pandas as pd +from bs4 import BeautifulSoup as Soup +from io import StringIO +import datetime + +client = MongoClient(host=os.environ.get("ATLAS_URI")) +ottoneu_db = client.ottoneu + +lambda_client = boto3.client('lambda') + +def lambda_handler(event, context): + print(event) + + if event.get('Records', None): + msg_map = list() + for record in event['Records']: + msg_map.append(json.loads(record['body'])) + else: + msg_map = [event] + + print(msg_map) + + if not msg_map or not msg_map[0].get('league_ids', None): + return { + 'statusCode': 400, + 'body': json.dumps('league_ids not present in event') + } + league_ids = [li for lis in msg_map for li in lis['league_ids']] + + leagues_col = ottoneu_db.leagues + + update_leagues = list() + + + print(f'League subset: {len(league_ids)}') + + for league_id in league_ids: + #print(f'Getting league_id: {league_id}' ) + try: + league_dict = dict() + league_dict['rosters'] = get_league_dict(league_id) + + league_dict['Last Updated'] = datetime.datetime.now() + + if not league_dict: + print(f'!!League_id {league_id} not valid') + continue + + #print(f'Roster length {len(league_dict)}') + + update_leagues.append(UpdateOne({'_id': league_id}, {'$set': league_dict}, upsert=True)) + except Exception as e: + print(f'Exception for league {league_id}') + print(e) + + if update_leagues: + try: + leagues_col.bulk_write(update_leagues, ordered=False) + except Exception as e: + print(e) + return { + 'statusCode': 500, + 'body': json.dumps('Error writing to db.') + } + + return { + 'statusCode': 200 + } + +def get_league_dict(lg_id: str): + '''Scrapes the /rosterexport page for a league (in csv format) and returns a DataFrame of the information. Index is Ottoneu Id''' + if lg_id: + roster_export_url = f'https://ottoneu.fangraphs.com/{lg_id}/rosterexport' + response = requests.get(roster_export_url) + + rost_soup = Soup(response.text, 'html.parser') + df = pd.read_csv(StringIO(rost_soup.contents[0])) + df = df[df['Salary'].notna()] + df = df[['ottoneu ID', 'TeamID', 'Team Name', 'Salary']] + df.set_index("ottoneu ID", inplace=True) + df.index = df.index.astype(str, copy = False) + return df.to_dict('index') + + return dict() \ No newline at end of file