From 62ed33df388ee0527d17ab3388d43459bd64e058 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 3 Jan 2025 15:41:40 +0000 Subject: [PATCH] Added submit-mlperf-results CM script for automatic mlperf result submissions (#98) * Update format.yml * Added submit-mlperf-results script for auto upload of mlperf results * [Automated Commit] Format Codebase --- script/submit-mlperf-results/_cm.yaml | 22 +++ script/submit-mlperf-results/customize.py | 194 ++++++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 script/submit-mlperf-results/_cm.yaml create mode 100644 script/submit-mlperf-results/customize.py diff --git a/script/submit-mlperf-results/_cm.yaml b/script/submit-mlperf-results/_cm.yaml new file mode 100644 index 000000000..b7a10ce83 --- /dev/null +++ b/script/submit-mlperf-results/_cm.yaml @@ -0,0 +1,22 @@ +alias: submit-mlperf-results +automation_alias: script +automation_uid: 5b4e0237da074764 +category: MLPerf benchmark support +default_env: + CM_MLPERF_SUBMISSION_URL: https://submissions-ui.mlcommons.org + +input_mapping: + input: CM_MLPERF_SUBMISSION_FILE + submitter_id: CM_MLPERF_SUBMITTER_ID +tags: +- submit +- mlperf +- results +- mlperf-results +- publish-results +- submission +uid: cc01f0a82bef4216 +variations: + inference: + env: + CM_MLPERF_BENCHMARK: "Inference" diff --git a/script/submit-mlperf-results/customize.py b/script/submit-mlperf-results/customize.py new file mode 100644 index 000000000..d39b233f3 --- /dev/null +++ b/script/submit-mlperf-results/customize.py @@ -0,0 +1,194 @@ +import requests +from cmind import utils +import cmind as cm +import os +import json + + +def preprocess(i): + + os_info = i['os_info'] + env = i['env'] + meta = i['meta'] + automation = i['automation'] + + server = env['CM_MLPERF_SUBMISSION_URL'] + benchmark = env['CM_MLPERF_BENCHMARK'] + submitter_id = env['CM_MLPERF_SUBMITTER_ID'] + file_path = env['CM_MLPERF_SUBMISSION_FILE'] + + r = get_signed_url(server, benchmark, submitter_id, file_path) + if r['return'] > 0: + return r + + signed_url = r['signed_url'] + submission_id = r['submission_id'] + + # print(signed_url) + # print(submission_id) + r = upload_file_to_signed_url(file_path, signed_url) + if r['return'] > 0: + return r + + r = trigger_submission_checker( + server, submitter_id, benchmark, submission_id) + if r['return'] > 0: + return r + + return {'return': 0} + + +def get_signed_url(server, benchmark, submitter_id, file_path): + # Define the URL + url = f"{server}/index/url" + + # Define the headers + headers = { + "Content-Type": "application/json" + } + + # Define the payload + payload = { + "submitter_id": submitter_id, + "benchmark": benchmark, + "filename": file_path + } + + try: + # Make the POST request + response = requests.post(url, json=payload, headers=headers) + + # Check the response status + if response.status_code == 200: + # print("Request successful!") + # print("Response:", response.json()) + pass + else: + # print(f"Request failed with status code {response.status_code}") + # print("Response:", response.text) + pass + + except requests.exceptions.RequestException as e: + return {"return": 1, + "error": f"An error occurred in connecting to the server: {e}"} + + response_json = response.json() + # print(response_json) + # response = json.loads(response_json) + try: + signed_url = response_json['signed_url'] + submission_id = response_json['submission_id'] + except Exception as e: + return { + "return": 1, "error": f"An error occurred while processing the response: {e}"} + + return {'return': 0, 'signed_url': signed_url, + 'submission_id': submission_id} + + +def upload_file_to_signed_url(file_path, signed_url): + """ + Uploads a file to a signed URL using HTTP PUT. + + Parameters: + file_path (str): The path to the file you want to upload. + signed_url (str): The pre-signed URL for uploading the file. + + Returns: + dict: A dictionary with 'status_code' and 'response' keys. + """ + headers = { + 'Content-Type': 'application/octet-stream', + 'Access-Control-Allow-Headers': '*' + } + + try: + # Open the file in binary mode + with open(file_path, 'rb') as file: + response = requests.put( + signed_url, + data=file, + headers=headers + ) + + if response.status_code in [200, 201, 204]: + print("File uploaded successfully!") + return { + 'return': 0 + } + else: + print( + f"Failed to upload file. Status code: {response.status_code}") + print("Response:", response.text) + + return { + 'return': response.status_code, + 'error': response.text + } + + except FileNotFoundError: + print("Error: File not found.") + return { + 'return': 400, + 'error': f'''File {file_path} not found''' + } + + except requests.exceptions.RequestException as e: + print(f"Request failed: {e}") + return { + 'return': 500, + 'error': str(e) + } + + +def trigger_submission_checker( + server_url, submitter_id, benchmark, submission_id): + """ + Sends a POST request with URL-encoded form data. + + Parameters: + server_url (str): The server endpoint URL (e.g., https://example.com/index). + submitter_id (str): The ID of the submitter. + benchmark (str): The benchmark identifier. + submission_id (str): The submission ID. + + Returns: + dict: A dictionary containing status code and response content. + """ + url = f"{server_url}/index" + headers = { + "Content-Type": "application/x-www-form-urlencoded" + } + payload = { + "submitter_id": submitter_id, + "benchmark": benchmark, + "submission_id": submission_id + } + + try: + # Make the POST request with URL-encoded data + response = requests.post(url, data=payload, headers=headers) + + if response.ok: + print("Submission Check Request successful!") + pass + else: + print( + f"Submission Check Request failed with status code: {response.status_code}") + print("Response:", response.text) + + return { + "return": 0, + "response": response.text + } + + except requests.exceptions.RequestException as e: + print("An error occurred:", e) + return { + "return": 500, + "error": str(e) + } + + +def postprocess(i): + return {'return': 0}