From 5c8791e7d29290f32206fa46376c96827fe7e89b Mon Sep 17 00:00:00 2001 From: e-koch Date: Tue, 8 Nov 2016 11:36:45 -0700 Subject: [PATCH 1/8] Updated aws_controller --- aws_controller | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_controller b/aws_controller index 3196cc3..d708812 160000 --- a/aws_controller +++ b/aws_controller @@ -1 +1 @@ -Subproject commit 3196cc389910ccb8e4eb91cce0a794aa8a709725 +Subproject commit d70881228923461533822fa302a53535dacc46e6 From 00a6aa78d4650771733f9ce6c6727502bcbf9989 Mon Sep 17 00:00:00 2001 From: e-koch Date: Tue, 8 Nov 2016 12:05:35 -0700 Subject: [PATCH 2/8] Add the tests folder into the repo --- .gitignore | 1 - tests/test_response.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 tests/test_response.txt diff --git a/.gitignore b/.gitignore index 2fc7391..453080f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ *.pyc -tests/ # Elastic Beanstalk Files .elasticbeanstalk/* !.elasticbeanstalk/*.cfg.yml diff --git a/tests/test_response.txt b/tests/test_response.txt new file mode 100644 index 0000000..ed68008 --- /dev/null +++ b/tests/test_response.txt @@ -0,0 +1 @@ +{"proc_name": "v404_test_20161107154342728278", "messages": {"receive_message": "Successfully read message.", "download_data": "Successfully downloaded data.", "execute": "Successfully executed command.", "upload_results": "Successfully uploaded results."}, "success": true} \ No newline at end of file From 249ba492d31448adf7e8eda1f1b4a871d7b8ea2b Mon Sep 17 00:00:00 2001 From: e-koch Date: Tue, 8 Nov 2016 12:14:38 -0700 Subject: [PATCH 3/8] Generate email after processing and set bucket's lifetime --- aws_exec.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/aws_exec.py b/aws_exec.py index 18690ee..ad3b8ea 100644 --- a/aws_exec.py +++ b/aws_exec.py @@ -3,9 +3,11 @@ from time import sleep, time from boto import sqs +from boto import ses from aws_controller.controller import WORKER_SCRIPT from aws_controller.launch_instance import launch +from aws_controller.upload_download_s3 import set_bucket_lifetime def run(timestamp, param_file, aws_file): @@ -107,6 +109,29 @@ def run(timestamp, param_file, aws_file): queue.delete() resp_queue.delete() + # Send out email to the user. + email_service = ses.connect_to_region(region, + aws_access_key_id=key, + aws_secret_access_key=secret) + + # The parameters should have the user's database values. + email_address = params["user_email"] + username = params["user_name"] + time_limit = params["time_limit"] + summary_url = params["summary_url"] + project_email = params["project_email"] + + subject = "AstroCompute Timing Script Job: {} Completion".format(proc_name) + body = email_body(username, proc_name, summary_url, time_limit, + project_email) + email_service.send_email(project_email, subject, body, email_address, + format='test') + + # Set the expiration on the S3 bucket. + set_bucket_lifetime(proc_name, days=time_limit, + aws_access={"aws_access_key_id": key, + "aws_secret_access_key": secret}) + # Return success/failure @@ -150,5 +175,33 @@ def json_message(params, proc_name): return json.dumps(params) +def email_body(username, job_name, url, time_limit, project_email): + ''' + Generate string for email body. + ''' + + email_str = \ + ''' + Hello USER,\n + \n + Your timing script job JOB is complete. The analysis results may be + downloaded from the link below. Note that data products are + automatically deleted after TIME days. Please retrieve the data + products as soon as possible!\n + \n + URL \n + \n + Questions or issues should be sent to EMAIL_HELP.\n + ''' + + email_str = email_str.replace("USER", username) + email_str = email_str.replace("JOB", job_name) + email_str = email_str.replace("URL", url) + email_str = email_str.replace("TIME", time_limit) + email_str = email_str.replace("EMAIL_HELP", project_email) + + return email_str + + if __name__ == "__main__": pass From e8c08d80ca647f33edb5b6454dd1393203568800 Mon Sep 17 00:00:00 2001 From: e-koch Date: Tue, 8 Nov 2016 12:28:19 -0700 Subject: [PATCH 4/8] Add user parameters for testing the email service --- param.txt | 2 +- test_aws_exec.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/param.txt b/param.txt index 536c988..33f184d 100644 --- a/param.txt +++ b/param.txt @@ -1,6 +1,6 @@ #param file for initial_clean.py, Aegean_ObjDet.py, and casa_timing_script.py ##common to all 3 scripts -target = '' +target = 'v404_test' obsDate = '' refFrequency ='GHz' intervalSizeH = 0 diff --git a/test_aws_exec.py b/test_aws_exec.py index 48a671c..bb27080 100644 --- a/test_aws_exec.py +++ b/test_aws_exec.py @@ -34,11 +34,18 @@ params = convert_param_format("param.txt", to="dict") + # Add in the user parameters. + params["user_email"] = "koch.eric.w@gmail.com" + params["user_name"] = "Dill Pickle" + params["time_limit"] = 1 + params["summary_url"] = "https://server-name" + params["project_email"] = "ualberta.astrocompute@gmail.com" start_time = timestring() print("Uploading at: " + human_time()) upload_to_s3(params['target'].lower() + "_" + start_time, - '/media/eric/Data_3/VLA/V404/v404_jun22_B_Cc7_bp.ms', + # '/media/eric/Data_3/VLA/V404/v404_jun22_B_Cc7_bp.ms', + '/Users/eric/Data/V404/v404_jun22_B_Cc7_bp.ms', key_prefix="data/", create_bucket=True) time.sleep(10) @@ -51,7 +58,8 @@ print("Downloading results at: " + human_time()) download_from_s3("data_products/*", params['target'].lower() + "_" + start_time, - output_dir="/media/eric/Data_3/VLA/V404/") + output_dir="/Users/eric/Data/V404/") + # output_dir="/media/eric/Data_3/VLA/V404/") except Exception as e: print("Failed at " + human_time()) From 17a1c46be44a5dea93f1504bfa9a96622f35d753 Mon Sep 17 00:00:00 2001 From: e-koch Date: Tue, 8 Nov 2016 13:31:51 -0700 Subject: [PATCH 5/8] Update aws_controller w/ file tar --- aws_controller | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_controller b/aws_controller index d708812..f78d0ad 160000 --- a/aws_controller +++ b/aws_controller @@ -1 +1 @@ -Subproject commit d70881228923461533822fa302a53535dacc46e6 +Subproject commit f78d0adc111486313a2efaded5c8b8deb5bd38fd From 9edd47ef1a3386fc041aa4dfcf109ff2f7f541b5 Mon Sep 17 00:00:00 2001 From: e-koch Date: Tue, 8 Nov 2016 13:37:17 -0700 Subject: [PATCH 6/8] Update aws_controller: Copy parameter file into outputs --- aws_controller | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_controller b/aws_controller index f78d0ad..8fcc16c 160000 --- a/aws_controller +++ b/aws_controller @@ -1 +1 @@ -Subproject commit f78d0adc111486313a2efaded5c8b8deb5bd38fd +Subproject commit 8fcc16ca9783a1d9185620340c26417d1e43a124 From 46b7e8559221d1ac1b0fed4792aa1e1f0761e34e Mon Sep 17 00:00:00 2001 From: e-koch Date: Tue, 8 Nov 2016 13:38:21 -0700 Subject: [PATCH 7/8] Ignore files within the tests folder (for now) --- .gitignore | 1 + tests/test_response.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 tests/test_response.txt diff --git a/.gitignore b/.gitignore index 453080f..27b382b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +tests/* # Elastic Beanstalk Files .elasticbeanstalk/* !.elasticbeanstalk/*.cfg.yml diff --git a/tests/test_response.txt b/tests/test_response.txt deleted file mode 100644 index ed68008..0000000 --- a/tests/test_response.txt +++ /dev/null @@ -1 +0,0 @@ -{"proc_name": "v404_test_20161107154342728278", "messages": {"receive_message": "Successfully read message.", "download_data": "Successfully downloaded data.", "execute": "Successfully executed command.", "upload_results": "Successfully uploaded results."}, "success": true} \ No newline at end of file From 41b7f24861f89f603718115e2ed33ea7c7b41700 Mon Sep 17 00:00:00 2001 From: e-koch Date: Tue, 8 Nov 2016 13:38:46 -0700 Subject: [PATCH 8/8] Minor fixes for email sending --- aws_exec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_exec.py b/aws_exec.py index ad3b8ea..bd4036c 100644 --- a/aws_exec.py +++ b/aws_exec.py @@ -125,7 +125,7 @@ def run(timestamp, param_file, aws_file): body = email_body(username, proc_name, summary_url, time_limit, project_email) email_service.send_email(project_email, subject, body, email_address, - format='test') + format='text') # Set the expiration on the S3 bucket. set_bucket_lifetime(proc_name, days=time_limit, @@ -197,7 +197,7 @@ def email_body(username, job_name, url, time_limit, project_email): email_str = email_str.replace("USER", username) email_str = email_str.replace("JOB", job_name) email_str = email_str.replace("URL", url) - email_str = email_str.replace("TIME", time_limit) + email_str = email_str.replace("TIME", str(time_limit)) email_str = email_str.replace("EMAIL_HELP", project_email) return email_str