Merge pull request #245 from DerekTBrown/feat-default-creds

feat: consolidate s3 logic, use default providers chain

Author: acjohnson

grafana_backup/s3_common.py

@@ -0,0 +1,39 @@
+import boto3
+from botocore.exceptions import NoCredentialsError, ClientError
+
+def get_boto_session(settings) -> boto3.Session:
+    aws_default_region = settings.get("AWS_DEFAULT_REGION")
+    aws_access_key_id = settings.get("AWS_ACCESS_KEY_ID")
+    aws_secret_access_key = settings.get("AWS_SECRET_ACCESS_KEY")
+
+    # If no credentials are provided, boto3 will use the default credentials provider chain.
+    if aws_access_key_id is None or aws_secret_access_key is None:
+        return boto3.Session(
+            region_name=aws_default_region,
+        )
+
+    # Otherwise, use the provided credentials.
+    return boto3.Session(
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+        region_name=aws_default_region,
+    )
+
+def get_s3_resource(settings):
+    session = get_boto_session(settings)
+    aws_endpoint_url = settings.get("AWS_ENDPOINT_URL")
+    s3 = session.resource(
+        service_name="s3",
+        endpoint_url=aws_endpoint_url,
+    )
+    return s3
+
+
+def get_s3_object(settings, s3_file_name):
+    aws_s3_bucket_name = settings.get('AWS_S3_BUCKET_NAME')
+    aws_s3_bucket_key = settings.get('AWS_S3_BUCKET_KEY')
+
+    s3 = get_s3_resource(settings)
+    s3_object = s3.Object(aws_s3_bucket_name, '{0}/{1}'.format(aws_s3_bucket_key, s3_file_name))
+
+    return s3_object

grafana_backup/s3_download.py

@@ -2,24 +2,13 @@
 from botocore.exceptions import NoCredentialsError, ClientError
 from io import BytesIO 
 
+from grafana_backup.s3_common import get_s3_object
 
 def main(args, settings):
     arg_archive_file = args.get("<archive_file>", None)
-    aws_s3_bucket_name = settings.get("AWS_S3_BUCKET_NAME")
-    aws_s3_bucket_key = settings.get("AWS_S3_BUCKET_KEY")
-    aws_default_region = settings.get("AWS_DEFAULT_REGION")
-    aws_access_key_id = settings.get("AWS_ACCESS_KEY_ID")
-    aws_secret_access_key = settings.get("AWS_SECRET_ACCESS_KEY")
-    aws_endpoint_url = settings.get("AWS_ENDPOINT_URL")
 
-    session = boto3.Session(
-        aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_default_region
-    )
-
-    s3 = session.resource(service_name="s3", endpoint_url=aws_endpoint_url)
-
-    s3_path = "{0}/{1}".format(aws_s3_bucket_key, arg_archive_file)
-    s3_object = s3.Object(aws_s3_bucket_name, s3_path)
+    aws_s3_bucket_name = settings.get('AWS_S3_BUCKET_NAME')
+    s3_object = get_s3_object(settings, s3_file_name=arg_archive_file)
 
     try:
         # .read() left off on purpose, tarfile.open() takes care of that part
@@ -28,7 +17,7 @@ def main(args, settings):
         print("Download from S3 was successful")
     except ClientError as e:
         if e.response["Error"]["Code"] == "NoSuchKey":
-            print("Error: Key {0} does not exist in bucket {1}".format(s3_path, aws_s3_bucket_name))
+            print("Error: Key {0} does not exist in bucket {1}".format(s3_object.key, aws_s3_bucket_name))
             return False
         raise e
     except NoCredentialsError:

grafana_backup/s3_upload.py

@@ -1,33 +1,16 @@
 import boto3
 from botocore.exceptions import NoCredentialsError
 
+from grafana_backup.s3_common import get_s3_object
 
 def main(args, settings):
-    aws_s3_bucket_name = settings.get('AWS_S3_BUCKET_NAME')
-    aws_s3_bucket_key = settings.get('AWS_S3_BUCKET_KEY')
-    aws_default_region = settings.get('AWS_DEFAULT_REGION')
-    aws_access_key_id = settings.get('AWS_ACCESS_KEY_ID')
-    aws_secret_access_key = settings.get('AWS_SECRET_ACCESS_KEY')
-    aws_endpoint_url = settings.get('AWS_ENDPOINT_URL')
-
     backup_dir = settings.get('BACKUP_DIR')
     timestamp = settings.get('TIMESTAMP')
 
     s3_file_name = '{0}.tar.gz'.format(timestamp)
     archive_file = '{0}/{1}'.format(backup_dir, s3_file_name)
 
-    session = boto3.Session(
-        aws_access_key_id=aws_access_key_id,
-        aws_secret_access_key=aws_secret_access_key,
-        region_name=aws_default_region
-    )
-
-    s3 = session.resource(
-        service_name='s3',
-        endpoint_url=aws_endpoint_url
-    )
-
-    s3_object = s3.Object(aws_s3_bucket_name, '{0}/{1}'.format(aws_s3_bucket_key, s3_file_name))
+    s3_object = get_s3_object(settings, s3_file_name=s3_file_name)
 
     try:
         s3_object.put(Body=open(archive_file, 'rb'))