Merge pull request #45 from seporaitis/v4_scheme

AWS V4 signature scheme

Author: mbrossard

CHANGELOG.md

@@ -1,5 +1,10 @@
+## 1.1.0 (2016-07-11)
+- #32: Add support for AWS v4 signature (@mbrossard)
+- #32: Add support for s3:// scheme (@asedge, @mbrossard)
+- #43: Add retries with exponential back-off (@bemehow, @mbrossard)
+
 ## 1.0.3 (2016-07-05)
-- Add support for delegated roles (@ToneD)
+- #44: Add support for delegated roles (@ToneD)
 
 ## 1.0.2 (2015-11-03)
-- Fix signature issue with python 2.7 (@mbrossard)
+- #34: Fix signature issue with python 2.7 (@mbrossard)

Makefile

@@ -1,7 +1,7 @@
-NAME				= yum-plugin-s3-iam
-VERSION			= 1.0.3
-RELEASE			= 1
-ARCH				= noarch
+NAME    = yum-plugin-s3-iam
+VERSION = 1.1.0
+RELEASE = 1
+ARCH    = noarch
 
 RPM_TOPDIR ?= $(shell rpm --eval '%{_topdir}')
 

README.md

@@ -33,8 +33,6 @@ use this plugin:
 Currently the plugin does not support:
 - Proxy server configuration
 - Multi-valued baseurl or mirrorlist
-- AWS version 4 signatures needed by S3 in some regions (see v4_scheme
-  branch for work in progress)
 
 ## Testing
 

s3iam.py

@@ -15,10 +15,12 @@
 
 import urllib2
 import urlparse
+import datetime
 import time
 import hashlib
 import hmac
 import json
+import re
 
 import yum
 import yum.config
@@ -31,7 +33,7 @@
 __email__ = "julius@seporaitis.net"
 __copyright__ = "Copyright 2012, Julius Seporaitis"
 __license__ = "Apache 2.0"
-__version__ = "1.0.3"
+__version__ = "1.1.0"
 
 
 __all__ = ['requires_api_version', 'plugin_type', 'CONDUIT',
@@ -40,53 +42,115 @@
 requires_api_version = '2.5'
 plugin_type = yum.plugins.TYPE_CORE
 CONDUIT = None
+DEFAULT_DELAY = 3
+DEFAULT_BACKOFF = 2
+BUFFER_SIZE = 1024 * 1024
+OPTIONAL_ATTRIBUTES = ['priority', 'base_persistdir', 'metadata_expire',
+                       'skip_if_unavailable', 'keepcache', 'priority']
+UNSUPPORTED_ATTRIBUTES = ['mirrorlist', 'proxy']
 
 
 def config_hook(conduit):
     yum.config.RepoConf.s3_enabled = yum.config.BoolOption(False)
+    yum.config.RepoConf.region = yum.config.Option()
     yum.config.RepoConf.key_id = yum.config.Option()
     yum.config.RepoConf.secret_key = yum.config.Option()
     yum.config.RepoConf.delegated_role = yum.config.Option()
+    yum.config.RepoConf.baseurl = yum.config.UrlListOption(
+        schemes=('http', 'https', 's3', 'ftp', 'file')
+    )
+    yum.config.RepoConf.backoff = yum.config.Option()
+    yum.config.RepoConf.delay = yum.config.Option()
+
+
+def parse_url(url):
+    # http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html
+    url = url[0] if isinstance(url, list) else url
+
+    # http[s]://<bucket>.s3.amazonaws.com
+    m = re.match(r'(http|https|s3)://([a-z0-9][a-z0-9-.]{1,61}[a-z0-9])[.]s3[.]amazonaws[.]com(.*)$', url)
+    if m:
+        return (m.group(2), None, m.group(3))
+
+    # http[s]://<bucket>.s3-<aws-region>.amazonaws.com
+    m = re.match(r'(http|https|s3)://([a-z0-9][a-z0-9-.]{1,61}[a-z0-9])[.]s3-([a-z0-9-]+)[.]amazonaws[.]com(.*)$', url)
+    if m:
+        return (m.group(2), m.group(3), m.group(4))
+
+    # http[s]://s3.amazonaws.com/<bucket>
+    m = re.match(r'(http|https|s3)://s3[.]amazonaws[.]com/([a-z0-9][a-z0-9-.]{1,61}[a-z0-9])(.*)$', url)
+    if m:
+        return (m.group(2), 'us-east-1', m.group(3))
+
+    # http[s]://s3-<region>.amazonaws.com/<bucket>
+    m = re.match(r'(http|https|s3)://s3-([a-z0-9-]+)[.]amazonaws[.]com/([a-z0-9][a-z0-9-.]{1,61}[a-z0-9])(.*)$', url)
+    if m:
+        return (m.group(3), m.group(2), m.group(4))
+
+    return (None, None, None)
+
+
+def replace_repo(repos, repo):
+    repos.delete(repo.id)
+    repos.add(S3Repository(repo.id, repo))
 
 
 def prereposetup_hook(conduit):
     """Plugin initialization hook. Setup the S3 repositories."""
 
     repos = conduit.getRepos()
-
     for repo in repos.listEnabled():
+        url = repo.baseurl
+        if(isinstance(url, list)):
+            if len(url) == 0:
+                continue
+            url = url[0]
+        if re.match(r'^s3://', url):
+            repo.s3_enabled = 1
         if isinstance(repo, YumRepository) and repo.s3_enabled:
-            new_repo = S3Repository(repo.id, repo.baseurl)
-            new_repo.name = repo.name
-            # new_repo.baseurl = repo.baseurl
-            new_repo.mirrorlist = repo.mirrorlist
-            new_repo.basecachedir = repo.basecachedir
-            new_repo.gpgcheck = repo.gpgcheck
-            new_repo.gpgkey = repo.gpgkey
-            new_repo.key_id = repo.key_id
-            new_repo.secret_key = repo.secret_key
-            new_repo.proxy = repo.proxy
-            new_repo.enablegroups = repo.enablegroups
-            if hasattr(repo, 'priority'):
-                new_repo.priority = repo.priority
-            if hasattr(repo, 'base_persistdir'):
-                new_repo.base_persistdir = repo.base_persistdir
-            if hasattr(repo, 'metadata_expire'):
-                new_repo.metadata_expire = repo.metadata_expire
-            if hasattr(repo, 'skip_if_unavailable'):
-                new_repo.skip_if_unavailable = repo.skip_if_unavailable
-
-            repos.delete(repo.id)
-            repos.add(new_repo)
+            replace_repo(repos, repo)
 
 
 class S3Repository(YumRepository):
     """Repository object for Amazon S3, using IAM Roles."""
 
-    def __init__(self, repoid, baseurl):
+    def __init__(self, repoid, repo):
         super(S3Repository, self).__init__(repoid)
+
+        bucket, region, path = parse_url(repo.baseurl)
+
+        if bucket is None:
+            msg = "s3iam: unable to parse url %s'" % repo.baseurl
+            raise yum.plugins.PluginYumExit(msg)
+
+        if region:
+            self.baseurl = "https://s3-%s.amazonaws.com/%s%s" % (region, bucket, path)
+        else:
+            self.baseurl = "https://%s.s3.amazonaws.com%s" % (bucket, path)
+
+        self.name = repo.name
+        self.region = repo.region if repo.region else region
+        self.basecachedir = repo.basecachedir
+        self.gpgcheck = repo.gpgcheck
+        self.gpgkey = repo.gpgkey
+        self.key_id = repo.key_id
+        self.secret_key = repo.secret_key
+        self.enablegroups = repo.enablegroups
+
+        self.retries = repo.retries
+        self.backoff = repo.backoff
+        self.delay = repo.delay
+
+        for attr in OPTIONAL_ATTRIBUTES:
+            if hasattr(repo, attr):
+                setattr(self, attr, getattr(repo, attr))
+
+        for attr in UNSUPPORTED_ATTRIBUTES:
+            if getattr(repo, attr):
+                msg = "%s: Unsupported attribute: %s." % (__file__, attr)
+                raise yum.plugins.PluginYumExit(msg)
+
         self.iamrole = None
-        self.baseurl = baseurl
         self.grabber = None
         self.enable()
 
@@ -118,11 +182,17 @@ def __init__(self, repo):
         """
         if isinstance(repo, basestring):
             self.baseurl = repo
+            self.region = None
+            self.retries = 0
         else:
+            self.region = repo.region
+            self.retries = repo.retries
+            self.backoff = DEFAULT_BACKOFF if repo.backoff is None else repo.backoff
+            self.delay = DEFAULT_DELAY if repo.delay is None else repo.delay
             if len(repo.baseurl) != 1:
-                raise yum.plugins.PluginYumExit("s3iam: repository '%s' "
-                                                "must have only one "
-                                                "'baseurl' value" % repo.id)
+                msg = "%s: repository '%s' must" % (__file__, repo.id)
+                msg += 'have only one baseurl value'
+                raise yum.plugins.PluginYumExit(msg)
             else:
                 self.baseurl = repo.baseurl[0]
         # Ensure urljoin doesn't ignore base path:
@@ -206,15 +276,13 @@ def get_instance_region(self):
                 response.close()
         self.region = data[:-1]
 
-    def _request(self, path):
+    def _request(self, path, timeval=None):
         url = urlparse.urljoin(self.baseurl, urllib2.quote(path))
         request = urllib2.Request(url)
-        if self.token:
-            request.add_header('x-amz-security-token', self.token)
-        signature = self.sign(request)
-        request.add_header('Authorization', "AWS {0}:{1}".format(
-            self.access_key,
-            signature))
+        if self.region:
+            self.signV4(request, timeval)
+        else:
+            self.signV2(request, timeval)
         return request
 
     def urlgrab(self, url, filename=None, **kwargs):
@@ -226,26 +294,38 @@ def urlgrab(self, url, filename=None, **kwargs):
                 filename = filename[1:]
 
         response = None
-        try:
-            out = open(filename, 'w+')
-            response = urllib2.urlopen(request)
-            buff = response.read(8192)
-            while buff:
-                out.write(buff)
-                buff = response.read(8192)
-        except urllib2.HTTPError, e:
-            # Wrap exception as URLGrabError so that YumRepository catches it
-            from urlgrabber.grabber import URLGrabError
-            new_e = URLGrabError(14, '%s on %s' % (e, url))
-            new_e.code = e.code
-            new_e.exception = e
-            new_e.url = url
-            raise new_e
-        finally:
-            if response:
-                response.close()
-            out.close()
-
+        retries = self.retries
+        delay = self.delay
+        out = open(filename, 'w+')
+        while retries > 0:
+            try:
+                response = urllib2.urlopen(request)
+                buff = response.read(BUFFER_SIZE)
+                while buff:
+                    out.write(buff)
+                    buff = response.read(BUFFER_SIZE)
+            except urllib2.HTTPError, e:
+                if retries > 0:
+                    time.sleep(delay)
+                    delay *= self.backoff
+                else:
+                    # Wrap exception as URLGrabError so that YumRepository catches it
+                    from urlgrabber.grabber import URLGrabError
+                    msg = '%s on %s tried' % (e, url)
+                    if self.retries > 0:
+                        msg += ' tried %d time(s)' % (self.retries)
+                        new_e = URLGrabError(14, msg)
+                        new_e.code = e.code
+                        new_e.exception = e
+                        new_e.url = url
+                        raise new_e
+            finally:
+                retries -= 1
+                if response:
+                    response.close()
+                    break
+
+        out.close()
         return filename
 
     def urlopen(self, url, **kwargs):
@@ -256,30 +336,19 @@ def urlread(self, url, limit=None, **kwargs):
         """urlread(url) return the contents of the file as a string."""
         return urllib2.urlopen(self._request(url)).read()
 
-    def sign(self, request, timeval=None):
+    def signV2(self, request, timeval=None):
         """Attach a valid S3 signature to request.
         request - instance of Request
         """
-        date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", timeval or time.gmtime())
+        t = timeval or time.gmtime()
+        date = time.strftime("%a, %d %b %Y %H:%M:%S +0000", t)
         request.add_header('Date', date)
-        host = request.get_host()
 
-        # TODO: bucket name finding is ugly, I should find a way to support
-        # both naming conventions: http://bucket.s3.amazonaws.com/ and
-        # http://s3.amazonaws.com/bucket/
-        try:
-            pos = host.find(".s3")
-            assert pos != -1
-            bucket = host[:pos]
-        except AssertionError:
-            raise yum.plugins.PluginYumExit(
-                "s3iam: baseurl hostname should be in format: "
-                "'<bucket>.s3<aws-region>.amazonaws.com'; "
-                "found '%s'" % host)
-
-        resource = "/%s%s" % (bucket, request.get_selector(), )
+        (bucket, ignore, path) = parse_url(request.get_full_url())
+        resource = '/' + bucket + path
         if self.token:
             amz_headers = 'x-amz-security-token:%s\n' % self.token
+            request.add_header('x-amz-security-token', self.token)
         else:
             amz_headers = ''
         sigstring = ("%(method)s\n\n\n%(date)s\n"
@@ -292,5 +361,57 @@ def sign(self, request, timeval=None):
             str(self.secret_key),
             str(sigstring),
             hashlib.sha1).digest()
-        signature = digest.encode('base64')
-        return signature.strip()
+        signature = digest.encode('base64').rstrip()
+
+        authorization = "AWS {0}:{1}".format(self.access_key, signature)
+        request.add_header('Authorization', authorization)
+
+    def derive(self, key, msg):
+        return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
+
+    def deriveKey(self, key, date, region, service):
+        kDate = self.derive(('AWS4' + key).encode('utf-8'), date)
+        kRegion = self.derive(kDate, region)
+        kService = self.derive(kRegion, service)
+        return self.derive(kService, 'aws4_request')
+
+    def signV4(self, request, timeval=None):
+        algorithm = 'AWS4-HMAC-SHA256'
+        t = datetime.datetime.utcnow()
+
+        amzdate = t.strftime('%Y%m%dT%H%M%SZ')
+        amz_headers = ('host:%s\nx-amz-date:%s\n' %
+                       (request.get_host(), amzdate))
+        signed_headers = 'host;x-amz-date'
+        if self.token:
+            amz_headers += 'x-amz-security-token:%s\n' % self.token
+            signed_headers += ';x-amz-security-token'
+            request.add_header('x-amz-security-token', self.token)
+
+        # Hash request
+        content_h = hashlib.sha256('').hexdigest()  # Empty content
+        req = ('GET\n%s\n\n%s\n%s\n%s' %
+               (request.get_selector(), amz_headers, signed_headers, content_h))
+        req_hash = hashlib.sha256(req).hexdigest()
+
+        # Assemble content to be signed
+        datestamp = t.strftime('%Y%m%d')
+        scope = datestamp + '/' + self.region + '/s3/aws4_request'
+        sign_content = '%s\n%s\n%s\n%s' % (algorithm, amzdate, scope, req_hash)
+
+        # Get derived key
+        signing_key = self.deriveKey(self.secret_key, datestamp,
+                                     self.region, 's3')
+
+        # Compute signature
+        signature = hmac.new(signing_key, (sign_content).encode('utf-8'),
+                             hashlib.sha256).hexdigest()
+
+        # Assemble 'Authorization' header value
+        credential = self.access_key + '/' + scope
+        auth = (('%s Credential=%s, SignedHeaders=%s, Signature=%s') %
+                (algorithm, credential, signed_headers, signature))
+
+        request.add_header('x-amz-content-sha256', content_h)
+        request.add_header('x-amz-date', amzdate)
+        request.add_header('Authorization', auth)