Add support for S3 repositories

Author: ktf

alibuild_helpers/args.py

@@ -234,7 +234,7 @@ def finaliseArgs(args, parser, star):
     if args.architecture == "slc7_x86-64" and not args.preferSystem:
       args.noSystem = True
       if not args.remoteStore:
-        args.remoteStore = "https://alicache.cern.ch/"
+        args.remoteStore = "https://s3.cern.ch/swift/v1/alibuild-repo"
 
     if args.remoteStore or args.writeStore:
       args.noSystem = True

alibuild_helpers/build.py

@@ -122,13 +122,26 @@ def getRetry(self, url, dest=None):
           os.rename(dest+".tmp", dest)  # we should not have errors here
           return True
         else:
-          # No destination specified: JSON request
-          resp = get(url, verify=not self.insecure, timeout=self.httpTimeoutSec)
-          if resp.status_code == 404:
-            # No need to retry any further
-            return None
-          resp.raise_for_status()
-          return resp.json()
+          # For CERN S3 we need to construct the JSON ourself...
+          s3Request = re.match("https://s3.cern.ch/swift/v1[/]+([^/]*)/(.*)$", url)
+          if s3Request:
+            [bucket, prefix] = s3Request.groups()
+            url = "https://s3.cern.ch/swift/v1/%s/?prefix=%s" % (bucket, prefix.strip("/"))
+            resp = get(url, verify=not self.insecure, timeout=self.httpTimeoutSec)
+            if resp.status_code == 404:
+              # No need to retry any further
+              return None
+            resp.raise_for_status()
+            resp = resp.text
+            return [{"name": basename(x), "type": "file"} for x in resp.split()]
+          else:
+            # No destination specified: JSON request
+            resp = get(url, verify=not self.insecure, timeout=self.httpTimeoutSec)
+            if resp.status_code == 404:
+              # No need to retry any further
+              return None
+            resp.raise_for_status()
+            return resp.json()
       except (RequestException,ValueError,PartialDownloadError) as e:
         if i == self.httpConnRetries-1:
           error("GET %s failed: %s" % (url, str(e)))

tests/test_args.py

@@ -66,7 +66,7 @@ class FakeExit(Exception):
   ((), "init zlib --dist ktf/alidist@dev"                                 , [("dist", {"repo": "ktf/alidist", "ver": "dev"})]),
   ((), "build zlib --remote-store rsync://test.local/"                    , [("noSystem", True)]),
   ((), "build zlib --remote-store rsync://test.local/::rw"                , [("noSystem", True), ("remoteStore", "rsync://test.local/"), ("writeStore", "rsync://test.local/")]),
-  ((), "build zlib --architecture slc7_x86-64"                            , [("noSystem", True), ("preferSystem", False), ("remoteStore", "https://alicache.cern.ch/")]),
+  ((), "build zlib --architecture slc7_x86-64"                            , [("noSystem", True), ("preferSystem", False), ("remoteStore", "https://s3.cern.ch/swift/v1/alibuild-repo")]),
   ((), "build zlib --architecture ubuntu1804_x86-64"                      , [("noSystem", False), ("preferSystem", False), ("remoteStore", "")]),
   ((), "build zlib -a slc7_x86-64 --docker-image alisw/slc7-builder"      , [("docker", True), ("dockerImage", "alisw/slc7-builder")]),
   ((), "build zlib -a slc7_x86-64 --docker"                               , [("docker", True), ("dockerImage", "alisw/slc7-builder")]),