add support for s3.url_style path|virtual (default: virtual)

src/S3Commands.cc

@@ -34,7 +34,7 @@
     return AWSv4Impl::canonicalizeQueryString( query_parameters );
 }
 
 // Takes in the configured `s3.service_url` and uses the bucket/object requested to generate
 // the virtual host URL, as well as the canonical URI (which is the path to the object).
 bool AmazonRequest::parseURL(	const std::string & url,
 				const std::string & bucket,
@@ -47,7 +47,12 @@
 
     auto j = url.find( "/", i + 3 );
     if( j == std::string::npos ) {
-        host = bucket + "." + substring( url, i + 3 );
+        if (style == "path") {
+            host = substring( url, i + 3 );
+        } else {
+            host = bucket + "." + substring( url, i + 3 );
+        }
+
         path = "/" + object;
         return true;
     }

src/S3Commands.hh

@@ -11,14 +11,16 @@ public:
         const std::string & skf,
         const std::string & b,
         const std::string & o,
+        const std::string & style,
         int sv = 4
     ) :
     HTTPRequest( s ),
     accessKeyFile(akf),
     secretKeyFile(skf),
     signatureVersion(sv),
     bucket(b),
-    object(o)
+    object(o),
+    style(style)
     { 
         requiresSignature = true;
         // Start off by parsing the hostUrl, which we use in conjunction with the bucket to fill in the host (for setting host header).
@@ -33,8 +35,11 @@ public:
 
         // Now that we have the host and canonicalURI, we can build the actual url we perform the curl against.
         // Using the previous example, we'd get a new hostUrl of "https://my-bucket.my-url.com:443/my-object".
-        hostUrl = protocol + "://" +
-            host + canonicalURI;
+        if (style == "path") {
+            hostUrl = protocol + "://" + host + "/" + b + canonicalURI;
+        } else {
+            hostUrl = protocol + "://" + host + canonicalURI;            
+        }
 
         // If we can, set the region based on the host.
         size_t secondDot = host.find( ".", 2 + 1 );
@@ -77,6 +82,7 @@ protected:
     std::string region;
     std::string service;
 
+    std::string style;
 private:
     bool createV4Signature( const std::string & payload, std::string & authorizationHeader, bool sendContentSHA = false );
 
@@ -91,9 +97,10 @@ public:
         const std::string & akf,
         const std::string & skf,
         const std::string & b,
-        const std::string & o
+        const std::string & o,
+        const std::string & style
     ) :
-    AmazonRequest(s, akf, skf, b, o){}
+    AmazonRequest(s, akf, skf, b, o, style){}
 
     virtual ~AmazonS3Upload();
 
@@ -111,9 +118,10 @@ public:
         const std::string & akf,
         const std::string & skf,
         const std::string & b,
-        const std::string & o
+        const std::string & o,
+        const std::string & style
     ) :
-    AmazonRequest(s, akf, skf, b, o){}
+    AmazonRequest(s, akf, skf, b, o, style){}
 
     virtual ~AmazonS3Download();
 
@@ -128,9 +136,10 @@ public:
         const std::string & akf,
         const std::string & skf,
         const std::string & b,
-        const std::string & o
+        const std::string & o,
+        const std::string & style
     ) :
-    AmazonRequest(s, akf, skf, b, o){}
+    AmazonRequest(s, akf, skf, b, o, style){}
 
     virtual ~AmazonS3Head();
 

src/S3File.cc

@@ -14,6 +14,7 @@
 #include <sstream>
 #include <vector>
 
+#include <algorithm>
 #include <filesystem>
 
 #include <map>
@@ -78,7 +79,7 @@
     return 0;
 }
 
 
 int
 S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env)
 {
@@ -95,6 +96,7 @@
     std::string configured_s3_service_url = m_oss->getS3ServiceURL();
     std::string configured_s3_access_key = m_oss->getS3AccessKeyFile();
     std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile();
+    std::string configured_s3_url_style = m_oss->getS3URLStyle();
 
 
     // We used to query S3 here to see if the object existed, but of course
@@ -105,11 +107,12 @@
     this->s3_service_url = configured_s3_service_url;
     this->s3_access_key = configured_s3_access_key;
     this->s3_secret_key = configured_s3_secret_key;
+    this->s3_url_style = configured_s3_url_style;
     return 0;
 }
 
 
 ssize_t
 S3File::Read(void *buffer, off_t offset, size_t size)
 {
     AmazonS3Download download(
@@ -117,7 +120,8 @@
         this->s3_access_key,
         this->s3_secret_key,
         this->s3_bucket_name,
-        this->s3_object_name
+        this->s3_object_name,
+        this->s3_url_style
     );
 
 
@@ -131,7 +135,7 @@
     return bytes.size();
 }
 
 
 int
 S3File::Fstat(struct stat *buff)
 {
@@ -140,7 +144,8 @@
         this->s3_access_key,
         this->s3_secret_key,
         this->s3_bucket_name,
-        this->s3_object_name
+        this->s3_object_name,
+        this->s3_url_style
     );
 
     if(! head.SendRequest()) {
@@ -169,10 +174,11 @@
             std::string attr = substring( line, 0, colon );
             std::string value = substring( line, colon + 1 );
             trim(value);
+            std::transform(attr.begin(), attr.end(), attr.begin(), ::tolower);
 
-            if( attr == "Content-Length" ) {
+            if( attr == "content-length" ) {
                 this->content_length = std::stol(value);
-            } else if( attr == "Last-Modified" ) {
+            } else if( attr == "last-modified" ) {
                 struct tm t;
                 char * eos = strptime( value.c_str(),
                     "%a, %d %b %Y %T %Z",
@@ -204,7 +210,7 @@
     return 0;
 }
 
 
 ssize_t
 S3File::Write(const void *buffer, off_t offset, size_t size)
 {
@@ -213,7 +219,8 @@
         this->s3_access_key,
         this->s3_secret_key,
         this->s3_bucket_name,
-        this->s3_object_name
+        this->s3_object_name,
+        this->s3_url_style
     );
 
     std::string payload( (char *)buffer, size );

src/S3File.hh

@@ -125,6 +125,7 @@ private:
     std::string s3_object_name;
     std::string s3_access_key;
     std::string s3_secret_key;
+    std::string s3_url_style;
 
     size_t content_length;
     time_t last_modified;

src/S3FileSystem.cc

@@ -66,7 +66,7 @@
     }
 
     char * temporary;
     std::string value;
     std::string attribute;
     Config.Attach(cfgFD);
     while ((temporary = Config.GetMyFirstWord())) {
@@ -85,9 +85,11 @@
             value, this->s3_access_key_file ) ) { Config.Close(); return false; }
         if(! handle_required_config( attribute, "s3.secret_key_file",
             value, this->s3_secret_key_file ) ) { Config.Close(); return false; }
+        if(! handle_required_config( attribute, "s3.url_style",
+            value, this->s3_url_style ) ) { Config.Close(); return false; }
     }
 
     if( this->s3_service_name.empty() ) {
         m_log.Emsg("Config", "s3.service_name not specified");
         return false;
     }
@@ -95,8 +97,15 @@
         m_log.Emsg("Config", "s3.region not specified");
         return false;
     }
+    if( this->s3_url_style.empty() ) {
+        this->s3_url_style = "virtual";
+    }
+    if( this->s3_url_style != "virtual" && this->s3_url_style != "path" ) {
+        m_log.Emsg("Config", "invalid s3.url_style specified");
+        return false;
+    }
 
     int retc = Config.LastError();
     if( retc ) {
         m_log.Emsg("Config", -retc, "read config file", configfn);
         Config.Close();

src/S3FileSystem.hh

@@ -60,6 +60,7 @@ public:
     const std::string & getS3ServiceName() const { return s3_service_name; }
     const std::string & getS3Region() const { return s3_region; }
     const std::string & getS3ServiceURL() const { return s3_service_url; }
+    const std::string & getS3URLStyle() const { return s3_url_style; }
 
     const std::string & getS3AccessKeyFile() const { return s3_access_key_file; }
     const std::string & getS3SecretKeyFile() const { return s3_secret_key_file; }
@@ -81,4 +82,6 @@ private:
 
     std::string s3_access_key_file;
     std::string s3_secret_key_file;
+
+    std::string s3_url_style;
 };