diff --git a/src/S3Commands.cc b/src/S3Commands.cc index a6f818d..0bb1526 100644 --- a/src/S3Commands.cc +++ b/src/S3Commands.cc @@ -47,7 +47,12 @@ bool AmazonRequest::parseURL( const std::string & url, auto j = url.find( "/", i + 3 ); if( j == std::string::npos ) { - host = bucket + "." + substring( url, i + 3 ); + if (style == "path") { + host = substring( url, i + 3 ); + } else { + host = bucket + "." + substring( url, i + 3 ); + } + path = "/" + object; return true; } diff --git a/src/S3Commands.hh b/src/S3Commands.hh index 0aa84b4..a7385ec 100644 --- a/src/S3Commands.hh +++ b/src/S3Commands.hh @@ -11,6 +11,7 @@ public: const std::string & skf, const std::string & b, const std::string & o, + const std::string & style, int sv = 4 ) : HTTPRequest( s ), @@ -18,7 +19,8 @@ public: secretKeyFile(skf), signatureVersion(sv), bucket(b), - object(o) + object(o), + style(style) { requiresSignature = true; // Start off by parsing the hostUrl, which we use in conjunction with the bucket to fill in the host (for setting host header). @@ -33,8 +35,11 @@ public: // Now that we have the host and canonicalURI, we can build the actual url we perform the curl against. // Using the previous example, we'd get a new hostUrl of "https://my-bucket.my-url.com:443/my-object". - hostUrl = protocol + "://" + - host + canonicalURI; + if (style == "path") { + hostUrl = protocol + "://" + host + "/" + b + canonicalURI; + } else { + hostUrl = protocol + "://" + host + canonicalURI; + } // If we can, set the region based on the host. size_t secondDot = host.find( ".", 2 + 1 ); @@ -77,6 +82,7 @@ protected: std::string region; std::string service; + std::string style; private: bool createV4Signature( const std::string & payload, std::string & authorizationHeader, bool sendContentSHA = false ); @@ -91,9 +97,10 @@ public: const std::string & akf, const std::string & skf, const std::string & b, - const std::string & o + const std::string & o, + const std::string & style ) : - AmazonRequest(s, akf, skf, b, o){} + AmazonRequest(s, akf, skf, b, o, style){} virtual ~AmazonS3Upload(); @@ -111,9 +118,10 @@ public: const std::string & akf, const std::string & skf, const std::string & b, - const std::string & o + const std::string & o, + const std::string & style ) : - AmazonRequest(s, akf, skf, b, o){} + AmazonRequest(s, akf, skf, b, o, style){} virtual ~AmazonS3Download(); @@ -128,9 +136,10 @@ public: const std::string & akf, const std::string & skf, const std::string & b, - const std::string & o + const std::string & o, + const std::string & style ) : - AmazonRequest(s, akf, skf, b, o){} + AmazonRequest(s, akf, skf, b, o, style){} virtual ~AmazonS3Head(); diff --git a/src/S3File.cc b/src/S3File.cc index 13c05f4..8d9f75e 100644 --- a/src/S3File.cc +++ b/src/S3File.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -95,6 +96,7 @@ S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) std::string configured_s3_service_url = m_oss->getS3ServiceURL(); std::string configured_s3_access_key = m_oss->getS3AccessKeyFile(); std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile(); + std::string configured_s3_url_style = m_oss->getS3URLStyle(); // We used to query S3 here to see if the object existed, but of course @@ -105,6 +107,7 @@ S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) this->s3_service_url = configured_s3_service_url; this->s3_access_key = configured_s3_access_key; this->s3_secret_key = configured_s3_secret_key; + this->s3_url_style = configured_s3_url_style; return 0; } @@ -117,7 +120,8 @@ S3File::Read(void *buffer, off_t offset, size_t size) this->s3_access_key, this->s3_secret_key, this->s3_bucket_name, - this->s3_object_name + this->s3_object_name, + this->s3_url_style ); @@ -140,7 +144,8 @@ S3File::Fstat(struct stat *buff) this->s3_access_key, this->s3_secret_key, this->s3_bucket_name, - this->s3_object_name + this->s3_object_name, + this->s3_url_style ); if(! head.SendRequest()) { @@ -169,10 +174,11 @@ S3File::Fstat(struct stat *buff) std::string attr = substring( line, 0, colon ); std::string value = substring( line, colon + 1 ); trim(value); + std::transform(attr.begin(), attr.end(), attr.begin(), ::tolower); - if( attr == "Content-Length" ) { + if( attr == "content-length" ) { this->content_length = std::stol(value); - } else if( attr == "Last-Modified" ) { + } else if( attr == "last-modified" ) { struct tm t; char * eos = strptime( value.c_str(), "%a, %d %b %Y %T %Z", @@ -213,7 +219,8 @@ S3File::Write(const void *buffer, off_t offset, size_t size) this->s3_access_key, this->s3_secret_key, this->s3_bucket_name, - this->s3_object_name + this->s3_object_name, + this->s3_url_style ); std::string payload( (char *)buffer, size ); diff --git a/src/S3File.hh b/src/S3File.hh index b7db206..ed8a8d4 100644 --- a/src/S3File.hh +++ b/src/S3File.hh @@ -125,6 +125,7 @@ private: std::string s3_object_name; std::string s3_access_key; std::string s3_secret_key; + std::string s3_url_style; size_t content_length; time_t last_modified; diff --git a/src/S3FileSystem.cc b/src/S3FileSystem.cc index 6bcaa04..3a6f9dc 100644 --- a/src/S3FileSystem.cc +++ b/src/S3FileSystem.cc @@ -85,6 +85,8 @@ S3FileSystem::Config(XrdSysLogger *lp, const char *configfn) value, this->s3_access_key_file ) ) { Config.Close(); return false; } if(! handle_required_config( attribute, "s3.secret_key_file", value, this->s3_secret_key_file ) ) { Config.Close(); return false; } + if(! handle_required_config( attribute, "s3.url_style", + value, this->s3_url_style ) ) { Config.Close(); return false; } } if( this->s3_service_name.empty() ) { @@ -95,6 +97,13 @@ S3FileSystem::Config(XrdSysLogger *lp, const char *configfn) m_log.Emsg("Config", "s3.region not specified"); return false; } + if( this->s3_url_style.empty() ) { + this->s3_url_style = "virtual"; + } + if( this->s3_url_style != "virtual" && this->s3_url_style != "path" ) { + m_log.Emsg("Config", "invalid s3.url_style specified"); + return false; + } int retc = Config.LastError(); if( retc ) { diff --git a/src/S3FileSystem.hh b/src/S3FileSystem.hh index 277d67f..065c611 100644 --- a/src/S3FileSystem.hh +++ b/src/S3FileSystem.hh @@ -60,6 +60,7 @@ public: const std::string & getS3ServiceName() const { return s3_service_name; } const std::string & getS3Region() const { return s3_region; } const std::string & getS3ServiceURL() const { return s3_service_url; } + const std::string & getS3URLStyle() const { return s3_url_style; } const std::string & getS3AccessKeyFile() const { return s3_access_key_file; } const std::string & getS3SecretKeyFile() const { return s3_secret_key_file; } @@ -81,4 +82,6 @@ private: std::string s3_access_key_file; std::string s3_secret_key_file; + + std::string s3_url_style; };