From 13727da1a95fd2ed8d17bb82babb4db5e906a972 Mon Sep 17 00:00:00 2001 From: "Nicholas J. Kisseberth" Date: Thu, 29 Feb 2024 14:25:23 -0800 Subject: [PATCH 1/2] add support for s3.url_style path|virtual (default: virtual) --- src/S3Commands.cc | 7 ++++++- src/S3Commands.hh | 27 ++++++++++++++++++--------- src/S3File.cc | 17 ++++++++++++----- src/S3File.hh | 1 + src/S3FileSystem.cc | 9 +++++++++ src/S3FileSystem.hh | 3 +++ 6 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/S3Commands.cc b/src/S3Commands.cc index a6f818d..0bb1526 100644 --- a/src/S3Commands.cc +++ b/src/S3Commands.cc @@ -47,7 +47,12 @@ bool AmazonRequest::parseURL( const std::string & url, auto j = url.find( "/", i + 3 ); if( j == std::string::npos ) { - host = bucket + "." + substring( url, i + 3 ); + if (style == "path") { + host = substring( url, i + 3 ); + } else { + host = bucket + "." + substring( url, i + 3 ); + } + path = "/" + object; return true; } diff --git a/src/S3Commands.hh b/src/S3Commands.hh index 0aa84b4..a7385ec 100644 --- a/src/S3Commands.hh +++ b/src/S3Commands.hh @@ -11,6 +11,7 @@ public: const std::string & skf, const std::string & b, const std::string & o, + const std::string & style, int sv = 4 ) : HTTPRequest( s ), @@ -18,7 +19,8 @@ public: secretKeyFile(skf), signatureVersion(sv), bucket(b), - object(o) + object(o), + style(style) { requiresSignature = true; // Start off by parsing the hostUrl, which we use in conjunction with the bucket to fill in the host (for setting host header). @@ -33,8 +35,11 @@ public: // Now that we have the host and canonicalURI, we can build the actual url we perform the curl against. // Using the previous example, we'd get a new hostUrl of "https://my-bucket.my-url.com:443/my-object". - hostUrl = protocol + "://" + - host + canonicalURI; + if (style == "path") { + hostUrl = protocol + "://" + host + "/" + b + canonicalURI; + } else { + hostUrl = protocol + "://" + host + canonicalURI; + } // If we can, set the region based on the host. size_t secondDot = host.find( ".", 2 + 1 ); @@ -77,6 +82,7 @@ protected: std::string region; std::string service; + std::string style; private: bool createV4Signature( const std::string & payload, std::string & authorizationHeader, bool sendContentSHA = false ); @@ -91,9 +97,10 @@ public: const std::string & akf, const std::string & skf, const std::string & b, - const std::string & o + const std::string & o, + const std::string & style ) : - AmazonRequest(s, akf, skf, b, o){} + AmazonRequest(s, akf, skf, b, o, style){} virtual ~AmazonS3Upload(); @@ -111,9 +118,10 @@ public: const std::string & akf, const std::string & skf, const std::string & b, - const std::string & o + const std::string & o, + const std::string & style ) : - AmazonRequest(s, akf, skf, b, o){} + AmazonRequest(s, akf, skf, b, o, style){} virtual ~AmazonS3Download(); @@ -128,9 +136,10 @@ public: const std::string & akf, const std::string & skf, const std::string & b, - const std::string & o + const std::string & o, + const std::string & style ) : - AmazonRequest(s, akf, skf, b, o){} + AmazonRequest(s, akf, skf, b, o, style){} virtual ~AmazonS3Head(); diff --git a/src/S3File.cc b/src/S3File.cc index 13c05f4..8d9f75e 100644 --- a/src/S3File.cc +++ b/src/S3File.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -95,6 +96,7 @@ S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) std::string configured_s3_service_url = m_oss->getS3ServiceURL(); std::string configured_s3_access_key = m_oss->getS3AccessKeyFile(); std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile(); + std::string configured_s3_url_style = m_oss->getS3URLStyle(); // We used to query S3 here to see if the object existed, but of course @@ -105,6 +107,7 @@ S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) this->s3_service_url = configured_s3_service_url; this->s3_access_key = configured_s3_access_key; this->s3_secret_key = configured_s3_secret_key; + this->s3_url_style = configured_s3_url_style; return 0; } @@ -117,7 +120,8 @@ S3File::Read(void *buffer, off_t offset, size_t size) this->s3_access_key, this->s3_secret_key, this->s3_bucket_name, - this->s3_object_name + this->s3_object_name, + this->s3_url_style ); @@ -140,7 +144,8 @@ S3File::Fstat(struct stat *buff) this->s3_access_key, this->s3_secret_key, this->s3_bucket_name, - this->s3_object_name + this->s3_object_name, + this->s3_url_style ); if(! head.SendRequest()) { @@ -169,10 +174,11 @@ S3File::Fstat(struct stat *buff) std::string attr = substring( line, 0, colon ); std::string value = substring( line, colon + 1 ); trim(value); + std::transform(attr.begin(), attr.end(), attr.begin(), ::tolower); - if( attr == "Content-Length" ) { + if( attr == "content-length" ) { this->content_length = std::stol(value); - } else if( attr == "Last-Modified" ) { + } else if( attr == "last-modified" ) { struct tm t; char * eos = strptime( value.c_str(), "%a, %d %b %Y %T %Z", @@ -213,7 +219,8 @@ S3File::Write(const void *buffer, off_t offset, size_t size) this->s3_access_key, this->s3_secret_key, this->s3_bucket_name, - this->s3_object_name + this->s3_object_name, + this->s3_url_style ); std::string payload( (char *)buffer, size ); diff --git a/src/S3File.hh b/src/S3File.hh index b7db206..ed8a8d4 100644 --- a/src/S3File.hh +++ b/src/S3File.hh @@ -125,6 +125,7 @@ private: std::string s3_object_name; std::string s3_access_key; std::string s3_secret_key; + std::string s3_url_style; size_t content_length; time_t last_modified; diff --git a/src/S3FileSystem.cc b/src/S3FileSystem.cc index 6bcaa04..3a6f9dc 100644 --- a/src/S3FileSystem.cc +++ b/src/S3FileSystem.cc @@ -85,6 +85,8 @@ S3FileSystem::Config(XrdSysLogger *lp, const char *configfn) value, this->s3_access_key_file ) ) { Config.Close(); return false; } if(! handle_required_config( attribute, "s3.secret_key_file", value, this->s3_secret_key_file ) ) { Config.Close(); return false; } + if(! handle_required_config( attribute, "s3.url_style", + value, this->s3_url_style ) ) { Config.Close(); return false; } } if( this->s3_service_name.empty() ) { @@ -95,6 +97,13 @@ S3FileSystem::Config(XrdSysLogger *lp, const char *configfn) m_log.Emsg("Config", "s3.region not specified"); return false; } + if( this->s3_url_style.empty() ) { + this->s3_url_style = "virtual"; + } + if( this->s3_url_style != "virtual" && this->s3_url_style != "path" ) { + m_log.Emsg("Config", "invalid s3.url_style specified"); + return false; + } int retc = Config.LastError(); if( retc ) { diff --git a/src/S3FileSystem.hh b/src/S3FileSystem.hh index 277d67f..065c611 100644 --- a/src/S3FileSystem.hh +++ b/src/S3FileSystem.hh @@ -60,6 +60,7 @@ public: const std::string & getS3ServiceName() const { return s3_service_name; } const std::string & getS3Region() const { return s3_region; } const std::string & getS3ServiceURL() const { return s3_service_url; } + const std::string & getS3URLStyle() const { return s3_url_style; } const std::string & getS3AccessKeyFile() const { return s3_access_key_file; } const std::string & getS3SecretKeyFile() const { return s3_secret_key_file; } @@ -81,4 +82,6 @@ private: std::string s3_access_key_file; std::string s3_secret_key_file; + + std::string s3_url_style; }; From af9b8579dc8ecaba6bba2f635907b7b92fbfcbad Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Thu, 29 Feb 2024 23:11:28 +0000 Subject: [PATCH 2/2] Update minor items --- src/S3File.cc | 4 ++-- src/S3FileSystem.cc | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/S3File.cc b/src/S3File.cc index 8d9f75e..47ad94e 100644 --- a/src/S3File.cc +++ b/src/S3File.cc @@ -6,6 +6,7 @@ #include "XrdVersion.hh" #include "S3FileSystem.hh" #include "S3File.hh" +#include "stl_string_utils.hh" #include @@ -14,7 +15,6 @@ #include #include -#include #include #include @@ -174,7 +174,7 @@ S3File::Fstat(struct stat *buff) std::string attr = substring( line, 0, colon ); std::string value = substring( line, colon + 1 ); trim(value); - std::transform(attr.begin(), attr.end(), attr.begin(), ::tolower); + toLower(attr); if( attr == "content-length" ) { this->content_length = std::stol(value); diff --git a/src/S3FileSystem.cc b/src/S3FileSystem.cc index 3a6f9dc..65e8bca 100644 --- a/src/S3FileSystem.cc +++ b/src/S3FileSystem.cc @@ -5,6 +5,7 @@ #include "S3FileSystem.hh" #include "S3Directory.hh" #include "S3File.hh" +#include "stl_string_utils.hh" #include #include @@ -98,10 +99,14 @@ S3FileSystem::Config(XrdSysLogger *lp, const char *configfn) return false; } if( this->s3_url_style.empty() ) { - this->s3_url_style = "virtual"; + m_log.Emsg("Config", "s3.url_style not specified"); + return false; + } else { + // We want this to be case-insensitive. + toLower( this->s3_url_style ); } if( this->s3_url_style != "virtual" && this->s3_url_style != "path" ) { - m_log.Emsg("Config", "invalid s3.url_style specified"); + m_log.Emsg("Config", "invalid s3.url_style specified. Must be 'virtual' or 'path'"); return false; }