From 730c356f85316245814e873c9fbbfc2f874c6a4e Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Tue, 27 Nov 2018 10:41:05 +0100 Subject: [PATCH 01/15] DMC-1125: add delegate option to xrootd TPC --- dist/etc/gfal2.d/http_plugin.conf | 5 +++++ src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp | 1 + 2 files changed, 6 insertions(+) diff --git a/dist/etc/gfal2.d/http_plugin.conf b/dist/etc/gfal2.d/http_plugin.conf index 69251467..2ae98f84 100644 --- a/dist/etc/gfal2.d/http_plugin.conf +++ b/dist/etc/gfal2.d/http_plugin.conf @@ -7,6 +7,10 @@ ENABLE_REMOTE_COPY=true ## Enable streamed copies (data passes via the client) ENABLE_STREAM_COPY=true +## Enable fallback copies: in case of TPC we start first with DEFAULT_COPY_MODE +## and we fallback in case of an error +ENABLE_FALLBACK_TPC_COPY=true + ## Default mode for http copy ## possible values are: 3rd pull, 3rd push or streamed ## By default all methods will be tried in case of failures starting from 3rd pull @@ -14,6 +18,7 @@ ENABLE_STREAM_COPY=true ## if streamed is set as detault only streamed transfer will be executed DEFAULT_COPY_MODE=3rd pull + # Enable or disable the SSL CA check INSECURE=false diff --git a/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp b/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp index 373119fb..b8e43bcd 100644 --- a/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp +++ b/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp @@ -235,6 +235,7 @@ int gfal_xrootd_3rd_copy_bulk(plugin_handle plugin_data, job.Set("thirdParty", "first"); } job.Set("tpcTimeout", gfalt_get_timeout(params, NULL)); + job.Set("delegate", true); #else XrdCl::JobDescriptor job; From f68caee99558b65f5cb8260da5eb4191bbfa4df7 Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Tue, 27 Nov 2018 10:43:54 +0100 Subject: [PATCH 02/15] DMC-1123: enable/disable copy fallbak --- src/plugins/http/gfal_http_copy.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/plugins/http/gfal_http_copy.cpp b/src/plugins/http/gfal_http_copy.cpp index c35728fb..7ff5afb3 100644 --- a/src/plugins/http/gfal_http_copy.cpp +++ b/src/plugins/http/gfal_http_copy.cpp @@ -93,6 +93,11 @@ static CopyMode get_default_copy_mode(gfal2_context_t context) return getCopyModeFromString(gfal2_get_opt_string_with_default(context, "HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL)); } +static bool is_http_3rdcopy_fallback_enabled(gfal2_context_t context) +{ + return gfal2_get_opt_boolean_with_default(context, "HTTP PLUGIN", "ENABLE_FALLBACK_TPC_COPY", TRUE); +} + static int gfal_http_exists(plugin_handle plugin_data, const char* url, GError** err) { @@ -578,7 +583,7 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, // Re-try different approaches int ret = 0; - while (copy_mode < HTTP_COPY_END) { + do { // The real, actual, copy plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_ENTER, @@ -614,16 +619,18 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, break; } else if (ret < 0) { + plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, - "ERROR: Copy failed with mode %s, will delete destination and retry with the next available mode: %s", + "ERROR: Copy failed with mode %s, with error: %s", CopyModeStr[copy_mode], nested_error->message); - // Delete any potential destination file. - gfal_http_copy_cleanup(plugin_data, dst, &nested_error); + // Delete any potential destination file. + gfal_http_copy_cleanup(plugin_data, dst, &nested_error); } copy_mode = (CopyMode)((int)copy_mode + 1); - } + + } while ((copy_mode < HTTP_COPY_END) && is_http_3rdcopy_fallback_enabled(context)); plugin_trigger_event(params, http_plugin_domain, From 6d1a24272831bd853d071952ee059ed1cfdc4320 Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Tue, 27 Nov 2018 13:09:48 +0100 Subject: [PATCH 03/15] bump release --- CMakeLists.txt | 2 +- packaging/debian/changelog | 5 +++++ packaging/rpm/gfal2.spec | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ebbad15..47ca6055 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,7 +45,7 @@ set (OUTPUT_NAME_TRANSFER "gfal_transfer") set (VERSION_MAJOR 2) set (VERSION_MINOR 16) -set (VERSION_PATCH 1) +set (VERSION_PATCH 2) set (VERSION_STRING ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}) add_definitions(-DVERSION="${VERSION_STRING}") diff --git a/packaging/debian/changelog b/packaging/debian/changelog index e8f8b33d..e19a4503 100644 --- a/packaging/debian/changelog +++ b/packaging/debian/changelog @@ -1,3 +1,8 @@ +gfal2 (2.16.2-1) unstable; urgency=low + * New version + + -- DMC Devel Tue, 27 Nov 2018 14:00:00 +0100 + gfal2 (2.16.1-1) unstable; urgency=low * New version diff --git a/packaging/rpm/gfal2.spec b/packaging/rpm/gfal2.spec index 6ae6d947..4f53e8c4 100644 --- a/packaging/rpm/gfal2.spec +++ b/packaging/rpm/gfal2.spec @@ -4,7 +4,7 @@ %bcond_with tests Name: gfal2 -Version: 2.16.1 +Version: 2.16.2 Release: 1%{?dist} Summary: Grid file access library 2.0 Group: Applications/Internet From 48537d551361875fffa7adf95503025bb6c5b8f3 Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Tue, 27 Nov 2018 18:10:46 +0100 Subject: [PATCH 04/15] disable delegation for now --- src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp b/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp index b8e43bcd..f809d682 100644 --- a/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp +++ b/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp @@ -235,7 +235,7 @@ int gfal_xrootd_3rd_copy_bulk(plugin_handle plugin_data, job.Set("thirdParty", "first"); } job.Set("tpcTimeout", gfalt_get_timeout(params, NULL)); - job.Set("delegate", true); + job.Set("delegate", false); #else XrdCl::JobDescriptor job; From 54729c6449060228c66531c8444f93f84713be22 Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Thu, 6 Dec 2018 16:45:28 +0100 Subject: [PATCH 05/15] DMC-1129: add copy_mode query parameter --- src/plugins/http/gfal_http_copy.cpp | 91 +++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 5 deletions(-) diff --git a/src/plugins/http/gfal_http_copy.cpp b/src/plugins/http/gfal_http_copy.cpp index 7ff5afb3..a59437f6 100644 --- a/src/plugins/http/gfal_http_copy.cpp +++ b/src/plugins/http/gfal_http_copy.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include "gfal_http_plugin.h" // An enumeration of the different HTTP third-party-copy strategies. @@ -40,12 +41,12 @@ const char* CopyModeStr[] = { GFAL_TRANSFER_TYPE_PULL, GFAL_TRANSFER_TYPE_PUSH, GFAL_TRANSFER_TYPE_STREAMED, NULL }; -static const CopyMode getCopyModeFromString(const char * copyModeStr) { - if (!strcmp(copyModeStr, GFAL_TRANSFER_TYPE_PULL)) +static const CopyMode get_copy_mode_from_string(const char * copy_mode_str) { + if (!strcmp(copy_mode_str, GFAL_TRANSFER_TYPE_PULL)) return HTTP_COPY_PULL; - if (!strcmp(copyModeStr, GFAL_TRANSFER_TYPE_PUSH)) + if (!strcmp(copy_mode_str, GFAL_TRANSFER_TYPE_PUSH)) return HTTP_COPY_PUSH; - if (!strcmp(copyModeStr, GFAL_TRANSFER_TYPE_STREAMED)) + if (!strcmp(copy_mode_str, GFAL_TRANSFER_TYPE_STREAMED)) return HTTP_COPY_STREAM; return HTTP_COPY_PULL; } @@ -62,6 +63,84 @@ struct PerfCallbackData { } }; +static bool set_copy_mode(gfal2_context_t context,const std::string& copy_mode) { + bool set = true; + GError *error = NULL; + if (copy_mode.compare ("push" ) == 0) { + gfal2_set_opt_boolean(context, "HTTP PLUGIN", "ENABLE_REMOTE_COPY", TRUE, &error); + gfal2_set_opt_string(context, "HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PUSH, &error); + gfal2_set_opt_boolean(context, "HTTP PLUGIN", "ENABLE_FALLBACK_TPC_COPY", FALSE, &error); + if (error != NULL) { + g_clear_error(&error); + set =false; + } + } else if (copy_mode.compare ("pull") == 0 ) { + gfal2_set_opt_boolean(context, "HTTP PLUGIN", "ENABLE_REMOTE_COPY", TRUE, &error); + gfal2_set_opt_string(context, "HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL, &error); + gfal2_set_opt_boolean(context, "HTTP PLUGIN", "ENABLE_FALLBACK_TPC_COPY", FALSE, &error); + if (error != NULL) { + g_clear_error(&error); + set =false; + } + } else { + set =false; + } + return set; +} + +static std::string extract_query_parameter(const std::string& query_param, const std::string& uristring) { + + std::string value = ""; + if (uristring.find(query_param) != std::string::npos) { + int pos = uristring.find(query_param); + if (uristring.find("=",pos+ query_param.length()) != std::string::npos) { + int pos_next_parameter = uristring.find('&',pos + query_param.length() +1); + if (pos_next_parameter != std::string::npos) { + value= uristring.substr(pos + query_param.length() + 1 ,pos_next_parameter); + } + else { + value= uristring.substr(pos + query_param.length() + 1 ); + } + } + } + return value; +} + +static void set_copy_mode_from_urls(gfal2_context_t context, const char * src_url, const char * dst_url) +{ + GError *error = NULL; + std::string copymode = ""; + gfal2_uri *uri = gfal2_parse_uri(src_url, &error); + if (error != NULL) { + g_clear_error(&error); + return; + } + bool done = false; + + if (uri->query != NULL) { + gfal2_log(G_LOG_LEVEL_DEBUG, "Source Query String %s", uri->query); + std::string uristring(uri->query); + copymode = extract_query_parameter("copy_mode", uristring); + if (copymode != "") { + gfal2_log(G_LOG_LEVEL_INFO,"Source copy mode is %s", copymode.c_str()); + done= set_copy_mode(context, copymode); + } + } + + if (!done) { + uri = gfal2_parse_uri(dst_url, &error); + if (uri->query != NULL) { + gfal2_log(G_LOG_LEVEL_DEBUG, "Destination Query String %s", uri->query); + std::string uristring(uri->query); + copymode = extract_query_parameter("copy_mode", uristring); + if (copymode != "") { + gfal2_log(G_LOG_LEVEL_INFO,"Destination copy mode is %s", copymode.c_str()); + done= set_copy_mode(context,copymode); + } + } + } + gfal2_free_uri(uri); +} static bool is_http_scheme(const char* url) { @@ -90,7 +169,7 @@ static bool is_http_streamed_enabled(gfal2_context_t context) static CopyMode get_default_copy_mode(gfal2_context_t context) { - return getCopyModeFromString(gfal2_get_opt_string_with_default(context, "HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL)); + return get_copy_mode_from_string(gfal2_get_opt_string_with_default(context, "HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL)); } static bool is_http_3rdcopy_fallback_enabled(gfal2_context_t context) @@ -572,6 +651,8 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, GFAL_EVENT_NONE, GFAL_EVENT_PREPARE_EXIT, "%s => %s", src_full, dst_full); + + set_copy_mode_from_urls(context,src_full, dst_full); // Initial copy mode CopyMode copy_mode = get_default_copy_mode(context); From cf886d57913dfe347c0073009eb1401ffcf4a691 Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Thu, 6 Dec 2018 16:48:24 +0100 Subject: [PATCH 06/15] DMC-1129: free uri in case of errors --- src/plugins/http/gfal_http_copy.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/http/gfal_http_copy.cpp b/src/plugins/http/gfal_http_copy.cpp index a59437f6..f1028a2d 100644 --- a/src/plugins/http/gfal_http_copy.cpp +++ b/src/plugins/http/gfal_http_copy.cpp @@ -113,6 +113,7 @@ static void set_copy_mode_from_urls(gfal2_context_t context, const char * src_ur gfal2_uri *uri = gfal2_parse_uri(src_url, &error); if (error != NULL) { g_clear_error(&error); + gfal2_free_uri(uri); return; } bool done = false; From 36b64ca58a39d581f06000e77b1da743b8f63367 Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Thu, 6 Dec 2018 23:16:17 +0100 Subject: [PATCH 07/15] DMC-1129: simplify parsing of query parameters --- src/plugins/http/gfal_http_copy.cpp | 82 +++++++++++++---------------- 1 file changed, 36 insertions(+), 46 deletions(-) diff --git a/src/plugins/http/gfal_http_copy.cpp b/src/plugins/http/gfal_http_copy.cpp index f1028a2d..2f0eacfd 100644 --- a/src/plugins/http/gfal_http_copy.cpp +++ b/src/plugins/http/gfal_http_copy.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include "gfal_http_plugin.h" // An enumeration of the different HTTP third-party-copy strategies. @@ -63,10 +62,10 @@ struct PerfCallbackData { } }; -static bool set_copy_mode(gfal2_context_t context,const std::string& copy_mode) { +static bool set_copy_mode(gfal2_context_t context,const char * copy_mode) { bool set = true; GError *error = NULL; - if (copy_mode.compare ("push" ) == 0) { + if (!strcmp(copy_mode,"push" )) { gfal2_set_opt_boolean(context, "HTTP PLUGIN", "ENABLE_REMOTE_COPY", TRUE, &error); gfal2_set_opt_string(context, "HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PUSH, &error); gfal2_set_opt_boolean(context, "HTTP PLUGIN", "ENABLE_FALLBACK_TPC_COPY", FALSE, &error); @@ -74,7 +73,7 @@ static bool set_copy_mode(gfal2_context_t context,const std::string& copy_mode) g_clear_error(&error); set =false; } - } else if (copy_mode.compare ("pull") == 0 ) { + } else if (!strcmp(copy_mode,"pull") ) { gfal2_set_opt_boolean(context, "HTTP PLUGIN", "ENABLE_REMOTE_COPY", TRUE, &error); gfal2_set_opt_string(context, "HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL, &error); gfal2_set_opt_boolean(context, "HTTP PLUGIN", "ENABLE_FALLBACK_TPC_COPY", FALSE, &error); @@ -88,59 +87,50 @@ static bool set_copy_mode(gfal2_context_t context,const std::string& copy_mode) return set; } -static std::string extract_query_parameter(const std::string& query_param, const std::string& uristring) { - - std::string value = ""; - if (uristring.find(query_param) != std::string::npos) { - int pos = uristring.find(query_param); - if (uristring.find("=",pos+ query_param.length()) != std::string::npos) { - int pos_next_parameter = uristring.find('&',pos + query_param.length() +1); - if (pos_next_parameter != std::string::npos) { - value= uristring.substr(pos + query_param.length() + 1 ,pos_next_parameter); - } - else { - value= uristring.substr(pos + query_param.length() + 1 ); +static void extract_query_parameter(const char* url, const char *key, char *value, size_t val_size) +{ + value[0] = '\0'; + + const char *str = strchr(url, '?'); + if (str == NULL) { + return; + } + + size_t key_len = strlen(key); + char **args = g_strsplit(str + 1, "&", 0); + int i; + for (i = 0; args[i] != NULL; ++i) { + if (strncmp(args[i], key, key_len) == 0) { + char *p = strchr(args[i], '='); + if (p) { + g_strlcpy(value, p + 1, val_size); + break; } - } - } - return value; -} + } + } + + g_strfreev(args); +} static void set_copy_mode_from_urls(gfal2_context_t context, const char * src_url, const char * dst_url) { GError *error = NULL; - std::string copymode = ""; - gfal2_uri *uri = gfal2_parse_uri(src_url, &error); - if (error != NULL) { - g_clear_error(&error); - gfal2_free_uri(uri); - return; - } + char copy_mode[64] = {0}; bool done = false; - - if (uri->query != NULL) { - gfal2_log(G_LOG_LEVEL_DEBUG, "Source Query String %s", uri->query); - std::string uristring(uri->query); - copymode = extract_query_parameter("copy_mode", uristring); - if (copymode != "") { - gfal2_log(G_LOG_LEVEL_INFO,"Source copy mode is %s", copymode.c_str()); - done= set_copy_mode(context, copymode); - } + extract_query_parameter(src_url, "copy_mode", copy_mode,sizeof(copy_mode)); + if (copy_mode != NULL) { + gfal2_log(G_LOG_LEVEL_INFO,"Source copy mode is %s", copy_mode); + done= set_copy_mode(context, copy_mode); } if (!done) { - uri = gfal2_parse_uri(dst_url, &error); - if (uri->query != NULL) { - gfal2_log(G_LOG_LEVEL_DEBUG, "Destination Query String %s", uri->query); - std::string uristring(uri->query); - copymode = extract_query_parameter("copy_mode", uristring); - if (copymode != "") { - gfal2_log(G_LOG_LEVEL_INFO,"Destination copy mode is %s", copymode.c_str()); - done= set_copy_mode(context,copymode); - } + extract_query_parameter(dst_url, "copy_mode", copy_mode,sizeof(copy_mode)); + if (copy_mode != NULL) { + gfal2_log(G_LOG_LEVEL_INFO,"Destination copy mode is %s", copy_mode); + set_copy_mode(context,copy_mode); + } } - gfal2_free_uri(uri); } static bool is_http_scheme(const char* url) From 67b8ef08315a823cd60aae90c1a6322c9432bd3c Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Thu, 6 Dec 2018 23:26:11 +0100 Subject: [PATCH 08/15] DMC-1129: correct check for empty string --- src/plugins/http/gfal_http_copy.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/http/gfal_http_copy.cpp b/src/plugins/http/gfal_http_copy.cpp index 2f0eacfd..b2cf82dc 100644 --- a/src/plugins/http/gfal_http_copy.cpp +++ b/src/plugins/http/gfal_http_copy.cpp @@ -118,14 +118,14 @@ static void set_copy_mode_from_urls(gfal2_context_t context, const char * src_ur char copy_mode[64] = {0}; bool done = false; extract_query_parameter(src_url, "copy_mode", copy_mode,sizeof(copy_mode)); - if (copy_mode != NULL) { + if (copy_mode[0] != '\0') { gfal2_log(G_LOG_LEVEL_INFO,"Source copy mode is %s", copy_mode); done= set_copy_mode(context, copy_mode); } if (!done) { extract_query_parameter(dst_url, "copy_mode", copy_mode,sizeof(copy_mode)); - if (copy_mode != NULL) { + if (copy_mode[0] != '\0') { gfal2_log(G_LOG_LEVEL_INFO,"Destination copy mode is %s", copy_mode); set_copy_mode(context,copy_mode); From f460a3ad4e2bff4af437b19f6a5129cf3421374a Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Mon, 17 Dec 2018 16:23:25 +0100 Subject: [PATCH 09/15] DMC-1131; set nstreams to 0 in case of ftp, so to disable GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK --- src/plugins/gridftp/gridftp_filecopy.cpp | 5 ++++- src/plugins/gridftp/gridftpwrapper.cpp | 4 ---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/plugins/gridftp/gridftp_filecopy.cpp b/src/plugins/gridftp/gridftp_filecopy.cpp index 102acaa0..daf48cf9 100644 --- a/src/plugins/gridftp/gridftp_filecopy.cpp +++ b/src/plugins/gridftp/gridftp_filecopy.cpp @@ -435,7 +435,10 @@ int gridftp_filecopy_copy_file_internal(GridFTPModule* module, if (nb_streams_from_conf !=0 ) { nbstream = nb_streams_from_conf; } - + //always set streams = 0 if source or dest are ftp, this disables the GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK + if (strncmp(src, "ftp:", 4) == 0 || strncmp(dst, "ftp:", 4) == 0) { + nbstream = 0; + } handler.session->set_nb_streams(nbstream); gfal2_log(G_LOG_LEVEL_DEBUG, diff --git a/src/plugins/gridftp/gridftpwrapper.cpp b/src/plugins/gridftp/gridftpwrapper.cpp index 39236a5d..a50c0c36 100644 --- a/src/plugins/gridftp/gridftpwrapper.cpp +++ b/src/plugins/gridftp/gridftpwrapper.cpp @@ -212,10 +212,6 @@ void GridFTPSession::set_dcau(bool dcau) void GridFTPSession::set_nb_streams(unsigned int nbstream) { - // plain FTP urls do not support mode - if (baseurl.compare(0, 3, "ftp") == 0) { - return; - } if (nbstream == 0) { parallelism.fixed.size = 1; parallelism.mode = GLOBUS_FTP_CONTROL_PARALLELISM_NONE; From 36b4375a75adc45112f969e9bdaf23f207ae6bb8 Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Mon, 7 Jan 2019 13:23:16 +0100 Subject: [PATCH 10/15] DMC-1125: enable tpc delegation --- src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp b/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp index f809d682..52d2999e 100644 --- a/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp +++ b/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp @@ -230,12 +230,12 @@ int gfal_xrootd_3rd_copy_bulk(plugin_handle plugin_data, if ((source_url.GetProtocol() == "root") && (dest_url.GetProtocol() == "root")) { job.Set("thirdParty", "only"); isThirdParty = true; + job.Set("delegate", true); } else { job.Set("thirdParty", "first"); } job.Set("tpcTimeout", gfalt_get_timeout(params, NULL)); - job.Set("delegate", false); #else XrdCl::JobDescriptor job; From 02abb932d869c4dd1f7dbe7861ca5697948867b2 Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Mon, 7 Jan 2019 16:26:02 +0100 Subject: [PATCH 11/15] DMC-1133: avoid trying streamed transfers if disabled, add copy mode to error messages --- src/plugins/http/gfal_http_copy.cpp | 31 +++++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/plugins/http/gfal_http_copy.cpp b/src/plugins/http/gfal_http_copy.cpp index b2cf82dc..ccad9fa4 100644 --- a/src/plugins/http/gfal_http_copy.cpp +++ b/src/plugins/http/gfal_http_copy.cpp @@ -646,22 +646,31 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, set_copy_mode_from_urls(context,src_full, dst_full); // Initial copy mode CopyMode copy_mode = get_default_copy_mode(context); - + + bool only_streaming = false; // If source is not even http, go straight to streamed // or if third party copy is disabled, go straight to streamed if (!is_http_scheme(src) || !is_http_3rdcopy_enabled(context)) { copy_mode = HTTP_COPY_STREAM; + only_streaming = true; } // Re-try different approaches int ret = 0; + + CopyMode end_copy_mode = HTTP_COPY_END; + + //if streaming is disabled stop the loop before + if (!is_http_streamed_enabled(context)) { + end_copy_mode = HTTP_COPY_STREAM; + } + do { // The real, actual, copy plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_ENTER, "%s => %s", src_full, dst_full); - gfal2_log(G_LOG_LEVEL_MESSAGE, - "Trying copying with mode %s", + gfal2_log(G_LOG_LEVEL_MESSAGE, "Trying copying with mode %s", CopyModeStr[copy_mode]); plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_TYPE, @@ -675,7 +684,8 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, checksum_mode, checksum_type, user_checksum, params, &nested_error); } - else { + else if (only_streaming) { + ret = -1; gfalt_set_error(&nested_error, http_plugin_domain, EIO, __func__, GFALT_ERROR_TRANSFER, "STREAMED DISABLED", "Trying to fallback to a streamed copy, but they are disabled"); @@ -691,18 +701,17 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, break; } else if (ret < 0) { - - plugin_trigger_event(params, http_plugin_domain, - GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, - "ERROR: Copy failed with mode %s, with error: %s", - CopyModeStr[copy_mode], nested_error->message); + g_prefix_error(&nested_error, "ERROR: Copy failed with mode %s, with error: ",CopyModeStr[copy_mode]); + plugin_trigger_event(params, http_plugin_domain, + GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, + nested_error->message); // Delete any potential destination file. - gfal_http_copy_cleanup(plugin_data, dst, &nested_error); + gfal_http_copy_cleanup(plugin_data, dst, &nested_error); } copy_mode = (CopyMode)((int)copy_mode + 1); - } while ((copy_mode < HTTP_COPY_END) && is_http_3rdcopy_fallback_enabled(context)); + } while ((copy_mode < end_copy_mode) && is_http_3rdcopy_fallback_enabled(context)); plugin_trigger_event(params, http_plugin_domain, From 7e013ffec9cca81d354ec87a872550812df3283e Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Tue, 8 Jan 2019 23:22:44 +0100 Subject: [PATCH 12/15] Revert "DMC-1133: avoid trying streamed transfers if disabled, add copy mode to error messages" This reverts commit 02abb932d869c4dd1f7dbe7861ca5697948867b2. --- src/plugins/http/gfal_http_copy.cpp | 31 ++++++++++------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/src/plugins/http/gfal_http_copy.cpp b/src/plugins/http/gfal_http_copy.cpp index ccad9fa4..b2cf82dc 100644 --- a/src/plugins/http/gfal_http_copy.cpp +++ b/src/plugins/http/gfal_http_copy.cpp @@ -646,31 +646,22 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, set_copy_mode_from_urls(context,src_full, dst_full); // Initial copy mode CopyMode copy_mode = get_default_copy_mode(context); - - bool only_streaming = false; + // If source is not even http, go straight to streamed // or if third party copy is disabled, go straight to streamed if (!is_http_scheme(src) || !is_http_3rdcopy_enabled(context)) { copy_mode = HTTP_COPY_STREAM; - only_streaming = true; } // Re-try different approaches int ret = 0; - - CopyMode end_copy_mode = HTTP_COPY_END; - - //if streaming is disabled stop the loop before - if (!is_http_streamed_enabled(context)) { - end_copy_mode = HTTP_COPY_STREAM; - } - do { // The real, actual, copy plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_ENTER, "%s => %s", src_full, dst_full); - gfal2_log(G_LOG_LEVEL_MESSAGE, "Trying copying with mode %s", + gfal2_log(G_LOG_LEVEL_MESSAGE, + "Trying copying with mode %s", CopyModeStr[copy_mode]); plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_TYPE, @@ -684,8 +675,7 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, checksum_mode, checksum_type, user_checksum, params, &nested_error); } - else if (only_streaming) { - ret = -1; + else { gfalt_set_error(&nested_error, http_plugin_domain, EIO, __func__, GFALT_ERROR_TRANSFER, "STREAMED DISABLED", "Trying to fallback to a streamed copy, but they are disabled"); @@ -701,17 +691,18 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, break; } else if (ret < 0) { - g_prefix_error(&nested_error, "ERROR: Copy failed with mode %s, with error: ",CopyModeStr[copy_mode]); - plugin_trigger_event(params, http_plugin_domain, - GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, - nested_error->message); + + plugin_trigger_event(params, http_plugin_domain, + GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, + "ERROR: Copy failed with mode %s, with error: %s", + CopyModeStr[copy_mode], nested_error->message); // Delete any potential destination file. - gfal_http_copy_cleanup(plugin_data, dst, &nested_error); + gfal_http_copy_cleanup(plugin_data, dst, &nested_error); } copy_mode = (CopyMode)((int)copy_mode + 1); - } while ((copy_mode < end_copy_mode) && is_http_3rdcopy_fallback_enabled(context)); + } while ((copy_mode < HTTP_COPY_END) && is_http_3rdcopy_fallback_enabled(context)); plugin_trigger_event(params, http_plugin_domain, From 108503ef343065ff08b4c7eea84318d3365743c7 Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Mon, 14 Jan 2019 15:50:51 +0100 Subject: [PATCH 13/15] Revert "Revert "DMC-1133: avoid trying streamed transfers if disabled, add copy mode to error messages"" This reverts commit 7e013ffec9cca81d354ec87a872550812df3283e. --- src/plugins/http/gfal_http_copy.cpp | 31 +++++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/plugins/http/gfal_http_copy.cpp b/src/plugins/http/gfal_http_copy.cpp index b2cf82dc..ccad9fa4 100644 --- a/src/plugins/http/gfal_http_copy.cpp +++ b/src/plugins/http/gfal_http_copy.cpp @@ -646,22 +646,31 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, set_copy_mode_from_urls(context,src_full, dst_full); // Initial copy mode CopyMode copy_mode = get_default_copy_mode(context); - + + bool only_streaming = false; // If source is not even http, go straight to streamed // or if third party copy is disabled, go straight to streamed if (!is_http_scheme(src) || !is_http_3rdcopy_enabled(context)) { copy_mode = HTTP_COPY_STREAM; + only_streaming = true; } // Re-try different approaches int ret = 0; + + CopyMode end_copy_mode = HTTP_COPY_END; + + //if streaming is disabled stop the loop before + if (!is_http_streamed_enabled(context)) { + end_copy_mode = HTTP_COPY_STREAM; + } + do { // The real, actual, copy plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_ENTER, "%s => %s", src_full, dst_full); - gfal2_log(G_LOG_LEVEL_MESSAGE, - "Trying copying with mode %s", + gfal2_log(G_LOG_LEVEL_MESSAGE, "Trying copying with mode %s", CopyModeStr[copy_mode]); plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_TYPE, @@ -675,7 +684,8 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, checksum_mode, checksum_type, user_checksum, params, &nested_error); } - else { + else if (only_streaming) { + ret = -1; gfalt_set_error(&nested_error, http_plugin_domain, EIO, __func__, GFALT_ERROR_TRANSFER, "STREAMED DISABLED", "Trying to fallback to a streamed copy, but they are disabled"); @@ -691,18 +701,17 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, break; } else if (ret < 0) { - - plugin_trigger_event(params, http_plugin_domain, - GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, - "ERROR: Copy failed with mode %s, with error: %s", - CopyModeStr[copy_mode], nested_error->message); + g_prefix_error(&nested_error, "ERROR: Copy failed with mode %s, with error: ",CopyModeStr[copy_mode]); + plugin_trigger_event(params, http_plugin_domain, + GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, + nested_error->message); // Delete any potential destination file. - gfal_http_copy_cleanup(plugin_data, dst, &nested_error); + gfal_http_copy_cleanup(plugin_data, dst, &nested_error); } copy_mode = (CopyMode)((int)copy_mode + 1); - } while ((copy_mode < HTTP_COPY_END) && is_http_3rdcopy_fallback_enabled(context)); + } while ((copy_mode < end_copy_mode) && is_http_3rdcopy_fallback_enabled(context)); plugin_trigger_event(params, http_plugin_domain, From e88ebf7b5d0bdfc9e7398af25f271170eb8a6f52 Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Fri, 18 Jan 2019 09:37:20 +0100 Subject: [PATCH 14/15] DMC-1133: validate that error string coming from davix is utf8 --- src/plugins/http/gfal_http_plugin.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plugins/http/gfal_http_plugin.cpp b/src/plugins/http/gfal_http_plugin.cpp index 790f891a..1582f55e 100644 --- a/src/plugins/http/gfal_http_plugin.cpp +++ b/src/plugins/http/gfal_http_plugin.cpp @@ -527,8 +527,11 @@ static int davix2errno(StatusCode::Code code) void davix2gliberr(const DavixError* daverr, GError** err) { + + std::string error_string= g_utf8_validate(daverr->getErrMsg().c_str(), daverr->getErrMsg().length(),NULL) ? + daverr->getErrMsg().c_str(): "Error string contains not UTF8 chars"; gfal2_set_error(err, http_plugin_domain, davix2errno(daverr->getStatus()), __func__, - "%s", daverr->getErrMsg().c_str()); + "%s", error_string.c_str()); } From f6b3da294635177d3dfecf133cf2d8d82c6eeff3 Mon Sep 17 00:00:00 2001 From: Andrea Manzi Date: Fri, 18 Jan 2019 09:38:41 +0100 Subject: [PATCH 15/15] better error string --- src/plugins/http/gfal_http_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/http/gfal_http_plugin.cpp b/src/plugins/http/gfal_http_plugin.cpp index 1582f55e..9bb62d09 100644 --- a/src/plugins/http/gfal_http_plugin.cpp +++ b/src/plugins/http/gfal_http_plugin.cpp @@ -529,7 +529,7 @@ void davix2gliberr(const DavixError* daverr, GError** err) { std::string error_string= g_utf8_validate(daverr->getErrMsg().c_str(), daverr->getErrMsg().length(),NULL) ? - daverr->getErrMsg().c_str(): "Error string contains not UTF8 chars"; + daverr->getErrMsg().c_str(): "Error string contains not valid UTF-8 chars"; gfal2_set_error(err, http_plugin_domain, davix2errno(daverr->getStatus()), __func__, "%s", error_string.c_str()); }