Fix merge

CMakeLists.txt

@@ -44,8 +44,8 @@ set (PROJECT_NAME_TRANSFER "gfal_transfer")
 set (OUTPUT_NAME_TRANSFER "gfal_transfer")
 
 set (VERSION_MAJOR  2)
-set (VERSION_MINOR  15)
-set (VERSION_PATCH  5)
+set (VERSION_MINOR  16)
+set (VERSION_PATCH  0)
 set (VERSION_STRING ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH})
 
 add_definitions(-DVERSION="${VERSION_STRING}")

dist/etc/gfal2.d/http_plugin.conf

@@ -7,7 +7,14 @@ ENABLE_REMOTE_COPY=true
 ## Enable streamed copies (data passes via the client)
 ENABLE_STREAM_COPY=true
 
-## Enable or disable the SSL CA check
+## Default mode for http copy
+## possible values are: 3rd pull, 3rd push or streamed 
+## By default all methods will be tried in case of failures starting from 3rd pull
+## if 3rd push is set as default only 3rd push and then streamed will be executed
+## if streamed is set as detault only streamed transfer will be executed
+DEFAULT_COPY_MODE=3rd pull
+
+# Enable or disable the SSL CA check
 INSECURE=false
 
 ## Force Davix log level
@@ -31,3 +38,10 @@ KEEP_ALIVE=true
 
 ## AWS Region
 #REGION=
+#
+[GCLOUD]
+
+## Path to the google JSON file path
+#JSON_AUTH_FILE=
+## google auth file as a string
+#JSON_AUTH_STRING=

packaging/debian/changelog

@@ -1,3 +1,8 @@
+gfal2 (2.16.0-1) unstable; urgency=low
+  * New version
+
+ -- DMC Devel <dmc-devel@cern.ch> Tue, 14 June 2018 16:00:00 +0100
+
 gfal2 (2.15.5-1) unstable; urgency=low
   * New version
 

packaging/rpm/gfal2.spec

@@ -4,17 +4,17 @@
 %bcond_with tests
 
 Name:               gfal2
-Version:            2.15.5
-Release:            2%{?dist}
+Version:            2.16.0
+Release:            1%{?dist}
 Summary:            Grid file access library 2.0
 Group:              Applications/Internet
 License:            ASL 2.0
 URL:                http://dmc.web.cern.ch/projects/gfal-2/home
-# git clone --depth=1 --branch master https://gitlab.cern.ch/dmc/gfal2.git gfal2-2.15.2
-# pushd gfal2-2.15.2
-# git checkout v2.15.2
+# git clone --depth=1 --branch master https://gitlab.cern.ch/dmc/gfal2.git gfal2-2.16.0
+# pushd gfal2-2.16.0
+# git checkout v2.16.0
 # popd
-# tar czf gfal2-2.15.2.tar.gz --exclude-vcs gfal2-2.15.2
+# tar czf gfal2-2.16.0.tar.gz --exclude-vcs gfal2-2.16.0
 Source0:            %{name}-%{version}.tar.gz
 BuildRoot:          %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
 
@@ -54,7 +54,7 @@ BuildRequires:      dcap-devel
 #gridftp plugin dependencies
 BuildRequires:      globus-gass-copy-devel
 #http plugin dependencies
-BuildRequires:      davix-devel >= 0.4.2
+BuildRequires:      davix-devel >= 0.6.8
 #xrootd plugin dependencies
 BuildRequires:      xrootd-client-devel >= 1:4.1.1
 # sftp plugin dependencies
@@ -169,7 +169,7 @@ the third party transfer support on the GSIFTP URLs.
 Summary:            Provides the HTTP/DAV support for %{name}
 Group:              Applications/Internet
 Requires:           %{name}%{?_isa} = %{version}-%{release}
-Requires:           davix-libs >= 0.4.2
+Requires:           davix-libs >= 0.6.8
 
 %description plugin-http
 Provides the HTTP (http[s]://) and WevDAV (dav[s]://) support for %{name}.
@@ -374,6 +374,9 @@ make DESTDIR=%{buildroot} install
 
 
 %changelog
+* Mon Sep 24 2018 Andrea Manzi <amanzi at cern.ch> - 2.16.0-1
+- New upstrem release
+
 * Thu Apr 05 2018 Andrea Manzi <amanzi at cern.ch> - 2.15.4-1
 - New upstrem release
 

src/core/common/gfal_common.c

@@ -67,9 +67,23 @@ static void gfal_setCredentialLocation(const char *where, gfal2_context_t handle
     gfal2_log(G_LOG_LEVEL_DEBUG, "Private key: %s", key);
 }
 
+static void gfal_setBearerToken(gfal2_context_t handle, const char *token)
+{
+    GError *error = NULL;
+    gfal2_set_opt_string(handle, "BEARER", "TOKEN", token, &error);
+    g_clear_error(&error);
+    gfal2_log(G_LOG_LEVEL_DEBUG, "Using BEARER token credentials from the env"); 
+}
+
 // Setup default credentials depending on the environment
 static void gfal_initCredentialLocation(gfal2_context_t handle)
 {
+    //check first if BEARER is on the env
+    const char *token = getenv("BEARER_TOKEN");
+    if (token != NULL) {
+        gfal_setBearerToken(handle, token);
+        return;
+    }
     // X509_USER_PROXY
     const char *proxy = getenv("X509_USER_PROXY");
     if (proxy != NULL) {

src/core/file/gfal_file_api.h

@@ -35,7 +35,7 @@
 #ifndef ENOATTR
 #define ENOATTR ENODATA
 #endif
-#if ( defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) ) || defined __APPLE__
+#if defined __APPLE__ || ( defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) )
 #include <sys/xattr.h>
 #else
 #include <attr/xattr.h>

src/core/posix/gfal_posix_api.h

@@ -37,7 +37,7 @@
 #ifndef ENOATTR
 #define ENOATTR ENODATA
 #endif
-#if ( defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) ) || defined __APPLE__
+#if defined __APPLE__  || ( defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) )
 #include <sys/xattr.h>
 #else
 #include <attr/xattr.h>

src/plugins/file/gfal_file_plugin_main.c

@@ -26,7 +26,7 @@
 #include <fcntl.h>
 #include <glib.h>
 #include <errno.h>
-#if ( defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) ) || defined __APPLE__
+#if defined __APPLE__ || ( defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) )
 #include <sys/xattr.h>
 #else
 #include <attr/xattr.h>

src/plugins/gridftp/gridftp_bulk.cpp

@@ -294,8 +294,12 @@ int gridftp_pipeline_transfer(plugin_handle plugin_data,
         &ftp_operation_attr_dst, handler.get_ftp_client_operationattr(), &cred_id_dst,
         context, udt, pairs->dsts[pairs->index], op_error);
 
+    int nbstreams = gfal2_get_opt_integer_with_default(context, GRIDFTP_CONFIG_GROUP,
+            GRIDFTP_CONFIG_NB_STREAM, 0);
 
-    int nbstreams = gfalt_get_nbstreams(pairs->params, NULL);
+    if (nbstreams == 0) {
+        nbstreams = gfalt_get_nbstreams(pairs->params, NULL);
+    }
     guint64 buffer_size = gfalt_get_tcp_buffer_size(pairs->params, NULL);
     globus_ftp_control_parallelism_t parallelism;
     globus_ftp_control_tcpbuffer_t tcp_buffer_size;

src/plugins/gridftp/gridftp_filecopy.cpp

@@ -413,7 +413,7 @@ int gridftp_filecopy_copy_file_internal(GridFTPModule* module,
     const time_t timeout = gfalt_get_timeout(params, &tmp_err);
     Gfal::gerror_to_cpp(&tmp_err);
 
-    const unsigned int nbstream = gfalt_get_nbstreams(params, &tmp_err);
+    unsigned int nbstream = gfalt_get_nbstreams(params, &tmp_err);
     Gfal::gerror_to_cpp(&tmp_err);
     const guint64 tcp_buffer_size = gfalt_get_tcp_buffer_size(params, &tmp_err);
     Gfal::gerror_to_cpp(&tmp_err);
@@ -427,7 +427,17 @@ int gridftp_filecopy_copy_file_internal(GridFTPModule* module,
     GridFTPSessionHandler handler(factory, src);
     GridFTPRequestState req(&handler, GRIDFTP_REQUEST_GASS);
 
+    const unsigned int nb_streams_from_conf = gfal2_get_opt_integer_with_default(
+          factory->get_gfal2_context(), GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_NB_STREAM, 0);
+
+
+    //if the number of streams in the config file is 0 use the one passed by paramameter
+    if (nb_streams_from_conf !=0 ) {
+        nbstream = nb_streams_from_conf;
+    }
+
     handler.session->set_nb_streams(nbstream);
+
     gfal2_log(G_LOG_LEVEL_DEBUG,
             "   [GridFTPFileCopyModule::filecopy] setup gsiftp number of streams to %d",
             nbstream);

src/plugins/gridftp/gridftp_plugin.h

@@ -35,6 +35,7 @@
 #define GRIDFTP_CONFIG_DELAY_PASSV    "DELAY_PASSV"
 #define GRIDFTP_CONFIG_ENABLE_PASV_PLUGIN "ENABLE_PASV_PLUGIN"
 #define GRIDFTP_CONFIG_BLOCK_SIZE     "BLOCK_SIZE"
+#define GRIDFTP_CONFIG_NB_STREAM      "RD_NB_STREAM"
 
 #define GRIDFTP_CONFIG_TRANSFER_CHECKSUM       "COPY_CHECKSUM_TYPE"
 #define GRIDFTP_CONFIG_TRANSFER_PERF_TIMEOUT   "PERF_MARKER_TIMEOUT"

src/plugins/http/gfal_http_copy.cpp

@@ -29,8 +29,7 @@
 // An enumeration of the different HTTP third-party-copy strategies.
 // NOTE: we loop through strategies from beginning to end, meaning the default
 // strategy should always be listed first.
-typedef enum {HTTP_COPY_DEFAULT=0,
-              HTTP_COPY_PULL=0,
+typedef enum {HTTP_COPY_PULL=0,
               HTTP_COPY_PUSH,
               HTTP_COPY_STREAM,
               HTTP_COPY_END} CopyMode;
@@ -40,6 +39,16 @@ const char* CopyModeStr[] = {
     GFAL_TRANSFER_TYPE_PULL, GFAL_TRANSFER_TYPE_PUSH, GFAL_TRANSFER_TYPE_STREAMED, NULL
 };
 
+static const CopyMode getCopyModeFromString(const char * copyModeStr) { 
+    if (!strcmp(copyModeStr, GFAL_TRANSFER_TYPE_PULL))  
+        return HTTP_COPY_PULL;
+    if (!strcmp(copyModeStr, GFAL_TRANSFER_TYPE_PUSH)) 
+        return HTTP_COPY_PUSH;
+     if (!strcmp(copyModeStr, GFAL_TRANSFER_TYPE_STREAMED))
+        return HTTP_COPY_STREAM;
+     return HTTP_COPY_PULL;
+}
+
 
 struct PerfCallbackData {
     gfalt_params_t     params;
@@ -55,7 +64,7 @@ struct PerfCallbackData {
 
 static bool is_http_scheme(const char* url)
 {
-    const char *schemes[] = {"http:", "https:", "dav:", "davs:", "s3:", "s3s:", NULL};
+    const char *schemes[] = {"http:", "https:", "dav:", "davs:", "s3:", "s3s:", "gcloud:", "gclouds:", NULL};
     const char *colon = strchr(url, ':');
     if (!colon)
         return false;
@@ -64,8 +73,8 @@ static bool is_http_scheme(const char* url)
         if (strncmp(url, schemes[i], scheme_len) == 0)
             return true;
     }
-    return false;}
-
+    return false;
+}
 
 static bool is_http_3rdcopy_enabled(gfal2_context_t context)
 {
@@ -78,6 +87,10 @@ static bool is_http_streamed_enabled(gfal2_context_t context)
     return gfal2_get_opt_boolean_with_default(context, "HTTP PLUGIN", "ENABLE_STREAM_COPY", TRUE);
 }
 
+static CopyMode get_default_copy_mode(gfal2_context_t context)
+{
+    return getCopyModeFromString(gfal2_get_opt_string_with_default(context, "HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL));
+}
 
 static int gfal_http_exists(plugin_handle plugin_data,
         const char* url, GError** err)
@@ -237,7 +250,7 @@ static std::string get_canonical_uri(const std::string& original)
     std::string scheme;
     char last_scheme;
 
-    if (original.compare(0, 2, "s3") == 0) {
+    if ((original.compare(0, 2, "s3") == 0)  || (original.compare(0, 6, "gcloud") == 0 )) {
         return original;
     }
 
@@ -289,7 +302,7 @@ static int gfal_http_third_party_copy(GfalHttpPluginData* davix,
 
     // dCache requires RequireChecksumVerification to be explicitly false if no checksums
     // are to be used
-    if (mode == HTTP_COPY_PULL && strncmp(dst, "s3", 2) != 0) {
+    if (mode == HTTP_COPY_PULL && strncmp(dst, "s3", 2) != 0 && strncmp(dst, "gcloud", 6) != 0) {
         if (!(gfalt_get_checksum_mode(params, err) & GFALT_CHECKSUM_TARGET)) {
             req_params.addHeader("RequireChecksumVerification", "false");
         }
@@ -398,7 +411,7 @@ static int gfal_http_streamed_copy(gfal2_context_t context,
         gfal2_propagate_prefixed_error(err, nested_err, __func__);
         return -1;
     }
-
+    
     int source_fd = gfal2_open(context, src, O_RDONLY, &nested_err);
     if (source_fd < 0) {
         gfal2_propagate_prefixed_error(err, nested_err, __func__);
@@ -420,7 +433,9 @@ static int gfal_http_streamed_copy(gfal2_context_t context,
 
     if (dst_uri.getProtocol() == "s3" || dst_uri.getProtocol() == "s3s")
         req_params.setProtocol(Davix::RequestProtocol::AwsS3);
-
+     else if (dst_uri.getProtocol() == "gcloud" ||  dst_uri.getProtocol() ==  "gclouds") {
+        req_params.setProtocol(Davix::RequestProtocol::Gcloud);
+    }
     // Set MD5 header on the PUT
     if (checksum_mode & GFALT_CHECKSUM_TARGET && strcasecmp(checksum_type, "md5") == 0 && user_checksum[0]) {
         req_params.addHeader("Content-MD5", user_checksum);
@@ -432,7 +447,6 @@ static int gfal_http_streamed_copy(gfal2_context_t context,
 
     request.setParameters(req_params);
     HttpStreamProvider provider(src, dst, context, source_fd, params);
-
     request.setRequestBody(gfal_http_streamed_provider, src_stat.st_size, &provider);
     request.executeRequest(&dav_error);
 
@@ -561,7 +575,7 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context,
                          "%s => %s", src_full, dst_full);
 
     // Initial copy mode
-    CopyMode copy_mode = HTTP_COPY_DEFAULT;
+    CopyMode copy_mode = get_default_copy_mode(context);
 
     // If source is not even http, go straight to streamed
     // or if third party copy is disabled, go straight to streamed
@@ -604,8 +618,10 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context,
         }
         else if (ret < 0) {
                 gfal2_log(G_LOG_LEVEL_WARNING,
-                        "Copy failed with mode %s, will retry with the next available mode: %s",
+                        "Copy failed with mode %s, will delete destination and retry with the next available mode: %s",
                         CopyModeStr[copy_mode], nested_error->message);
+                // Delete any potential destination file.
+                gfal_http_copy_cleanup(plugin_data, dst, &nested_error);
         }
 
         copy_mode = (CopyMode)((int)copy_mode + 1);