Merge branch 'develop'

CMakeLists.txt

@@ -45,7 +45,7 @@ set (OUTPUT_NAME_TRANSFER "gfal_transfer")
 
 set (VERSION_MAJOR  2)
 set (VERSION_MINOR  16)
-set (VERSION_PATCH  0)
+set (VERSION_PATCH  1)
 set (VERSION_STRING ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH})
 
 add_definitions(-DVERSION="${VERSION_STRING}")

RELEASE-NOTES

@@ -1,5 +1,39 @@
 GFAL2 RELEASE-NOTES
 ===================
+## Wed Oct 31 2018 Andrea Manzi <amanzi at cern.ch> - 2.16.1
+###Bug
+- [DMC-1099] - Checksum comparison in Xrootd is wrongly enabled for FTS transfers
+- [DMC-1106] - HTTP to HTTP streamed transfers timeout after 300 secs
+- [DMC-1107] - The instant throughut for HTTP Streamed transfers is wrongly calculated
+- [DMC-1109] - HTTP TPC failures are not logged at INFO level
+- [DMC-1110] - RequireChecksumVerification set to false only for HTTP TPC PULL
+- [DMC-1112] - Setting BEARER_TOKEN breaks pre-signed URLa
+
+###Task
+- [DMC-1095] - Refactor http streamed copy to support s3 multipart upload via Davix
+
+## Mon Seo 28 2018 Andrea Manzi <amanzi at cern.ch> - 2.16.0
+### Bug 
+- [DMC-1048] - Xrootd plugin: the query for the EOS-CTA extended attribute "sys.retrieve.error" should be asynchronous
+- [DMC-1055] - The Xrootd checksum preset should not contain leading 0s
+- [DMC-1057] - xattr.h is in glibc-headers >= 2.27
+- [DMC-1066] - gfal-sum does not honour HTTP 202 responses
+- [DMC-1067] - Xrootd plugin does not pass the checksum type
+- [DMC-1068] - Remove file after error in HTTP COPY
+- [DMC-1069] - Work with endpoints that support HTTP macaroons but not for COPY
+- [DMC-1084] - TPC with destination/source http are not working with dCache
+
+### Task
+
+- [DMC-1046] - add support for gCloud in gfal2 http plugin
+- [DMC-1060] - xrootd plugin should clean destination on failure
+- [DMC-1064] - Add possibility to pass a BEARER token via ENV var
+
+### Improvement
+
+- [DMC-1028] - RD_NB_STREAM in /etc/gfal2.d/gsiftp_plugin.conf has no effect
+- [DMC-1074] - Add DEFAULT_COPY_MODE to gfal2 http conf
+
 ## Mon May 28 2018 Andrea Manzi <amanzi at cern.ch> - 2.15.5
 ### Bug
 - [DMC-1023] - Xrootd plugin:The checksum string provided by the user is not correctly parsed

doc/src/mainpage.c

@@ -1,8 +1,9 @@
 /**
 	\mainpage GFAL 2.0  Documentation
-    \author Data Management Clients support, CERN IT-SDC-ID ( dmc-support@cern.ch )
+    \author Data Management Clients support, CERN IT-ST ( dmc-support@cern.ch )
 	\author Devresse Adrien ( adrien.devresse@cern.ch )
 	\author Alejandro Alvarez Ayllon  ( Alejandro.Alvarez.Ayllon@cern.ch )
+        \author Andres Manzi ( andrea.manzi.cern.ch)
 
 <h2>GFAL 2.0 Wiki:</h2>
 - <a href="http://dmc.web.cern.ch/">Data Management Clients web page</a>
@@ -35,7 +36,7 @@ Main file : gfal_api.h
 
 <h2>How to compile locally GFAL 2.0</h2>
 - Compile :
-	- " 1. svn export http://svnweb.cern.ch/guest/lcgutil/gfal2/trunk gfal2 "
+	- " 1. git clone  https://gitlab.cern.ch/dmc/gfal2 "
 	- " 2. cd gfal2 "
 	- " 3. mkdir build; cd build"
 	- " 4. cmake −D CMAKE_INSTALL_PREFIX=/usr"
@@ -46,7 +47,6 @@ Main file : gfal_api.h
 	- " 5.  make; make test"	
 	
 - make RPMS :
-	- ./packaging/bin/packager_rpm.sh ./packaging/rpm/specs/ ./
-	- mock -r [mycfg] RPMS/gfal2-.....
+	- cd packaging/; make rpm
 	
 */

packaging/debian/changelog

@@ -1,3 +1,8 @@
+gfal2 (2.16.1-1) unstable; urgency=low
+  * New version
+
+ -- DMC Devel <dmc-devel@cern.ch> Thu, 04 Oct 2018 16:00:00 +0100
+
 gfal2 (2.16.0-1) unstable; urgency=low
   * New version
 

packaging/rpm/gfal2.spec

@@ -4,7 +4,7 @@
 %bcond_with tests
 
 Name:               gfal2
-Version:            2.16.0
+Version:            2.16.1
 Release:            1%{?dist}
 Summary:            Grid file access library 2.0
 Group:              Applications/Internet

readme.html

@@ -32,11 +32,14 @@ <h2>Description</h2>
 			Differents file systems are supported by a collections of GFAL 2 plugins, this is a non complete list:
 		</p>
 		<ul>
+			<li>GRIDFTP with gsiftp://</li>
 			<li>LFC (Logical File Catalog), with lfn://</li>
 			<li>SRM (Storage Resource Manager) with srm://</li>
 			<li>RFIO (Remote File I/O) with rfio://</li>
 			<li>DCAP (Remote FTP for dCache ) with gsidcap://</li>
-			<li>HTTP/Webdav with cluster, third party copy and client side credential support.</li>
+			<li>HTTP/Webdav with cluster, third party copy and client side credential support</li>
+			<li>XRootD protocll with root://</li>
+			<li>S3 and Google Cloud, with s3:// and gcloud://.</li>
 		</ul>
 
 		</p>
@@ -47,21 +50,22 @@ <h2>Installation</h2>
 			support for a given protocol, its corresponding plugin needs to be installed.
 		</p>
 		<ul>
+			<li>gfal2-plugin-gridftp</li>
 			<li>gfal2-plugin-lfc</li>
 			<li>gfal2-plugin-srm</li>
 			<li>gfal2-plugin-rfio</li>
 			<li>gfal2-plugin-dcap</li>
 			<li>gfal2-plugin-http</li>
+                        <li>gfal2-plugin-xrootd</li>
+			<li>gfal2-plugin-file</li>
+			<li>gfal2-plugin-mock</li>
 		</ul>
 		<p>
 			For convenience we provide a package called gfal2-all, which will trigger the installation of all these packages.<br />
 			gfal2-devel is required only for development.<br />
 			A typical installation can be done as easy as bellow:
 		</p>
 		<pre>yum install gfal2-all gfal2-devel</pre>
-		<p>
-			Shipped separatly there is also available a plugin for XRootD (root://) protocol: gfal2-plugin-xrootd
-		</p>
 
 		<h2>API Documentation</h2>
 		<p>
@@ -86,7 +90,7 @@ <h2>Contact</h2>
 		</p>
 		<dl>
 			<dt>Tracker</dt>
-			<dd><a href="https://its.cern.ch/jira/browse/DMC/component/10808">https://its.cern.ch/jira/browse/DMC/component/10808</a></dd>
+			<dd><a href="https://its.cern.ch/jira/projects/DMC/">https://its.cern.ch/jira/projects/DMC/</a></dd>
 			<dt>Mailing list</dt>
 			<dd><a href="mailto:dmc-support@cern.ch">dmc-support@cern.ch</a></dd>
 		</dl>

src/core/file/gfal_file_api.h

@@ -35,11 +35,16 @@
 #ifndef ENOATTR
 #define ENOATTR ENODATA
 #endif
-#if defined __APPLE__ || ( defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) )
+#if defined __APPLE__ 
 #include <sys/xattr.h>
-#else
+#else 
+#if defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27)
+#include <sys/xattr.h>
+#else 
 #include <attr/xattr.h>
 #endif
+#endif
+
 #include <glib.h>
 
 #include <common/gfal_common.h>

src/core/posix/gfal_posix_api.h

@@ -37,11 +37,15 @@
 #ifndef ENOATTR
 #define ENOATTR ENODATA
 #endif
-#if defined __APPLE__  || ( defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) )
+#if defined __APPLE__ 
+#include <sys/xattr.h>
+#else
+#if defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27)
 #include <sys/xattr.h>
 #else
 #include <attr/xattr.h>
 #endif
+#endif
 
 #include <common/gfal_common.h>
 #include <common/gfal_constants.h>

src/plugins/file/gfal_file_plugin_main.c

@@ -26,11 +26,15 @@
 #include <fcntl.h>
 #include <glib.h>
 #include <errno.h>
-#if defined __APPLE__ || ( defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) )
+#if defined __APPLE__ 
+#include <sys/xattr.h>
+#else
+#if defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27)
 #include <sys/xattr.h>
 #else
 #include <attr/xattr.h>
 #endif
+#endif
 #include <zlib.h>
 
 #include <gfal_plugins_api.h>

src/plugins/http/gfal_http_copy.cpp

@@ -20,6 +20,7 @@
 
 #include <davix.hpp>
 #include <copy/davixcopy.hpp>
+#include <status/davixstatusrequest.hpp>
 #include <unistd.h>
 #include <checksums/checksums.h>
 #include <cstdio>
@@ -302,11 +303,17 @@ static int gfal_http_third_party_copy(GfalHttpPluginData* davix,
 
     // dCache requires RequireChecksumVerification to be explicitly false if no checksums
     // are to be used
-    if (mode == HTTP_COPY_PULL && strncmp(dst, "s3", 2) != 0 && strncmp(dst, "gcloud", 6) != 0) {
-        if (!(gfalt_get_checksum_mode(params, err) & GFALT_CHECKSUM_TARGET)) {
-            req_params.addHeader("RequireChecksumVerification", "false");
-        }
-        g_clear_error(err);
+    gfalt_checksum_mode_t checksum_mode = gfalt_get_checksum_mode(params, err);
+    if (*err) {
+	g_clear_error(err);
+    } else {
+        if (mode == HTTP_COPY_PUSH) {
+	    if ((checksum_mode == GFALT_CHECKSUM_SOURCE) || (checksum_mode == GFALT_CHECKSUM_NONE) )
+                req_params.addHeader("RequireChecksumVerification", "false");
+	} else if (mode == HTTP_COPY_PULL) {
+		if ((checksum_mode == GFALT_CHECKSUM_TARGET) || (checksum_mode == GFALT_CHECKSUM_NONE) )
+		    req_params.addHeader("RequireChecksumVerification", "false");
+	} 
     }
 
     // add timeout
@@ -354,7 +361,7 @@ struct HttpStreamProvider {
 
 
 static dav_ssize_t gfal_http_streamed_provider(void *userdata,
-        char *buffer, dav_size_t buflen)
+        void *buffer, dav_size_t buflen)
 {
     GError* error = NULL;
     HttpStreamProvider* data = static_cast<HttpStreamProvider*>(userdata);
@@ -380,7 +387,7 @@ static dav_ssize_t gfal_http_streamed_provider(void *userdata,
             data->perf.bytes_transfered += data->read_instant;
             data->perf.transfer_time = now - data->start;
             data->perf.average_baudrate = data->perf.bytes_transfered / data->perf.transfer_time;
-            data->perf.instant_baudrate = data->read_instant / now - data->last_update;
+            data->perf.instant_baudrate = data->read_instant / (now - data->last_update);
 
             data->last_update = now;
             data->read_instant = 0;
@@ -396,6 +403,7 @@ static dav_ssize_t gfal_http_streamed_provider(void *userdata,
 }
 
 
+
 static int gfal_http_streamed_copy(gfal2_context_t context,
         GfalHttpPluginData* davix,
         const char* src, const char* dst,
@@ -419,54 +427,44 @@ static int gfal_http_streamed_copy(gfal2_context_t context,
     }
 
     Davix::Uri dst_uri(dst);
-
-    Davix::DavixError* dav_error = NULL;
-    Davix::PutRequest request(davix->context, dst_uri, &dav_error);
-    if (dav_error != NULL) {
-        davix2gliberr(dav_error, err);
-        Davix::DavixError::clearError(&dav_error);
-        return -1;
-    }
-
     Davix::RequestParams req_params;
-    davix->get_params(&req_params, dst_uri);
 
-    if (dst_uri.getProtocol() == "s3" || dst_uri.getProtocol() == "s3s")
-        req_params.setProtocol(Davix::RequestProtocol::AwsS3);
-     else if (dst_uri.getProtocol() == "gcloud" ||  dst_uri.getProtocol() ==  "gclouds") {
-        req_params.setProtocol(Davix::RequestProtocol::Gcloud);
-    }
-    // Set MD5 header on the PUT
-    if (checksum_mode & GFALT_CHECKSUM_TARGET && strcasecmp(checksum_type, "md5") == 0 && user_checksum[0]) {
-        req_params.addHeader("Content-MD5", user_checksum);
-    }
+    davix->get_params(&req_params, dst_uri);
     //add timeout
     struct timespec opTimeout;
     opTimeout.tv_sec = gfalt_get_timeout(params, NULL);
     req_params.setOperationTimeout(&opTimeout);
 
-    request.setParameters(req_params);
+    // Set MD5 header on the PUT
+    if (checksum_mode & GFALT_CHECKSUM_TARGET && strcasecmp(checksum_type, "md5") == 0 && user_checksum[0]) {
+    	req_params.addHeader("Content-MD5", user_checksum);
+    }
+
+    if (dst_uri.getProtocol() == "s3" || dst_uri.getProtocol() == "s3s")
+    	req_params.setProtocol(Davix::RequestProtocol::AwsS3);
+    else if (dst_uri.getProtocol() == "gcloud" ||  dst_uri.getProtocol() ==  "gclouds")
+    	req_params.setProtocol(Davix::RequestProtocol::Gcloud);
+
+    Davix::DavFile dest(davix->context,req_params, dst_uri );
+
     HttpStreamProvider provider(src, dst, context, source_fd, params);
-    request.setRequestBody(gfal_http_streamed_provider, src_stat.st_size, &provider);
-    request.executeRequest(&dav_error);
+
+    try {
+    	dest.put(&req_params, std::bind(&gfal_http_streamed_provider,&provider,
+        		  std::placeholders::_1, std::placeholders::_2), src_stat.st_size);
+
+    } catch(Davix::DavixException &ex) {
+    	Davix::DavixError* daverr = NULL;
+        ex.toDavixError(&daverr);
+        davix2gliberr(daverr, err);
+        Davix::DavixError::clearError(&daverr);
+    }
 
     gfal2_close(context, source_fd, &nested_err);
     // Throw away this error
     if (nested_err)
         g_error_free(nested_err);
 
-    if (dav_error != NULL) {
-        davix2gliberr(dav_error, err);
-        Davix::DavixError::clearError(&dav_error);
-        return -1;
-    }
-
-    // Double check the HTTP code
-    if (request.getRequestCode() >= 400) {
-        http2gliberr(err, request.getRequestCode(), __func__, "Failed to PUT the file");
-    }
-
-    gfal2_log(G_LOG_LEVEL_DEBUG, "HTTP code %d", request.getRequestCode());
     return *err == NULL ? 0 : -1;
 }
 
@@ -569,11 +567,6 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context,
                          GFAL_EVENT_NONE, GFAL_EVENT_PREPARE_EXIT,
                          "%s => %s", src_full, dst_full);
 
-    // The real, actual, copy
-    plugin_trigger_event(params, http_plugin_domain,
-                         GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_ENTER,
-                         "%s => %s", src_full, dst_full);
-
     // Initial copy mode
     CopyMode copy_mode = get_default_copy_mode(context);
 
@@ -586,6 +579,10 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context,
     // Re-try different approaches
     int ret = 0;
     while (copy_mode < HTTP_COPY_END) {
+        // The real, actual, copy
+        plugin_trigger_event(params, http_plugin_domain,
+                                     GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_ENTER,
+                               "%s => %s", src_full, dst_full);
         gfal2_log(G_LOG_LEVEL_MESSAGE,
             "Trying copying with mode %s",
             CopyModeStr[copy_mode]);
@@ -617,8 +614,9 @@ int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context,
             break;
         }
         else if (ret < 0) {
-                gfal2_log(G_LOG_LEVEL_WARNING,
-                        "Copy failed with mode %s, will delete destination and retry with the next available mode: %s",
+               plugin_trigger_event(params, http_plugin_domain,
+                         GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT,
+                        "ERROR: Copy failed with mode %s, will delete destination and retry with the next available mode: %s",
                         CopyModeStr[copy_mode], nested_error->message);
                 // Delete any potential destination file.
                 gfal_http_copy_cleanup(plugin_data, dst, &nested_error);