Merge branch '2.9.4rc': 2.9.6 release

CHANGES.md

@@ -1,6 +1,12 @@
 # NCBI External Developer Release:
 
 
+## NCBI VDB 2.9.6
+**March 18, 2019**
+
+  **prefetch, **vfs**: fixed regression that prevented re-download of incomplete files  
+
+
 ## NCBI VDB 2.9.4-1
 **March 4, 2019**
 

build/Makefile.vers

@@ -23,4 +23,4 @@
 # ===========================================================================
 
 # NCBI-VDB and library version
-VERSION = 2.9.4
+VERSION = 2.9.6

interfaces/vfs/services-priv.h

@@ -65,6 +65,8 @@ rc_t KService1Search ( const struct KNSManager * mgr, const char * cgi,
     const char * acc, const struct Kart ** result );
 
 
+rc_t KSrvRespFileGetHttp ( const KSrvRespFile * self,
+                           const struct VPath ** path );
 
 #ifdef __cplusplus
 }

libs/blast/blast-mgr.c

@@ -50,7 +50,7 @@
 #include <stdio.h> /* fprintf */
 #include <string.h> /* memset */
 
-#define TOOLKIT "sratoolkit2_9_4"
+#define TOOLKIT "sratoolkit2_9_6"
 
 /******************************************************************************/
 

libs/kdb/libkdb.vers.h

@@ -24,4 +24,4 @@
 *
 */
 
-#define LIBKDB_VERS 0x0207001D
+#define LIBKDB_VERS 0x0207001E

libs/klib/release-vers.h

@@ -28,7 +28,7 @@
 
 
 /* Version of current SRA Toolkit Release */
-#define RELEASE_VERS 0x02090004
+#define RELEASE_VERS 0x02090006
 
 
 /* Type of Version of current SRA Toolkit Release is one of:
@@ -41,4 +41,4 @@
 #define RELEASE_TYPE 'r'
 
 /* Revision of Version of current SRA Toolkit Release */
-#define RELEASE_REVISION 1
+#define RELEASE_REVISION 0

libs/ncbi-vdb/libncbi-vdb.vers

@@ -1 +1 @@
-2.9.4
+2.9.6

libs/vdb/libvdb.vers.h

@@ -24,4 +24,4 @@
 *
 */
 
-#define LIBVDB_VERS 0x0207001D
+#define LIBVDB_VERS 0x0207001E

libs/vfs/json-response.c

@@ -69,6 +69,10 @@ typedef struct {
 
     int64_t size;
 
+    const VPath * http; /* http path from path[]
+when all path[] are alternative ways to get the same acc by different protocols:
+data received by names protocol-3.0 */
+
     const VPath * path [ MAX_PATHS ];
 
     const VPath * local;
@@ -366,6 +370,7 @@ static rc_t LocationsRelease ( Locations * self ) {
     RELEASE ( VPath, self -> local );
     RELEASE ( VPath, self -> cache );
     RELEASE ( VPath, self -> mapping );
+    RELEASE(VPath, self->http);
     free(self->cType);
     free ( self -> name );
 
@@ -389,24 +394,47 @@ static bool LocationsEmpty ( const Locations * self ) {
     return true;
 }
 
-static rc_t LocationsAddVPath ( Locations * self, const VPath * path,
-                                const VPath * mapping )
-{
+static rc_t LocationsSetHttp(Locations * self, const VPath * path) {
     rc_t rc = 0;
 
+    char scheme[6] = "";
+
+    assert( self );
+
+    if (self->http != NULL)
+        return 0;
+
+    rc = VPathReadScheme(path, scheme, sizeof scheme, NULL);
+    if (rc != 0)
+        return rc;
+
+    if (scheme[0] != 'h' ||
+        scheme[1] != 't' ||
+        scheme[2] != 't' ||
+        scheme[3] != 'p')
+    {
+        return 0;
+    }
+
+    rc = VPathAddRef(path);
+    if (rc != 0)
+        return rc;
+
+    self->http = path;
+    return rc;
+}
+
+static rc_t LocationsAddVPath ( Locations * self, const VPath * path,
+                            const VPath * mapping, bool setHttp, uint64_t osize)
+{
     int i = 0;
-    char scheme [ 6 ] = "";
 
     if ( self == NULL )
         return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
 
     if ( path == NULL )
         return 0;
 
-    rc = VPathReadScheme ( path, scheme, sizeof scheme, NULL );
-    if ( rc != 0 )
-        return rc;
-
     for ( i = 0; i < MAX_PATHS; ++ i ) {
         if ( self -> path [ i ] == NULL ) {
             rc_t rc = VPathAddRef ( path );
@@ -422,7 +450,12 @@ static rc_t LocationsAddVPath ( Locations * self, const VPath * path,
                 self->mapping = (VPath *) mapping;
             }
 
-            return 0;
+            if (setHttp) {
+                self->size = osize;
+                rc = LocationsSetHttp(self, path);
+            }
+
+            return rc;
         }
     }
 
@@ -628,13 +661,13 @@ static rc_t ItemAddFormat ( Item * self, const char * cType, const Data * dad,
 }
 
 rc_t ItemAddVPath ( Item * self, const char * type,
-                    const VPath * path, const VPath * mapping )
+    const VPath * path, const VPath * mapping, bool setHttp, uint64_t osize )
 {
     rc_t rc = 0;
     Locations * l = NULL;
     rc = ItemAddFormat ( self, type, NULL, & l );
     if ( rc == 0 )
-        rc = LocationsAddVPath ( l, path, mapping);
+        rc = LocationsAddVPath ( l, path, mapping, setHttp, osize);
     return rc;
 }
 
@@ -1151,7 +1184,7 @@ rc_t Response4AppendUrl ( Response4 * self, const char * url ) {
         rc = ItemAddFormat ( item, "", NULL, & l );
 
     if ( rc == 0 )
-        rc = LocationsAddVPath ( l, path, NULL );
+        rc = LocationsAddVPath ( l, path, NULL, false, 0 );
 
     RELEASE ( VPath, path );
 
@@ -1443,7 +1476,7 @@ static rc_t LocationsAddLink ( Locations * self, const KJsonValue * node,
         return rc;
     }
 
-    rc = LocationsAddVPath ( self, path, NULL );
+    rc = LocationsAddVPath ( self, path, NULL, false, 0);
 
     RELEASE ( VPath, path );
 
@@ -2494,6 +2527,23 @@ rc_t KSrvRespFileGetId ( const KSrvRespFile * self, uint64_t * id,
     return 0;
 }
 
+rc_t KSrvRespFileGetHttp ( const KSrvRespFile * self,
+                            const VPath ** path )
+{
+    rc_t rc = 0;
+
+    assert ( self && self -> file && path );
+
+    * path = NULL;
+
+    rc = VPathAddRef ( self -> file -> http );
+
+    if ( rc == 0 )
+        * path = self -> file -> http;
+
+    return rc;
+}
+
 rc_t KSrvRespFileGetCache ( const KSrvRespFile * self,
                             const VPath ** path )
 {

libs/vfs/json-response.h

@@ -54,7 +54,7 @@ rc_t Response4GetRc      ( const Response4 * self, rc_t * rc );
 rc_t ContainerAdd ( Container * self, const char * acc, int64_t id,
                     Item ** newItem, const struct Data * data );
 rc_t ItemAddVPath(Item * self, const char * type, const struct VPath * path,
-                                                  const struct VPath * mapping);
+                    const struct VPath * mapping, bool setHttp, uint64_t osize);
 rc_t ItemSetTicket ( Item * self, const struct String * ticket );
 rc_t Response4GetKSrvRespObjCount ( const Response4 * self, uint32_t * n );
 rc_t Response4GetKSrvRespObjByIdx ( const Response4 * self, uint32_t i,

libs/vfs/path-priv.h

@@ -215,7 +215,7 @@ rc_t VPathSetGetCache ( const VPathSet * self, const struct VPath ** path );
 rc_t VPathSetGetLocal ( const VPathSet * self, const struct VPath ** path );
 
 /* name resolver response row converted into VDB objects */
-typedef struct {
+typedef struct {          /*       hdbcache */
     struct VPath * fasp ; struct VPath * vcFasp;
     struct VPath * file ; struct VPath * vcFile;
     struct VPath * http ; struct VPath * vcHttp;
@@ -225,6 +225,7 @@ typedef struct {
     const struct KSrvError * error;
     char * reqId;
     char * respId;
+    uint64_t osize; /*size of VPath object */
 } EVPath;
 
 rc_t VPathSetMake

libs/vfs/remote-services.c

@@ -1760,6 +1760,8 @@ static rc_t EVPathInit ( EVPath * self, const STyped * src,
                                      rcMemory, rcExhausted );
             }
 
+            self->osize = src->osize;
+
             return rc;
         }
         rc = RC ( rcVFS, rcQuery, rcResolving, rcError, rcUnexpected );
@@ -3857,10 +3859,13 @@ rc_t KServiceProcessStream ( KService * self, KStream * stream )
                 const VPath * vdbcache = NULL;
                 const KSrvError * error = NULL;
                 const VPath * mapping = NULL;
+                uint64_t osize = 0;
                 rc = KSrvResponseGetPath ( self -> resp .list, i, pp [ p ],
                                            & path, & vdbcache, & error );
                 if (rc == 0)
                     rc = KSrvResponseGetMapping(self->resp.list, i, &mapping);
+                if (rc == 0)
+                    rc = KSrvResponseGetOSize(self->resp.list, i, &osize);
                 if (rc == 0) {
                     if (path != NULL) {
                         String ticket;
@@ -3870,13 +3875,15 @@ rc_t KServiceProcessStream ( KService * self, KStream * stream )
                         if (r == 0)
                             rc = ItemSetTicket(file, &ticket);
                         if (rc == 0)
-                            rc = ItemAddVPath(file, "sra", path, mapping);
+                            rc = ItemAddVPath(file, "sra", path, mapping,
+                                true, osize);
                         RELEASE(VPath, path);
                         if (rc != 0)
                             break;
                     }
                     if (vdbcache != NULL) {
-                        rc = ItemAddVPath(file, "vdbcache", vdbcache, NULL);
+                        rc = ItemAddVPath(file, "vdbcache", vdbcache, NULL,
+                            true, 0);
                         RELEASE(VPath, vdbcache);
                         if (rc != 0)
                             break;

libs/vfs/services-priv.h

@@ -77,6 +77,9 @@ rc_t KSrvResponseGetIds ( const struct KSrvResponse * self, uint32_t idx,
 rc_t KSrvResponseGetMapping(const KSrvResponse * self, uint32_t idx,
                             const struct VPath ** mapping);
 
+rc_t KSrvResponseGetOSize(const KSrvResponse * self, uint32_t idx,
+    uint64_t * osize);
+
 rc_t KSrvResponseGetR4 ( const struct KSrvResponse * self,
                          struct Response4 ** r );
 rc_t KSrvResponseSetR4 ( struct KSrvResponse * self,

libs/vfs/srv-response.c

@@ -49,6 +49,9 @@ struct VPathSet {
     const VPath * http;
     const VPath * https;
     const VPath * s3;
+    uint64_t osize; /*size of VPath object */
+
+    /* vdbcache */
     const VPath * cacheFasp;
     const VPath * cacheFile;
     const VPath * cacheHttp;
@@ -383,6 +386,8 @@ rc_t VPathSetMake ( VPathSet ** self, const EVPath * src,
                                      rcMemory, rcExhausted );
             }
         }
+
+        p->osize = src->osize;
     }
 
     if ( rc == 0 ) {
@@ -668,6 +673,26 @@ rc_t KSrvResponseGetMapping(const KSrvResponse * self, uint32_t idx,
     return rc;
 }
 
+rc_t KSrvResponseGetOSize(const KSrvResponse * self, uint32_t idx,
+    uint64_t * osize)
+{
+    rc_t rc = 0;
+    const VPathSet * s = NULL;
+    if (osize == NULL)
+        return RC(rcVFS, rcQuery, rcExecuting, rcParam, rcNull);
+    *osize = 0;
+    if (self == NULL)
+        return RC(rcVFS, rcQuery, rcExecuting, rcSelf, rcNull);
+    s = (VPathSet *)VectorGet(&self->list, idx);
+    if (s != NULL) {
+        if (s->error != NULL)
+            return 0;
+        if (rc == 0)
+            * osize = s->osize;
+    }
+    return rc;
+}
+
 rc_t KSrvResponseGetPath ( const KSrvResponse * self, uint32_t idx,
     VRemoteProtocols p, const VPath ** aPath, const VPath ** vdbcache,
     const KSrvError ** error )

test/vdb/kfg/linux/test-dependencies.kfg

@@ -1,3 +1,4 @@
+repository/remote/main/CGI/resolver-cgi = "https://trace.ncbi.nlm.nih.gov/Traces/names/names.fcgi"
 repository/site/main/tracearc/apps/sra/volumes/ncbi = "sra17:sra16:sra15:sra14:sra13:sra12:sra11:sra10:sra9:sra8:sra7:sra6:sra5:sra4:sra3:sra2:sra1:sra0"
 repository/site/main/tracearc/apps/refseq/volumes/refseq = "refseq"
 repository/site/main/tracearc/root = "/netmnt/traces04"

test/vdb/kfg/mac/test-dependencies.kfg

@@ -1,3 +1,4 @@
+repository/remote/main/CGI/resolver-cgi = "https://trace.ncbi.nlm.nih.gov/Traces/names/names.fcgi"
 repository/site/main/tracearc/apps/sra/volumes/ncbi = "sra17:sra16:sra15:sra14:sra13:sra12:sra11:sra10:sra9:sra8:sra7:sra6:sra5:sra4:sra3:sra2:sra1:sra0"
 repository/site/main/tracearc/apps/refseq/volumes/refseq = "refseq"
 repository/site/main/tracearc/root = "/net/traces04"

test/vdb/test-dependencies.cpp

@@ -141,18 +141,18 @@ FIXTURE_TEST_CASE(TestNoDeps, RefseqFixture) {
     const VDatabase *db = NULL;
     const char SRR600096[] = "SRR600096";
     REQUIRE_RC(VDBManagerOpenDBRead(mgr, &db, NULL, SRR600096));
-    RELEASE(VDatabase, db);
+//  RELEASE(VDatabase, db);
 
     VPath* acc = NULL;
     REQUIRE_RC(VFSManagerMakePath(vmgr, &acc, SRR600096));
     const VPath *local = NULL;
-    REQUIRE_RC(VResolverLocal(resolver, acc, &local));
-    RELEASE(VPath, acc);
+/*  REQUIRE_RC(VResolverLocal(resolver, acc, &local));
+    RELEASE(VPath, acc); noved to cldn */
     const String *s = NULL;
-    REQUIRE_RC(VPathMakeString(local, &s));
+/*  REQUIRE_RC(VPathMakeString(local, &s));
     REQUIRE(s && s->addr);
 
-    REQUIRE_RC(VDBManagerOpenDBRead(mgr, &db, NULL, s->addr));
+    REQUIRE_RC(VDBManagerOpenDBRead(mgr, &db, NULL, s->addr)); */
 
     const VDBDependencies *dep = NULL;
 
@@ -326,18 +326,18 @@ FIXTURE_TEST_CASE(TestManyYesDep, RefseqFixture) {
     const VDatabase *db = NULL;
     const char SRR543323[] = "SRR543323";
     REQUIRE_RC(VDBManagerOpenDBRead(mgr, &db, NULL, SRR543323));
-    RELEASE(VDatabase, db);
+//  RELEASE(VDatabase, db);
 
     VPath* acc = NULL;
     REQUIRE_RC(VFSManagerMakePath(vmgr, &acc, SRR543323));
     const VPath *local = NULL;
-    REQUIRE_RC(VResolverLocal(resolver, acc, &local));
+//  REQUIRE_RC(VResolverLocal(resolver, acc, &local)); noved to cldn
     RELEASE(VPath, acc);
     const String *s = NULL;
-    REQUIRE_RC(VPathMakeString(local, &s));
+/*  REQUIRE_RC(VPathMakeString(local, &s));
     REQUIRE(s && s->addr);
 
-    REQUIRE_RC(VDBManagerOpenDBRead(mgr, &db, NULL, s->addr));
+    REQUIRE_RC(VDBManagerOpenDBRead(mgr, &db, NULL, s->addr)); */
 
     const VDBDependencies *dep = NULL;