diff --git a/CHANGES.md b/CHANGES.md index b3c768d5f..58de7f2c9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,12 @@ # NCBI External Developer Release: +## NCBI VDB 2.9.6 +**March 18, 2019** + + **prefetch, **vfs**: fixed regression that prevented re-download of incomplete files + + ## NCBI VDB 2.9.4-1 **March 4, 2019** diff --git a/build/Makefile.vers b/build/Makefile.vers index dab514a06..838a65ca4 100644 --- a/build/Makefile.vers +++ b/build/Makefile.vers @@ -23,4 +23,4 @@ # =========================================================================== # NCBI-VDB and library version -VERSION = 2.9.4 +VERSION = 2.9.6 diff --git a/interfaces/vfs/services-priv.h b/interfaces/vfs/services-priv.h index 4d04d256c..1bb54f86d 100644 --- a/interfaces/vfs/services-priv.h +++ b/interfaces/vfs/services-priv.h @@ -65,6 +65,8 @@ rc_t KService1Search ( const struct KNSManager * mgr, const char * cgi, const char * acc, const struct Kart ** result ); +rc_t KSrvRespFileGetHttp ( const KSrvRespFile * self, + const struct VPath ** path ); #ifdef __cplusplus } diff --git a/libs/blast/blast-mgr.c b/libs/blast/blast-mgr.c index 61362056c..8e9c66b69 100644 --- a/libs/blast/blast-mgr.c +++ b/libs/blast/blast-mgr.c @@ -50,7 +50,7 @@ #include /* fprintf */ #include /* memset */ -#define TOOLKIT "sratoolkit2_9_4" +#define TOOLKIT "sratoolkit2_9_6" /******************************************************************************/ diff --git a/libs/kdb/libkdb.vers.h b/libs/kdb/libkdb.vers.h index 35ae19fbd..b4b2bd13f 100644 --- a/libs/kdb/libkdb.vers.h +++ b/libs/kdb/libkdb.vers.h @@ -24,4 +24,4 @@ * */ -#define LIBKDB_VERS 0x0207001D +#define LIBKDB_VERS 0x0207001E diff --git a/libs/klib/release-vers.h b/libs/klib/release-vers.h index 439bdbc4e..eaaaac951 100644 --- a/libs/klib/release-vers.h +++ b/libs/klib/release-vers.h @@ -28,7 +28,7 @@ /* Version of current SRA Toolkit Release */ -#define RELEASE_VERS 0x02090004 +#define RELEASE_VERS 0x02090006 /* Type of Version of current SRA Toolkit Release is one of: @@ -41,4 +41,4 @@ #define RELEASE_TYPE 'r' /* Revision of Version of current SRA Toolkit Release */ -#define RELEASE_REVISION 1 +#define RELEASE_REVISION 0 diff --git a/libs/ncbi-vdb/libncbi-vdb.vers b/libs/ncbi-vdb/libncbi-vdb.vers index 93ea0c959..23ae1b5dc 100644 --- a/libs/ncbi-vdb/libncbi-vdb.vers +++ b/libs/ncbi-vdb/libncbi-vdb.vers @@ -1 +1 @@ -2.9.4 +2.9.6 diff --git a/libs/vdb/libvdb.vers.h b/libs/vdb/libvdb.vers.h index 4cec6040b..99d76a92d 100644 --- a/libs/vdb/libvdb.vers.h +++ b/libs/vdb/libvdb.vers.h @@ -24,4 +24,4 @@ * */ -#define LIBVDB_VERS 0x0207001D +#define LIBVDB_VERS 0x0207001E diff --git a/libs/vfs/json-response.c b/libs/vfs/json-response.c index aec173ba4..36dece412 100644 --- a/libs/vfs/json-response.c +++ b/libs/vfs/json-response.c @@ -69,6 +69,10 @@ typedef struct { int64_t size; + const VPath * http; /* http path from path[] +when all path[] are alternative ways to get the same acc by different protocols: +data received by names protocol-3.0 */ + const VPath * path [ MAX_PATHS ]; const VPath * local; @@ -366,6 +370,7 @@ static rc_t LocationsRelease ( Locations * self ) { RELEASE ( VPath, self -> local ); RELEASE ( VPath, self -> cache ); RELEASE ( VPath, self -> mapping ); + RELEASE(VPath, self->http); free(self->cType); free ( self -> name ); @@ -389,13 +394,40 @@ static bool LocationsEmpty ( const Locations * self ) { return true; } -static rc_t LocationsAddVPath ( Locations * self, const VPath * path, - const VPath * mapping ) -{ +static rc_t LocationsSetHttp(Locations * self, const VPath * path) { rc_t rc = 0; + char scheme[6] = ""; + + assert( self ); + + if (self->http != NULL) + return 0; + + rc = VPathReadScheme(path, scheme, sizeof scheme, NULL); + if (rc != 0) + return rc; + + if (scheme[0] != 'h' || + scheme[1] != 't' || + scheme[2] != 't' || + scheme[3] != 'p') + { + return 0; + } + + rc = VPathAddRef(path); + if (rc != 0) + return rc; + + self->http = path; + return rc; +} + +static rc_t LocationsAddVPath ( Locations * self, const VPath * path, + const VPath * mapping, bool setHttp, uint64_t osize) +{ int i = 0; - char scheme [ 6 ] = ""; if ( self == NULL ) return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull ); @@ -403,10 +435,6 @@ static rc_t LocationsAddVPath ( Locations * self, const VPath * path, if ( path == NULL ) return 0; - rc = VPathReadScheme ( path, scheme, sizeof scheme, NULL ); - if ( rc != 0 ) - return rc; - for ( i = 0; i < MAX_PATHS; ++ i ) { if ( self -> path [ i ] == NULL ) { rc_t rc = VPathAddRef ( path ); @@ -422,7 +450,12 @@ static rc_t LocationsAddVPath ( Locations * self, const VPath * path, self->mapping = (VPath *) mapping; } - return 0; + if (setHttp) { + self->size = osize; + rc = LocationsSetHttp(self, path); + } + + return rc; } } @@ -628,13 +661,13 @@ static rc_t ItemAddFormat ( Item * self, const char * cType, const Data * dad, } rc_t ItemAddVPath ( Item * self, const char * type, - const VPath * path, const VPath * mapping ) + const VPath * path, const VPath * mapping, bool setHttp, uint64_t osize ) { rc_t rc = 0; Locations * l = NULL; rc = ItemAddFormat ( self, type, NULL, & l ); if ( rc == 0 ) - rc = LocationsAddVPath ( l, path, mapping); + rc = LocationsAddVPath ( l, path, mapping, setHttp, osize); return rc; } @@ -1151,7 +1184,7 @@ rc_t Response4AppendUrl ( Response4 * self, const char * url ) { rc = ItemAddFormat ( item, "", NULL, & l ); if ( rc == 0 ) - rc = LocationsAddVPath ( l, path, NULL ); + rc = LocationsAddVPath ( l, path, NULL, false, 0 ); RELEASE ( VPath, path ); @@ -1443,7 +1476,7 @@ static rc_t LocationsAddLink ( Locations * self, const KJsonValue * node, return rc; } - rc = LocationsAddVPath ( self, path, NULL ); + rc = LocationsAddVPath ( self, path, NULL, false, 0); RELEASE ( VPath, path ); @@ -2494,6 +2527,23 @@ rc_t KSrvRespFileGetId ( const KSrvRespFile * self, uint64_t * id, return 0; } +rc_t KSrvRespFileGetHttp ( const KSrvRespFile * self, + const VPath ** path ) +{ + rc_t rc = 0; + + assert ( self && self -> file && path ); + + * path = NULL; + + rc = VPathAddRef ( self -> file -> http ); + + if ( rc == 0 ) + * path = self -> file -> http; + + return rc; +} + rc_t KSrvRespFileGetCache ( const KSrvRespFile * self, const VPath ** path ) { diff --git a/libs/vfs/json-response.h b/libs/vfs/json-response.h index 2efade6bf..272ffdc8d 100644 --- a/libs/vfs/json-response.h +++ b/libs/vfs/json-response.h @@ -54,7 +54,7 @@ rc_t Response4GetRc ( const Response4 * self, rc_t * rc ); rc_t ContainerAdd ( Container * self, const char * acc, int64_t id, Item ** newItem, const struct Data * data ); rc_t ItemAddVPath(Item * self, const char * type, const struct VPath * path, - const struct VPath * mapping); + const struct VPath * mapping, bool setHttp, uint64_t osize); rc_t ItemSetTicket ( Item * self, const struct String * ticket ); rc_t Response4GetKSrvRespObjCount ( const Response4 * self, uint32_t * n ); rc_t Response4GetKSrvRespObjByIdx ( const Response4 * self, uint32_t i, diff --git a/libs/vfs/path-priv.h b/libs/vfs/path-priv.h index 988ba3cd1..cb02b800f 100644 --- a/libs/vfs/path-priv.h +++ b/libs/vfs/path-priv.h @@ -215,7 +215,7 @@ rc_t VPathSetGetCache ( const VPathSet * self, const struct VPath ** path ); rc_t VPathSetGetLocal ( const VPathSet * self, const struct VPath ** path ); /* name resolver response row converted into VDB objects */ -typedef struct { +typedef struct { /* hdbcache */ struct VPath * fasp ; struct VPath * vcFasp; struct VPath * file ; struct VPath * vcFile; struct VPath * http ; struct VPath * vcHttp; @@ -225,6 +225,7 @@ typedef struct { const struct KSrvError * error; char * reqId; char * respId; + uint64_t osize; /*size of VPath object */ } EVPath; rc_t VPathSetMake diff --git a/libs/vfs/remote-services.c b/libs/vfs/remote-services.c index 6e9e825a4..79bc15ef9 100644 --- a/libs/vfs/remote-services.c +++ b/libs/vfs/remote-services.c @@ -1760,6 +1760,8 @@ static rc_t EVPathInit ( EVPath * self, const STyped * src, rcMemory, rcExhausted ); } + self->osize = src->osize; + return rc; } rc = RC ( rcVFS, rcQuery, rcResolving, rcError, rcUnexpected ); @@ -3857,10 +3859,13 @@ rc_t KServiceProcessStream ( KService * self, KStream * stream ) const VPath * vdbcache = NULL; const KSrvError * error = NULL; const VPath * mapping = NULL; + uint64_t osize = 0; rc = KSrvResponseGetPath ( self -> resp .list, i, pp [ p ], & path, & vdbcache, & error ); if (rc == 0) rc = KSrvResponseGetMapping(self->resp.list, i, &mapping); + if (rc == 0) + rc = KSrvResponseGetOSize(self->resp.list, i, &osize); if (rc == 0) { if (path != NULL) { String ticket; @@ -3870,13 +3875,15 @@ rc_t KServiceProcessStream ( KService * self, KStream * stream ) if (r == 0) rc = ItemSetTicket(file, &ticket); if (rc == 0) - rc = ItemAddVPath(file, "sra", path, mapping); + rc = ItemAddVPath(file, "sra", path, mapping, + true, osize); RELEASE(VPath, path); if (rc != 0) break; } if (vdbcache != NULL) { - rc = ItemAddVPath(file, "vdbcache", vdbcache, NULL); + rc = ItemAddVPath(file, "vdbcache", vdbcache, NULL, + true, 0); RELEASE(VPath, vdbcache); if (rc != 0) break; diff --git a/libs/vfs/services-priv.h b/libs/vfs/services-priv.h index 32e9ff448..2da8be000 100644 --- a/libs/vfs/services-priv.h +++ b/libs/vfs/services-priv.h @@ -77,6 +77,9 @@ rc_t KSrvResponseGetIds ( const struct KSrvResponse * self, uint32_t idx, rc_t KSrvResponseGetMapping(const KSrvResponse * self, uint32_t idx, const struct VPath ** mapping); +rc_t KSrvResponseGetOSize(const KSrvResponse * self, uint32_t idx, + uint64_t * osize); + rc_t KSrvResponseGetR4 ( const struct KSrvResponse * self, struct Response4 ** r ); rc_t KSrvResponseSetR4 ( struct KSrvResponse * self, diff --git a/libs/vfs/srv-response.c b/libs/vfs/srv-response.c index 6b8bc8394..595ecd16b 100644 --- a/libs/vfs/srv-response.c +++ b/libs/vfs/srv-response.c @@ -49,6 +49,9 @@ struct VPathSet { const VPath * http; const VPath * https; const VPath * s3; + uint64_t osize; /*size of VPath object */ + + /* vdbcache */ const VPath * cacheFasp; const VPath * cacheFile; const VPath * cacheHttp; @@ -383,6 +386,8 @@ rc_t VPathSetMake ( VPathSet ** self, const EVPath * src, rcMemory, rcExhausted ); } } + + p->osize = src->osize; } if ( rc == 0 ) { @@ -668,6 +673,26 @@ rc_t KSrvResponseGetMapping(const KSrvResponse * self, uint32_t idx, return rc; } +rc_t KSrvResponseGetOSize(const KSrvResponse * self, uint32_t idx, + uint64_t * osize) +{ + rc_t rc = 0; + const VPathSet * s = NULL; + if (osize == NULL) + return RC(rcVFS, rcQuery, rcExecuting, rcParam, rcNull); + *osize = 0; + if (self == NULL) + return RC(rcVFS, rcQuery, rcExecuting, rcSelf, rcNull); + s = (VPathSet *)VectorGet(&self->list, idx); + if (s != NULL) { + if (s->error != NULL) + return 0; + if (rc == 0) + * osize = s->osize; + } + return rc; +} + rc_t KSrvResponseGetPath ( const KSrvResponse * self, uint32_t idx, VRemoteProtocols p, const VPath ** aPath, const VPath ** vdbcache, const KSrvError ** error ) diff --git a/test/vdb/kfg/linux/test-dependencies.kfg b/test/vdb/kfg/linux/test-dependencies.kfg index 3a0e575b8..04730d009 100644 --- a/test/vdb/kfg/linux/test-dependencies.kfg +++ b/test/vdb/kfg/linux/test-dependencies.kfg @@ -1,3 +1,4 @@ +repository/remote/main/CGI/resolver-cgi = "https://trace.ncbi.nlm.nih.gov/Traces/names/names.fcgi" repository/site/main/tracearc/apps/sra/volumes/ncbi = "sra17:sra16:sra15:sra14:sra13:sra12:sra11:sra10:sra9:sra8:sra7:sra6:sra5:sra4:sra3:sra2:sra1:sra0" repository/site/main/tracearc/apps/refseq/volumes/refseq = "refseq" repository/site/main/tracearc/root = "/netmnt/traces04" diff --git a/test/vdb/kfg/mac/test-dependencies.kfg b/test/vdb/kfg/mac/test-dependencies.kfg index 407f738de..a0fd87942 100644 --- a/test/vdb/kfg/mac/test-dependencies.kfg +++ b/test/vdb/kfg/mac/test-dependencies.kfg @@ -1,3 +1,4 @@ +repository/remote/main/CGI/resolver-cgi = "https://trace.ncbi.nlm.nih.gov/Traces/names/names.fcgi" repository/site/main/tracearc/apps/sra/volumes/ncbi = "sra17:sra16:sra15:sra14:sra13:sra12:sra11:sra10:sra9:sra8:sra7:sra6:sra5:sra4:sra3:sra2:sra1:sra0" repository/site/main/tracearc/apps/refseq/volumes/refseq = "refseq" repository/site/main/tracearc/root = "/net/traces04" diff --git a/test/vdb/test-dependencies.cpp b/test/vdb/test-dependencies.cpp index 3ea7c30c7..ea72dad48 100644 --- a/test/vdb/test-dependencies.cpp +++ b/test/vdb/test-dependencies.cpp @@ -141,18 +141,18 @@ FIXTURE_TEST_CASE(TestNoDeps, RefseqFixture) { const VDatabase *db = NULL; const char SRR600096[] = "SRR600096"; REQUIRE_RC(VDBManagerOpenDBRead(mgr, &db, NULL, SRR600096)); - RELEASE(VDatabase, db); +// RELEASE(VDatabase, db); VPath* acc = NULL; REQUIRE_RC(VFSManagerMakePath(vmgr, &acc, SRR600096)); const VPath *local = NULL; - REQUIRE_RC(VResolverLocal(resolver, acc, &local)); - RELEASE(VPath, acc); +/* REQUIRE_RC(VResolverLocal(resolver, acc, &local)); + RELEASE(VPath, acc); noved to cldn */ const String *s = NULL; - REQUIRE_RC(VPathMakeString(local, &s)); +/* REQUIRE_RC(VPathMakeString(local, &s)); REQUIRE(s && s->addr); - REQUIRE_RC(VDBManagerOpenDBRead(mgr, &db, NULL, s->addr)); + REQUIRE_RC(VDBManagerOpenDBRead(mgr, &db, NULL, s->addr)); */ const VDBDependencies *dep = NULL; @@ -326,18 +326,18 @@ FIXTURE_TEST_CASE(TestManyYesDep, RefseqFixture) { const VDatabase *db = NULL; const char SRR543323[] = "SRR543323"; REQUIRE_RC(VDBManagerOpenDBRead(mgr, &db, NULL, SRR543323)); - RELEASE(VDatabase, db); +// RELEASE(VDatabase, db); VPath* acc = NULL; REQUIRE_RC(VFSManagerMakePath(vmgr, &acc, SRR543323)); const VPath *local = NULL; - REQUIRE_RC(VResolverLocal(resolver, acc, &local)); +// REQUIRE_RC(VResolverLocal(resolver, acc, &local)); noved to cldn RELEASE(VPath, acc); const String *s = NULL; - REQUIRE_RC(VPathMakeString(local, &s)); +/* REQUIRE_RC(VPathMakeString(local, &s)); REQUIRE(s && s->addr); - REQUIRE_RC(VDBManagerOpenDBRead(mgr, &db, NULL, s->addr)); + REQUIRE_RC(VDBManagerOpenDBRead(mgr, &db, NULL, s->addr)); */ const VDBDependencies *dep = NULL;