Skip to content

Commit

Permalink
[io] Extract tkey walk logic from TFile::Map()
Browse files Browse the repository at this point in the history
  • Loading branch information
@silverweed
silverweed committed Jan 30, 2025
1 parent caa8c54 commit 0491d72
Show file tree
Hide file tree
Showing 2 changed files with 153 additions and 96 deletions.
34 changes: 34 additions & 0 deletions io/io/inc/TFile.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

#include <atomic>
#include <string>
#include <cstdint>

#include "Compression.h"
#include "TDirectoryFile.h"
Expand Down Expand Up @@ -50,6 +51,35 @@ class TProcessID;
class TStopwatch;
class TFilePrefetch;

namespace Detail {
struct TKeyMapNode {
enum EType {
kError,
kGap,
kKey
};

std::uint64_t fAddr = 0;
EType fType = kError;
std::uint32_t fLen = 0;

// these are only valid for Keys
Version_t fKeyVersion = 0;
Int_t fObjLen = 0;
Int_t fDatime = 0;
Short_t fKeyLen = 0;
Short_t fCycle = 0;
Long64_t fSeekKey = 0;
Long64_t fSeekPdir = 0;
std::string fClassName;
std::string fKeyName;
std::string fKeyTitle;

TKeyMapNode() = default;
TKeyMapNode(std::uint64_t addr, EType type, std::uint32_t len = 0) : fAddr(addr), fType(type), fLen(len) {}
};
} // namespace Detail

class TFile : public TDirectoryFile {
friend class TDirectoryFile;
friend class TFilePrefetch;
Expand Down Expand Up @@ -263,6 +293,10 @@ class TFile : public TDirectoryFile {
virtual void MakeFree(Long64_t first, Long64_t last);
virtual void MakeProject(const char *dirname, const char *classes="*",
Option_t *option="new"); // *MENU*

/// Traverses all TKeys in the TFile and returns information about them.
std::vector<Detail::TKeyMapNode> WalkTKeys();

virtual void Map(Option_t *opt); // *MENU*
virtual void Map() { Map(""); }; // *MENU*
virtual Bool_t Matches(const char *name);
Expand Down
215 changes: 119 additions & 96 deletions io/io/src/TFile.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ The structure of a directory is shown in TDirectoryFile::TDirectoryFile
#include "TGlobal.h"
#include "ROOT/RConcurrentHashColl.hxx"
#include <memory>
#include <inttypes.h>

#ifdef R__FBSD
#include <sys/extattr.h>
Expand Down Expand Up @@ -1568,128 +1569,150 @@ void TFile::Map(Option_t *opt)
{
TString options(opt);
options.ToLower();
bool forComp = options.Contains("forcomp");
bool extended = options.Contains("extended");
const bool forComp = options.Contains("forcomp");
const bool extended = options.Contains("extended");

Short_t keylen,cycle;
UInt_t datime;
Int_t nbytes,date,time,objlen;
date = 0;
time = 0;
Long64_t seekkey,seekpdir;
char *buffer;
char nwhc;
Long64_t idcur = fBEGIN;
const unsigned char nDigits = std::log10(fEND) + 1;

const auto tkeyInfos = WalkTKeys();
for (const auto &key : tkeyInfos) {
switch (key.fType) {
case Detail::TKeyMapNode::kError:
Printf("Address = %" PRIu64 "\tNbytes = %u\t=====E R R O R=======", key.fAddr, key.fLen);
break;

case Detail::TKeyMapNode::kGap:
Printf("Address = %" PRIu64 "\tNbytes = %d\t=====G A P===========", key.fAddr, -key.fLen);
break;

case Detail::TKeyMapNode::kKey:
TString extrainfo;
if (extended)
extrainfo.Form(" name: %-16s title: %s", key.fKeyName.c_str(), key.fKeyTitle.c_str());

if (forComp) {
// Printing to help compare two files.
if (key.fObjLen != static_cast<Int_t>(key.fLen) - key.fKeyLen) {
Float_t cx = static_cast<float>(key.fObjLen + key.fKeyLen) / key.fLen;
Printf("At:%-*" PRIu64 " N=%-8u K=%-3d O=%-8d %-14s CX = %5.2f %s", nDigits + 1, key.fAddr, key.fLen,
key.fKeyLen, key.fObjLen, key.fClassName.c_str(), cx, extrainfo.Data());
} else {
Printf("At:%-*" PRIu64 " N=%-8u K=%-3d O=%-8d %-14s CX = 1 %s", nDigits + 1, key.fAddr, key.fLen,
key.fKeyLen, key.fObjLen, key.fClassName.c_str(), extrainfo.Data());
}
} else {
Int_t date, time;
TDatime::GetDateTime(key.fDatime, date, time);
if (key.fObjLen != static_cast<Int_t>(key.fLen) - key.fKeyLen) {
Float_t cx = static_cast<float>(key.fObjLen + key.fKeyLen) / key.fLen;
Printf("%d/%06d At:%-*" PRIu64 " N=%-8u %-14s CX = %5.2f %s", date, time, nDigits + 1, key.fAddr,
key.fLen, key.fClassName.c_str(), cx, extrainfo.Data());
} else {
Printf("%d/%06d At:%-*" PRIu64 " N=%-8u %-14s %s", date, time, nDigits + 1, key.fAddr,
key.fLen, key.fClassName.c_str(), extrainfo.Data());
}
}
}
}

constexpr Int_t nwheader = 512;
if (!forComp) {
Int_t datime = tkeyInfos.empty() ? 0 : tkeyInfos.back().fDatime;
Int_t date, time;
TDatime::GetDateTime(datime, date, time);
Printf("%d/%06d At:%-*lld N=%-8d %-14s", date, time, nDigits + 1, fEND, 1, "END");
} else {
Printf("At:%-*lld N=%-8d K= O= %-14s", nDigits + 1, fEND, 1, "END");
}
}

char header[nwheader];
char classname[512];
char keyname[512];
char keytitle[512];
TString extrainfo;
std::vector<Detail::TKeyMapNode> TFile::WalkTKeys()
{
std::vector<Detail::TKeyMapNode> res;

unsigned char nDigits = std::log10(fEND) + 1;
constexpr int headerSize = 512;

while (idcur < fEND) {
std::uint64_t idcur = fBEGIN;
const std::uint64_t end = fEND;
while (idcur < end) {
Seek(idcur);
Int_t nread = nwheader;
if (idcur+nread >= fEND) nread = fEND-idcur-1;
auto nread = headerSize;
if (idcur + nread >= end)
nread = fEND - idcur - 1;

char header[headerSize];
if (ReadBuffer(header, nread)) {
// ReadBuffer returns kTRUE in case of failure.
Warning("Map","%s: failed to read the key data from disk at %lld.",
GetName(),idcur);
res.push_back({idcur, Detail::TKeyMapNode::kError});
break;
}

buffer=header;
char *buffer = header;
Int_t nbytes;
frombuf(buffer, &nbytes);
if (!nbytes) {
Printf("Address = %lld\tNbytes = %d\t=====E R R O R=======", idcur, nbytes);
date = 0; time = 0;
res.push_back({idcur, Detail::TKeyMapNode::kError});
break;
}
if (nbytes < 0) {
Printf("Address = %lld\tNbytes = %d\t=====G A P===========", idcur, nbytes);
// free slot
res.push_back({idcur, Detail::TKeyMapNode::kGap, static_cast<std::uint32_t>(-nbytes)});
idcur -= nbytes;
Seek(idcur);
continue;
}
Version_t versionkey;
frombuf(buffer, &versionkey);
frombuf(buffer, &objlen);
frombuf(buffer, &datime);
frombuf(buffer, &keylen);
frombuf(buffer, &cycle);
if (versionkey > 1000) {
frombuf(buffer, &seekkey);
frombuf(buffer, &seekpdir);
} else {
Int_t skey,sdir;
frombuf(buffer, &skey); seekkey = (Long64_t)skey;
frombuf(buffer, &sdir); seekpdir = (Long64_t)sdir;
}
frombuf(buffer, &nwhc);
if ( ((buffer-header) + nwhc) > nwheader ) // Don't read past the end of the part of the key we have read.
nwhc = nwheader - (buffer-header);
for (int i = 0;i < nwhc; i++) frombuf(buffer, &classname[i]);
classname[(int)nwhc] = '\0'; //cast to avoid warning with gcc3.4
if (idcur == fSeekFree) strlcpy(classname,"FreeSegments",512);
if (idcur == fSeekInfo) strlcpy(classname,"StreamerInfo",512);
if (idcur == fSeekKeys) strlcpy(classname,"KeysList",512);

if (extended) {
if ( (buffer-header) >= nwheader )
nwhc = 0;
else {
frombuf(buffer, &nwhc);
if (nwhc < 0)
nwhc = 0;
else if ( ((buffer-header) + nwhc) > nwheader ) // Don't read past the end of the part of the key we have read.
nwhc = nwheader - (buffer-header);
}
for (int i = 0;i < nwhc; i++) frombuf(buffer, &keyname[i]);
keyname[(int)nwhc] = '\0'; //cast to avoid warning with gcc3.4

if ( (buffer-header) >= nwheader )
nwhc = 0;
else {
frombuf(buffer, &nwhc);
if (nwhc < 0)
nwhc = 0;
else if ( ((buffer-header) + nwhc) > nwheader ) // Don't read past the end of the part of the key we have read.
nwhc = nwheader - (buffer-header);
}
for (int i = 0;i < nwhc; i++) frombuf(buffer, &keytitle[i]);
keytitle[(int)nwhc] = '\0'; //cast to avoid warning with gcc3.4

extrainfo.Form(" name: %-16s title: %s", keyname, keytitle);
}
auto &node = res.emplace_back(idcur, Detail::TKeyMapNode::kKey, nbytes);

TDatime::GetDateTime(datime, date, time);
if (!forComp) {
if (objlen != nbytes - keylen) {
Float_t cx = Float_t(objlen + keylen) / Float_t(nbytes);
Printf("%d/%06d At:%-*lld N=%-8d %-14s CX = %5.2f %s", date, time, nDigits + 1, idcur, nbytes, classname,
cx, extrainfo.Data());
} else {
Printf("%d/%06d At:%-*lld N=%-8d %-14s %s", date, time, nDigits + 1, idcur, nbytes, classname, extrainfo.Data());
}
frombuf(buffer, &node.fKeyVersion);
frombuf(buffer, &node.fObjLen);
frombuf(buffer, &node.fDatime);
frombuf(buffer, &node.fKeyLen);
frombuf(buffer, &node.fCycle);
if (node.fKeyVersion > 1000) {
frombuf(buffer, &node.fSeekKey);
frombuf(buffer, &node.fSeekPdir);
} else {
// Printing to help compare two files.
if (objlen != nbytes - keylen) {
Float_t cx = Float_t(objlen + keylen) / Float_t(nbytes);
Printf("At:%-*lld N=%-8d K=%-3d O=%-8d %-14s CX = %5.2f %s", nDigits+1, idcur, nbytes, keylen, objlen, classname, cx, extrainfo.Data());
Int_t skey, sdir;
frombuf(buffer, &skey);
frombuf(buffer, &sdir);
node.fSeekKey = static_cast<Long64_t>(skey);
node.fSeekPdir = static_cast<Long64_t>(sdir);
}

const auto readString = [&buffer, &header] (bool skipCheck = false) {
char stringLen;
char str[256];
if (!skipCheck && ((buffer - header) >= headerSize)) {
stringLen = 0;
} else {
Printf("At:%-*lld N=%-8d K=%-3d O=%-8d %-14s CX = 1 %s", nDigits+1, idcur, nbytes, keylen, objlen, classname, extrainfo.Data());
frombuf(buffer, &stringLen);
if (stringLen < 0)
stringLen = 0;
else if ((buffer - header) + stringLen > headerSize)
stringLen = headerSize - (buffer - header);
}
}
for (int i = 0; i < stringLen; ++i)
frombuf(buffer, &str[i]);
str[static_cast<int>(stringLen)] = 0;

return std::string(str, stringLen);
};

node.fClassName = readString(true);

if (idcur == static_cast<std::uint64_t>(fSeekFree))
node.fClassName = "FreeSegments";
else if (idcur == static_cast<std::uint64_t>(fSeekInfo))
node.fClassName = "StreamerInfo";
else if (idcur == static_cast<std::uint64_t>(fSeekKeys))
node.fClassName = "KeysList";

node.fKeyName = readString();
node.fKeyTitle = readString();

idcur += nbytes;
}
if (!forComp)
Printf("%d/%06d At:%-*lld N=%-8d %-14s",date,time, nDigits+1, idcur,1,"END");
else
Printf("At:%-*lld N=%-8d K= O= %-14s", nDigits+1, idcur,1,"END");

return res;
}

////////////////////////////////////////////////////////////////////////////////
Expand Down

0 comments on commit 0491d72

Please sign in to comment.