From ab243af7d49d0f3f66e3691fc6ff035c23387d9c Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Sun, 8 Feb 2026 17:34:45 +0100 Subject: [PATCH 1/8] io: Correct handling of byte count pos in TBufferFile::ReadObjectAny --- io/io/src/TBufferFile.cxx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index 92bacafdcd231..fbe038bff88e5 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -2593,7 +2593,8 @@ void *TBufferFile::ReadObjectAny(const TClass *clCast) InitMap(); // before reading object save start position - UInt_t startpos = UInt_t(fBufCur-fBuffer); + ULong64_t startpos = static_cast(fBufCur-fBuffer); + ULong64_t cntpos = startpos <= kMaxCountPosition ? startpos : kOverflowPosition; // attempt to load next object as TClass clCast UInt_t tag; // either tag or byte count @@ -2613,7 +2614,7 @@ void *TBufferFile::ReadObjectAny(const TClass *clCast) Error("ReadObject", "got object of wrong class! requested %s but got %s", clCast->GetName(), clRef->GetName()); - CheckByteCount(startpos, tag, (TClass *)nullptr); // avoid mis-leading byte count error message + CheckByteCount(cntpos, tag, (TClass *)nullptr); // avoid mis-leading byte count error message return 0; // We better return at this point } baseOffset = 0; // For now we do not support requesting from a class that is the base of one of the class for which there is transformation to .... @@ -2628,7 +2629,7 @@ void *TBufferFile::ReadObjectAny(const TClass *clCast) //we cannot mix a compiled class with an emulated class in the inheritance Error("ReadObject", "trying to read an emulated class (%s) to store in a compiled pointer (%s)", clRef->GetName(),clCast->GetName()); - CheckByteCount(startpos, tag, (TClass *)nullptr); // avoid mis-leading byte count error message + CheckByteCount(cntpos, tag, (TClass *)nullptr); // avoid mis-leading byte count error message return 0; } } @@ -2640,7 +2641,7 @@ void *TBufferFile::ReadObjectAny(const TClass *clCast) obj = (char *) (Longptr_t)fMap->GetValue(startpos+kMapOffset); if (obj == (void*) -1) obj = nullptr; if (obj) { - CheckByteCount(startpos, tag, (TClass *)nullptr); + CheckByteCount(cntpos, tag, (TClass *)nullptr); return (obj + baseOffset); } } @@ -2652,7 +2653,7 @@ void *TBufferFile::ReadObjectAny(const TClass *clCast) MapObject((TObject*) -1, startpos+kMapOffset); else MapObject((void*)nullptr, nullptr, fMapCount); - CheckByteCount(startpos, tag, (TClass *)nullptr); + CheckByteCount(cntpos, tag, (TClass *)nullptr); return 0; } @@ -2711,7 +2712,7 @@ void *TBufferFile::ReadObjectAny(const TClass *clCast) // let the object read itself clRef->Streamer( obj, *this, clOnfile ); - CheckByteCount(startpos, tag, clRef); + CheckByteCount(cntpos, tag, clRef); } return obj+baseOffset; From 8cfe2ddb4ffc0d4b9e153eb0f401289250c59d54 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Wed, 10 Dec 2025 13:18:38 -0600 Subject: [PATCH 2/8] io: add test for long range references For now, the testing of the long range references is disabled. --- io/io/test/InnerReferencesTests.cxx | 133 ++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 io/io/test/InnerReferencesTests.cxx diff --git a/io/io/test/InnerReferencesTests.cxx b/io/io/test/InnerReferencesTests.cxx new file mode 100644 index 0000000000000..eef112d289e92 --- /dev/null +++ b/io/io/test/InnerReferencesTests.cxx @@ -0,0 +1,133 @@ +#include "TBufferFile.h" +#include "TClass.h" +#include "TMacro.h" +#include "TNamed.h" +#include "TProtoClass.h" + +#include "gtest/gtest.h" + +namespace { + +struct ReadResult { + int fError = 0; + TObject *fObj = nullptr; +}; + +void Update(int &errors, std::unique_ptr &obj, const ReadResult &res) +{ + errors += res.fError; + obj.reset(res.fObj); +} + +ReadResult ReadAndCheck(TBuffer &b, TClass *cl, TObject *ident = nullptr) +{ + ReadResult result; + + b >> result.fObj; + + EXPECT_NE(result.fObj, nullptr) + << "Failed to read object of class '" << cl->GetName() << "'"; + if (!result.fObj) { + result.fError = 1; + return result; + } + + EXPECT_EQ(result.fObj->IsA(), cl) + << "Expected class '" << cl->GetName() + << "' but read '" << result.fObj->IsA()->GetName() << "'"; + if (result.fObj->IsA() != cl) { + result.fError = 2; + return result; + } + + if (ident) { + EXPECT_EQ(ident, result.fObj); + if (ident != result.fObj) { + result.fError = 3; + return result; + } + } + + return result; +} + +} // anonymous namespace + +TEST(TBufferFileInnerReferences, LargeOffsetsAndReferences) +{ + int errors = 0; + + auto n0 = std::make_unique("n0", "At start"); + auto n1 = std::make_unique("n1", "Below 1G"); + auto n2 = std::make_unique("n2", "Over 1G"); + auto m1 = std::make_unique("m1", "Below 1G"); + auto m2 = std::make_unique("m2", "Over 1G"); + auto c1 = std::make_unique(); // Only over 1G + auto c2 = std::make_unique(); // Also over 1G + + // TBufferFile currently rejects sizes larger than 2GB. + // SetBufferOffset does not check against the size, + // so we can provide and use a larger buffer. + std::vector databuffer{}; + databuffer.reserve(4ull * 1024 * 1024 * 1024); + TBufferFile b(TBuffer::kWrite, 2ull * 1024 * 1024 * 1024 - 100, databuffer.data(), false /* don't adopt */); + + b << n0.get(); + b.SetBufferOffset(512ull * 1024 * 1024); + b << n1.get(); + b << m1.get(); + b.SetBufferOffset(1536ull * 1024 * 1024); + b << n2.get(); + b << m2.get(); + b << c1.get(); + b << c2.get(); + + // Those should all be references. + b << n0.get(); + b << n1.get(); + b << m1.get(); + b << n2.get(); + b << m2.get(); + b << c1.get(); + b << c2.get(); + + // To make a copy instead of using the const references: + auto bytecounts = b.GetByteCounts(); + // Rewind. Other code uses Reset instead of SetBufferOffset + b.SetReadMode(); + b.Reset(); + b.SetByteCounts(std::move(bytecounts)); + + std::unique_ptr rn0; + std::unique_ptr rn1; + std::unique_ptr rn2; + std::unique_ptr rm1; + std::unique_ptr rm2; + std::unique_ptr rc1; + std::unique_ptr rc2; + + Update(errors, rn0, ReadAndCheck(b, n0->IsA())); + + b.SetBufferOffset(512ull * 1024 * 1024); + Update(errors, rn1, ReadAndCheck(b, n1->IsA())); + Update(errors, rm1, ReadAndCheck(b, m1->IsA())); + + b.SetBufferOffset(1536ull * 1024 * 1024); + Update(errors, rn2, ReadAndCheck(b, n2->IsA())); + Update(errors, rm2, ReadAndCheck(b, m2->IsA())); + Update(errors, rc1, ReadAndCheck(b, c1->IsA())); + Update(errors, rc2, ReadAndCheck(b, c2->IsA())); + + // Reference and Class name below 1G + errors += ReadAndCheck(b, n0->IsA(), rn0.get()).fError; + errors += ReadAndCheck(b, n1->IsA(), rn1.get()).fError; + errors += ReadAndCheck(b, m1->IsA(), rm1.get()).fError; + if (0) { // These require implementing proper support for long range references. + errors += ReadAndCheck(b, n2->IsA(), rn2.get()).fError; // Reference over 1G + errors += ReadAndCheck(b, m2->IsA(), rm2.get()).fError; // Reference over 1G + errors += ReadAndCheck(b, c1->IsA(), rc1.get()).fError; // Class and reference over 1G + errors += ReadAndCheck(b, c2->IsA(), rc2.get()).fError; // Class and reference over 1G + } + + EXPECT_EQ(errors, 0); +} From db6ab45524a2424498120329490bd201a1b6ce06 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 12 Dec 2025 14:31:31 -0600 Subject: [PATCH 3/8] io: add comment about 32/64 bits --- io/io/src/TBufferFile.cxx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index fbe038bff88e5..a4f8e66c1bc74 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -2753,6 +2753,7 @@ void TBufferFile::WriteObjectClass(const void *actualObjectStart, const TClass * UInt_t objIdx = UInt_t(idx); // save index of already stored object + // FIXME/TRUNCATION: potential truncation from 64 to 32 bits *this << objIdx; } else { @@ -2816,6 +2817,7 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, UInt_t *objTag) return cl; } UInt_t bcnt, tag, startpos = 0; + // FIXME/TRUNCATION: potential truncation from 64 to 32 bits *this >> bcnt; if (!(bcnt & kByteCountMask) || bcnt == kNewClassTag) { tag = bcnt; @@ -2909,6 +2911,9 @@ void TBufferFile::WriteClass(const TClass *cl) UInt_t clIdx = UInt_t(idx); // save index of already stored class + // FIXME/TRUNCATION: potential truncation from 64 to 32 bits + // FIXME/INCORRECTNESS: if clIdx > 0x3FFFFFFF the control bit (2nd highest bit) will be wrong + // FIXME/INCORRECTNESS: similarly if clIdx > kClassMask (2GB) the code will be wrong *this << (clIdx | kClassMask); } else { From da28f8f219e9204d351b48b6a654f3485e68074b Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 15 Dec 2025 13:08:25 -0600 Subject: [PATCH 4/8] NFC io: white space --- io/io/src/TBufferFile.cxx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index a4f8e66c1bc74..070041b0076f1 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -2834,7 +2834,8 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, UInt_t *objTag) // in case tag is object tag return tag if (!(tag & kClassMask)) { - if (objTag) *objTag = tag; + if (objTag) + *objTag = tag; return 0; } @@ -2885,10 +2886,12 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, UInt_t *objTag) } // return bytecount in objTag - if (objTag) *objTag = (bcnt & ~kByteCountMask); + if (objTag) + *objTag = (bcnt & ~kByteCountMask); // case of unknown class - if (!cl) cl = (TClass*)-1; + if (!cl) + cl = (TClass*)-1; return cl; } From 3495676d96ca6acc9a096ede3e3175fc20ba3860 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 15 Dec 2025 13:10:08 -0600 Subject: [PATCH 5/8] io: Add support for long range references (objects and class names) References for class names and objects are stored in 64bits whenever they are located past 1GB. This is the 'only' option as there are no control bits left to distinguish between a 32bits and a 64bits reference. --- io/io/inc/TBufferFile.h | 2 +- io/io/src/TBufferFile.cxx | 88 +++++++++++++++++++++++++++------------ 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/io/io/inc/TBufferFile.h b/io/io/inc/TBufferFile.h index 143f8ac374a69..873e1ba085988 100644 --- a/io/io/inc/TBufferFile.h +++ b/io/io/inc/TBufferFile.h @@ -71,7 +71,7 @@ class TBufferFile : public TBufferIO { Long64_t CheckByteCount(ULong64_t startpos, ULong64_t bcnt, const TClass *clss, const char* classname); void CheckCount(UInt_t offset) override; - UInt_t CheckObject(UInt_t offset, const TClass *cl, Bool_t readClass = kFALSE); + UInt_t CheckObject(ULong64_t offset, const TClass *cl, Bool_t readClass = kFALSE); UInt_t ReserveByteCount(); void WriteObjectClass(const void *actualObjStart, const TClass *actualClass, Bool_t cacheReuse) override; diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index 070041b0076f1..c1b44274aa692 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -74,6 +74,9 @@ constexpr Version_t kByteCountVMask = 0x4000; // OR the version byte coun constexpr Version_t kMaxVersion = 0x3FFF; // highest possible version number constexpr Int_t kMapOffset = 2; // first 2 map entries are taken by null obj and self obj +// constexpr ULong64_t kMaxLongRange = 0x0FFFFFFFFFFFFFFE; // We reserve the 4 highest bits for flags, currently only 2 are in use. +constexpr ULong64_t kLongRangeClassMask = 0x8000000000000000; // OR the class index with this +// constexpr ULong64_t kLongRangeByteCountMask = 0x4000000000000000; // OR the byte count with this //////////////////////////////////////////////////////////////////////////////// /// Thread-safe check on StreamerInfos of a TClass @@ -2816,11 +2819,28 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, UInt_t *objTag) cl = (TClass*)-1; return cl; } - UInt_t bcnt, tag, startpos = 0; + UInt_t bcnt; + Long64_t tag64, startpos = 0; + const bool shortRange = (fBufCur - fBuffer) <= kMaxMapCount; + bool isNewClassTag = false; + // FIXME/TRUNCATION: potential truncation from 64 to 32 bits *this >> bcnt; - if (!(bcnt & kByteCountMask) || bcnt == kNewClassTag) { - tag = bcnt; + if (bcnt == kNewClassTag) { + isNewClassTag = true; + tag64 = 0; + bcnt = 0; + } else if (!(bcnt & kByteCountMask)) { + if (R__likely(shortRange)) { + tag64 = bcnt; + } else { + // Two implementation choices: + // 1) rewind and read full 64-bit value + // 2) use the already read 32-bits, read the rest and combine + UInt_t low32; + *this >> low32; + tag64 = (static_cast(bcnt) << 32) | low32; + } bcnt = 0; } else { fVersion = 1; @@ -2828,18 +2848,27 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, UInt_t *objTag) // count and will not (can not) call CheckByteCount. if (objTag) fByteCountStack.push_back(fBufCur - fBuffer); - startpos = UInt_t(fBufCur-fBuffer); - *this >> tag; + startpos = static_cast(fBufCur - fBuffer); + if (R__likely(shortRange)) { + UInt_t tag; + *this >> tag; + tag64 = tag; + } else { + *this >> tag64; + } } + const bool isClassTag = shortRange ? (tag64 & kClassMask) : (tag64 & kLongRangeClassMask); + // in case tag is object tag return tag - if (!(tag & kClassMask)) { + if (!isClassTag) { + // FIXME/TRUNCATION: potential truncation from 64 to 32 bits if (objTag) - *objTag = tag; + *objTag = tag64; return 0; } - if (tag == kNewClassTag) { + if (isNewClassTag) { // got a new class description followed by a new object // (class can be 0 if class dictionary is not found, in that @@ -2858,14 +2887,14 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, UInt_t *objTag) } else { // got a tag to an already seen class - UInt_t clTag = (tag & ~kClassMask); + ULong64_t clTag = shortRange ? (tag64 & ~kClassMask) : (tag64 & ~kLongRangeClassMask); if (fVersion > 0) { clTag += fDisplacement; clTag = CheckObject(clTag, clReq, kTRUE); } else { if (clTag == 0 || clTag > (UInt_t)fMap->GetSize()) { - Error("ReadClass", "illegal class tag=%d (0GetSize()); // exception } @@ -2907,22 +2936,29 @@ void TBufferFile::WriteClass(const TClass *cl) ULong_t hash = Void_Hash(cl); UInt_t slot; - if ((idx = (ULongptr_t)fMap->GetValue(hash, (Longptr_t)cl,slot)) != 0) { - - // truncation is OK the value we did put in the map is an 30-bit offset - // and not a pointer - UInt_t clIdx = UInt_t(idx); - - // save index of already stored class - // FIXME/TRUNCATION: potential truncation from 64 to 32 bits - // FIXME/INCORRECTNESS: if clIdx > 0x3FFFFFFF the control bit (2nd highest bit) will be wrong - // FIXME/INCORRECTNESS: similarly if clIdx > kClassMask (2GB) the code will be wrong - *this << (clIdx | kClassMask); + if ((idx = (ULongptr_t)fMap->GetValue(hash, (Longptr_t)cl, slot)) != 0) { + const bool shortRange = (fBufCur - fBuffer) <= kMaxMapCount; + if (R__likely(shortRange)) { + // truncation is OK the value we did put in the map is an 30-bit offset + // and not a pointer + UInt_t clIdx = UInt_t(idx); + // save index of already stored class + // FIXME/TRUNCATION: potential truncation from 64 to 32 bits + // FIXME/INCORRECTNESS: if clIdx > 0x3FFFFFFF the control bit (2nd highest bit) will be wrong + // FIXME/INCORRECTNESS: similarly if clIdx > kClassMask (2GB) the code will be wrong + *this << (clIdx | kClassMask); + } else { + // The 64-bit value is stored highest bytes first in the buffer, + // so when reading just the first 32-bits we get the control bits in place. + // This is needed so that the reader can distinguish between references, + // bytecounts, and new class definitions. + ULong64_t clIdx = static_cast(idx); + *this << (clIdx | kLongRangeClassMask); + } } else { - // offset in buffer where class info is written - UInt_t offset = UInt_t(fBufCur-fBuffer); + Long64_t offset = (static_cast(fBufCur-fBuffer)); // save new class tag *this << kNewClassTag; @@ -3398,7 +3434,7 @@ void TBufferFile::CheckCount(UInt_t offset) /// object is -1 then it really does not exist and we return 0. If the object /// exists just return the offset. -UInt_t TBufferFile::CheckObject(UInt_t offset, const TClass *cl, Bool_t readClass) +UInt_t TBufferFile::CheckObject(ULong64_t offset, const TClass *cl, Bool_t readClass) { // in position 0 we always have the reference to the null object if (!offset) return offset; @@ -3451,8 +3487,8 @@ UInt_t TBufferFile::CheckObject(UInt_t offset, const TClass *cl, Bool_t readClas // mark object as really not available fMap->Remove(offset); fMap->Add(offset, -1); - Warning("CheckObject", "reference to object of unavailable class %s, offset=%d" - " pointer will be 0", cl ? cl->GetName() : "TObject",offset); + Warning("CheckObject", "reference to object of unavailable class %s, offset=%llu" + " pointer will be 0", cl ? cl->GetName() : "TObject", offset); offset = 0; } From d2262b7d26df76535119bba0205105486f096ed2 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 15 Dec 2025 14:30:53 -0600 Subject: [PATCH 6/8] io: 64 bit/long range references TBufferFile::ReadClass --- core/base/inc/TBuffer.h | 2 +- core/base/src/TString.cxx | 2 +- core/cont/src/TArray.cxx | 2 +- io/io/inc/TBufferFile.h | 2 +- io/io/inc/TBufferJSON.h | 2 +- io/io/src/TBufferFile.cxx | 18 ++++++++++++++---- io/io/src/TBufferJSON.cxx | 2 +- io/io/src/TContainerConverters.cxx | 4 ++-- io/sql/inc/TBufferSQL2.h | 2 +- io/sql/src/TBufferSQL2.cxx | 2 +- io/xml/inc/TBufferXML.h | 2 +- io/xml/src/TBufferXML.cxx | 2 +- 12 files changed, 26 insertions(+), 16 deletions(-) diff --git a/core/base/inc/TBuffer.h b/core/base/inc/TBuffer.h index edafce36448f7..793529b385056 100644 --- a/core/base/inc/TBuffer.h +++ b/core/base/inc/TBuffer.h @@ -150,7 +150,7 @@ class TBuffer : public TObject { virtual TVirtualArray *PopDataCache(); virtual void PushDataCache(TVirtualArray *); - virtual TClass *ReadClass(const TClass *cl = nullptr, UInt_t *objTag = nullptr) = 0; + virtual TClass *ReadClass(const TClass *cl = nullptr, ULong64_t *objTag = nullptr) = 0; virtual void WriteClass(const TClass *cl) = 0; virtual TObject *ReadObject(const TClass *cl) = 0; diff --git a/core/base/src/TString.cxx b/core/base/src/TString.cxx index 0224eb6461d8e..40861a02e0c21 100644 --- a/core/base/src/TString.cxx +++ b/core/base/src/TString.cxx @@ -1375,7 +1375,7 @@ TString *TString::ReadString(TBuffer &b, const TClass *clReq) // Before reading object save start position UInt_t startpos = UInt_t(b.Length()); - UInt_t tag; + ULong64_t tag; TClass *clRef = b.ReadClass(clReq, &tag); TString *a; diff --git a/core/cont/src/TArray.cxx b/core/cont/src/TArray.cxx index a5283e5523d53..16f9149f943c4 100644 --- a/core/cont/src/TArray.cxx +++ b/core/cont/src/TArray.cxx @@ -47,7 +47,7 @@ TArray *TArray::ReadArray(TBuffer &b, const TClass *clReq) // Before reading object save start position UInt_t startpos = UInt_t(b.Length()); - UInt_t tag; + ULong64_t tag; TClass *clRef = b.ReadClass(clReq, &tag); TArray *a; diff --git a/io/io/inc/TBufferFile.h b/io/io/inc/TBufferFile.h index 873e1ba085988..a11408ead08cf 100644 --- a/io/io/inc/TBufferFile.h +++ b/io/io/inc/TBufferFile.h @@ -112,7 +112,7 @@ class TBufferFile : public TBufferIO { char *ReadString(char *s, Long64_t max) override; void WriteString(const char *s) override; - TClass *ReadClass(const TClass *cl = nullptr, UInt_t *objTag = nullptr) override; + TClass *ReadClass(const TClass *cl = nullptr, ULong64_t *objTag = nullptr) override; void WriteClass(const TClass *cl) override; TObject *ReadObject(const TClass *cl) override; diff --git a/io/io/inc/TBufferJSON.h b/io/io/inc/TBufferJSON.h index 8ab2d521e6a1d..011b1a7ae8743 100644 --- a/io/io/inc/TBufferJSON.h +++ b/io/io/inc/TBufferJSON.h @@ -99,7 +99,7 @@ class TBufferJSON final : public TBufferText { // suppress class writing/reading - TClass *ReadClass(const TClass *cl = nullptr, UInt_t *objTag = nullptr) final; + TClass *ReadClass(const TClass *cl = nullptr, ULong64_t *objTag = nullptr) final; void WriteClass(const TClass *cl) final; // redefined virtual functions of TBuffer diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index c1b44274aa692..a837f25df815f 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -2600,7 +2600,7 @@ void *TBufferFile::ReadObjectAny(const TClass *clCast) ULong64_t cntpos = startpos <= kMaxCountPosition ? startpos : kOverflowPosition; // attempt to load next object as TClass clCast - UInt_t tag; // either tag or byte count + ULong64_t tag; // either tag or byte count TClass *clRef = ReadClass(clCast, &tag); TClass *clOnfile = nullptr; Int_t baseOffset = 0; @@ -2808,7 +2808,7 @@ void TBufferFile::WriteObjectClass(const void *actualObjectStart, const TClass * /// \param[in] clReq Can be used to cross check if the actually read object is of the requested class. /// \param[in] objTag Set in case the object is a reference to an already read object. -TClass *TBufferFile::ReadClass(const TClass *clReq, UInt_t *objTag) +TClass *TBufferFile::ReadClass(const TClass *clReq, ULong64_t *objTag) { R__ASSERT(IsReading()); @@ -2852,16 +2852,26 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, UInt_t *objTag) if (R__likely(shortRange)) { UInt_t tag; *this >> tag; + isNewClassTag = (tag == kNewClassTag); tag64 = tag; } else { - *this >> tag64; + UInt_t high32, low32; + *this >> high32; + if (high32 == kNewClassTag) { + isNewClassTag = true; + tag64 = 0; + } else { + // continue reading low 32-bits + *this >> low32; + tag64 = (static_cast(high32) << 32) | low32; + } } } const bool isClassTag = shortRange ? (tag64 & kClassMask) : (tag64 & kLongRangeClassMask); // in case tag is object tag return tag - if (!isClassTag) { + if (!isClassTag && !isNewClassTag) { // FIXME/TRUNCATION: potential truncation from 64 to 32 bits if (objTag) *objTag = tag64; diff --git a/io/io/src/TBufferJSON.cxx b/io/io/src/TBufferJSON.cxx index 23ff6091ea355..e0a297bc3d626 100644 --- a/io/io/src/TBufferJSON.cxx +++ b/io/io/src/TBufferJSON.cxx @@ -2533,7 +2533,7 @@ void TBufferJSON::PerformPostProcessing(TJSONStackObj *stack, const TClass *obj_ //////////////////////////////////////////////////////////////////////////////// /// suppressed function of TBuffer -TClass *TBufferJSON::ReadClass(const TClass *, UInt_t *) +TClass *TBufferJSON::ReadClass(const TClass *, ULong64_t *) { return nullptr; } diff --git a/io/io/src/TContainerConverters.cxx b/io/io/src/TContainerConverters.cxx index c5f8fe65c4a5f..ff464ba3227c8 100644 --- a/io/io/src/TContainerConverters.cxx +++ b/io/io/src/TContainerConverters.cxx @@ -105,7 +105,7 @@ void TConvertClonesArrayToProxy::operator()(TBuffer &b, void *pmember, Int_t siz UInt_t startpos = b.Length(); // attempt to load next object as TClass clCast - UInt_t tag; // either tag or byte count + ULong64_t tag; // either tag or byte count TClass *clRef = b.ReadClass(TClonesArray::Class(), &tag); if (clRef==0) { @@ -122,7 +122,7 @@ void TConvertClonesArrayToProxy::operator()(TBuffer &b, void *pmember, Int_t siz b.GetMappedObject( tag, objptr, clRef); if ( objptr == (void*)-1 ) { Error("TConvertClonesArrayToProxy", - "Object can not be found in the buffer's map (at %d)",tag); + "Object can not be found in the buffer's map (at %llu)", tag); continue; } if ( objptr == 0 ) { diff --git a/io/sql/inc/TBufferSQL2.h b/io/sql/inc/TBufferSQL2.h index 1c7c7e696adae..4f7fc0af223ba 100644 --- a/io/sql/inc/TBufferSQL2.h +++ b/io/sql/inc/TBufferSQL2.h @@ -139,7 +139,7 @@ class TBufferSQL2 final : public TBufferText { // suppress class writing/reading - TClass *ReadClass(const TClass *cl = nullptr, UInt_t *objTag = nullptr) final; + TClass *ReadClass(const TClass *cl = nullptr, ULong64_t *objTag = nullptr) final; void WriteClass(const TClass *cl) final; // redefined virtual functions of TBuffer diff --git a/io/sql/src/TBufferSQL2.cxx b/io/sql/src/TBufferSQL2.cxx index cf43d319f9a73..6e7556c7a6be4 100644 --- a/io/sql/src/TBufferSQL2.cxx +++ b/io/sql/src/TBufferSQL2.cxx @@ -784,7 +784,7 @@ void TBufferSQL2::WorkWithElement(TStreamerElement *elem, Int_t /* comp_type */) //////////////////////////////////////////////////////////////////////////////// /// Suppressed function of TBuffer -TClass *TBufferSQL2::ReadClass(const TClass *, UInt_t *) +TClass *TBufferSQL2::ReadClass(const TClass *, ULong64_t *) { return nullptr; } diff --git a/io/xml/inc/TBufferXML.h b/io/xml/inc/TBufferXML.h index 4210e1eb159dc..ace668dfa4ef7 100644 --- a/io/xml/inc/TBufferXML.h +++ b/io/xml/inc/TBufferXML.h @@ -67,7 +67,7 @@ class TBufferXML final : public TBufferText, public TXMLSetup { // suppress class writing/reading - TClass *ReadClass(const TClass *cl = nullptr, UInt_t *objTag = nullptr) final; + TClass *ReadClass(const TClass *cl = nullptr, ULong64_t *objTag = nullptr) final; void WriteClass(const TClass *cl) final; // redefined virtual functions of TBuffer diff --git a/io/xml/src/TBufferXML.cxx b/io/xml/src/TBufferXML.cxx index 8e4330dd11c0e..37cd0424123c4 100644 --- a/io/xml/src/TBufferXML.cxx +++ b/io/xml/src/TBufferXML.cxx @@ -1400,7 +1400,7 @@ void TBufferXML::BeforeIOoperation() //////////////////////////////////////////////////////////////////////////////// /// Function to read class from buffer, used in old-style streamers -TClass *TBufferXML::ReadClass(const TClass *, UInt_t *) +TClass *TBufferXML::ReadClass(const TClass *, ULong64_t *) { const char *clname = nullptr; From 9bfe65ffb7300ffd16a90cdb6f897df86320bd0b Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Tue, 16 Dec 2025 11:05:07 -0600 Subject: [PATCH 7/8] io: Properly handle nullptr in longRange section. Reference in the longRange section (>1GB) are now tagged so that we can differentiate nullptr (stored as only 32 bits) from short reference stored in the longRange section (i.e. those are stored in 64 bits but have only zeros in the high bit). --- io/io/src/TBufferFile.cxx | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index a837f25df815f..37c0637da15d6 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -74,9 +74,10 @@ constexpr Version_t kByteCountVMask = 0x4000; // OR the version byte coun constexpr Version_t kMaxVersion = 0x3FFF; // highest possible version number constexpr Int_t kMapOffset = 2; // first 2 map entries are taken by null obj and self obj -// constexpr ULong64_t kMaxLongRange = 0x0FFFFFFFFFFFFFFE; // We reserve the 4 highest bits for flags, currently only 2 are in use. +constexpr ULong64_t kMaxLongRange = 0x0FFFFFFFFFFFFFFE; // We reserve the 4 highest bits for flags, currently only 2 are in use. constexpr ULong64_t kLongRangeClassMask = 0x8000000000000000; // OR the class index with this // constexpr ULong64_t kLongRangeByteCountMask = 0x4000000000000000; // OR the byte count with this +constexpr ULong64_t kLongRangeRefMask = 0x2000000000000000; // OR the reference index with this //////////////////////////////////////////////////////////////////////////////// /// Thread-safe check on StreamerInfos of a TClass @@ -2750,14 +2751,25 @@ void TBufferFile::WriteObjectClass(const void *actualObjectStart, const TClass * ULong_t hash = Void_Hash(actualObjectStart); if ((idx = (ULongptr_t)fMap->GetValue(hash, (Longptr_t)actualObjectStart, slot)) != 0) { - - // truncation is OK the value we did put in the map is an 30-bit offset - // and not a pointer - UInt_t objIdx = UInt_t(idx); + const bool shortRange = (fBufCur - fBuffer) <= kMaxMapCount; // save index of already stored object // FIXME/TRUNCATION: potential truncation from 64 to 32 bits - *this << objIdx; + if (R__likely(shortRange)) { + // truncation is OK the value we did put in the map is an 30-bit offset + // and not a pointer + UInt_t objIdx = UInt_t(idx); + *this << objIdx; + } else { + // The 64-bit value is stored highest bytes first in the buffer, + // so when reading just the first 32-bits we get the control bits in place. + // This is needed so that the reader can distinguish between references, + // bytecounts, and new class definitions. + ULong64_t objIdx = static_cast(idx); + // FIXME: verify that objIdx is guaranteed to fit in 60-bits, i.e. objIdx <= kMaxLongRange + assert(objIdx <= kMaxLongRange); + *this << (objIdx | kLongRangeRefMask); + } } else { @@ -2833,13 +2845,17 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, ULong64_t *objTag) } else if (!(bcnt & kByteCountMask)) { if (R__likely(shortRange)) { tag64 = bcnt; - } else { + } else if (bcnt & ((kLongRangeRefMask|kLongRangeClassMask) >> 32)) { // Two implementation choices: // 1) rewind and read full 64-bit value // 2) use the already read 32-bits, read the rest and combine UInt_t low32; *this >> low32; tag64 = (static_cast(bcnt) << 32) | low32; + tag64 &= ~kLongRangeRefMask; + } else { + R__ASSERT(bcnt == 0); // isn't it? If true we could return 0 early with (*objTag=0) + tag64 = bcnt; } bcnt = 0; } else { @@ -2860,10 +2876,14 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, ULong64_t *objTag) if (high32 == kNewClassTag) { isNewClassTag = true; tag64 = 0; - } else { + } else if (high32 & ((kLongRangeRefMask|kLongRangeClassMask) >> 32)) { // continue reading low 32-bits *this >> low32; tag64 = (static_cast(high32) << 32) | low32; + tag64 &= ~kLongRangeRefMask; + } else { + R__ASSERT(high32 == 0); // isn't it? If true we could return 0 early with (*objTag=0) + tag64 = high32; } } } @@ -2871,6 +2891,7 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, ULong64_t *objTag) const bool isClassTag = shortRange ? (tag64 & kClassMask) : (tag64 & kLongRangeClassMask); // in case tag is object tag return tag + // NOTE: if we return early for reference for longRange, this would be only for shortRange if (!isClassTag && !isNewClassTag) { // FIXME/TRUNCATION: potential truncation from 64 to 32 bits if (objTag) @@ -2964,6 +2985,8 @@ void TBufferFile::WriteClass(const TClass *cl) // This is needed so that the reader can distinguish between references, // bytecounts, and new class definitions. ULong64_t clIdx = static_cast(idx); + // FIXME: verify that clIdx is guaranteed to fit in 60-bits, i.e. clIdx <= kMaxLongRange + assert(clIdx <= kMaxLongRange); *this << (clIdx | kLongRangeClassMask); } } else { From 52b3714a23cd56f39d75d45de01320e32474338b Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Wed, 4 Feb 2026 19:27:25 +0100 Subject: [PATCH 8/8] io: Enable long range reference test --- io/io/test/InnerReferencesTests.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/io/test/InnerReferencesTests.cxx b/io/io/test/InnerReferencesTests.cxx index eef112d289e92..6b2dd248bec07 100644 --- a/io/io/test/InnerReferencesTests.cxx +++ b/io/io/test/InnerReferencesTests.cxx @@ -122,7 +122,7 @@ TEST(TBufferFileInnerReferences, LargeOffsetsAndReferences) errors += ReadAndCheck(b, n0->IsA(), rn0.get()).fError; errors += ReadAndCheck(b, n1->IsA(), rn1.get()).fError; errors += ReadAndCheck(b, m1->IsA(), rm1.get()).fError; - if (0) { // These require implementing proper support for long range references. + if (1) { // These require implementing proper support for long range references. errors += ReadAndCheck(b, n2->IsA(), rn2.get()).fError; // Reference over 1G errors += ReadAndCheck(b, m2->IsA(), rm2.get()).fError; // Reference over 1G errors += ReadAndCheck(b, c1->IsA(), rc1.get()).fError; // Class and reference over 1G