Skip to content

Commit d42589f

Browse files
linusculinuscu
committed
Improve serialization base to handle differing data formats better. Integrate automatic serialization base for sequential cache data.
1 parent fb2f366 commit d42589f

File tree

5 files changed

+182
-52
lines changed

5 files changed

+182
-52
lines changed

‎packages/nodegraph/include/nodegraph/core/NodeGraphGroup.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ namespace l::nodegraph {
3535

3636
class NodeGraphGroup : public l::serialization::SerializationBase {
3737
public:
38-
NodeGraphGroup() {}
38+
NodeGraphGroup() : SerializationBase(0, 0, true, false, true, true, true) {}
3939
~NodeGraphGroup() {
4040
LOG(LogInfo) << "Node group destroyed";
4141
}

‎packages/serialization/include/serialization/SerializationBase.h

Lines changed: 39 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@ namespace l::serialization {
3333
}
3434

3535
extern const int32_t kHeaderIdentifier;
36+
extern const int32_t kTinyHeaderIdentifier;
3637

3738
struct HeaderValidity {
38-
3939
friend zpp::serializer::access;
4040
template <typename Archive, typename Self>
4141
static void serialize(Archive& archive, Self& self) {
@@ -50,26 +50,48 @@ namespace l::serialization {
5050
int32_t mVersion;
5151
};
5252

53+
struct TinyHeaderValidity {
54+
friend zpp::serializer::access;
55+
template <typename Archive, typename Self>
56+
static void serialize(Archive& archive, Self& self) {
57+
archive(self.mHeader);
58+
}
59+
60+
bool Peek(std::vector<unsigned char>& data);
61+
bool IsIdentifierValid();
62+
bool IsVersionValid(int32_t minVersion, int32_t latestVersion);
63+
64+
int32_t mHeader;
65+
};
66+
5367
class SerializationBase : public zpp::serializer::polymorphic {
5468
public:
5569
using SaveArchive = zpp::serializer::archive<zpp::serializer::lazy_vector_memory_output_archive>;
5670
using LoadArchive = zpp::serializer::archive<zpp::serializer::memory_view_input_archive>;
5771

72+
// default creator for loading of existing data and should be able to handle all cases
5873
SerializationBase() :
5974
mIdentifier(0),
6075
mVersion(0),
6176
mLatestVersion(mVersion),
6277
mUseIdentifier(false),
6378
mUseVersion(false),
64-
mUseFiletype(false)
79+
mUseFiletype(false),
80+
mUseTinyHeader(false),
81+
mExpectIdentifier(false),
82+
mExpectVersion(false)
6583
{}
66-
SerializationBase(int32_t minimumVersion, int32_t latestVersion, bool useVersion = true, bool useFiletype = false) :
84+
SerializationBase(int32_t minimumVersion, int32_t latestVersion, bool useVersion = true, bool useFiletype = false, bool useIdentifier = false, bool expectIdentifier = false, bool expectVersion = false, bool useTinyHeader = false) :
6785
mIdentifier(0),
6886
mVersion(minimumVersion),
6987
mLatestVersion(latestVersion),
70-
mUseIdentifier(false),
71-
mUseVersion(useVersion),
72-
mUseFiletype(useFiletype) {
88+
mUseIdentifier(useIdentifier),
89+
mUseVersion(useVersion || useTinyHeader),
90+
mUseFiletype(useFiletype),
91+
mUseTinyHeader(useTinyHeader),
92+
mExpectIdentifier(expectIdentifier),
93+
mExpectVersion(expectVersion)
94+
{
7395
if (useVersion) {
7496
ASSERT(minimumVersion <= mLatestVersion);
7597
}
@@ -88,34 +110,12 @@ namespace l::serialization {
88110
template <typename Archive, typename Self>
89111
static void serialize(Archive& archive, Self& self) {
90112
if constexpr (std::is_base_of<SaveArchive, Archive>{}) {
91-
int32_t* p = const_cast<int32_t*>(&self.mVersion);
92-
*p = self.mLatestVersion;
93-
94-
if (self.mUseIdentifier) {
95-
archive(kHeaderIdentifier);
96-
}
97-
if (self.mUseVersion) {
98-
archive(self.mVersion);
99-
}
100-
if (self.mUseFiletype) {
101-
archive(self.mFiletype);
102-
}
103-
self.Save(archive);
113+
auto& saveArchive = *reinterpret_cast<SaveArchive*>(&archive);
114+
self.SaveHandler(saveArchive);
104115
}
105116
if constexpr (std::is_base_of<LoadArchive, Archive>{}) {
106-
if (self.mUseIdentifier) {
107-
int32_t fileIdentifier;
108-
archive(fileIdentifier);
109-
ASSERT(fileIdentifier == kHeaderIdentifier);
110-
}
111-
if (self.mUseVersion) {
112-
archive(self.mVersion);
113-
}
114-
if (self.mUseFiletype) {
115-
archive(self.mFiletype);
116-
}
117-
self.Load(archive);
118-
self.UpgradeToLatest();
117+
auto& loadArchive = *reinterpret_cast<LoadArchive*>(&archive);
118+
self.LoadHandler(loadArchive);
119119
}
120120
}
121121

@@ -135,6 +135,13 @@ namespace l::serialization {
135135
bool mUseVersion = true;
136136
bool mUseFiletype = false;
137137

138+
bool mUseTinyHeader = false;
139+
140+
bool mExpectIdentifier = false;
141+
bool mExpectVersion = false;
142+
143+
void SaveHandler(SaveArchive& saveArchive) const;
144+
void LoadHandler(LoadArchive& loadArchive);
138145
void UpgradeToLatest();
139146
};
140147

‎packages/serialization/source/common/SerializationBase.cpp

Lines changed: 115 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
namespace l::serialization {
1212
const int32_t kHeaderIdentifier = 0x00defa00; // storage base file identifier
1313

14+
const int32_t kTinyHeaderIdentifier = 0xdf000000; // tiny identifier
15+
const int32_t kTinyHeaderVersionMask = 0x00fff000;
16+
1417
bool HeaderValidity::Peek(std::vector<unsigned char>& data) {
1518
if (data.size() < 4 * 2) { // identifier + version = 8 bytes
1619
return false;
@@ -30,6 +33,22 @@ namespace l::serialization {
3033
return mVersion >= 0 && mVersion <= latestVersion;
3134
}
3235

36+
bool TinyHeaderValidity::Peek(std::vector<unsigned char>& data) {
37+
if (data.size() < 4) { // identifier + version = 8 bytes
38+
return false;
39+
}
40+
zpp::serializer::memory_input_archive_peek inPeek(data);
41+
inPeek(*this);
42+
return true;
43+
}
44+
45+
bool TinyHeaderValidity::IsIdentifierValid() {
46+
return (mHeader & kTinyHeaderIdentifier) == kTinyHeaderIdentifier;
47+
}
48+
bool TinyHeaderValidity::IsVersionValid(int32_t minVersion, int32_t latestVersion) {
49+
auto version = (mHeader & kTinyHeaderVersionMask) >> 3;
50+
return version >= minVersion && version <= latestVersion;
51+
}
3352
/******************************************************************************/
3453

3554
SerializationBase& SerializationBase::operator=(SerializationBase&& other) noexcept {
@@ -40,6 +59,9 @@ namespace l::serialization {
4059
mUseIdentifier = other.mUseIdentifier;
4160
mUseVersion = other.mUseVersion;
4261
mUseFiletype = other.mUseFiletype;
62+
mUseTinyHeader = other.mUseTinyHeader;
63+
mExpectIdentifier = other.mExpectIdentifier;
64+
mExpectVersion = other.mExpectVersion;
4365
return *this;
4466
}
4567
SerializationBase& SerializationBase::operator=(const SerializationBase& other) noexcept {
@@ -50,6 +72,9 @@ namespace l::serialization {
5072
mUseIdentifier = other.mUseIdentifier;
5173
mUseVersion = other.mUseVersion;
5274
mUseFiletype = other.mUseFiletype;
75+
mUseTinyHeader = other.mUseTinyHeader;
76+
mExpectIdentifier = other.mExpectIdentifier;
77+
mExpectVersion = other.mExpectVersion;
5378
return *this;
5479
}
5580
SerializationBase::SerializationBase(SerializationBase&& other) noexcept {
@@ -60,24 +85,57 @@ namespace l::serialization {
6085
}
6186

6287
void SerializationBase::LoadArchiveData(std::vector<unsigned char>& data) {
63-
HeaderValidity headerValidity;
64-
bool peekSuccessful = headerValidity.Peek(data);
65-
if (peekSuccessful && headerValidity.IsIdentifierValid()) {
66-
mUseIdentifier = true;
67-
mUseVersion = true;
88+
// data should have identifier, but doesn't, or should not have identifier - don't load identifier (this is fine since we can determine if identifier is present or not)
89+
// data should not have version - fine
90+
// data should have version but doesn't - error (we can't know this)
91+
// data should have version and seems to have version - load version (it is possible for false positives when checking for version in data so must be sure data contain version in this case)
92+
93+
// data should have identifier, and does - load identifier
94+
// no - (data should not have version - fine)
95+
// no - (data should have version but doesn't - error (we can't know this)) - it must be there
96+
// data should have version and does - load version
97+
98+
if (mUseTinyHeader) {
99+
68100
}
69101
else {
70-
mUseIdentifier = false;
71-
if (mUseVersion) {
72-
ASSERT(peekSuccessful && headerValidity.IsVersionValid(mLatestVersion));
102+
HeaderValidity headerValidity;
103+
bool peekSuccessful = headerValidity.Peek(data);
104+
if (mExpectIdentifier) {
105+
if (!peekSuccessful || !headerValidity.IsIdentifierValid()) {
106+
LOG(LogError) << "Expected serialization identifier: " << headerValidity.mIdentifier << " (version: " << headerValidity.mVersion << ")";
107+
return;
108+
}
109+
if (!peekSuccessful || !headerValidity.IsVersionValid(mLatestVersion)) {
110+
LOG(LogError) << "Expected serialization version: " << headerValidity.mVersion << " (identifier: " << headerValidity.mIdentifier << ")";
111+
return;
112+
}
113+
mUseIdentifier = true; // if identifier is expected, we should continue using it even if user didn't not set it
114+
mExpectIdentifier = true;
115+
mExpectVersion = true;
116+
mUseVersion = true;
73117
}
74-
}
118+
else if (peekSuccessful && headerValidity.IsIdentifierValid()) {
119+
mUseIdentifier = true;
120+
mExpectIdentifier = true;
121+
122+
if (!headerValidity.IsVersionValid(mLatestVersion)) {
123+
LOG(LogError) << "Expected serialization version: " << headerValidity.mVersion;
124+
return;
125+
}
75126

76-
zpp::serializer::memory_input_archive in(data);
77-
in(*this);
78-
79-
mUseIdentifier = true;
80-
mUseVersion = true;
127+
mExpectVersion = true;
128+
mUseVersion = true;
129+
}
130+
else if (mExpectVersion) {
131+
if (!peekSuccessful || !headerValidity.IsVersionValid(mLatestVersion)) {
132+
return;
133+
}
134+
mUseVersion = true; // if version is expected, we should continue using it even if user didn't not set it
135+
}
136+
zpp::serializer::memory_input_archive in(data);
137+
in(*this);
138+
}
81139
}
82140

83141
void SerializationBase::GetArchiveData(std::vector<unsigned char>& data) {
@@ -89,6 +147,49 @@ namespace l::serialization {
89147
return mVersion;
90148
}
91149

150+
void SerializationBase::SaveHandler(SaveArchive& saveArchive) const {
151+
if (mUseTinyHeader) {
152+
153+
}
154+
else {
155+
if (mUseIdentifier) {
156+
saveArchive(kHeaderIdentifier);
157+
}
158+
if (mUseVersion) {
159+
if (mVersion != mLatestVersion) { // make sure version is latest version
160+
int32_t* p = const_cast<int32_t*>(&mVersion);
161+
*p = mLatestVersion;
162+
}
163+
saveArchive(mVersion);
164+
}
165+
if (mUseFiletype) {
166+
saveArchive(mFiletype);
167+
}
168+
}
169+
Save(saveArchive);
170+
}
171+
172+
void SerializationBase::LoadHandler(LoadArchive& loadArchive) {
173+
if (mUseTinyHeader) {
174+
175+
}
176+
else {
177+
if (mExpectIdentifier) {
178+
int32_t fileIdentifier;
179+
loadArchive(fileIdentifier);
180+
ASSERT(fileIdentifier == kHeaderIdentifier);
181+
}
182+
if (mExpectVersion) {
183+
loadArchive(mVersion);
184+
}
185+
if (mUseFiletype) {
186+
loadArchive(mFiletype);
187+
}
188+
}
189+
Load(loadArchive);
190+
UpgradeToLatest();
191+
}
192+
92193
void SerializationBase::UpgradeToLatest() {
93194
mLatestVersion = mVersion > mLatestVersion ? mVersion : mLatestVersion;
94195
if (mUseVersion) {
@@ -99,5 +200,4 @@ namespace l::serialization {
99200
mVersion = mVersion < mLatestVersion ? mLatestVersion : mVersion;
100201
}
101202

102-
103203
}

‎packages/serialization/tests/common/SerializationBaseTest.cpp

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ using namespace serialization;
1010

1111
class TestData : public SerializationBase {
1212
public:
13-
TestData(int32_t version, int a, bool useVersion) : SerializationBase(version, 5, useVersion, false), mA(a) {}
13+
TestData(int32_t version, int a, bool useVersion, bool useIdentifier, bool expectVersion, bool expectIdentifier) : SerializationBase(version, 5, useVersion, false, useIdentifier, expectIdentifier, expectVersion), mA(a) {}
1414
virtual ~TestData() = default;
1515

1616
friend zpp::serializer::access;
@@ -88,7 +88,7 @@ TEST(SerializationBase, Basics) {
8888
std::vector<unsigned char> dataHeaderAndVersion;
8989

9090
{ // load simple data
91-
TestData storage(3, 10, false);
91+
TestData storage(3, 10, true, true, false, false);
9292
storage.LoadArchiveData(dataNoVersion);
9393
TEST_TRUE(dataNoVersion.empty(), "");
9494
TEST_TRUE(storage.GetVersion() == 5, "");
@@ -101,8 +101,13 @@ TEST(SerializationBase, Basics) {
101101
TEST_TRUE(dataHeaderAndVersion.at(1) == 0xfa, "");
102102
TEST_TRUE(dataHeaderAndVersion.at(2) == 0xde, "");
103103
TEST_TRUE(dataHeaderAndVersion.at(3) == 0, "");
104+
TEST_TRUE(dataHeaderAndVersion.at(4) == 5, "");
105+
TEST_TRUE(dataHeaderAndVersion.at(5) == 0, "");
106+
TEST_TRUE(dataHeaderAndVersion.at(6) == 0, "");
107+
TEST_TRUE(dataHeaderAndVersion.at(7) == 0, "");
108+
TEST_TRUE(dataHeaderAndVersion.at(8) == 10, "");
104109

105-
TestData storage2(0, 0, false);
110+
TestData storage2(0, 0, true, true, true, true);
106111
storage2.LoadArchiveData(dataHeaderAndVersion);
107112
TEST_TRUE(storage2.GetVersion() == 5, "");
108113
TEST_TRUE(storage2.mA == 10, "");
@@ -117,7 +122,7 @@ TEST(SerializationBase, Basics) {
117122
dataHeaderAndVersion.clear();
118123
}
119124
{ // have to explicitly set use version to true for versioned only data
120-
TestData storage(0, 0, true);
125+
TestData storage(0, 0, true, true, true, false);
121126
storage.LoadArchiveData(dataOnlyVersion);
122127
TEST_TRUE(dataOnlyVersion.empty(), "");
123128
TEST_TRUE(storage.GetVersion() == 5, "");
@@ -132,8 +137,10 @@ TEST(SerializationBase, Basics) {
132137
TEST_TRUE(dataHeaderAndVersion.at(3) == 0, "");
133138
dataHeaderAndVersion.clear();
134139
}
140+
{
135141

136142

143+
}
137144
return 0;
138145
}
139146

‎packages/storage/include/storage/SequentialCache.h

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,15 @@ namespace l::filecache {
127127
if (!mData) {
128128
return false;
129129
}
130+
131+
if constexpr (std::is_base_of_v<l::serialization::SerializationBase, T>) {
132+
auto sb = reinterpret_cast<l::serialization::SerializationBase*>(mData.get());
133+
if (sb != nullptr) {
134+
sb->GetArchiveData(data);
135+
return true;
136+
}
137+
}
138+
130139
zpp::serializer::memory_output_archive out(data);
131140
out(*this);
132141

@@ -139,6 +148,13 @@ namespace l::filecache {
139148
if (!mData) {
140149
mData = std::make_unique<T>();
141150
}
151+
if constexpr (std::is_base_of_v<l::serialization::SerializationBase, T>) {
152+
auto sb = reinterpret_cast<l::serialization::SerializationBase*>(mData.get());
153+
if (sb != nullptr) {
154+
sb->LoadArchiveData(data);
155+
return true;
156+
}
157+
}
142158

143159
zpp::serializer::memory_input_archive in(data);
144160
in(*this);

0 commit comments

Comments
 (0)