Skip to content

Commit

Permalink
Reducing stack-size of hash-map allocated first-byte-mappings.
Browse files Browse the repository at this point in the history
  • Loading branch information
@RealTimeChris
RealTimeChris committed Feb 2, 2025
1 parent 27e0345 commit 17c4821
Show file tree
Hide file tree
Showing 5 changed files with 203 additions and 194 deletions.
55 changes: 21 additions & 34 deletions Benchmarks/Tests.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ namespace tests {
static constexpr bnch_swt::string_literal testNameRead{ testName + "-Read-Raw-Json-Data" };
static constexpr bnch_swt::string_literal testNameWrite{ testName + "-Write-Raw-Json-Data" };
static constexpr bool partialRead{ std::is_same_v<test_data_type, partial_test<partial_test_struct>> || std::is_same_v<test_data_type, twitter_partial_message> };
static constexpr bool knownOrder{ !std::is_same_v<test_data_type, twitter_partial_message> && !std::is_same_v<test_data_type, abc_test<abc_test_struct>> };
static constexpr bool knownOrder{ true };
std::vector<std::string> newStrings{ iterations };
results_data r{ jsonifierLibraryName, testName, jsonifierCommitUrl };
jsonifier::jsonifier_core parser{};
Expand Down Expand Up @@ -166,7 +166,7 @@ namespace tests {
static constexpr bnch_swt::string_literal testNameRead{ testName + "-Read" };
static constexpr bnch_swt::string_literal testNameWrite{ testName + "-Write" };
static constexpr bool partialRead{ std::is_same_v<test_data_type, partial_test<partial_test_struct>> || std::is_same_v<test_data_type, twitter_partial_message> };
static constexpr bool knownOrder{ !std::is_same_v<test_data_type, twitter_partial_message> && !std::is_same_v<test_data_type, abc_test<abc_test_struct>> };
static constexpr bool knownOrder{ true };
results_data r{ jsonifierLibraryName, testName, jsonifierCommitUrl };
jsonifier::jsonifier_core parser{};
std::vector<std::vector<std::string>> newStrings{ iterations };
Expand Down Expand Up @@ -217,7 +217,7 @@ namespace tests {
});
r.readResult = result<result_type::read>{ "teal", readResult };
r.writeResult = result<result_type::write>{ "steelblue", writeResult };
bnch_swt::file_loader::saveFile(static_cast<std::string>(parser.serializeJson(testDatas[0])),
bnch_swt::file_loader::saveFile(static_cast<std::string>(parser.serializeJson(testDatas)),
jsonOutPath.operator std::string() + "/" + testName.operator std::string() + "-jsonifier.json");
return r;
}
Expand All @@ -230,7 +230,7 @@ namespace tests {
static constexpr bnch_swt::string_literal testNameRead{ testName + "-Read" };
static constexpr bnch_swt::string_literal testNameWrite{ testName + "-Write" };
static constexpr bool partialRead{ std::is_same_v<test_data_type, partial_test<partial_test_struct>> || std::is_same_v<test_data_type, twitter_partial_message> };
static constexpr bool knownOrder{ !std::is_same_v<test_data_type, twitter_partial_message> && !std::is_same_v<test_data_type, abc_test<abc_test_struct>> };
static constexpr bool knownOrder{ true };
results_data r{ jsonifierLibraryName, testName, jsonifierCommitUrl };
jsonifier::jsonifier_core parser{};
std::vector<std::string> newStrings{ iterations };
Expand Down Expand Up @@ -278,7 +278,7 @@ namespace tests {
static constexpr bnch_swt::string_literal testNameRead{ testName + "-Read" };
static constexpr bnch_swt::string_literal testNameWrite{ testName + "-Write" };
static constexpr bool partialRead{ std::is_same_v<test_data_type, partial_test<partial_test_struct>> || std::is_same_v<test_data_type, twitter_partial_message> };
static constexpr bool knownOrder{ !std::is_same_v<test_data_type, twitter_partial_message> && !std::is_same_v<test_data_type, abc_test<abc_test_struct>> };
static constexpr bool knownOrder{ true };
results_data r{ jsonifierLibraryName, testName, jsonifierCommitUrl };
jsonifier::jsonifier_core parser{};
std::vector<std::string> newStrings{ iterations };
Expand Down Expand Up @@ -422,7 +422,8 @@ namespace tests {
testDatas[x].resize(testDataNew[x].size());
}
size_t currentIndex{};
bnch_swt::performance_metrics readResult = bnch_swt::benchmark_stage<testNameRead, iterations, measuredIterations>::template runBenchmark<glazeLibraryName, "teal">([&]() mutable {
bnch_swt::performance_metrics readResult =
bnch_swt::benchmark_stage<testNameRead, iterations, measuredIterations>::template runBenchmark<glazeLibraryName, "teal">([&]() mutable {
size_t newSize{};
for (size_t x = 0; x < testDatas[currentIndex].size(); ++x) {
get_ref_t<decltype(testDatas[currentIndex][x])> reference = testDatas[currentIndex][x];
Expand All @@ -439,7 +440,7 @@ namespace tests {
}
++currentIndex;
return newSize;
});
});
for (size_t x = 0; x < iterations; ++x) {
for (size_t y = 0; y < testDataNew[x].size(); ++y) {
newStrings[x][y] = std::string{};
Expand All @@ -462,7 +463,7 @@ namespace tests {
r.readResult = result<result_type::read>{ "dodgerblue", readResult };
r.writeResult = result<result_type::write>{ "skyblue", writeResult };
std::string newString{};
auto newResult = glz::write_json(testDatas[0], newString);
auto newResult = glz::write_json(testDatas, newString);
( void )newResult;
bnch_swt::file_loader::saveFile(newString, jsonOutPath.operator std::string() + "/" + testName.operator std::string() + "-glaze.json");
return r;
Expand Down Expand Up @@ -669,7 +670,7 @@ namespace tests {
});
r.readResult = result<result_type::read>{ "cadetblue", readResult };
std::string newString{};
auto newResult = glz::write_json(testDatas[0], newString);
auto newResult = glz::write_json(testDatas, newString);
( void )newResult;
bnch_swt::file_loader::saveFile(newString, jsonOutPath.operator std::string() + "/" + testName.operator std::string() + "-simdjson.json");
return r;
Expand Down Expand Up @@ -994,21 +995,13 @@ In contrast, hash-based solutions offer a viable alternative by circumventing th
JSONIFIER_CLANG_INLINE void testFunction() {
std::string jsonDataNew{};
jsonifier::jsonifier_core parser{};
std::vector<test_struct> jsonDataSmallNewer{ maxIterations };
for (size_t x = 0; x < maxIterations; ++x) {
jsonDataSmallNewer[x] = test_generator::generateTestStruct();
}
std::vector<test<test_struct>> jsonDataNewer{ maxIterations };
for (size_t x = 0; x < maxIterations; ++x) {
jsonDataNewer[x] = test_generator::generateTest();
}
parser.serializeJson<jsonifier::serialize_options{ .prettify = true }>(jsonDataSmallNewer[0], jsonDataNew);
bnch_swt::file_loader::saveFile(jsonDataNew, jsonOutPath.operator std::string() + "/Json Test (Small) (Prettified).json");
std::string jsonMinifiedData{ parser.minifyJson(jsonDataNew) };
bnch_swt::file_loader::saveFile(jsonMinifiedData, jsonOutPath.operator std::string() + "/Json Test (Small) (Minified).json");
parser.serializeJson<jsonifier::serialize_options{ .prettify = true }>(jsonDataNewer[0], jsonDataNew);
bnch_swt::file_loader::saveFile(jsonDataNew, jsonOutPath.operator std::string() + "/Json Test (Prettified).json");
jsonMinifiedData = parser.minifyJson(jsonDataNew);
std::string jsonMinifiedData = parser.minifyJson(jsonDataNew);
bnch_swt::file_loader::saveFile(jsonMinifiedData, jsonOutPath.operator std::string() + "/Json Test (Minified).json");
bnch_swt::file_loader::saveFile(jsonDataNew, jsonOutPath.operator std::string() + "/Partial Test (Prettified).json");
bnch_swt::file_loader::saveFile(jsonMinifiedData, jsonOutPath.operator std::string() + "/Partial Test (Minified).json");
Expand All @@ -1023,51 +1016,45 @@ In contrast, hash-based solutions offer a viable alternative by circumventing th
static_cast<std::string>(section001) };
test_results testResults{};
{
std::vector<std::vector<double>> doubleData{ test_generator::generateValues<double>(maxIterations, 2500) };
parser.serializeJson<jsonifier::serialize_options{ .prettify = true }>(doubleData[0], jsonDataNew);
std::vector<std::vector<double>> doubleData{ test_generator::generateValues<double>(maxIterations, 100) };
parser.serializeJson<jsonifier::serialize_options{ .prettify = true }>(doubleData, jsonDataNew);
bnch_swt::file_loader::saveFile(jsonDataNew, jsonOutPath.operator std::string() + "/Double Test.json");
testResults = json_tests_helper<test_type::parse_and_serialize, std::vector<std::vector<double>>, false, maxIterations, "Double Test">::run(doubleData);
newerString += testResults.markdownResults;
benchmark_data.emplace_back(testResults);
}
{
std::vector<std::vector<std::string>> stringData{ test_generator::generateValues<std::string>(maxIterations, 2500) };
parser.serializeJson<jsonifier::serialize_options{ .prettify = true }>(stringData[0], jsonDataNew);
std::vector<std::vector<std::string>> stringData{ test_generator::generateValues<std::string>(maxIterations, 100) };
parser.serializeJson<jsonifier::serialize_options{ .prettify = true }>(stringData, jsonDataNew);
bnch_swt::file_loader::saveFile(jsonDataNew, jsonOutPath.operator std::string() + "/String Test.json");
testResults = json_tests_helper<test_type::parse_and_serialize, std::vector<std::vector<std::string>>, false, maxIterations, "String Test">::run(stringData);
newerString += testResults.markdownResults;
benchmark_data.emplace_back(testResults);
}
{
std::vector<std::vector<uint64_t>> uintData{ test_generator::generateValues<uint64_t>(maxIterations, 2500) };
parser.serializeJson<jsonifier::serialize_options{ .prettify = true }>(uintData[0], jsonDataNew);
std::vector<std::vector<uint64_t>> uintData{ test_generator::generateValues<uint64_t>(maxIterations, 100) };
parser.serializeJson<jsonifier::serialize_options{ .prettify = true }>(uintData, jsonDataNew);
bnch_swt::file_loader::saveFile(jsonDataNew, jsonOutPath.operator std::string() + "/Uint Test.json");
testResults = json_tests_helper<test_type::parse_and_serialize, std::vector<std::vector<uint64_t>>, false, maxIterations, "Uint Test">::run(uintData);
newerString += testResults.markdownResults;
benchmark_data.emplace_back(testResults);
}
{
std::vector<std::vector<int64_t>> intData{ test_generator::generateValues<int64_t>(maxIterations, 2500) };
parser.serializeJson<jsonifier::serialize_options{ .prettify = true }>(intData[0], jsonDataNew);
std::vector<std::vector<int64_t>> intData{ test_generator::generateValues<int64_t>(maxIterations, 100) };
parser.serializeJson<jsonifier::serialize_options{ .prettify = true }>(intData, jsonDataNew);
bnch_swt::file_loader::saveFile(jsonDataNew, jsonOutPath.operator std::string() + "/Int Test.json");
testResults = json_tests_helper<test_type::parse_and_serialize, std::vector<std::vector<int64_t>>, false, maxIterations, "Int Test">::run(intData);
newerString += testResults.markdownResults;
benchmark_data.emplace_back(testResults);
}
{
std::vector<std::vector<bool>> boolData{ test_generator::generateValues<bool>(maxIterations, 2500) };
parser.serializeJson<jsonifier::serialize_options{ .prettify = true }>(boolData[0], jsonDataNew);
std::vector<std::vector<bool>> boolData{ test_generator::generateValues<bool>(maxIterations, 100) };
parser.serializeJson<jsonifier::serialize_options{ .prettify = true }>(boolData, jsonDataNew);
bnch_swt::file_loader::saveFile(jsonDataNew, jsonOutPath.operator std::string() + "/Bool Test.json");
testResults = json_tests_helper<test_type::parse_and_serialize, std::vector<std::vector<bool>>, false, maxIterations, "Bool Test">::run(boolData);
newerString += testResults.markdownResults;
benchmark_data.emplace_back(testResults);
}
testResults = json_tests_helper<test_type::parse_and_serialize, test_struct, false, maxIterations, "Json Test (Small) (Prettified)">::run(jsonDataSmallNewer);
newerString += testResults.markdownResults;
benchmark_data.emplace_back(testResults);
testResults = json_tests_helper<test_type::parse_and_serialize, test_struct, true, maxIterations, "Json Test (Small) (Minified)">::run(jsonDataSmallNewer);
newerString += testResults.markdownResults;
benchmark_data.emplace_back(testResults);
testResults = json_tests_helper<test_type::parse_and_serialize, test<test_struct>, false, maxIterations, "Json Test (Prettified)">::run(jsonDataNewer);
newerString += testResults.markdownResults;
benchmark_data.emplace_back(testResults);
Expand Down
4 changes: 0 additions & 4 deletions Include/jsonifier/Utilities/Error.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,6 @@ namespace jsonifier::internal {
return returnValues;
}() };

static bool isNumberType(uint8_t c) noexcept {
return numberTable[c];
}

static string convertChar(char value) {
switch (value) {
[[unlikely]] case '\b': { return R"(\b)"; }
Expand Down
20 changes: 15 additions & 5 deletions Include/jsonifier/Utilities/HashMap.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -584,9 +584,19 @@ namespace jsonifier::internal {
return mappings;
}

template<size_t firstCharCount>
template<size_t firstCharCount> constexpr char getMaxFirstByte(const array<first_bytes, firstCharCount>& keys) noexcept {
char returnValue{};
for (size_t x = 0; x < firstCharCount; ++x) {
if (keys[x].value > returnValue) {
returnValue = keys[x].value;
}
}
return returnValue;
}

template<char maxFirstByte, size_t firstCharCount>
constexpr static auto generateMappingsForFirstBytes(const array<first_bytes, firstCharCount>& keys, const array<uint8_t, 256>& uniqueIndices) noexcept {
constexpr size_t flattenedSize = 256ull * 256ull;
constexpr size_t flattenedSize = (maxFirstByte + 1) * 256ull;
std::array<uint8_t, flattenedSize> flattenedMappings{};
std::fill(flattenedMappings.data(), flattenedMappings.data() + flattenedMappings.size(), flattenedMappings.size() - 1);

Expand All @@ -599,7 +609,7 @@ namespace jsonifier::internal {
const auto& keyNew = keys[x].rootPtr[y].key;
if (uniqueIndex < keyNew.size()) {
uint8_t keyChar = static_cast<uint8_t>(keyNew[uniqueIndex]);
size_t flattenedIdx = firstByte * 256ull + keyChar;
size_t flattenedIdx = firstByte * maxFirstByte + keyChar;
flattenedMappings[flattenedIdx] = keys[x].rootPtr[y].oldIndex;
}
}
Expand Down Expand Up @@ -653,8 +663,8 @@ namespace jsonifier::internal {
return hashData<value_type>.storageSize;
} else if constexpr (hashData<value_type>.type == hash_map_type::first_byte_and_unique_index) {
static constexpr auto uniqueFirstByteCount{ countFirstBytes(tupleReferencesByFirstByte<value_type>) };
static constexpr auto mappings{ generateMappingsForFirstBytes(collectFirstBytes<uniqueFirstByteCount>(tupleReferencesByFirstByte<value_type>),
hashData<value_type>.uniqueIndices) };
static constexpr auto firstBytes{ collectFirstBytes<uniqueFirstByteCount>(tupleReferencesByFirstByte<value_type>) };
static constexpr auto mappings{ generateMappingsForFirstBytes<getMaxFirstByte(firstBytes)>(firstBytes, hashData<value_type>.uniqueIndices) };
const uint8_t firstByte = static_cast<uint8_t>(iter[0]);
const uint8_t uniqueIdx = hashData<value_type>.uniqueIndices[firstByte];
if JSONIFIER_LIKELY (checkForEnd(iter, end, uniqueIdx)) {
Expand Down
Loading

0 comments on commit 17c4821

Please sign in to comment.