diff --git a/src/cxon/lang/common/cio/char.hxx b/src/cxon/lang/common/cio/char.hxx index d25f7081..3e3fa3bc 100644 --- a/src/cxon/lang/common/cio/char.hxx +++ b/src/cxon/lang/common/cio/char.hxx @@ -505,15 +505,15 @@ namespace cxon { namespace cio { namespace chr { __m128i const v4 = _mm_set1_epi8('\''); std::size_t n = l - f; while (n >= 16) { - __m128i r0 = _mm_loadu_si128((__m128i const*)f); + __m128i const r0 = _mm_loadu_si128((__m128i const*)f); __m128i r1; CXON_IF_CONSTEXPR (is_quoted_key_context::value || X::string::del == '\"') r1 = _mm_cmpeq_epi8(r0, v1); else CXON_IF_CONSTEXPR (is_quoted_key_context::value || X::string::del == '\'') r1 = _mm_cmpeq_epi8(r0, v4); - __m128i r2 = _mm_cmpeq_epi8(r0, v2); - __m128i r3 = _mm_cmpeq_epi8(_mm_min_epu8(r0, v3), r0); - __m128i r4 = _mm_or_si128(_mm_or_si128(r1, r2), r3); + __m128i const r2 = _mm_cmpeq_epi8(r0, v2); + __m128i const r3 = _mm_cmpeq_epi8(r0, _mm_min_epu8(r0, v3)); + __m128i const r4 = _mm_or_si128(_mm_or_si128(r1, r2), r3); if (int r = _mm_movemask_epi8(r4)) { int e = ffs(r); if (a != (f += e) && !poke(o, a, f, cx)) return false; diff --git a/test/data/json/tests.cf b/test/data/json/tests.cf index ad1309ff..f40274ae 100644 --- a/test/data/json/tests.cf +++ b/test/data/json/tests.cf @@ -23,9 +23,13 @@ tests = { label = "time-native" group = ["set-time-4"] } - diff = { - label = "diff" - group = ["set-diff-1" "set-diff-2" "set-diff-3"] + diff-node = { + label = "diff-node" + group = ["set-diff-node-1" "set-diff-node-2" "set-diff-node-3"] + } + diff-native = { + label = "diff-native" + group = ["set-diff-native-1" "set-diff-native-2" "set-diff-native-3"] } } @@ -458,19 +462,19 @@ sets = { "set.6/random-object.json" ] } - set-diff-1 = { + set-diff-node-1 = { label = "https://github.com/minimaxir/big-list-of-naughty-strings" group = [ "set.3/blns.json" ] } - set-diff-2 = { + set-diff-node-2 = { label = "https://github.com/github/gemoji" group = [ "set.4/emoji.json" ] } - set-diff-3 = { + set-diff-node-3 = { label = "https://github.com/lemire/simdjson" group = [ "set.5/apache_builds.json" @@ -496,4 +500,44 @@ sets = { //"set.5/update-center.json" ] } + set-diff-native-1 = { + label = "https://github.com/minimaxir/big-list-of-naughty-strings" + group = [ + "set.3/blns.json" + ] + } + set-diff-native-2 = { + label = "https://github.com/github/gemoji" + group = [ + // redundant default value + //"set.4/emoji.json" + ] + } + set-diff-native-3 = { + label = "https://github.com/lemire/simdjson" + group = [ + "set.5/apache_builds.json" + "set.5/canada.json" + "set.5/citm_catalog.json" + // different key order for same objects + //"set.5/github_events.json" + // redundant escapes + //"set.5/gsoc-2018.json" + "set.5/instruments.json" + // number precision + //"set.5/marine_ik.json" + // number precision + //"set.5/mesh.json" + //"set.5/mesh.pretty.json" + // number precision + //"set.5/numbers.json" + "set.5/random.json" + // number precision + //"set.5/twitter.json" + // redundant escapes + number precision + //"set.5/twitterescaped.json" + // redundant escapes + //"set.5/update-center.json" + ] + } } diff --git a/test/makefile b/test/makefile index 0c865f79..518f3e05 100644 --- a/test/makefile +++ b/test/makefile @@ -29,17 +29,19 @@ check-all: build-json build-json-node build-cbor build-cbor-node @(cd $(outdir) && ./check-json) @(cd $(outdir) && ./check-json-node) @(cd $(outdir) && ./check-json-node $(datdir)/json/tests.cf pass fail) - @(cd $(outdir) && ./check-json-node $(datdir)/json/tests.cf diff | xargs -n2 diff -q) + @(cd $(outdir) && ./check-json-node $(datdir)/json/tests.cf diff-node | xargs -n2 diff -q) + @(cd $(outdir) && ./check-json-node $(datdir)/json/tests.cf diff-native | xargs -n2 diff -q) @(cd $(outdir) && ./check-cbor) @(cd $(outdir) && ./check-cbor-node) @(cd $(outdir) && ./check-cbor-node $(datdir)/cbor/tests.cf test-vector) - @(cd $(outdir) && ./check-cbor-node $(datdir)/cbor/tests.cf round-trip | xargs -n2 diff -q) + @(cd $(outdir) && ./check-cbor-node $(datdir)/cbor/tests.cf round-trip | xargs -n2 diff -q) check-json: build-json @(cd $(outdir) && ./check-json) check-json-node: build-json-node @(cd $(outdir) && ./check-json-node) @(cd $(outdir) && ./check-json-node $(datdir)/json/tests.cf pass fail) - @(cd $(outdir) && ./check-json-node $(datdir)/json/tests.cf diff | xargs -n2 diff -q) + @(cd $(outdir) && ./check-json-node $(datdir)/json/tests.cf diff-node | xargs -n2 diff -q) + @(cd $(outdir) && ./check-json-node $(datdir)/json/tests.cf diff-native | xargs -n2 diff -q) check-cbor: build-cbor @(cd $(outdir) && ./check-cbor) check-cbor-node: build-cbor-node diff --git a/test/src/cbor/cbor.node-main.cxx b/test/src/cbor/cbor.node-main.cxx index 09668471..62202ea3 100644 --- a/test/src/cbor/cbor.node-main.cxx +++ b/test/src/cbor/cbor.node-main.cxx @@ -253,8 +253,8 @@ namespace test { namespace kind { // round-trip ++err, std::fprintf(stderr, "error: cannot be opened: '%s'\n", file.c_str()); continue; } - std::string const f0 = name(file) + ".ct(0).json"; - std::string const f1 = name(file) + ".ct(1).json"; + std::string const f0 = "cbor.round-trip." + name(file) + ".(0).json"; + std::string const f1 = "cbor.round-trip." + name(file) + ".(1).json"; { // 0 std::ofstream os(f0, std::ofstream::binary); if (!os) { diff --git a/test/src/json/json.node-diff.cxx b/test/src/json/json.node-diff.cxx new file mode 100644 index 00000000..76f795df --- /dev/null +++ b/test/src/json/json.node-diff.cxx @@ -0,0 +1,165 @@ +// Copyright (c) 2017-2024 oknenavin. +// Licensed under the MIT license. See LICENSE file in the library root for full license information. +// +// SPDX-License-Identifier: MIT + +#ifndef CXON_TIME_ONLY + +#include "json.node.hxx" +#include "json.node-time.native.hxx" + +#include "cxon/json.hxx" +#include "cxon/lib/node.ordered.hxx" +#include "cxon/lang/json/tidy.hxx" + +#include +#include +#include + +//////////////////////////////////////////////////////////////////////////////// + +namespace test { namespace kind { + + using node = cxon::json::ordered_node; + + static std::string name(const std::string& p) { + auto f = p.find_last_of("/\\"); f = f != p.npos ? f + 1 : 0; + auto l = p.rfind('.'); if (l == p.npos) l = p.size(); + return std::string(p, f, l - f); + }; + + template + int diff(test& test, const char* prefix) { + std::string json; + { // read + std::ifstream is(test.source, std::ifstream::binary); + if (!is) { + test.error = "cannot be opened"; + return 1; + } + json.assign(std::istreambuf_iterator(is), std::istreambuf_iterator()); + } + { // tidy + std::ofstream os(prefix + name(test.source) + ".(0).json", std::ofstream::binary); + if (!os) { + test.error = prefix + name(test.source) + ".(0).json" + ": cannot be opened"; + return 1; + } + cxon::json::tidy(std::ostreambuf_iterator(os), json); + } + T result; + { // from + auto const r = cxon::from_bytes(result, json); + if (!r) { + test.error = format_error(r, json.cbegin()); + return 1; + } + } + { // to + std::ofstream os(prefix + name(test.source) + ".(1).json", std::ofstream::binary); + if (!os) { + test.error = prefix + name(test.source) + ".(1).json" + "cannot be opened"; + return 1; + } + auto const w = cxon::to_bytes(cxon::json::make_indenter(std::ostreambuf_iterator(os)), result); + if (!w) { + test.error += w.ec.category().name(), + test.error += ": " + w.ec.message(); + } + } + return 0; + } + + int diff_node(cases& cases) { + int err = 0; + + char const*const prefix = "json.node."; + + for (auto& c : cases) { + err += diff(c, prefix); + } + std::size_t fc = 0; + for (auto& c : cases) { + if (!c.error.empty()) { + ++fc, std::fprintf(stderr, "%s:\n\tfailed: %s\n", c.source.c_str(), c.error.c_str()), std::fflush(stderr); + } + } + if (!fc) { + for (auto& c : cases) { + std::fprintf(stdout, "%s %s ", (prefix + name(c.source) + ".(0).json").c_str(), (prefix + name(c.source) + ".(1).json").c_str()), std::fflush(stdout); + } + std::fputc('\n', stdout); + } + else { + std::fprintf(stdout, "%-21s: %zu of %4zu failed\n", "cxon/json/node/diff", fc, cases.size()), std::fflush(stdout); + } + + return err; + } + + using executor = int (*)(test&, const char*); + + static std::map executors_ = { + { "blns.json", &diff }, + { "emoji.json", &diff }, + { "apache_builds.json", &diff }, + { "canada.json", &diff }, +# ifdef CXON_HAS_LIB_STD_OPTIONAL + { "citm_catalog.json", &diff }, +# endif +# ifdef CXON_HAS_LIB_STD_OPTIONAL + { "github_events.json", &diff }, +# endif + { "gsoc-2018.json", &diff }, +# ifdef CXON_HAS_LIB_STD_OPTIONAL + { "instruments.json", &diff }, +# endif + { "marine_ik.json", &diff }, + { "mesh.json", &diff }, + { "mesh.pretty.json", &diff }, + { "numbers.json", &diff }, + { "random.json", &diff }, +# ifdef CXON_HAS_LIB_STD_OPTIONAL + { "twitter.json", &diff }, + { "twitterescaped.json", &diff }, +# endif + { "update-center.json", &diff } + }; + + int diff_native(cases& cases) { + int err = 0; + + char const*const prefix = "json.native."; + + for (auto& c : cases) { + + auto i = executors_.find(c.source.substr(c.source.rfind('/') + 1)); + if (i == executors_.end()) { + ++err, c.error = name(c.source) + ": unknown format"; + continue; + } + else + err += i->second(c, prefix); + } + std::size_t fc = 0; + for (auto& c : cases) { + if (!c.error.empty()) { + ++fc, std::fprintf(stderr, "%s:\n\tfailed: %s\n", c.source.c_str(), c.error.c_str()), std::fflush(stderr); + } + } + if (!fc) { + for (auto& c : cases) { + std::fprintf(stdout, "%s %s ", (prefix + name(c.source) + ".(0).json").c_str(), (prefix + name(c.source) + ".(1).json").c_str()), std::fflush(stdout); + } + std::fputc('\n', stdout); + } + else { + std::fprintf(stdout, "%-21s: %zu of %4zu failed\n", "cxon/json/node/diff", fc, cases.size()), std::fflush(stdout); + } + + return err; + } + +}} + +#endif // CXON_TIME_ONLY diff --git a/test/src/json/json.node-main.cxx b/test/src/json/json.node-main.cxx index 0c7e56bb..1fc98809 100644 --- a/test/src/json/json.node-main.cxx +++ b/test/src/json/json.node-main.cxx @@ -10,7 +10,6 @@ #include "cxon/lib/std/vector.hxx" #include "cxon/lib/std/map.hxx" #include "cxon/lib/node.ordered.hxx" -#include "cxon/lang/json/tidy.hxx" #include #include @@ -19,7 +18,6 @@ //////////////////////////////////////////////////////////////////////////////// -using node = cxon::json::ordered_node; namespace test { @@ -91,8 +89,10 @@ int main(int argc, char *argv[]) { err += test::kind::pass(sets["pass"]); if (!sets["fail"].empty()) err += test::kind::fail(sets["fail"]); - if (!sets["diff"].empty()) - err += test::kind::diff(sets["diff"]); + if (!sets["diff-node"].empty()) + err += test::kind::diff_node(sets["diff-node"]); + if (!sets["diff-native"].empty()) + err += test::kind::diff_native(sets["diff-native"]); # endif if (!sets["time-node"].empty()) test::kind::time(sets["time-node"], test::kind::time_cxon_node); @@ -116,6 +116,8 @@ CXON_JSON_CLS_BARE(test::config) namespace test { namespace kind { + using node = cxon::json::ordered_node; + int pass(cases& cases) { int err = 0; @@ -172,72 +174,6 @@ CXON_JSON_CLS_BARE(test::config) return err; } - int diff(cases& cases) { - int err = 0; - - static auto const name = [](const std::string& p) { - auto f = p.find_last_of("/\\"); f = f != p.npos ? f + 1 : 0; - auto l = p.rfind('.'); if (l == p.npos) l = p.size(); - return std::string(p, f, l - f); - }; - for (auto& c : cases) { - std::string json; - { // read - std::ifstream is(c.source, std::ifstream::binary); - if (!is) { - ++err, c.error = "cannot be opened"; - continue; - } - json.assign(std::istreambuf_iterator(is), std::istreambuf_iterator()); - } - { // tidy - std::ofstream os(name(c.source) + ".jt(0).json", std::ofstream::binary); - if (!os) { - ++err, c.error = name(c.source) + ".jt(0).json" + ": cannot be opened"; - continue; - } - cxon::json::tidy(std::ostreambuf_iterator(os), json); - } - node result; - { // from - auto const r = cxon::from_bytes(result, json); - if (!r) { - ++err, c.error = format_error(r, json.cbegin()); - continue; - } - } - { // to - std::ofstream os(name(c.source) + ".jt(1).json", std::ofstream::binary); - if (!os) { - ++err, c.error = name(c.source) + ".jt(1).json" + "cannot be opened"; - continue; - } - auto const w = cxon::to_bytes(cxon::json::make_indenter(std::ostreambuf_iterator(os)), result); - if (!w) { - ++err, c.error += w.ec.category().name(), - c.error += ": " + w.ec.message(); - } - } - } - std::size_t fc = 0; - for (auto& c : cases) { - if (!c.error.empty()) { - ++fc, std::fprintf(stderr, "%s:\n\tfailed: %s\n", c.source.c_str(), c.error.c_str()), std::fflush(stderr); - } - } - if (!fc) { - for (auto& c : cases) { - std::fprintf(stdout, "%s %s ", (name(c.source) + ".jt(0).json").c_str(), (name(c.source) + ".jt(1).json").c_str()), std::fflush(stdout); - } - std::fputc('\n', stdout); - } - else { - std::fprintf(stdout, "%-21s: %zu of %4zu failed\n", "cxon/json/node/diff", fc, cases.size()), std::fflush(stdout); - } - - return err; - } - }} #endif // CXON_TIME_ONLY diff --git a/test/src/json/json.node-time.native.cxx b/test/src/json/json.node-time.native.cxx index ba3e0fb8..c8558dd6 100644 --- a/test/src/json/json.node-time.native.cxx +++ b/test/src/json/json.node-time.native.cxx @@ -3,19 +3,15 @@ // // SPDX-License-Identifier: MIT - #include "json.node-time.hxx" #include "json.node-time.native.hxx" #include "cxon/lib/node.ordered.hxx" -#include #include //////////////////////////////////////////////////////////////////////////////// -using node = cxon::json::ordered_node; - namespace test { namespace kind { template @@ -52,7 +48,7 @@ namespace test { namespace kind { )); } - using executor = void (*)(test& t); + using executor = void (*)(test&); static std::map executors_ = { // set 1 (time-s1.in) diff --git a/test/src/json/json.node.hxx b/test/src/json/json.node.hxx index ffb71c60..c06fa276 100644 --- a/test/src/json/json.node.hxx +++ b/test/src/json/json.node.hxx @@ -36,7 +36,8 @@ namespace test { namespace kind { int pass(cases& cases); int fail(cases& cases); - int diff(cases& cases); + int diff_node(cases& cases); + int diff_native(cases& cases); }}