Skip to content

Benchmarks

Benchmarks #373

Workflow file for this run

name: Benchmarks
on:
push:
branches:
- develop
- master
- release/*
pull_request:
workflow_dispatch:
schedule:
- cron: '37 3 * * 6'
jobs:
# gcc ########################################################################
gcc:
runs-on: ubuntu-22.04
steps:
- name: Checkout cxon
uses: actions/checkout@v3
- name: Checkout Boost
uses: actions/checkout@v3
with:
repository: boostorg/boost
ref: boost-1.81.0
path: .dep/boost
- name: Checkout RapidJSON
uses: actions/checkout@v3
with:
repository: tencent/rapidjson
ref: master
path: .dep/rapidjson
- name: Checkout nlohmann/json
uses: actions/checkout@v3
with:
repository: nlohmann/json
ref: master
path: .dep/nlohmann
- name: Checkout fastfloat/fast_float
uses: actions/checkout@v3
with:
repository: fastfloat/fast_float
ref: main
path: .dep/fast_float
- name: Compiler setup
run: |
sudo apt-get update
sudo apt-get install g++-12
- name: Boost setup
run: |
cd .dep/boost
git submodule update --init tools/build
git submodule update --init tools/boostdep
git submodule update --init libs/config
git submodule update --init libs/json
python tools/boostdep/depinst/depinst.py json
./bootstrap.sh
./b2 headers
- name: Time setup
run: mkdir test/.time
- name: Time JSON (default)
env:
CXX: g++-12
CXXFLAGS: -std=c++20 -DCXON_TIME_ONLY -I ../.dep/boost -DCXON_TIME_BOOST_JSON -I ../.dep/rapidjson/include -DCXON_TIME_RAPIDJSON -I ../.dep/nlohmann/single_include -DCXON_TIME_NLOHMANN
run: |
COMPILER_VERSION=`$CXX -dumpversion`
make clean && make -j build-json-node
# S1
echo "# `date -u --iso-8601=seconds` / g++-$COMPILER_VERSION (`uname -sm`) / node (default)" > test/.time/g++.head.default.json.node-s1.data
make -s time-json-node-s1 | tee -a test/.time/g++.head.default.json.node-s1.data
echo "# `date -u --iso-8601=seconds` / g++-$COMPILER_VERSION (`uname -sm`) / native (default)" > test/.time/g++.head.default.json.native-s1.data
make -s time-json-native-s1 | tee -a test/.time/g++.head.default.json.native-s1.data
# S2
echo "# `date -u --iso-8601=seconds` / g++-$COMPILER_VERSION (`uname -sm`) / node (default)" > test/.time/g++.head.default.json.node-s2.data
make -s time-json-node-s2 | tee -a test/.time/g++.head.default.json.node-s2.data
echo "# `date -u --iso-8601=seconds` / g++-$COMPILER_VERSION (`uname -sm`) / native (default)" > test/.time/g++.head.default.json.native-s2.data
make -s time-json-native-s2 | tee -a test/.time/g++.head.default.json.native-s2.data
- name: Time JSON (fast_float)
env:
CXX: g++-12
CXXFLAGS: -std=c++20 -DCXON_TIME_ONLY -I ../.dep/fast_float/include -DCXON_USE_FAST_FLOAT -I ../.dep/boost -DCXON_TIME_BOOST_JSON -I ../.dep/rapidjson/include -DCXON_TIME_RAPIDJSON -I ../.dep/nlohmann/single_include -DCXON_TIME_NLOHMANN
run: |
COMPILER_VERSION=`$CXX -dumpversion`
make clean && make -j build-json-node
# S1
echo "# `date -u --iso-8601=seconds` / g++-$COMPILER_VERSION (`uname -sm`) / node (fast_float)" > test/.time/g++.head.fast_float.json.node-s1.data
make -s time-json-node-s1 | tee -a test/.time/g++.head.fast_float.json.node-s1.data
echo "# `date -u --iso-8601=seconds` / g++-$COMPILER_VERSION (`uname -sm`) / native (fast_float)" > test/.time/g++.head.fast_float.json.native-s1.data
make -s time-json-native-s1 | tee -a test/.time/g++.head.fast_float.json.native-s1.data
# S2
echo "# `date -u --iso-8601=seconds` / g++-$COMPILER_VERSION (`uname -sm`) / node (fast_float)" > test/.time/g++.head.fast_float.json.node-s2.data
make -s time-json-node-s2 | tee -a test/.time/g++.head.fast_float.json.node-s2.data
echo "# `date -u --iso-8601=seconds` / g++-$COMPILER_VERSION (`uname -sm`) / native (fast_float)" > test/.time/g++.head.fast_float.json.native-s2.data
make -s time-json-native-s2 | tee -a test/.time/g++.head.fast_float.json.native-s2.data
- name: Time CBOR (default)
env:
CXX: g++-12
CXXFLAGS: -std=c++20 -DCXON_TIME_ONLY
run: |
COMPILER_VERSION=`$CXX -dumpversion`
make clean && make -j build-cbor-node
echo "# `date -u --iso-8601=seconds` / g++-$COMPILER_VERSION (`uname -sm`)" > test/.time/g++.head.cbor.data
make -s time-cbor | tee -a test/.time/g++.head.cbor.data
- name: Space setup
run: |
chmod +x test/benchmark-space.sh
mkdir test/.space
- name: Space JSON (default)
env:
CXX: g++-12
CPPFLAGS: -I ../.dep/boost -I ../.dep/rapidjson/include -I ../.dep/nlohmann/single_include
CXXFLAGS: -std=c++20 -O2 -s -DNDEBUG
run: |
COMPILER_VERSION=`$CXX -dumpversion`
cd test
echo "# `date -u --iso-8601=seconds` / g++-$COMPILER_VERSION $CXXFLAGS (`uname -sm`)" > .space/g++.head.default.json.data
./benchmark-space.sh | tee -a .space/g++.head.default.json.data
- name: Upload Artifacts
uses: actions/upload-artifact@v3
with:
name: benchmark-data
path: |
test/.time/*.data
test/.space/*.data
# clang ######################################################################
clang:
runs-on: ubuntu-22.04
steps:
- name: Checkout cxon
uses: actions/checkout@v3
- name: Checkout Boost
uses: actions/checkout@v3
with:
repository: boostorg/boost
ref: boost-1.81.0
path: .dep/boost
- name: Checkout RapidJSON
uses: actions/checkout@v3
with:
repository: tencent/rapidjson
ref: master
path: .dep/rapidjson
- name: Checkout nlohmann/json
uses: actions/checkout@v3
with:
repository: nlohmann/json
ref: master
path: .dep/nlohmann
- name: Checkout fastfloat/fast_float
uses: actions/checkout@v3
with:
repository: fastfloat/fast_float
ref: main
path: .dep/fast_float
- name: Compiler setup
run: |
sudo apt-get update
sudo apt-get install clang-14
- name: Boost setup
run: |
cd .dep/boost
git submodule update --init tools/build
git submodule update --init tools/boostdep
git submodule update --init libs/config
git submodule update --init libs/json
python tools/boostdep/depinst/depinst.py json
./bootstrap.sh
./b2 headers
- name: Time setup
run: mkdir test/.time
- name: Time JSON (default)
env:
CXX: clang++-14
CXXFLAGS: -std=c++20 -DCXON_TIME_ONLY -I ../.dep/boost -DCXON_TIME_BOOST_JSON -I ../.dep/rapidjson/include -DCXON_TIME_RAPIDJSON -I ../.dep/nlohmann/single_include -DCXON_TIME_NLOHMANN
run: |
COMPILER_VERSION=`$CXX -dumpversion`
make clean && make -j build-json-node
# S1
echo "# `date -u --iso-8601=seconds` / clang++-$COMPILER_VERSION (`uname -sm`) / node (default)" > test/.time/clang++.head.default.json.node-s1.data
make -s time-json-node-s1 | tee -a test/.time/clang++.head.default.json.node-s1.data
echo "# `date -u --iso-8601=seconds` / clang++-$COMPILER_VERSION (`uname -sm`) / native (default)" > test/.time/clang++.head.default.json.native-s1.data
make -s time-json-native-s1 | tee -a test/.time/clang++.head.default.json.native-s1.data
# S2
echo "# `date -u --iso-8601=seconds` / clang++-$COMPILER_VERSION (`uname -sm`) / node (default)" > test/.time/clang++.head.default.json.node-s2.data
make -s time-json-node-s2 | tee -a test/.time/clang++.head.default.json.node-s2.data
echo "# `date -u --iso-8601=seconds` / clang++-$COMPILER_VERSION (`uname -sm`) / native (default)" > test/.time/clang++.head.default.json.native-s2.data
make -s time-json-native-s2 | tee -a test/.time/clang++.head.default.json.native-s2.data
- name: Time JSON (fast_float)
env:
CXX: clang++-14
CXXFLAGS: -std=c++20 -DCXON_TIME_ONLY -I ../.dep/fast_float/include -DCXON_USE_FAST_FLOAT -I ../.dep/boost -DCXON_TIME_BOOST_JSON -I ../.dep/rapidjson/include -DCXON_TIME_RAPIDJSON -I ../.dep/nlohmann/single_include -DCXON_TIME_NLOHMANN
run: |
COMPILER_VERSION=`$CXX -dumpversion`
make clean && make -j build-json-node
# S1
echo "# `date -u --iso-8601=seconds` / clang++-$COMPILER_VERSION (`uname -sm`) / node (fast_float)" > test/.time/clang++.head.fast_float.json.node-s1.data
make -s time-json-node-s1 | tee -a test/.time/clang++.head.fast_float.json.node-s1.data
echo "# `date -u --iso-8601=seconds` / clang++-$COMPILER_VERSION (`uname -sm`) / native (fast_float)" > test/.time/clang++.head.fast_float.json.native-s1.data
make -s time-json-native-s1 | tee -a test/.time/clang++.head.fast_float.json.native-s1.data
# S2
echo "# `date -u --iso-8601=seconds` / clang++-$COMPILER_VERSION (`uname -sm`) / node (fast_float)" > test/.time/clang++.head.fast_float.json.node-s2.data
make -s time-json-node-s2 | tee -a test/.time/clang++.head.fast_float.json.node-s2.data
echo "# `date -u --iso-8601=seconds` / clang++-$COMPILER_VERSION (`uname -sm`) / native (fast_float)" > test/.time/clang++.head.fast_float.json.native-s2.data
make -s time-json-native-s2 | tee -a test/.time/clang++.head.fast_float.json.native-s2.data
- name: Time CBOR (default)
env:
CXX: clang++-14
CXXFLAGS: -std=c++20 -DCXON_TIME_ONLY
run: |
COMPILER_VERSION=`$CXX -dumpversion`
make clean && make -j build-cbor-node
echo "# `date -u --iso-8601=seconds` / clang++-$COMPILER_VERSION (`uname -sm`)" > test/.time/clang++.head.cbor.data
make -s time-cbor | tee -a test/.time/clang++.head.cbor.data
- name: Space setup
run: |
chmod +x test/benchmark-space.sh
mkdir test/.space
- name: Space JSON (default)
env:
CXX: clang++-14
CPPFLAGS: -I ../.dep/boost -I ../.dep/rapidjson/include -I ../.dep/nlohmann/single_include
CXXFLAGS: -std=c++20 -O2 -s -DNDEBUG
run: |
COMPILER_VERSION=`$CXX -dumpversion`
cd test
echo "# `date -u --iso-8601=seconds` / clang++-$COMPILER_VERSION $CXXFLAGS (`uname -sm`)" > .space/clang++.head.default.json.data
./benchmark-space.sh | tee -a .space/clang++.head.default.json.data
- name: Upload Artifacts
uses: actions/upload-artifact@v3
with:
name: benchmark-data
path: |
test/.time/*.data
test/.space/*.data
# msvc ######################################################################
msvc:
runs-on: windows-latest
steps:
- name: Checkout cxon
uses: actions/checkout@v3
- name: Checkout Boost
uses: actions/checkout@v3
with:
repository: boostorg/boost
ref: boost-1.81.0
path: .dep/boost
- name: Checkout RapidJSON
uses: actions/checkout@v3
with:
repository: tencent/rapidjson
ref: master
path: .dep/rapidjson
- name: Checkout nlohmann/json
uses: actions/checkout@v3
with:
repository: nlohmann/json
ref: master
path: .dep/nlohmann
- name: Checkout fastfloat/fast_float
uses: actions/checkout@v3
with:
repository: fastfloat/fast_float
ref: main
path: .dep/fast_float
- name: Boost setup
shell: cmd
run: |
cd .dep/boost
git submodule update --init tools/build
git submodule update --init tools/boostdep
git submodule update --init libs/config
git submodule update --init libs/json
python tools/boostdep/depinst/depinst.py json
.\bootstrap.bat
#.\b2 # nothing runs after bootstrap.bat by some reason
- name: Boost finalization
shell: cmd
run: |
cd .dep/boost
.\b2
- name: Time setup
shell: bash
run: |
mkdir test/.time
echo > test/.root
- name: Time JSON (default)
shell: bash
env:
CXXFLAGS: -EHsc -Oi -Ot -GR- -GS- -GL -D CXON_TIME_ONLY -I ../../.dep/boost -D CXON_TIME_BOOST_JSON -I ../../.dep/rapidjson/include -D CXON_TIME_RAPIDJSON -I ../../.dep/nlohmann/single_include -D CXON_TIME_NLOHMANN
LDFLAGS: -LTCG /LIBPATH:../../.dep/boost/stage/lib
run: |
cd test
#cmake --build .bld --target clean
cmake -S . -B .bld -D CMAKE_CXX_STANDARD=20 -D CMAKE_CXX_FLAGS="$CXXFLAGS" -DCMAKE_EXE_LINKER_FLAGS="$LDFLAGS"
cmake --build .bld --config Release --target cxon-json-node --parallel
COMPILER_VERSION=`grep -Erho 'CMAKE_CXX_COMPILER_VERSION ".*"' .bld | sed -E 's/.*"(.*)"/\1/' | head -n1`
cd .out
# S1
echo "# `date -u --iso-8601=seconds` / msvc++-$COMPILER_VERSION (`uname -sm`) / node (default)" > ../.time/msvc++.head.default.json.node-s1.data
./cxon-json-node ../data/json/tests.cf time-node-s1 | tee -a ../.time/msvc++.head.default.json.node-s1.data
echo "# `date -u --iso-8601=seconds` / msvc++-$COMPILER_VERSION (`uname -sm`) / native (default)" > ../.time/msvc++.head.default.json.native-s1.data
./cxon-json-node ../data/json/tests.cf time-native-s1 | tee -a ../.time/msvc++.head.default.json.native-s1.data
# S2
echo "# `date -u --iso-8601=seconds` / msvc++-$COMPILER_VERSION (`uname -sm`) / node (default)" > ../.time/msvc++.head.default.json.node-s2.data
./cxon-json-node ../data/json/tests.cf time-node-s2 | tee -a ../.time/msvc++.head.default.json.node-s2.data
echo "# `date -u --iso-8601=seconds` / msvc++-$COMPILER_VERSION (`uname -sm`) / native (default)" > ../.time/msvc++.head.default.json.native-s2.data
./cxon-json-node ../data/json/tests.cf time-native-s2 | tee -a ../.time/msvc++.head.default.json.native-s2.data
- name: Time JSON (fast_float)
shell: bash
env:
CXXFLAGS: -EHsc -Oi -Ot -GR- -GS- -GL -D CXON_TIME_ONLY -I ../../.dep/fast_float/include -D CXON_USE_FAST_FLOAT -I ../../.dep/boost -D CXON_TIME_BOOST_JSON -I ../../.dep/rapidjson/include -D CXON_TIME_RAPIDJSON -I ../../.dep/nlohmann/single_include -D CXON_TIME_NLOHMANN
LDFLAGS: -LTCG /LIBPATH:../../.dep/boost/stage/lib
run: |
cd test
cmake --build .bld --target clean
cmake -S . -B .bld -D CMAKE_CXX_STANDARD=20 -D CMAKE_CXX_FLAGS="$CXXFLAGS" -DCMAKE_EXE_LINKER_FLAGS="$LDFLAGS"
cmake --build .bld --config Release --target cxon-json-node --parallel
COMPILER_VERSION=`grep -Erho 'CMAKE_CXX_COMPILER_VERSION ".*"' .bld | sed -E 's/.*"(.*)"/\1/' | head -n1`
cd .out
# S1
echo "# `date -u --iso-8601=seconds` / msvc++-$COMPILER_VERSION (`uname -sm`) / node (fast_float)" > ../.time/msvc++.head.fast_float.json.node-s1.data
./cxon-json-node ../data/json/tests.cf time-node-s1 | tee -a ../.time/msvc++.head.fast_float.json.node-s1.data
echo "# `date -u --iso-8601=seconds` / msvc++-$COMPILER_VERSION (`uname -sm`) / native (fast_float)" > ../.time/msvc++.head.fast_float.json.native-s1.data
./cxon-json-node ../data/json/tests.cf time-native-s1 | tee -a ../.time/msvc++.head.fast_float.json.native-s1.data
# S2
echo "# `date -u --iso-8601=seconds` / msvc++-$COMPILER_VERSION (`uname -sm`) / node (fast_float)" > ../.time/msvc++.head.fast_float.json.node-s2.data
./cxon-json-node ../data/json/tests.cf time-node-s2 | tee -a ../.time/msvc++.head.fast_float.json.node-s2.data
echo "# `date -u --iso-8601=seconds` / msvc++-$COMPILER_VERSION (`uname -sm`) / native (fast_float)" > ../.time/msvc++.head.fast_float.json.native-s2.data
./cxon-json-node ../data/json/tests.cf time-native-s2 | tee -a ../.time/msvc++.head.fast_float.json.native-s2.data
- name: Upload Artifacts
uses: actions/upload-artifact@v3
with:
name: benchmark-data
path: |
test/.root
test/.time/*.data
# xcode ######################################################################
xcode:
runs-on: macos-latest
steps:
- name: Checkout cxon
uses: actions/checkout@v3
- name: Checkout Boost
uses: actions/checkout@v3
with:
repository: boostorg/boost
ref: boost-1.81.0
path: .dep/boost
- name: Checkout RapidJSON
uses: actions/checkout@v3
with:
repository: tencent/rapidjson
ref: master
path: .dep/rapidjson
- name: Checkout nlohmann/json
uses: actions/checkout@v3
with:
repository: nlohmann/json
ref: master
path: .dep/nlohmann
- name: Checkout fastfloat/fast_float
uses: actions/checkout@v3
with:
repository: fastfloat/fast_float
ref: main
path: .dep/fast_float
- name: Boost setup
run: |
cd .dep/boost
git submodule update --init tools/build
git submodule update --init tools/boostdep
git submodule update --init libs/config
git submodule update --init libs/json
python tools/boostdep/depinst/depinst.py json
./bootstrap.sh
./b2 headers
- name: Time setup
run: |
mkdir test/.time
echo > test/.root
- name: Time JSON (default)
run: |
cd test
#cmake --build .bld --target clean
cmake -S . -B .bld -D CMAKE_BUILD_TYPE=Release -D CMAKE_CXX_STANDARD=20 -D CMAKE_CXX_FLAGS="-DCXON_TIME_ONLY -I ../../.dep/boost -DCXON_TIME_BOOST_JSON -I ../../.dep/rapidjson/include -DCXON_TIME_RAPIDJSON -I ../../.dep/nlohmann/single_include -DCXON_TIME_NLOHMANN"
cmake --build .bld --target cxon-json-node --parallel
COMPILER_VERSION=`grep -Erho 'CMAKE_CXX_COMPILER_VERSION ".*"' .bld | sed -E 's/.*"(.*)"/\1/' | head -n1`
cd .out
# S1
echo "# `date -u +'%Y-%m-%dT%H:%M:%S+00:00'` / clang++-$COMPILER_VERSION (`uname -sm`) / node (default)" > ../.time/xcode.head.default.json.node-s1.data
./cxon-json-node ../data/json/tests.cf time-node-s1 | tee -a ../.time/xcode.head.default.json.node-s1.data
echo "# `date -u +'%Y-%m-%dT%H:%M:%S+00:00'` / clang++-$COMPILER_VERSION (`uname -sm`) / native (default)" > ../.time/xcode.head.default.json.native-s1.data
./cxon-json-node ../data/json/tests.cf time-native-s1 | tee -a ../.time/xcode.head.default.json.native-s1.data
# S2
echo "# `date -u +'%Y-%m-%dT%H:%M:%S+00:00'` / clang++-$COMPILER_VERSION (`uname -sm`) / node (default)" > ../.time/xcode.head.default.json.node-s2.data
./cxon-json-node ../data/json/tests.cf time-node-s2 | tee -a ../.time/xcode.head.default.json.node-s2.data
echo "# `date -u +'%Y-%m-%dT%H:%M:%S+00:00'` / clang++-$COMPILER_VERSION (`uname -sm`) / native (default)" > ../.time/xcode.head.default.json.native-s2.data
./cxon-json-node ../data/json/tests.cf time-native-s2 | tee -a ../.time/xcode.head.default.json.native-s2.data
- name: Time JSON (fast_float)
run: |
cd test
cmake --build .bld --target clean
cmake -S . -B .bld -D CMAKE_BUILD_TYPE=Release -D CMAKE_CXX_STANDARD=20 -D CMAKE_CXX_FLAGS="-DCXON_TIME_ONLY -I ../../.dep/fast_float/include -DCXON_USE_FAST_FLOAT -I ../../.dep/boost -DCXON_TIME_BOOST_JSON -I ../../.dep/rapidjson/include -DCXON_TIME_RAPIDJSON -I ../../.dep/nlohmann/single_include -DCXON_TIME_NLOHMANN"
cmake --build .bld --target cxon-json-node --parallel
COMPILER_VERSION=`grep -Erho 'CMAKE_CXX_COMPILER_VERSION ".*"' .bld | sed -E 's/.*"(.*)"/\1/' | head -n1`
cd .out
# S1
echo "# `date -u +'%Y-%m-%dT%H:%M:%S+00:00'` / clang++-$COMPILER_VERSION (`uname -sm`) / node (fast_float)" > ../.time/xcode.head.fast_float.json.node-s1.data
./cxon-json-node ../data/json/tests.cf time-node-s1 | tee -a ../.time/xcode.head.fast_float.json.node-s1.data
echo "# `date -u +'%Y-%m-%dT%H:%M:%S+00:00'` / clang++-$COMPILER_VERSION (`uname -sm`) / native (fast_float)" > ../.time/xcode.head.fast_float.json.native-s1.data
./cxon-json-node ../data/json/tests.cf time-native-s1 | tee -a ../.time/xcode.head.fast_float.json.native-s1.data
# S2
echo "# `date -u +'%Y-%m-%dT%H:%M:%S+00:00'` / clang++-$COMPILER_VERSION (`uname -sm`) / node (fast_float)" > ../.time/xcode.head.fast_float.json.node-s2.data
./cxon-json-node ../data/json/tests.cf time-node-s2 | tee -a ../.time/xcode.head.fast_float.json.node-s2.data
echo "# `date -u +'%Y-%m-%dT%H:%M:%S+00:00'` / clang++-$COMPILER_VERSION (`uname -sm`) / native (fast_float)" > ../.time/xcode.head.fast_float.json.native-s2.data
./cxon-json-node ../data/json/tests.cf time-node-s2 | tee -a ../.time/xcode.head.fast_float.json.native-s2.data
- name: Upload Artifacts
uses: actions/upload-artifact@v3
with:
name: benchmark-data
path: |
test/.root
test/.time/*.data
# commit #####################################################################
commit:
needs: [gcc, clang, msvc, xcode]
runs-on: ubuntu-latest
steps:
- name: Checkout cxon
uses: actions/checkout@v3
- name: Checkout workflows-data
uses: actions/checkout@v3
with:
repository: oknenavin/workflows-data
token: ${{ secrets.WORKFLOWS_DATA_TOKEN }}
ref: master # branch-links
path: .dep/workflows-data
- name: Download Artifacts
uses: actions/download-artifact@v3
with:
name: benchmark-data
path: test/.data
- name: Commit
run: |
data=.dep/workflows-data/cxon/benchmarks/data
for i in test/.data/.time/*.data; do
name=`basename $i`
cp -f $i $data/time/$name
cat $data/time/${name/head/tail} >> $i && mv -f $i $data/time/${name/head/tail}
done
for i in test/.data/.space/*.data; do
name=`basename $i`
cp -f $i $data/space/$name
cat $data/space/${name/head/tail} >> $i && mv -f $i $data/space/${name/head/tail}
done
cd $data
git config user.name "oknenavin" && git config user.email ${{ secrets.CXON_MAIL }}
git pull
git add . && git commit -a -m "<${{ github.event.head_commit.message }}" -m "<- `date -u --iso-8601=seconds` / `git rev-parse --short $GITHUB_SHA`"
git push