Skip to content

Commit

Permalink
Merge pull request #20 from m-peko/add-benchmark
Browse files Browse the repository at this point in the history
Add benchmark
  • Loading branch information
m-peko authored Nov 15, 2020
2 parents 6015bfb + 3155ed2 commit 3069839
Show file tree
Hide file tree
Showing 14 changed files with 709 additions and 11 deletions.
4 changes: 4 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,7 @@
path = googletest
url = https://github.com/google/googletest.git
ignore = dirty
[submodule "googlebench"]
path = googlebench
url = https://github.com/google/benchmark.git
ignore = dirty
5 changes: 4 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ matrix:

# Enable code coverage
- ENABLE_COVERAGE=True

after_success:
# Upload reports to Codecov
- bash <(curl -s https://codecov.io/bash) -x gcov-8
Expand All @@ -157,8 +158,10 @@ script:
# Generate build files
- cmake .. -DBITFLAGS_CPP_VERSION:INTEGER=${CPP_VERSION-17} -DBITFLAGS_ENABLE_COVERAGE:BOOL=${ENABLE_COVERAGE-False}

# Compile examples and tests
# Compile samples
# - cmake --build . --target samples

# Compile tests
- cmake --build . --target tests

# Execute tests
Expand Down
39 changes: 33 additions & 6 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ if (DEFINED PROJECT_NAME)
endif ()

set (BITFLAGS_VERSION_MAJOR 1)
set (BITFLAGS_VERSION_MINOR 2)
set (BITFLAGS_VERSION_MINOR 4)
set (BITFLAGS_VERSION_PATCH 0)

set (BITFLAGS_VERSION "${BITFLAGS_VERSION_MAJOR}.${BITFLAGS_VERSION_MINOR}.${BITFLAGS_VERSION_PATCH}")
Expand Down Expand Up @@ -39,6 +39,9 @@ elseif (BITFLAGS_CPP_VERSION EQUAL 14)
message (STATUS "Using C++14")
elseif (BITFLAGS_CPP_VERSION EQUAL 11)
message (STATUS "Using C++11")
else ()
message (STATUS "Using default C++17")
set (BITFLAGS_CPP_VERSION 17)
endif ()

set (CMAKE_CXX_STANDARD ${BITFLAGS_CPP_VERSION})
Expand All @@ -59,8 +62,9 @@ else ()
endif ()
endif ()

option (BITFLAGS_BUILD_SAMPLES "Build examples" ON)
option (BITFLAGS_BUILD_TESTS "Build tests" ON)
option (BITFLAGS_BUILD_SAMPLES "Build examples" ON)
option (BITFLAGS_BUILD_BENCHMARK "Build benchmark" OFF)

include (GNUInstallDirs)

Expand All @@ -74,10 +78,6 @@ target_include_directories (
$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>
)

if (BITFLAGS_BUILD_SAMPLES)
add_subdirectory (samples)
endif ()

if (BITFLAGS_BUILD_TESTS)
# Only include googletest if the git submodule has been fetched
if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/googletest/CMakeLists.txt")
Expand All @@ -98,6 +98,33 @@ if (BITFLAGS_BUILD_TESTS)
endif ()
endif ()

if (BITFLAGS_BUILD_SAMPLES)
add_subdirectory (samples)
endif ()

if (BITFLAGS_BUILD_BENCHMARK)
# Only include googlebench if the git submodule has been fetched
if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/googlebench/CMakeLists.txt")
# Enable benchmark and add the benchmark directory
message (STATUS "Benchmark has been enabled")
set (GOOGLEBENCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/googlebench)
set (GOOGLEBENCH_INCLUDE ${GOOGLEBENCH_ROOT}/googlebench/include)
set (GOOGLEBENCH_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/googlebench)
set (GOOGLEBENCH_LIBRARY ${GOOGLEBENCH_BINARY_DIR}/googlebench)

set (BENCHMARK_ENABLE_GTEST_TESTS OFF)

if (NOT CMAKE_BUILD_TYPE MATCHES Debug)
add_definitions(-DNDEBUG)
endif()

add_subdirectory (googlebench)
add_subdirectory (benchmark)
else ()
message (STATUS "googlebench git submodule is absent. Run `git submodule init && git submodule update` to get it")
endif ()
endif ()

if (NOT BITFLAGS_SUBPROJECT)
include (CMakePackageConfigHelpers)

Expand Down
13 changes: 13 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ int main() {
* [Set and Remove Specific Flag](#set-and-remove-specific-flag)
* [Toggle Flags](#toggle-flags)
* [Clear Flags](#clear-flags)
* [Benchmark](#benchmark)
* [Building Samples and Tests](#building-samples-and-tests)
* [Compiler Compatibility](#compiler-compatibility)
* [Contributing](#contributing)
Expand Down Expand Up @@ -354,6 +355,18 @@ std::cout << flags.contains(Flags::flag_b) << std::endl; // false
std::cout << flags.contains(Flags::flag_c) << std::endl; // false
```
## Benchmark
As you can see from the following chart, using `raw_flag`s is as fast as using `std::bitset`. However, using ordinary `flag`s (i.e. flags with string representation) is a bit slower (as it is expected because of additional feature of having string representation).
<img src="img/benchmark.png"/>
If you want to run the benchmark yourself, you can use `plot.py` script like:
```bash
$ python3 plot.py --benchmarks-dir <benchmark-json-dir>
```

## Building Samples and Tests

```bash
Expand Down
4 changes: 4 additions & 0 deletions benchmark/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
cmake_minimum_required (VERSION 3.8)

set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/src)
add_subdirectory (src)
33 changes: 33 additions & 0 deletions benchmark/src/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
cmake_minimum_required (VERSION 3.8)

# Use bitflags's include directories + Google Benchmark include directories
include_directories (
${PROJECT_SOURCE_DIR}/include/
${GOOGLEBENCH_INCLUDE}
)

link_directories (
${GOOGLEBENCH_LIBRARY}
)

# Link against GoogleBenchmark and bitflags
link_libraries (
benchmark
bitflags
)

add_custom_target (benchmarks)

macro (create_benchmark benchmark_name)
string (REPLACE "/" "_" binary_name ${benchmark_name})
set (binary_name "${binary_name}_benchmark")
add_executable (${binary_name} EXCLUDE_FROM_ALL "${benchmark_name}_benchmark.cpp")
add_test (${benchmark_name} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${binary_name})
add_dependencies (benchmarks ${binary_name})
endmacro ()

# Benchmarks

create_benchmark (bitset)
create_benchmark (bitflags)
create_benchmark (raw_bitflags)
174 changes: 174 additions & 0 deletions benchmark/src/bitflags_benchmark.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
/*
* Copyright (c) 2020, Marin Peko
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/

#include <benchmark/benchmark.h>
#include <bitflags/bitflags.hpp>

BEGIN_BITFLAGS(Flags)
FLAG(none)
FLAG(flag_a)
FLAG(flag_b)
FLAG(flag_c)
END_BITFLAGS(Flags)

void OperatorNot(benchmark::State& state) {
Flags flags;

for (auto _ : state) {
flags = ~Flags::none;
benchmark::DoNotOptimize(flags);
}
}

BENCHMARK(OperatorNot);

void OperatorAnd(benchmark::State& state) {
Flags flags;

for (auto _ : state) {
flags &= Flags::flag_a;
benchmark::DoNotOptimize(flags);
}
}

BENCHMARK(OperatorAnd);

void OperatorOr(benchmark::State& state) {
Flags flags;

for (auto _ : state) {
flags |= Flags::flag_a;
benchmark::DoNotOptimize(flags);
}
}

BENCHMARK(OperatorOr);

void OperatorXor(benchmark::State& state) {
Flags flags;

for (auto _ : state) {
flags ^= Flags::flag_a;
benchmark::DoNotOptimize(flags);
}
}

BENCHMARK(OperatorXor);

void Empty(benchmark::State& state) {
Flags flags;

for (auto _ : state) {
flags = Flags::empty();
benchmark::DoNotOptimize(flags);
}
}

BENCHMARK(Empty);

void All(benchmark::State& state) {
Flags flags;

for (auto _ : state) {
flags = Flags::all();
benchmark::DoNotOptimize(flags);
}
}

BENCHMARK(All);

void IsEmpty(benchmark::State& state) {
Flags flags;

for (auto _ : state) {
[[maybe_unused]] bool is_empty = flags.is_empty();
benchmark::DoNotOptimize(flags);
benchmark::DoNotOptimize(is_empty);
}
}

BENCHMARK(IsEmpty);

void IsAll(benchmark::State& state) {
Flags flags;

for (auto _ : state) {
[[maybe_unused]] bool is_all = flags.is_all();
benchmark::DoNotOptimize(flags);
benchmark::DoNotOptimize(is_all);
}
}

BENCHMARK(IsAll);

void Set(benchmark::State& state) {
Flags flags;

for (auto _ : state) {
flags.set(Flags::flag_a);
benchmark::DoNotOptimize(flags);
}
}

BENCHMARK(Set);

void Remove(benchmark::State& state) {
Flags flags;

for (auto _ : state) {
flags.remove(Flags::flag_a);
benchmark::DoNotOptimize(flags);
}
}

BENCHMARK(Remove);

void Toggle(benchmark::State& state) {
Flags flags;

for (auto _ : state) {
flags.toggle(Flags::flag_a);
benchmark::DoNotOptimize(flags);
}
}

BENCHMARK(Toggle);

void Clear(benchmark::State& state) {
Flags flags;

for (auto _ : state) {
flags.clear();
benchmark::DoNotOptimize(flags);
}
}

BENCHMARK(Clear);

BENCHMARK_MAIN();
Loading

0 comments on commit 3069839

Please sign in to comment.