From 260bffcff98d250512b2762d87e7b1ab01401181 Mon Sep 17 00:00:00 2001 From: Alexander Bigerl Date: Fri, 15 Jan 2021 13:41:31 +0100 Subject: [PATCH] Dependencies updated and build process cleaned up (#75) * updated dependencie * use clang buildable sparql-parser-base/0.2.1 * all clang Dockerfile * load googletest via FetchContent only if tests are enabled * updated ignore files * added clang format * updated third-party libraries, removed unused; added licenses * updated serd to 30.8 --- .clang-format | 68 + .dockerignore | 165 +- .gitignore | 167 +- CMakeLists.txt | 26 +- Dockerfile | 55 +- conanfile.txt | 10 +- src/exec/TentrisServer.cpp | 3 +- src/exec/TentrisTerminal.cpp | 3 +- src/exec/config/ExecutableConfig.hpp | 4 +- src/exec/config/ServerConfig.hpp | 2 +- src/exec/config/TerminalConfig.hpp | 2 +- src/lib/tentris/store/SPARQL/ParsedSPARQL.hpp | 5 +- tests/CMakeLists.txt | 17 +- .../RapidJSON/include/rapidjson/allocators.h | 2 +- .../include/rapidjson/cursorstreamwrapper.h | 2 +- .../RapidJSON/include/rapidjson/document.h | 29 +- .../include/rapidjson/encodedstream.h | 2 +- .../RapidJSON/include/rapidjson/encodings.h | 2 +- .../RapidJSON/include/rapidjson/error/en.h | 2 +- .../RapidJSON/include/rapidjson/error/error.h | 2 +- .../include/rapidjson/filereadstream.h | 2 +- .../include/rapidjson/filewritestream.h | 2 +- thirdparty/RapidJSON/include/rapidjson/fwd.h | 2 +- .../include/rapidjson/internal/biginteger.h | 2 +- .../include/rapidjson/internal/clzll.h | 2 +- .../include/rapidjson/internal/diyfp.h | 2 +- .../include/rapidjson/internal/dtoa.h | 2 +- .../include/rapidjson/internal/ieee754.h | 2 +- .../include/rapidjson/internal/itoa.h | 2 +- .../include/rapidjson/internal/meta.h | 2 +- .../include/rapidjson/internal/pow10.h | 2 +- .../include/rapidjson/internal/regex.h | 2 +- .../include/rapidjson/internal/stack.h | 2 +- .../include/rapidjson/internal/strfunc.h | 2 +- .../include/rapidjson/internal/strtod.h | 2 +- .../include/rapidjson/internal/swap.h | 2 +- .../include/rapidjson/istreamwrapper.h | 2 +- .../include/rapidjson/memorybuffer.h | 2 +- .../include/rapidjson/memorystream.h | 2 +- .../include/rapidjson/ostreamwrapper.h | 2 +- .../RapidJSON/include/rapidjson/pointer.h | 2 +- .../include/rapidjson/prettywriter.h | 2 +- .../RapidJSON/include/rapidjson/rapidjson.h | 2 +- .../RapidJSON/include/rapidjson/reader.h | 2 +- .../RapidJSON/include/rapidjson/stream.h | 2 +- .../include/rapidjson/stringbuffer.h | 2 +- .../RapidJSON/include/rapidjson/writer.h | 2 +- thirdparty/RapidJSON/license.txt | 57 + thirdparty/{cxxopts => csv-parser}/LICENSE | 12 +- thirdparty/csv-parser/csv.hpp | 1508 +-- thirdparty/cxxopts/.gitignore | 9 - thirdparty/cxxopts/.travis.yml | 70 - thirdparty/cxxopts/BUILD | 8 - thirdparty/cxxopts/CHANGELOG.md | 98 - thirdparty/cxxopts/CMakeLists.txt | 109 - thirdparty/cxxopts/INSTALL | 23 - thirdparty/cxxopts/README.md | 224 - thirdparty/cxxopts/WORKSPACE | 0 thirdparty/cxxopts/cxxopts-config.cmake.in | 4 - thirdparty/cxxopts/include/cxxopts.hpp | 2228 ---- thirdparty/cxxopts/src/.gitignore | 1 - thirdparty/cxxopts/src/CMakeLists.txt | 24 - thirdparty/cxxopts/src/example.cpp | 168 - thirdparty/cxxopts/test/.gitignore | 1 - thirdparty/cxxopts/test/CMakeLists.txt | 35 - .../test/add-subdirectory-test/CMakeLists.txt | 11 - thirdparty/cxxopts/test/catch.hpp | 10460 ---------------- .../test/find-package-test/CMakeLists.txt | 11 - thirdparty/cxxopts/test/link_a.cpp | 6 - thirdparty/cxxopts/test/link_b.cpp | 1 - thirdparty/cxxopts/test/main.cpp | 2 - thirdparty/cxxopts/test/options.cpp | 776 -- 72 files changed, 1347 insertions(+), 15121 deletions(-) create mode 100644 .clang-format create mode 100644 thirdparty/RapidJSON/license.txt rename thirdparty/{cxxopts => csv-parser}/LICENSE (87%) delete mode 100644 thirdparty/cxxopts/.gitignore delete mode 100644 thirdparty/cxxopts/.travis.yml delete mode 100644 thirdparty/cxxopts/BUILD delete mode 100644 thirdparty/cxxopts/CHANGELOG.md delete mode 100644 thirdparty/cxxopts/CMakeLists.txt delete mode 100644 thirdparty/cxxopts/INSTALL delete mode 100644 thirdparty/cxxopts/README.md delete mode 100644 thirdparty/cxxopts/WORKSPACE delete mode 100644 thirdparty/cxxopts/cxxopts-config.cmake.in delete mode 100644 thirdparty/cxxopts/include/cxxopts.hpp delete mode 100644 thirdparty/cxxopts/src/.gitignore delete mode 100644 thirdparty/cxxopts/src/CMakeLists.txt delete mode 100644 thirdparty/cxxopts/src/example.cpp delete mode 100644 thirdparty/cxxopts/test/.gitignore delete mode 100644 thirdparty/cxxopts/test/CMakeLists.txt delete mode 100644 thirdparty/cxxopts/test/add-subdirectory-test/CMakeLists.txt delete mode 100644 thirdparty/cxxopts/test/catch.hpp delete mode 100644 thirdparty/cxxopts/test/find-package-test/CMakeLists.txt delete mode 100644 thirdparty/cxxopts/test/link_a.cpp delete mode 100644 thirdparty/cxxopts/test/link_b.cpp delete mode 100644 thirdparty/cxxopts/test/main.cpp delete mode 100644 thirdparty/cxxopts/test/options.cpp diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..4ae554ea --- /dev/null +++ b/.clang-format @@ -0,0 +1,68 @@ +# Generated from CLion C/C++ Code Style settings +BasedOnStyle: LLVM +Language: Cpp +Standard: c++20 +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignOperands: true +AllowAllArgumentsOnNextLine: false +AllowAllConstructorInitializersOnNextLine: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: Always +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Always +AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: true +AlwaysBreakAfterReturnType: None +AlwaysBreakTemplateDeclarations: Yes +BreakBeforeBraces: Custom +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: Never + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: false + SplitEmptyRecord: true +BreakBeforeBinaryOperators: None +BreakBeforeTernaryOperators: true +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +ColumnLimit: 0 +CompactNamespaces: false +ContinuationIndentWidth: 8 +IndentCaseLabels: true +IndentPPDirectives: None +IndentWidth: 4 +KeepEmptyLinesAtTheStartOfBlocks: true +MaxEmptyLinesToKeep: 2 +NamespaceIndentation: All +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PointerAlignment: Right +ReflowComments: false +SpaceAfterCStyleCast: true +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 0 +SpacesInAngles: false +SpacesInCStyleCastParentheses: false +SpacesInContainerLiterals: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +TabWidth: 4 +UseTab: ForContinuationAndIndentation diff --git a/.dockerignore b/.dockerignore index 7caf5d04..dbbc0138 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,165 @@ +## project specific # log files tentris_* -# cmake build folders -./cmake* \ No newline at end of file + +# Created by https://www.toptal.com/developers/gitignore/api/c++,conan,jetbrains+all,cmake +# Edit at https://www.toptal.com/developers/gitignore?templates=c++,conan,jetbrains+all,cmake + +### C++ ### +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Linker files +*.ilk + +# Debugger Files +*.pdb + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +### CMake ### +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps +CMakeUserPresets.json + +### CMake Patch ### +# External projects +*-prefix/ + +### Conan ### +# Conan build information +conan.lock +conanbuildinfo.* +conaninfo.txt +graph_info.json + +### JetBrains+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### JetBrains+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + +.idea/ + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +# Sonarlint plugin +.idea/sonarlint + +# End of https://www.toptal.com/developers/gitignore/api/c++,conan,jetbrains+all,cmake + +# docu folder +/docu/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index cbedc289..dbbc0138 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,165 @@ -build/ -cmake-* +## project specific +# log files +tentris_* + +# Created by https://www.toptal.com/developers/gitignore/api/c++,conan,jetbrains+all,cmake +# Edit at https://www.toptal.com/developers/gitignore?templates=c++,conan,jetbrains+all,cmake + +### C++ ### +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Linker files +*.ilk + +# Debugger Files +*.pdb + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +### CMake ### +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps +CMakeUserPresets.json + +### CMake Patch ### +# External projects +*-prefix/ + +### Conan ### +# Conan build information +conan.lock +conanbuildinfo.* +conaninfo.txt +graph_info.json + +### JetBrains+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### JetBrains+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + .idea/ -tests/googletest \ No newline at end of file + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +# Sonarlint plugin +.idea/sonarlint + +# End of https://www.toptal.com/developers/gitignore/api/c++,conan,jetbrains+all,cmake + +# docu folder +/docu/ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 53ce17db..5c048c84 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,9 +13,7 @@ if (NOT EXISTS ${CMAKE_BINARY_DIR}/CMakeCache.txt) endif () endif () -if(NOT DEFINED ${TENTRIS_MARCH}) - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -march=x86-64") -else() +if(DEFINED ${TENTRIS_MARCH}) set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -march=${TENTRIS_MARCH}") endif() @@ -32,15 +30,19 @@ if (TENTRIS_BUILD_WITH_TCMALLOC) endif() set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -g -O0") - - # Lightweight C++ command line option parser https://github.com/jarro2783/cxxopts -add_library(cxxopts INTERFACE) -target_include_directories(cxxopts INTERFACE - thirdparty/cxxopts/include - ) - - +include(FetchContent) +FetchContent_Declare( + cxxopts + GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git + GIT_TAG v2.2.1 + GIT_SHALLOW TRUE +) +set(CXXOPTS_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +set(CXXOPTS_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(CXXOPTS_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) +set(CXXOPTS_ENABLE_WARNINGS OFF CACHE BOOL "" FORCE) +FetchContent_MakeAvailable(cxxopts) add_library(csv-parser INTERFACE) target_include_directories(csv-parser INTERFACE @@ -53,7 +55,6 @@ target_include_directories(rapidjson INTERFACE thirdparty/RapidJSON/include ) -find_package(absl REQUIRED) find_package(tsl-hopscotch-map REQUIRED) find_package(fmt REQUIRED) find_package(hypertrie REQUIRED) @@ -80,7 +81,6 @@ target_link_libraries(tentris cxxopts rapidjson sparql-parser-base::sparql-parser-base - absl::absl tsl-hopscotch-map::tsl-hopscotch-map fmt::fmt hypertrie::hypertrie diff --git a/Dockerfile b/Dockerfile index 85eeed64..66fc1ffb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,29 +6,24 @@ ARG TENTRIS_MARCH="x86-64" RUN apt-get -qq update && \ apt-get -qq install -y make cmake uuid-dev git openjdk-11-jdk python3-pip python3-setuptools python3-wheel libstdc++-10-dev clang-11 g++-10 pkg-config google-perftools libgoogle-perftools-dev + +ARG CXX="clang++-11" +ARG CC="clang-11" # we need serd as static library. Not available from ubuntu repos -RUN ln -s /usr/bin/python3 /usr/bin/python && \ - git clone --quiet --branch v0.30.2 https://gitlab.com/drobilla/serd.git && \ - cd serd && \ - git submodule update --quiet --init --recursive && \ +RUN ln -s /usr/bin/python3 /usr/bin/python +RUN git clone --quiet --branch v0.30.8 https://gitlab.com/drobilla/serd.git +WORKDIR serd +RUN git submodule update --quiet --init --recursive && \ ./waf configure --static && \ - ./waf install &&\ - rm /usr/bin/python + ./waf install +RUN rm /usr/bin/python +WORKDIR / # install and configure conan RUN pip3 install conan && \ conan user && \ conan profile new --detect default && \ - conan profile update settings.sparql-parser-base:compiler.version=10 default && \ - conan profile update settings.sparql-parser-base:compiler.libcxx=libstdc++11 default && \ - conan profile update settings.sparql-parser-base:compiler=gcc default &&\ - conan profile update env.sparql-parser-base:CXX=/usr/bin/g++-10 default && \ - conan profile update env.sparql-parser-base:CC=/usr/bin/gcc-10 default && \ - conan profile update settings.compiler=clang default &&\ - conan profile update settings.compiler.version=11 default && \ - conan profile update settings.compiler.libcxx=libstdc++11 default && \ - conan profile update env.CXX=/usr/bin/clang++-11 default && \ - conan profile update env.CC=/usr/bin/clang-11 default + conan profile update settings.compiler.libcxx=libstdc++11 default # add conan repositories RUN conan remote add tsl https://api.bintray.com/conan/tessil/tsl @@ -37,24 +32,22 @@ RUN conan remote add stiffstream https://api.bintray.com/conan/stiffstream/publi RUN conan remote add dice-group https://api.bintray.com/conan/dice-group/tentris # build and cache dependencies via conan -COPY conanfile.txt /conan_cache/conanfile.txt -RUN cd /conan_cache && conan install . --build=missing --profile default > conan_build.log +WORKDIR /conan_cache +COPY conanfile.txt conanfile.txt +RUN conan install . --build=missing --profile default > conan_build.log # import project files -COPY thirdparty /tentris/thirdparty/ -COPY src /tentris/src/ -COPY CMakeLists.txt /tentris/CMakeLists.txt -COPY conanfile.txt /tentris/conanfile.txt +WORKDIR /tentris +COPY thirdparty thirdparty +COPY src src +COPY CMakeLists.txt CMakeLists.txt +COPY conanfile.txt conanfile.txt ##build -# import and build depenedencies via conan -RUN mkdir /tentris/build && cd /tentris/build && \ - conan install .. --build=missing -# build tentris_server with clang++-11 again -RUN export CXX="clang++-11" && export CC="clang-11" && \ - cd /tentris/build && \ - cmake -DCMAKE_BUILD_TYPE=Release -DTENTRIS_BUILD_WITH_TCMALLOC=true -DTENTRIS_STATIC=true .. && \ - make -j $(nproc) +WORKDIR /tentris/build +RUN conan install .. --build=missing +RUN cmake -DCMAKE_BUILD_TYPE=Release -DTENTRIS_BUILD_WITH_TCMALLOC=true -DTENTRIS_STATIC=true .. +RUN make -j $(nproc) FROM scratch WORKDIR /tentris @@ -62,4 +55,6 @@ COPY --from=builder /tentris/build/tentris_server /tentris_server COPY --from=builder /tentris/build/tentris_terminal /tentris_terminal COPY --from=builder /tentris/build/ids2hypertrie /ids2hypertrie COPY --from=builder /tentris/build/rdf2ids /rdf2ids +COPY LICENSE LICENSE +COPY README.MD README.MD ENTRYPOINT ["/tentris_server"] diff --git a/conanfile.txt b/conanfile.txt index fbb16ece..1f322e3c 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -1,18 +1,18 @@ [requires] -boost/1.74.0 -fmt/7.1.2 -gtest/1.8.1 -abseil/20200225.2 +boost/1.75.0 +fmt/7.1.3 +abseil/20200225.2 # remove when rdf-parser is updated restinio/0.6.12 hypertrie/0.6.0-rc13@dice-group/stable rdf-parser/0.10@dice-group/stable -sparql-parser-base/0.1.0@dice-group/stable +sparql-parser-base/0.2.1@dice-group/stable [options] restinio:asio=boost restinio:with_zlib=True boost:shared=False *:shared=False +sparql-parser-base:sparql_version=1.0 [generators] diff --git a/src/exec/TentrisServer.cpp b/src/exec/TentrisServer.cpp index c02076bc..ee786863 100644 --- a/src/exec/TentrisServer.cpp +++ b/src/exec/TentrisServer.cpp @@ -53,8 +53,7 @@ int main(int argc, char *argv[]) { using namespace fmt::literals; using namespace tentris::logging; - auto const_argv = const_cast(argv); - ServerConfig cfg{argc, const_argv}; + ServerConfig cfg{argc, argv}; init_logging(cfg.logstdout, cfg.logfile, cfg.logfiledir, cfg.loglevel); diff --git a/src/exec/TentrisTerminal.cpp b/src/exec/TentrisTerminal.cpp index 2be26962..45c1115f 100644 --- a/src/exec/TentrisTerminal.cpp +++ b/src/exec/TentrisTerminal.cpp @@ -239,8 +239,7 @@ void commandlineInterface(QueryExecutionPackage_cache &querypackage_cache) { int main(int argc, char *argv[]) { - auto const_argv = const_cast(argv); - cfg = TerminalConfig{argc, const_argv}; + cfg = TerminalConfig{argc, argv}; tentris::logging::init_logging(cfg.logstdout, cfg.logfile, cfg.logfiledir, cfg.loglevel); TripleStore triplestore{}; diff --git a/src/exec/config/ExecutableConfig.hpp b/src/exec/config/ExecutableConfig.hpp index 1906d2df..5a4d9a26 100644 --- a/src/exec/config/ExecutableConfig.hpp +++ b/src/exec/config/ExecutableConfig.hpp @@ -82,7 +82,7 @@ struct ExecutableConfig { public: - ExecutableConfig(int argc, const char ** &argv) : ExecutableConfig{} { + ExecutableConfig(int argc, char ** &argv) : ExecutableConfig{} { initConfig(argc, argv); } @@ -92,7 +92,7 @@ struct ExecutableConfig { * @param argv array of char arrays with arguments */ - void initConfig(int argc, const char **&argv) { + void initConfig(int argc, char **&argv) { try { cxxopts::ParseResult arguments = options.parse(argc, argv); parseArguments(arguments); diff --git a/src/exec/config/ServerConfig.hpp b/src/exec/config/ServerConfig.hpp index 12d71492..1a9df00d 100644 --- a/src/exec/config/ServerConfig.hpp +++ b/src/exec/config/ServerConfig.hpp @@ -28,7 +28,7 @@ struct ServerConfig : public ExecutableConfig { cxxopts::value()->default_value("{}"_format(std::thread::hardware_concurrency()))); } - ServerConfig(int argc, const char **&argv) : ServerConfig{} { + ServerConfig(int argc, char **&argv) : ServerConfig{} { initConfig(argc, argv); } diff --git a/src/exec/config/TerminalConfig.hpp b/src/exec/config/TerminalConfig.hpp index df7e0aea..8be3b66d 100644 --- a/src/exec/config/TerminalConfig.hpp +++ b/src/exec/config/TerminalConfig.hpp @@ -19,7 +19,7 @@ struct TerminalConfig : public ExecutableConfig { } - TerminalConfig(int argc, const char **&argv) : TerminalConfig{} { + TerminalConfig(int argc, char **&argv) : TerminalConfig{} { initConfig(argc, argv); } diff --git a/src/lib/tentris/store/SPARQL/ParsedSPARQL.hpp b/src/lib/tentris/store/SPARQL/ParsedSPARQL.hpp index 294a2cbf..0dd10a0e 100644 --- a/src/lib/tentris/store/SPARQL/ParsedSPARQL.hpp +++ b/src/lib/tentris/store/SPARQL/ParsedSPARQL.hpp @@ -31,7 +31,8 @@ namespace tentris::store::sparql { namespace { using Subscript = einsum::internal::Subscript; - using SparqlParser =Dice::tentris::sparql::parser::SparqlParser; + namespace parser = Dice::sparql_parser::base; + using SparqlParser = parser::SparqlParser; using namespace fmt::literals; } @@ -72,7 +73,7 @@ namespace tentris::store::sparql { class ParsedSPARQL { - using SparqlLexer = Dice::tentris::sparql::parser::SparqlLexer; + using SparqlLexer = parser::SparqlLexer; using ANTLRInputStream =antlr4::ANTLRInputStream; using CommonTokenStream = antlr4::CommonTokenStream; using QueryContext = SparqlParser::QueryContext; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 46d7711a..45673ba3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,13 +1,24 @@ cmake_minimum_required(VERSION 3.13) # since 3.10 gtest_discover_tests is included set(CMAKE_CXX_STANDARD 20) +include(FetchContent) +FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG release-1.8.1 + GIT_SHALLOW TRUE +) +set(BUILD_GMOCK OFF CACHE BOOL "" FORCE) +set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) +FetchContent_MakeAvailable(googletest) +find_package(GTest REQUIRED) include(GoogleTest) - -# add the exectutable for all tests +# add the executable for all tests add_executable(tests Tests.cpp) target_link_libraries(tests - ${CONAN_LIBS} + GTest::GTest + GTest::Main tentris ) diff --git a/thirdparty/RapidJSON/include/rapidjson/allocators.h b/thirdparty/RapidJSON/include/rapidjson/allocators.h index 0b8f5e14..44ec5295 100644 --- a/thirdparty/RapidJSON/include/rapidjson/allocators.h +++ b/thirdparty/RapidJSON/include/rapidjson/allocators.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/cursorstreamwrapper.h b/thirdparty/RapidJSON/include/rapidjson/cursorstreamwrapper.h index 52c11a7c..fd6513db 100644 --- a/thirdparty/RapidJSON/include/rapidjson/cursorstreamwrapper.h +++ b/thirdparty/RapidJSON/include/rapidjson/cursorstreamwrapper.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/document.h b/thirdparty/RapidJSON/include/rapidjson/document.h index 68aaae7e..028235ec 100644 --- a/thirdparty/RapidJSON/include/rapidjson/document.h +++ b/thirdparty/RapidJSON/include/rapidjson/document.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -289,12 +289,14 @@ class GenericMemberIterator; //! non-const GenericMemberIterator template class GenericMemberIterator { +public: //! use plain pointer as iterator type typedef GenericMember* Iterator; }; //! const GenericMemberIterator template class GenericMemberIterator { +public: //! use plain const pointer as iterator type typedef const GenericMember* Iterator; }; @@ -2001,17 +2003,18 @@ class GenericValue { // Initial flags of different types. kNullFlag = kNullType, - kTrueFlag = kTrueType | kBoolFlag, - kFalseFlag = kFalseType | kBoolFlag, - kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, - kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, - kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, - kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, - kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, - kNumberAnyFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag, - kConstStringFlag = kStringType | kStringFlag, - kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, - kShortStringFlag = kStringType | kStringFlag | kCopyFlag | kInlineStrFlag, + // These casts are added to suppress the warning on MSVC about bitwise operations between enums of different types. + kTrueFlag = static_cast(kTrueType) | static_cast(kBoolFlag), + kFalseFlag = static_cast(kFalseType) | static_cast(kBoolFlag), + kNumberIntFlag = static_cast(kNumberType) | static_cast(kNumberFlag | kIntFlag | kInt64Flag), + kNumberUintFlag = static_cast(kNumberType) | static_cast(kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag), + kNumberInt64Flag = static_cast(kNumberType) | static_cast(kNumberFlag | kInt64Flag), + kNumberUint64Flag = static_cast(kNumberType) | static_cast(kNumberFlag | kUint64Flag), + kNumberDoubleFlag = static_cast(kNumberType) | static_cast(kNumberFlag | kDoubleFlag), + kNumberAnyFlag = static_cast(kNumberType) | static_cast(kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag), + kConstStringFlag = static_cast(kStringType) | static_cast(kStringFlag), + kCopyStringFlag = static_cast(kStringType) | static_cast(kStringFlag | kCopyFlag), + kShortStringFlag = static_cast(kStringType) | static_cast(kStringFlag | kCopyFlag | kInlineStrFlag), kObjectFlag = kObjectType, kArrayFlag = kArrayType, @@ -2609,6 +2612,7 @@ class GenericArray { GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } ~GenericArray() {} + operator ValueType&() const { return value_; } SizeType Size() const { return value_.Size(); } SizeType Capacity() const { return value_.Capacity(); } bool Empty() const { return value_.Empty(); } @@ -2664,6 +2668,7 @@ class GenericObject { GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } ~GenericObject() {} + operator ValueType&() const { return value_; } SizeType MemberCount() const { return value_.MemberCount(); } SizeType MemberCapacity() const { return value_.MemberCapacity(); } bool ObjectEmpty() const { return value_.ObjectEmpty(); } diff --git a/thirdparty/RapidJSON/include/rapidjson/encodedstream.h b/thirdparty/RapidJSON/include/rapidjson/encodedstream.h index 223601c0..cf046b89 100644 --- a/thirdparty/RapidJSON/include/rapidjson/encodedstream.h +++ b/thirdparty/RapidJSON/include/rapidjson/encodedstream.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/encodings.h b/thirdparty/RapidJSON/include/rapidjson/encodings.h index 0b244679..50ad18bd 100644 --- a/thirdparty/RapidJSON/include/rapidjson/encodings.h +++ b/thirdparty/RapidJSON/include/rapidjson/encodings.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/error/en.h b/thirdparty/RapidJSON/include/rapidjson/error/en.h index 2db838bf..37a62ebc 100644 --- a/thirdparty/RapidJSON/include/rapidjson/error/en.h +++ b/thirdparty/RapidJSON/include/rapidjson/error/en.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/error/error.h b/thirdparty/RapidJSON/include/rapidjson/error/error.h index 9311d2f0..71f6ec4d 100644 --- a/thirdparty/RapidJSON/include/rapidjson/error/error.h +++ b/thirdparty/RapidJSON/include/rapidjson/error/error.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/filereadstream.h b/thirdparty/RapidJSON/include/rapidjson/filereadstream.h index 6b343707..f8bb43cb 100644 --- a/thirdparty/RapidJSON/include/rapidjson/filereadstream.h +++ b/thirdparty/RapidJSON/include/rapidjson/filereadstream.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/filewritestream.h b/thirdparty/RapidJSON/include/rapidjson/filewritestream.h index 8b48fee1..5d89588c 100644 --- a/thirdparty/RapidJSON/include/rapidjson/filewritestream.h +++ b/thirdparty/RapidJSON/include/rapidjson/filewritestream.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/fwd.h b/thirdparty/RapidJSON/include/rapidjson/fwd.h index b74a2b81..d62f77f0 100644 --- a/thirdparty/RapidJSON/include/rapidjson/fwd.h +++ b/thirdparty/RapidJSON/include/rapidjson/fwd.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/biginteger.h b/thirdparty/RapidJSON/include/rapidjson/internal/biginteger.h index 8eb87c7c..12455788 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/biginteger.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/biginteger.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/clzll.h b/thirdparty/RapidJSON/include/rapidjson/internal/clzll.h index 47bb7ab1..8fc5118a 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/clzll.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/clzll.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/diyfp.h b/thirdparty/RapidJSON/include/rapidjson/internal/diyfp.h index 8f7d853a..a40797ec 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/diyfp.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/diyfp.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/dtoa.h b/thirdparty/RapidJSON/include/rapidjson/internal/dtoa.h index bf2e9b2e..621402fd 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/dtoa.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/dtoa.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/ieee754.h b/thirdparty/RapidJSON/include/rapidjson/internal/ieee754.h index c2684ba2..68c9e966 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/ieee754.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/ieee754.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/itoa.h b/thirdparty/RapidJSON/include/rapidjson/internal/itoa.h index 9b1c45cc..9fe8c932 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/itoa.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/itoa.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/meta.h b/thirdparty/RapidJSON/include/rapidjson/internal/meta.h index d401edf8..27092dc0 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/meta.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/meta.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/pow10.h b/thirdparty/RapidJSON/include/rapidjson/internal/pow10.h index 02f475d7..eae1a43e 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/pow10.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/pow10.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/regex.h b/thirdparty/RapidJSON/include/rapidjson/internal/regex.h index af7e06de..6446c403 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/regex.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/regex.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/stack.h b/thirdparty/RapidJSON/include/rapidjson/internal/stack.h index 45dca6a8..73abd706 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/stack.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/stack.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/strfunc.h b/thirdparty/RapidJSON/include/rapidjson/internal/strfunc.h index 226439a7..baecb6cc 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/strfunc.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/strfunc.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/strtod.h b/thirdparty/RapidJSON/include/rapidjson/internal/strtod.h index dfca22b6..d61a67a4 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/strtod.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/strtod.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/internal/swap.h b/thirdparty/RapidJSON/include/rapidjson/internal/swap.h index 666e49f9..2cf92f93 100644 --- a/thirdparty/RapidJSON/include/rapidjson/internal/swap.h +++ b/thirdparty/RapidJSON/include/rapidjson/internal/swap.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/istreamwrapper.h b/thirdparty/RapidJSON/include/rapidjson/istreamwrapper.h index c4950b9d..01437ec0 100644 --- a/thirdparty/RapidJSON/include/rapidjson/istreamwrapper.h +++ b/thirdparty/RapidJSON/include/rapidjson/istreamwrapper.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/memorybuffer.h b/thirdparty/RapidJSON/include/rapidjson/memorybuffer.h index 39bee1de..ffbc41ed 100644 --- a/thirdparty/RapidJSON/include/rapidjson/memorybuffer.h +++ b/thirdparty/RapidJSON/include/rapidjson/memorybuffer.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/memorystream.h b/thirdparty/RapidJSON/include/rapidjson/memorystream.h index 1d71d8a4..77af6c99 100644 --- a/thirdparty/RapidJSON/include/rapidjson/memorystream.h +++ b/thirdparty/RapidJSON/include/rapidjson/memorystream.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/ostreamwrapper.h b/thirdparty/RapidJSON/include/rapidjson/ostreamwrapper.h index 6f4667c0..11ed4d33 100644 --- a/thirdparty/RapidJSON/include/rapidjson/ostreamwrapper.h +++ b/thirdparty/RapidJSON/include/rapidjson/ostreamwrapper.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/pointer.h b/thirdparty/RapidJSON/include/rapidjson/pointer.h index b8143b63..90e5903b 100644 --- a/thirdparty/RapidJSON/include/rapidjson/pointer.h +++ b/thirdparty/RapidJSON/include/rapidjson/pointer.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/prettywriter.h b/thirdparty/RapidJSON/include/rapidjson/prettywriter.h index 94eeb69d..fe45df1d 100644 --- a/thirdparty/RapidJSON/include/rapidjson/prettywriter.h +++ b/thirdparty/RapidJSON/include/rapidjson/prettywriter.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/rapidjson.h b/thirdparty/RapidJSON/include/rapidjson/rapidjson.h index c5f4d65f..78aa89a0 100644 --- a/thirdparty/RapidJSON/include/rapidjson/rapidjson.h +++ b/thirdparty/RapidJSON/include/rapidjson/rapidjson.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/reader.h b/thirdparty/RapidJSON/include/rapidjson/reader.h index 30e45e1f..09ace4eb 100644 --- a/thirdparty/RapidJSON/include/rapidjson/reader.h +++ b/thirdparty/RapidJSON/include/rapidjson/reader.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/stream.h b/thirdparty/RapidJSON/include/rapidjson/stream.h index 7f2643e4..1fd70915 100644 --- a/thirdparty/RapidJSON/include/rapidjson/stream.h +++ b/thirdparty/RapidJSON/include/rapidjson/stream.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/stringbuffer.h b/thirdparty/RapidJSON/include/rapidjson/stringbuffer.h index 4e38b82c..82ad3ca6 100644 --- a/thirdparty/RapidJSON/include/rapidjson/stringbuffer.h +++ b/thirdparty/RapidJSON/include/rapidjson/stringbuffer.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/include/rapidjson/writer.h b/thirdparty/RapidJSON/include/rapidjson/writer.h index 51dd86d5..8b389219 100644 --- a/thirdparty/RapidJSON/include/rapidjson/writer.h +++ b/thirdparty/RapidJSON/include/rapidjson/writer.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/thirdparty/RapidJSON/license.txt b/thirdparty/RapidJSON/license.txt new file mode 100644 index 00000000..995dca48 --- /dev/null +++ b/thirdparty/RapidJSON/license.txt @@ -0,0 +1,57 @@ +Tencent is pleased to support the open source community by making RapidJSON available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. + +If you have downloaded a copy of the RapidJSON binary from Tencent, please note that the RapidJSON binary is licensed under the MIT License. +If you have downloaded a copy of the RapidJSON source code from Tencent, please note that RapidJSON source code is licensed under the MIT License, except for the third-party components listed below which are subject to different license terms. Your integration of RapidJSON into your own projects may require compliance with the MIT License, as well as the other licenses applicable to the third-party components included within RapidJSON. To avoid the problematic JSON license in your own projects, it's sufficient to exclude the bin/jsonchecker/ directory, as it's the only code under the JSON license. +A copy of the MIT License is included in this file. + +Other dependencies and licenses: + +Open Source Software Licensed Under the BSD License: +-------------------------------------------------------------------- + +The msinttypes r29 +Copyright (c) 2006-2013 Alexander Chemeris +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +* Neither the name of copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Open Source Software Licensed Under the JSON License: +-------------------------------------------------------------------- + +json.org +Copyright (c) 2002 JSON.org +All Rights Reserved. + +JSON_checker +Copyright (c) 2002 JSON.org +All Rights Reserved. + + +Terms of the JSON License: +--------------------------------------------------- + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Terms of the MIT License: +-------------------------------------------------------------------- + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/thirdparty/cxxopts/LICENSE b/thirdparty/csv-parser/LICENSE similarity index 87% rename from thirdparty/cxxopts/LICENSE rename to thirdparty/csv-parser/LICENSE index 324a2035..da835973 100644 --- a/thirdparty/cxxopts/LICENSE +++ b/thirdparty/csv-parser/LICENSE @@ -1,4 +1,6 @@ -Copyright (c) 2014 Jarryd Beck +MIT License + +Copyright (c) 2017-2019 Vincent La Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -7,13 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/thirdparty/csv-parser/csv.hpp b/thirdparty/csv-parser/csv.hpp index 12604e02..a3d164ea 100644 --- a/thirdparty/csv-parser/csv.hpp +++ b/thirdparty/csv-parser/csv.hpp @@ -6556,7 +6556,7 @@ namespace csv { size_t decimal = (size_t)(((double)std::abs(value) - (double)integral) * 100000); result += "."; - result += (decimal == 0) ? "0" : to_string(integral); + result += (decimal == 0) ? "0" : to_string(decimal); return result; } @@ -6574,6 +6574,9 @@ namespace csv { * @tparam OutputStream The output stream, e.g. `std::ofstream`, `std::stringstream` * @tparam Delim The delimiter character * @tparam Quote The quote character + * @tparam Flush True: flush after every writing function, + * false: you need to flush explicitly if needed. + * In both cases the destructor will flush. * * @par Hint * Use the aliases csv::CSVWriter to write CSV @@ -6586,7 +6589,7 @@ namespace csv { * @par Example w/ std::tuple * @snippet test_write_csv.cpp CSV Writer Tuple Example */ - template + template class DelimWriter { public: /** Construct a DelimWriter over the specified output stream @@ -6603,6 +6606,13 @@ namespace csv { */ DelimWriter(const std::string& filename) : DelimWriter(std::ifstream(filename)) {}; + /** Destructor will flush remaining data + * + */ + ~DelimWriter() { + out.flush(); + } + /** Format a sequence of strings and write to CSV according to RFC 4180 * * @warning This does not check to make sure row lengths are consistent @@ -6618,7 +6628,7 @@ namespace csv { if (i + 1 != Size) out << Delim; } - out << std::endl; + end_out(); return *this; } @@ -6649,10 +6659,17 @@ namespace csv { i++; } - out << std::endl; + end_out(); return *this; } + /** Flushes the written data + * + */ + void flush() { + out.flush(); + } + private: template< typename T, @@ -6732,7 +6749,13 @@ namespace csv { template typename std::enable_if::type write_tuple(const std::tuple& record) { (void)record; - out << std::endl; + end_out(); + } + + /** Ends a line in 'out' and flushes, if Flush is true.*/ + void end_out() { + out << '\n'; + IF_CONSTEXPR(Flush) out.flush(); } OutputStream & out; @@ -6746,19 +6769,19 @@ namespace csv { * @note Use `csv::make_csv_writer()` to in instatiate this class over * an actual output stream. */ - template - using CSVWriter = DelimWriter; + template + using CSVWriter = DelimWriter; /** Class for writing tab-separated values files -* + * * @sa csv::DelimWriter::write_row() * @sa csv::DelimWriter::operator<<() * * @note Use `csv::make_tsv_writer()` to in instatiate this class over * an actual output stream. */ - template - using TSVWriter = DelimWriter; + template + using TSVWriter = DelimWriter; /** Return a csv::CSVWriter over the output stream */ template @@ -6766,282 +6789,184 @@ namespace csv { return CSVWriter(out, quote_minimal); } + /** Return a buffered csv::CSVWriter over the output stream (does not auto flush) */ + template + inline CSVWriter make_csv_writer_buffered(OutputStream& out, bool quote_minimal=true) { + return CSVWriter(out, quote_minimal); + } + /** Return a csv::TSVWriter over the output stream */ template inline TSVWriter make_tsv_writer(OutputStream& out, bool quote_minimal=true) { return TSVWriter(out, quote_minimal); } + + /** Return a buffered csv::TSVWriter over the output stream (does not auto flush) */ + template + inline TSVWriter make_tsv_writer_buffered(OutputStream& out, bool quote_minimal=true) { + return TSVWriter(out, quote_minimal); + } ///@} } +/** @file + * Defines an input iterator for csv::CSVReader + */ + namespace csv { - namespace internals { - CSV_INLINE size_t get_file_size(csv::string_view filename) { - std::ifstream infile(std::string(filename), std::ios::binary); - const auto start = infile.tellg(); - infile.seekg(0, std::ios::end); - const auto end = infile.tellg(); + /** Return an iterator to the first row in the reader */ + CSV_INLINE CSVReader::iterator CSVReader::begin() { + if (this->records.empty()) { + this->read_csv_worker = std::thread(&CSVReader::read_csv, this, internals::ITERATION_CHUNK_SIZE); + this->read_csv_worker.join(); - return end - start; + // Still empty => return end iterator + if (this->records.empty()) return this->end(); } - CSV_INLINE std::string get_csv_head(csv::string_view filename) { - return get_csv_head(filename, get_file_size(filename)); - } + CSVReader::iterator ret(this, std::move(this->records.pop_front())); + return ret; + } - CSV_INLINE std::string get_csv_head(csv::string_view filename, size_t file_size) { - const size_t bytes = 500000; + /** A placeholder for the imaginary past the end row in a CSV. + * Attempting to deference this will lead to bad things. + */ + CSV_INLINE HEDLEY_CONST CSVReader::iterator CSVReader::end() const noexcept { + return CSVReader::iterator(); + } - std::error_code error; - size_t length = std::min((size_t)file_size, bytes); - auto mmap = mio::make_mmap_source(std::string(filename), 0, length, error); + ///////////////////////// + // CSVReader::iterator // + ///////////////////////// - if (error) { - throw std::runtime_error("Cannot open file " + std::string(filename)); - } + CSV_INLINE CSVReader::iterator::iterator(CSVReader* _daddy, CSVRow&& _row) : + daddy(_daddy) { + row = std::move(_row); + } - return std::string(mmap.begin(), mmap.end()); + /** Advance the iterator by one row. If this CSVReader has an + * associated file, then the iterator will lazily pull more data from + * that file until the end of file is reached. + * + * @note This iterator does **not** block the thread responsible for parsing CSV. + * + */ + CSV_INLINE CSVReader::iterator& CSVReader::iterator::operator++() { + if (!daddy->read_row(this->row)) { + this->daddy = nullptr; // this == end() } -#ifdef _MSC_VER -#pragma region IBasicCVParser -#endif - CSV_INLINE IBasicCSVParser::IBasicCSVParser( - const CSVFormat& format, - const ColNamesPtr& col_names - ) : _col_names(col_names) { - if (format.no_quote) { - _parse_flags = internals::make_parse_flags(format.get_delim()); - } - else { - _parse_flags = internals::make_parse_flags(format.get_delim(), format.quote_char); - } + return *this; + } - _ws_flags = internals::make_ws_flags( - format.trim_chars.data(), format.trim_chars.size() - ); + /** Post-increment iterator */ + CSV_INLINE CSVReader::iterator CSVReader::iterator::operator++(int) { + auto temp = *this; + if (!daddy->read_row(this->row)) { + this->daddy = nullptr; // this == end() } - CSV_INLINE void IBasicCSVParser::end_feed() { - using internals::ParseFlags; + return temp; + } +} +/** @file + * Defines an object used to store CSV format settings + */ - bool empty_last_field = this->data_ptr - && this->data_ptr->_data - && !this->data_ptr->data.empty() - && parse_flag(this->data_ptr->data.back()) == ParseFlags::DELIMITER; +#include +#include - // Push field - if (this->field_length > 0 || empty_last_field) { - this->push_field(); - } - // Push row - if (this->current_row.size() > 0) - this->push_row(); - } +namespace csv { + CSV_INLINE CSVFormat& CSVFormat::delimiter(char delim) { + this->possible_delimiters = { delim }; + this->assert_no_char_overlap(); + return *this; + } - CSV_INLINE void IBasicCSVParser::parse_field() noexcept { - using internals::ParseFlags; - auto& in = this->data_ptr->data; + CSV_INLINE CSVFormat& CSVFormat::delimiter(const std::vector & delim) { + this->possible_delimiters = delim; + this->assert_no_char_overlap(); + return *this; + } - // Trim off leading whitespace - while (data_pos < in.size() && ws_flag(in[data_pos])) - data_pos++; + CSV_INLINE CSVFormat& CSVFormat::quote(char quote) { + this->no_quote = false; + this->quote_char = quote; + this->assert_no_char_overlap(); + return *this; + } - if (field_start == UNINITIALIZED_FIELD) - field_start = (int)(data_pos - current_row_start()); + CSV_INLINE CSVFormat& CSVFormat::trim(const std::vector & chars) { + this->trim_chars = chars; + this->assert_no_char_overlap(); + return *this; + } - // Optimization: Since NOT_SPECIAL characters tend to occur in contiguous - // sequences, use the loop below to avoid having to go through the outer - // switch statement as much as possible - while (data_pos < in.size() && compound_parse_flag(in[data_pos]) == ParseFlags::NOT_SPECIAL) - data_pos++; + CSV_INLINE CSVFormat& CSVFormat::column_names(const std::vector& names) { + this->col_names = names; + this->header = -1; + return *this; + } - field_length = data_pos - (field_start + current_row_start()); + CSV_INLINE CSVFormat& CSVFormat::header_row(int row) { + if (row < 0) this->variable_column_policy = VariableColumnPolicy::KEEP; - // Trim off trailing whitespace, this->field_length constraint matters - // when field is entirely whitespace - for (size_t j = data_pos - 1; ws_flag(in[j]) && this->field_length > 0; j--) - this->field_length--; - } + this->header = row; + this->col_names = {}; + return *this; + } - CSV_INLINE void IBasicCSVParser::push_field() - { - // Update - if (field_has_double_quote) { - fields->emplace_back( - field_start == UNINITIALIZED_FIELD ? 0 : (unsigned int)field_start, - field_length, - true - ); - field_has_double_quote = false; + CSV_INLINE void CSVFormat::assert_no_char_overlap() + { + auto delims = std::set( + this->possible_delimiters.begin(), this->possible_delimiters.end()), + trims = std::set( + this->trim_chars.begin(), this->trim_chars.end()); - } - else { - fields->emplace_back( - field_start == UNINITIALIZED_FIELD ? 0 : (unsigned int)field_start, - field_length - ); - } + // Stores intersection of possible delimiters and trim characters + std::vector intersection = {}; - current_row.row_length++; + // Find which characters overlap, if any + std::set_intersection( + delims.begin(), delims.end(), + trims.begin(), trims.end(), + std::back_inserter(intersection)); - // Reset field state - field_start = UNINITIALIZED_FIELD; - field_length = 0; + // Make sure quote character is not contained in possible delimiters + // or whitespace characters + if (delims.find(this->quote_char) != delims.end() || + trims.find(this->quote_char) != trims.end()) { + intersection.push_back(this->quote_char); } - /** @return The number of characters parsed that belong to complete rows */ - CSV_INLINE size_t IBasicCSVParser::parse() - { - using internals::ParseFlags; - - this->quote_escape = false; - this->data_pos = 0; - this->current_row_start() = 0; - this->trim_utf8_bom(); + if (!intersection.empty()) { + std::string err_msg = "There should be no overlap between the quote character, " + "the set of possible delimiters " + "and the set of whitespace characters. Offending characters: "; - auto& in = this->data_ptr->data; - while (this->data_pos < in.size()) { - switch (compound_parse_flag(in[this->data_pos])) { - case ParseFlags::DELIMITER: - this->push_field(); - this->data_pos++; - break; + // Create a pretty error message with the list of overlapping + // characters + for (size_t i = 0; i < intersection.size(); i++) { + err_msg += "'"; + err_msg += intersection[i]; + err_msg += "'"; - case ParseFlags::NEWLINE: - this->data_pos++; + if (i + 1 < intersection.size()) + err_msg += ", "; + } - // Catches CRLF (or LFLF) - if (this->data_pos < in.size() && parse_flag(in[this->data_pos]) == ParseFlags::NEWLINE) - this->data_pos++; + throw std::runtime_error(err_msg + '.'); + } + } +} - // End of record -> Write record - this->push_field(); - this->push_row(); - - // Reset - this->current_row = CSVRow(data_ptr, this->data_pos, fields->size()); - break; - - case ParseFlags::NOT_SPECIAL: - this->parse_field(); - break; - - case ParseFlags::QUOTE_ESCAPE_QUOTE: - if (data_pos + 1 == in.size()) return this->current_row_start(); - else if (data_pos + 1 < in.size()) { - auto next_ch = parse_flag(in[data_pos + 1]); - if (next_ch >= ParseFlags::DELIMITER) { - quote_escape = false; - data_pos++; - break; - } - else if (next_ch == ParseFlags::QUOTE) { - // Case: Escaped quote - data_pos += 2; - this->field_length += 2; - this->field_has_double_quote = true; - break; - } - } - - // Case: Unescaped single quote => not strictly valid but we'll keep it - this->field_length++; - data_pos++; - - break; - - default: // Quote (currently not quote escaped) - if (this->field_length == 0) { - quote_escape = true; - data_pos++; - break; - } - - // Case: Unescaped quote - this->field_length++; - data_pos++; - - break; - } - } - - return this->current_row_start(); - } - - CSV_INLINE void IBasicCSVParser::push_row() { - current_row.row_length = fields->size() - current_row.fields_start; - this->_records->push_back(std::move(current_row)); - } - - CSV_INLINE void IBasicCSVParser::reset_data_ptr() { - this->data_ptr = std::make_shared(); - this->data_ptr->parse_flags = this->_parse_flags; - this->data_ptr->col_names = this->_col_names; - this->fields = &(this->data_ptr->fields); - } - - CSV_INLINE void IBasicCSVParser::trim_utf8_bom() { - auto& data = this->data_ptr->data; - - if (!this->unicode_bom_scan && data.size() >= 3) { - if (data[0] == '\xEF' && data[1] == '\xBB' && data[2] == '\xBF') { - this->data_pos += 3; // Remove BOM from input string - this->_utf8_bom = true; - } - - this->unicode_bom_scan = true; - } - } -#ifdef _MSC_VER -#pragma endregion -#endif - -#ifdef _MSC_VER -#pragma region Specializations -#endif - CSV_INLINE void MmapParser::next(size_t bytes = ITERATION_CHUNK_SIZE) { - // Reset parser state - this->field_start = UNINITIALIZED_FIELD; - this->field_length = 0; - this->reset_data_ptr(); - - // Create memory map - size_t length = std::min(this->source_size - this->mmap_pos, bytes); - std::error_code error; - this->data_ptr->_data = std::make_shared>(mio::make_mmap_source(this->_filename, this->mmap_pos, length, error)); - this->mmap_pos += length; - if (error) throw error; - - auto mmap_ptr = (mio::basic_mmap_source*)(this->data_ptr->_data.get()); - - // Create string view - this->data_ptr->data = csv::string_view(mmap_ptr->data(), mmap_ptr->length()); - - // Parse - this->current_row = CSVRow(this->data_ptr); - size_t remainder = this->parse(); - - if (this->mmap_pos == this->source_size || no_chunk()) { - this->_eof = true; - this->end_feed(); - } - - this->mmap_pos -= (length - remainder); - } -#ifdef _MSC_VER -#pragma endregion -#endif - } -} - -namespace csv { - namespace internals { - CSV_INLINE std::vector ColNames::get_col_names() const { - return this->col_names; - } +namespace csv { + namespace internals { + CSV_INLINE std::vector ColNames::get_col_names() const { + return this->col_names; + } CSV_INLINE void ColNames::set_col_names(const std::vector& cnames) { this->col_names = cnames; @@ -7065,95 +6990,82 @@ namespace csv { } } -/** @file - * Defines an object used to store CSV format settings - */ - -#include -#include +#include +#include namespace csv { - CSV_INLINE CSVFormat& CSVFormat::delimiter(char delim) { - this->possible_delimiters = { delim }; - this->assert_no_char_overlap(); - return *this; + /** Shorthand function for parsing an in-memory CSV string + * + * @return A collection of CSVRow objects + * + * @par Example + * @snippet tests/test_read_csv.cpp Parse Example + */ + CSV_INLINE CSVReader parse(csv::string_view in, CSVFormat format) { + std::stringstream stream(in.data()); + return CSVReader(stream, format); } - CSV_INLINE CSVFormat& CSVFormat::delimiter(const std::vector & delim) { - this->possible_delimiters = delim; - this->assert_no_char_overlap(); - return *this; - } + /** Parses a CSV string with no headers + * + * @return A collection of CSVRow objects + */ + CSV_INLINE CSVReader parse_no_header(csv::string_view in) { + CSVFormat format; + format.header_row(-1); - CSV_INLINE CSVFormat& CSVFormat::quote(char quote) { - this->no_quote = false; - this->quote_char = quote; - this->assert_no_char_overlap(); - return *this; + return parse(in, format); } - CSV_INLINE CSVFormat& CSVFormat::trim(const std::vector & chars) { - this->trim_chars = chars; - this->assert_no_char_overlap(); - return *this; + /** Parse a RFC 4180 CSV string, returning a collection + * of CSVRow objects + * + * @par Example + * @snippet tests/test_read_csv.cpp Escaped Comma + * + */ + CSV_INLINE CSVReader operator ""_csv(const char* in, size_t n) { + return parse(csv::string_view(in, n)); } - CSV_INLINE CSVFormat& CSVFormat::column_names(const std::vector& names) { - this->col_names = names; - this->header = -1; - return *this; + /** A shorthand for csv::parse_no_header() */ + CSV_INLINE CSVReader operator ""_csv_no_header(const char* in, size_t n) { + return parse_no_header(csv::string_view(in, n)); } - CSV_INLINE CSVFormat& CSVFormat::header_row(int row) { - if (row < 0) this->variable_column_policy = VariableColumnPolicy::KEEP; - - this->header = row; - this->col_names = {}; - return *this; + /** + * Find the position of a column in a CSV file or CSV_NOT_FOUND otherwise + * + * @param[in] filename Path to CSV file + * @param[in] col_name Column whose position we should resolve + * @param[in] format Format of the CSV file + */ + CSV_INLINE int get_col_pos( + csv::string_view filename, + csv::string_view col_name, + const CSVFormat& format) { + CSVReader reader(filename, format); + return reader.index_of(col_name); } - CSV_INLINE void CSVFormat::assert_no_char_overlap() - { - auto delims = std::set( - this->possible_delimiters.begin(), this->possible_delimiters.end()), - trims = std::set( - this->trim_chars.begin(), this->trim_chars.end()); - - // Stores intersection of possible delimiters and trim characters - std::vector intersection = {}; - - // Find which characters overlap, if any - std::set_intersection( - delims.begin(), delims.end(), - trims.begin(), trims.end(), - std::back_inserter(intersection)); - - // Make sure quote character is not contained in possible delimiters - // or whitespace characters - if (delims.find(this->quote_char) != delims.end() || - trims.find(this->quote_char) != trims.end()) { - intersection.push_back(this->quote_char); - } - - if (!intersection.empty()) { - std::string err_msg = "There should be no overlap between the quote character, " - "the set of possible delimiters " - "and the set of whitespace characters. Offending characters: "; - - // Create a pretty error message with the list of overlapping - // characters - for (size_t i = 0; i < intersection.size(); i++) { - err_msg += "'"; - err_msg += intersection[i]; - err_msg += "'"; + /** Get basic information about a CSV file + * @include programs/csv_info.cpp + */ + CSV_INLINE CSVFileInfo get_file_info(const std::string& filename) { + CSVReader reader(filename); + CSVFormat format = reader.get_format(); + for (auto it = reader.begin(); it != reader.end(); ++it); - if (i + 1 < intersection.size()) - err_msg += ", "; - } + CSVFileInfo info = { + filename, + reader.get_col_names(), + format.get_delim(), + reader.n_rows(), + reader.get_col_names().size() + }; - throw std::runtime_error(err_msg + '.'); - } + return info; } } /** @file @@ -7169,8 +7081,9 @@ namespace csv { for (size_t i = 0; i < row.size(); i++) { ret << row[i]; if (i + 1 < row.size()) ret << delim; - else ret << std::endl; + else ret << '\n'; } + ret.flush(); return ret.str(); } @@ -7307,7 +7220,7 @@ namespace csv { * \snippet tests/test_read_csv.cpp CSVField Example * */ - CSV_INLINE CSVReader::CSVReader(csv::string_view filename, CSVFormat format) : _format(format) { + CSV_INLINE CSVReader::CSVReader(csv::string_view filename, CSVFormat format) : _format(format) { auto head = internals::get_csv_head(filename); using Parser = internals::MmapParser; @@ -7464,180 +7377,377 @@ namespace csv { } } /** @file - * Defines an input iterator for csv::CSVReader + * Calculates statistics from CSV files */ +#include namespace csv { - /** Return an iterator to the first row in the reader */ - CSV_INLINE CSVReader::iterator CSVReader::begin() { - if (this->records.empty()) { - this->read_csv_worker = std::thread(&CSVReader::read_csv, this, internals::ITERATION_CHUNK_SIZE); - this->read_csv_worker.join(); + /** Calculate statistics for an arbitrarily large file. When this constructor + * is called, CSVStat will process the entire file iteratively. Once finished, + * methods like get_mean(), get_counts(), etc... can be used to retrieve statistics. + */ + CSV_INLINE CSVStat::CSVStat(csv::string_view filename, CSVFormat format) : + reader(filename, format) { + this->calc(); + } - // Still empty => return end iterator - if (this->records.empty()) return this->end(); - } + /** Calculate statistics for a CSV stored in a std::stringstream */ + CSV_INLINE CSVStat::CSVStat(std::stringstream& stream, CSVFormat format) : + reader(stream, format) { + this->calc(); + } - CSVReader::iterator ret(this, std::move(this->records.pop_front())); + /** Return current means */ + CSV_INLINE std::vector CSVStat::get_mean() const { + std::vector ret; + for (size_t i = 0; i < this->get_col_names().size(); i++) { + ret.push_back(this->rolling_means[i]); + } return ret; } - /** A placeholder for the imaginary past the end row in a CSV. - * Attempting to deference this will lead to bad things. - */ - CSV_INLINE HEDLEY_CONST CSVReader::iterator CSVReader::end() const noexcept { - return CSVReader::iterator(); + /** Return current variances */ + CSV_INLINE std::vector CSVStat::get_variance() const { + std::vector ret; + for (size_t i = 0; i < this->get_col_names().size(); i++) { + ret.push_back(this->rolling_vars[i]/(this->n[i] - 1)); + } + return ret; } - ///////////////////////// - // CSVReader::iterator // - ///////////////////////// - - CSV_INLINE CSVReader::iterator::iterator(CSVReader* _daddy, CSVRow&& _row) : - daddy(_daddy) { - row = std::move(_row); + /** Return current mins */ + CSV_INLINE std::vector CSVStat::get_mins() const { + std::vector ret; + for (size_t i = 0; i < this->get_col_names().size(); i++) { + ret.push_back(this->mins[i]); + } + return ret; } - /** Advance the iterator by one row. If this CSVReader has an - * associated file, then the iterator will lazily pull more data from - * that file until the end of file is reached. - * - * @note This iterator does **not** block the thread responsible for parsing CSV. - * - */ - CSV_INLINE CSVReader::iterator& CSVReader::iterator::operator++() { - if (!daddy->read_row(this->row)) { - this->daddy = nullptr; // this == end() + /** Return current maxes */ + CSV_INLINE std::vector CSVStat::get_maxes() const { + std::vector ret; + for (size_t i = 0; i < this->get_col_names().size(); i++) { + ret.push_back(this->maxes[i]); } - - return *this; + return ret; } - /** Post-increment iterator */ - CSV_INLINE CSVReader::iterator CSVReader::iterator::operator++(int) { - auto temp = *this; - if (!daddy->read_row(this->row)) { - this->daddy = nullptr; // this == end() + /** Get counts for each column */ + CSV_INLINE std::vector CSVStat::get_counts() const { + std::vector ret; + for (size_t i = 0; i < this->get_col_names().size(); i++) { + ret.push_back(this->counts[i]); } + return ret; + } - return temp; + /** Get data type counts for each column */ + CSV_INLINE std::vector CSVStat::get_dtypes() const { + std::vector ret; + for (size_t i = 0; i < this->get_col_names().size(); i++) { + ret.push_back(this->dtypes[i]); + } + return ret; } -} -/** @file - * Defines the data type used for storing information about a CSV row - */ -#include -#include + CSV_INLINE void CSVStat::calc() { + constexpr size_t CALC_CHUNK_SIZE = 5000; -namespace csv { - namespace internals { - CSV_INLINE RawCSVField& CSVFieldList::operator[](size_t n) const { - const size_t page_no = n / _single_buffer_capacity; - const size_t buffer_idx = (page_no < 1) ? n : n % _single_buffer_capacity; - return this->buffers[page_no][buffer_idx]; - } + while (true) { + /** Chunk rows */ + for (auto& row : reader) { + if (this->records.size() < CALC_CHUNK_SIZE) { + this->records.push_back(std::move(row)); + } + } - CSV_INLINE void CSVFieldList::allocate() { - RawCSVField * buffer = new RawCSVField[_single_buffer_capacity]; - buffers.push_back(buffer); - _current_buffer_size = 0; - _back = &(buffers.back()[0]); - } - } + /** Go through all records and calculate specified statistics */ + for (size_t i = 0; i < this->get_col_names().size(); i++) { + dtypes.push_back({}); + counts.push_back({}); + rolling_means.push_back(0); + rolling_vars.push_back(0); + mins.push_back(NAN); + maxes.push_back(NAN); + n.push_back(0); + } - /** Return a CSVField object corrsponding to the nth value in the row. - * - * @note This method performs bounds checking, and will throw an - * `std::runtime_error` if n is invalid. - * - * @complexity - * Constant, by calling csv::CSVRow::get_csv::string_view() - * - */ - CSV_INLINE CSVField CSVRow::operator[](size_t n) const { - return CSVField(this->get_field(n)); - } + std::vector pool; - /** Retrieve a value by its associated column name. If the column - * specified can't be round, a runtime error is thrown. - * - * @complexity - * Constant. This calls the other CSVRow::operator[]() after - * converting column names into indices using a hash table. - * - * @param[in] col_name The column to look for - */ - CSV_INLINE CSVField CSVRow::operator[](const std::string& col_name) const { - auto & col_names = this->data->col_names; - auto col_pos = col_names->index_of(col_name); - if (col_pos > -1) { - return this->operator[](col_pos); - } + // Start threads + for (size_t i = 0; i < this->get_col_names().size(); i++) + pool.push_back(std::thread(&CSVStat::calc_worker, this, i)); - throw std::runtime_error("Can't find a column named " + col_name); + // Block until done + for (auto& th : pool) + th.join(); + + if (reader.eof()) break; + } } - CSV_INLINE CSVRow::operator std::vector() const { - std::vector ret; - for (size_t i = 0; i < size(); i++) - ret.push_back(std::string(this->get_field(i))); + CSV_INLINE void CSVStat::calc_worker(const size_t &i) { + /** Worker thread for CSVStat::calc() which calculates statistics for one column. + * + * @param[in] i Column index + */ - return ret; - } + auto current_record = this->records.begin(); - CSV_INLINE csv::string_view CSVRow::get_field(size_t index) const - { - using internals::ParseFlags; + for (size_t processed = 0; current_record != this->records.end(); processed++) { + if (current_record->size() == this->get_col_names().size()) { + auto current_field = (*current_record)[i]; - if (index >= this->size()) - throw std::runtime_error("Index out of bounds."); + // Optimization: Don't count() if there's too many distinct values in the first 1000 rows + if (processed < 1000 || this->counts[i].size() <= 500) + this->count(current_field, i); - const size_t field_index = this->fields_start + index; - auto& field = this->data->fields[field_index]; - auto field_str = csv::string_view(this->data->data).substr(this->data_start + field.start); + this->dtype(current_field, i); - if (field.has_double_quote) { - auto& value = this->data->double_quote_fields[field_index]; - if (value.empty()) { - bool prev_ch_quote = false; - for (size_t i = 0; i < field.length; i++) { - if (this->data->parse_flags[field_str[i] + 128] == ParseFlags::QUOTE) { - if (prev_ch_quote) { - prev_ch_quote = false; - continue; - } - else { - prev_ch_quote = true; - } - } + // Numeric Stuff + if (current_field.is_num()) { + long double x_n = current_field.get(); - value += field_str[i]; + // This actually calculates mean AND variance + this->variance(x_n, i); + this->min_max(x_n, i); } } + else if (this->reader.get_format().get_variable_column_policy() == VariableColumnPolicy::THROW) { + throw std::runtime_error("Line has different length than the others " + internals::format_row(*current_record)); + } - return csv::string_view(value); + ++current_record; } - - return field_str.substr(0, field.length); - } - -#ifdef _MSC_VER -#pragma region CSVRow Iterator -#endif - /** Return an iterator pointing to the first field. */ - CSV_INLINE CSVRow::iterator CSVRow::begin() const { - return CSVRow::iterator(this, 0); } - /** Return an iterator pointing to just after the end of the CSVRow. - * - * @warning Attempting to dereference the end iterator results - * in dereferencing a null pointer. - */ - CSV_INLINE CSVRow::iterator CSVRow::end() const noexcept { - return CSVRow::iterator(this, (int)this->size()); + CSV_INLINE void CSVStat::dtype(CSVField& data, const size_t &i) { + /** Given a record update the type counter + * @param[in] record Data observation + * @param[out] i The column index that should be updated + */ + + auto type = data.type(); + if (this->dtypes[i].find(type) != + this->dtypes[i].end()) { + // Increment count + this->dtypes[i][type]++; + } else { + // Initialize count + this->dtypes[i].insert(std::make_pair(type, 1)); + } + } + + CSV_INLINE void CSVStat::count(CSVField& data, const size_t &i) { + /** Given a record update the frequency counter + * @param[in] record Data observation + * @param[out] i The column index that should be updated + */ + + auto item = data.get(); + + if (this->counts[i].find(item) != + this->counts[i].end()) { + // Increment count + this->counts[i][item]++; + } else { + // Initialize count + this->counts[i].insert(std::make_pair(item, 1)); + } + } + + CSV_INLINE void CSVStat::min_max(const long double &x_n, const size_t &i) { + /** Update current minimum and maximum + * @param[in] x_n Data observation + * @param[out] i The column index that should be updated + */ + if (std::isnan(this->mins[i])) + this->mins[i] = x_n; + if (std::isnan(this->maxes[i])) + this->maxes[i] = x_n; + + if (x_n < this->mins[i]) + this->mins[i] = x_n; + else if (x_n > this->maxes[i]) + this->maxes[i] = x_n; + } + + CSV_INLINE void CSVStat::variance(const long double &x_n, const size_t &i) { + /** Given a record update rolling mean and variance for all columns + * using Welford's Algorithm + * @param[in] x_n Data observation + * @param[out] i The column index that should be updated + */ + long double& current_rolling_mean = this->rolling_means[i]; + long double& current_rolling_var = this->rolling_vars[i]; + long double& current_n = this->n[i]; + long double delta; + long double delta2; + + current_n++; + + if (current_n == 1) { + current_rolling_mean = x_n; + } else { + delta = x_n - current_rolling_mean; + current_rolling_mean += delta/current_n; + delta2 = x_n - current_rolling_mean; + current_rolling_var += delta*delta2; + } + } + + /** Useful for uploading CSV files to SQL databases. + * + * Return a data type for each column such that every value in a column can be + * converted to the corresponding data type without data loss. + * @param[in] filename The CSV file + * + * \return A mapping of column names to csv::DataType enums + */ + CSV_INLINE std::unordered_map csv_data_types(const std::string& filename) { + CSVStat stat(filename); + std::unordered_map csv_dtypes; + + auto col_names = stat.get_col_names(); + auto temp = stat.get_dtypes(); + + for (size_t i = 0; i < stat.get_col_names().size(); i++) { + auto& col = temp[i]; + auto& col_name = col_names[i]; + + if (col[DataType::CSV_STRING]) + csv_dtypes[col_name] = DataType::CSV_STRING; + else if (col[DataType::CSV_INT64]) + csv_dtypes[col_name] = DataType::CSV_INT64; + else if (col[DataType::CSV_INT32]) + csv_dtypes[col_name] = DataType::CSV_INT32; + else if (col[DataType::CSV_INT16]) + csv_dtypes[col_name] = DataType::CSV_INT16; + else if (col[DataType::CSV_INT8]) + csv_dtypes[col_name] = DataType::CSV_INT8; + else + csv_dtypes[col_name] = DataType::CSV_DOUBLE; + } + + return csv_dtypes; + } +} +/** @file + * Defines the data type used for storing information about a CSV row + */ + +#include +#include + +namespace csv { + namespace internals { + CSV_INLINE RawCSVField& CSVFieldList::operator[](size_t n) const { + const size_t page_no = n / _single_buffer_capacity; + const size_t buffer_idx = (page_no < 1) ? n : n % _single_buffer_capacity; + return this->buffers[page_no][buffer_idx]; + } + + CSV_INLINE void CSVFieldList::allocate() { + RawCSVField * buffer = new RawCSVField[_single_buffer_capacity]; + buffers.push_back(buffer); + _current_buffer_size = 0; + _back = &(buffers.back()[0]); + } + } + + /** Return a CSVField object corrsponding to the nth value in the row. + * + * @note This method performs bounds checking, and will throw an + * `std::runtime_error` if n is invalid. + * + * @complexity + * Constant, by calling csv::CSVRow::get_csv::string_view() + * + */ + CSV_INLINE CSVField CSVRow::operator[](size_t n) const { + return CSVField(this->get_field(n)); + } + + /** Retrieve a value by its associated column name. If the column + * specified can't be round, a runtime error is thrown. + * + * @complexity + * Constant. This calls the other CSVRow::operator[]() after + * converting column names into indices using a hash table. + * + * @param[in] col_name The column to look for + */ + CSV_INLINE CSVField CSVRow::operator[](const std::string& col_name) const { + auto & col_names = this->data->col_names; + auto col_pos = col_names->index_of(col_name); + if (col_pos > -1) { + return this->operator[](col_pos); + } + + throw std::runtime_error("Can't find a column named " + col_name); + } + + CSV_INLINE CSVRow::operator std::vector() const { + std::vector ret; + for (size_t i = 0; i < size(); i++) + ret.push_back(std::string(this->get_field(i))); + + return ret; + } + + CSV_INLINE csv::string_view CSVRow::get_field(size_t index) const + { + using internals::ParseFlags; + + if (index >= this->size()) + throw std::runtime_error("Index out of bounds."); + + const size_t field_index = this->fields_start + index; + auto& field = this->data->fields[field_index]; + auto field_str = csv::string_view(this->data->data).substr(this->data_start + field.start); + + if (field.has_double_quote) { + auto& value = this->data->double_quote_fields[field_index]; + if (value.empty()) { + bool prev_ch_quote = false; + for (size_t i = 0; i < field.length; i++) { + if (this->data->parse_flags[field_str[i] + 128] == ParseFlags::QUOTE) { + if (prev_ch_quote) { + prev_ch_quote = false; + continue; + } + else { + prev_ch_quote = true; + } + } + + value += field_str[i]; + } + } + + return csv::string_view(value); + } + + return field_str.substr(0, field.length); + } + +#ifdef _MSC_VER +#pragma region CSVRow Iterator +#endif + /** Return an iterator pointing to the first field. */ + CSV_INLINE CSVRow::iterator CSVRow::begin() const { + return CSVRow::iterator(this, 0); + } + + /** Return an iterator pointing to just after the end of the CSVRow. + * + * @warning Attempting to dereference the end iterator results + * in dereferencing a null pointer. + */ + CSV_INLINE CSVRow::iterator CSVRow::end() const noexcept { + return CSVRow::iterator(this, (int)this->size()); } CSV_INLINE CSVRow::reverse_iterator CSVRow::rbegin() const noexcept { @@ -7978,340 +8088,266 @@ namespace csv { return ret; } } -/** @file - * Calculates statistics from CSV files - */ - -#include namespace csv { - /** Calculate statistics for an arbitrarily large file. When this constructor - * is called, CSVStat will process the entire file iteratively. Once finished, - * methods like get_mean(), get_counts(), etc... can be used to retrieve statistics. - */ - CSV_INLINE CSVStat::CSVStat(csv::string_view filename, CSVFormat format) : - reader(filename, format) { - this->calc(); - } - - /** Calculate statistics for a CSV stored in a std::stringstream */ - CSV_INLINE CSVStat::CSVStat(std::stringstream& stream, CSVFormat format) : - reader(stream, format) { - this->calc(); - } + namespace internals { + CSV_INLINE size_t get_file_size(csv::string_view filename) { + std::ifstream infile(std::string(filename), std::ios::binary); + const auto start = infile.tellg(); + infile.seekg(0, std::ios::end); + const auto end = infile.tellg(); - /** Return current means */ - CSV_INLINE std::vector CSVStat::get_mean() const { - std::vector ret; - for (size_t i = 0; i < this->get_col_names().size(); i++) { - ret.push_back(this->rolling_means[i]); + return end - start; } - return ret; - } - /** Return current variances */ - CSV_INLINE std::vector CSVStat::get_variance() const { - std::vector ret; - for (size_t i = 0; i < this->get_col_names().size(); i++) { - ret.push_back(this->rolling_vars[i]/(this->n[i] - 1)); + CSV_INLINE std::string get_csv_head(csv::string_view filename) { + return get_csv_head(filename, get_file_size(filename)); } - return ret; - } - /** Return current mins */ - CSV_INLINE std::vector CSVStat::get_mins() const { - std::vector ret; - for (size_t i = 0; i < this->get_col_names().size(); i++) { - ret.push_back(this->mins[i]); - } - return ret; - } + CSV_INLINE std::string get_csv_head(csv::string_view filename, size_t file_size) { + const size_t bytes = 500000; - /** Return current maxes */ - CSV_INLINE std::vector CSVStat::get_maxes() const { - std::vector ret; - for (size_t i = 0; i < this->get_col_names().size(); i++) { - ret.push_back(this->maxes[i]); - } - return ret; - } + std::error_code error; + size_t length = std::min((size_t)file_size, bytes); + auto mmap = mio::make_mmap_source(std::string(filename), 0, length, error); - /** Get counts for each column */ - CSV_INLINE std::vector CSVStat::get_counts() const { - std::vector ret; - for (size_t i = 0; i < this->get_col_names().size(); i++) { - ret.push_back(this->counts[i]); - } - return ret; - } + if (error) { + throw std::runtime_error("Cannot open file " + std::string(filename)); + } - /** Get data type counts for each column */ - CSV_INLINE std::vector CSVStat::get_dtypes() const { - std::vector ret; - for (size_t i = 0; i < this->get_col_names().size(); i++) { - ret.push_back(this->dtypes[i]); + return std::string(mmap.begin(), mmap.end()); } - return ret; - } - - CSV_INLINE void CSVStat::calc() { - constexpr size_t CALC_CHUNK_SIZE = 5000; - while (true) { - /** Chunk rows */ - for (auto& row : reader) { - if (this->records.size() < CALC_CHUNK_SIZE) { - this->records.push_back(std::move(row)); - } +#ifdef _MSC_VER +#pragma region IBasicCVParser +#endif + CSV_INLINE IBasicCSVParser::IBasicCSVParser( + const CSVFormat& format, + const ColNamesPtr& col_names + ) : _col_names(col_names) { + if (format.no_quote) { + _parse_flags = internals::make_parse_flags(format.get_delim()); } - - /** Go through all records and calculate specified statistics */ - for (size_t i = 0; i < this->get_col_names().size(); i++) { - dtypes.push_back({}); - counts.push_back({}); - rolling_means.push_back(0); - rolling_vars.push_back(0); - mins.push_back(NAN); - maxes.push_back(NAN); - n.push_back(0); + else { + _parse_flags = internals::make_parse_flags(format.get_delim(), format.quote_char); } - std::vector pool; + _ws_flags = internals::make_ws_flags( + format.trim_chars.data(), format.trim_chars.size() + ); + } - // Start threads - for (size_t i = 0; i < this->get_col_names().size(); i++) - pool.push_back(std::thread(&CSVStat::calc_worker, this, i)); + CSV_INLINE void IBasicCSVParser::end_feed() { + using internals::ParseFlags; - // Block until done - for (auto& th : pool) - th.join(); + bool empty_last_field = this->data_ptr + && this->data_ptr->_data + && !this->data_ptr->data.empty() + && parse_flag(this->data_ptr->data.back()) == ParseFlags::DELIMITER; - if (reader.eof()) break; + // Push field + if (this->field_length > 0 || empty_last_field) { + this->push_field(); + } + + // Push row + if (this->current_row.size() > 0) + this->push_row(); } - } - CSV_INLINE void CSVStat::calc_worker(const size_t &i) { - /** Worker thread for CSVStat::calc() which calculates statistics for one column. - * - * @param[in] i Column index - */ + CSV_INLINE void IBasicCSVParser::parse_field() noexcept { + using internals::ParseFlags; + auto& in = this->data_ptr->data; - auto current_record = this->records.begin(); + // Trim off leading whitespace + while (data_pos < in.size() && ws_flag(in[data_pos])) + data_pos++; - for (size_t processed = 0; current_record != this->records.end(); processed++) { - if (current_record->size() == this->get_col_names().size()) { - auto current_field = (*current_record)[i]; + if (field_start == UNINITIALIZED_FIELD) + field_start = (int)(data_pos - current_row_start()); - // Optimization: Don't count() if there's too many distinct values in the first 1000 rows - if (processed < 1000 || this->counts[i].size() <= 500) - this->count(current_field, i); + // Optimization: Since NOT_SPECIAL characters tend to occur in contiguous + // sequences, use the loop below to avoid having to go through the outer + // switch statement as much as possible + while (data_pos < in.size() && compound_parse_flag(in[data_pos]) == ParseFlags::NOT_SPECIAL) + data_pos++; - this->dtype(current_field, i); + field_length = data_pos - (field_start + current_row_start()); - // Numeric Stuff - if (current_field.is_num()) { - long double x_n = current_field.get(); + // Trim off trailing whitespace, this->field_length constraint matters + // when field is entirely whitespace + for (size_t j = data_pos - 1; ws_flag(in[j]) && this->field_length > 0; j--) + this->field_length--; + } + + CSV_INLINE void IBasicCSVParser::push_field() + { + // Update + if (field_has_double_quote) { + fields->emplace_back( + field_start == UNINITIALIZED_FIELD ? 0 : (unsigned int)field_start, + field_length, + true + ); + field_has_double_quote = false; - // This actually calculates mean AND variance - this->variance(x_n, i); - this->min_max(x_n, i); - } } - else if (this->reader.get_format().get_variable_column_policy() == VariableColumnPolicy::THROW) { - throw std::runtime_error("Line has different length than the others " + internals::format_row(*current_record)); + else { + fields->emplace_back( + field_start == UNINITIALIZED_FIELD ? 0 : (unsigned int)field_start, + field_length + ); } - ++current_record; - } - } + current_row.row_length++; - CSV_INLINE void CSVStat::dtype(CSVField& data, const size_t &i) { - /** Given a record update the type counter - * @param[in] record Data observation - * @param[out] i The column index that should be updated - */ - - auto type = data.type(); - if (this->dtypes[i].find(type) != - this->dtypes[i].end()) { - // Increment count - this->dtypes[i][type]++; - } else { - // Initialize count - this->dtypes[i].insert(std::make_pair(type, 1)); + // Reset field state + field_start = UNINITIALIZED_FIELD; + field_length = 0; } - } - CSV_INLINE void CSVStat::count(CSVField& data, const size_t &i) { - /** Given a record update the frequency counter - * @param[in] record Data observation - * @param[out] i The column index that should be updated - */ + /** @return The number of characters parsed that belong to complete rows */ + CSV_INLINE size_t IBasicCSVParser::parse() + { + using internals::ParseFlags; - auto item = data.get(); + this->quote_escape = false; + this->data_pos = 0; + this->current_row_start() = 0; + this->trim_utf8_bom(); - if (this->counts[i].find(item) != - this->counts[i].end()) { - // Increment count - this->counts[i][item]++; - } else { - // Initialize count - this->counts[i].insert(std::make_pair(item, 1)); - } - } + auto& in = this->data_ptr->data; + while (this->data_pos < in.size()) { + switch (compound_parse_flag(in[this->data_pos])) { + case ParseFlags::DELIMITER: + this->push_field(); + this->data_pos++; + break; - CSV_INLINE void CSVStat::min_max(const long double &x_n, const size_t &i) { - /** Update current minimum and maximum - * @param[in] x_n Data observation - * @param[out] i The column index that should be updated - */ - if (std::isnan(this->mins[i])) - this->mins[i] = x_n; - if (std::isnan(this->maxes[i])) - this->maxes[i] = x_n; - - if (x_n < this->mins[i]) - this->mins[i] = x_n; - else if (x_n > this->maxes[i]) - this->maxes[i] = x_n; - } + case ParseFlags::NEWLINE: + this->data_pos++; - CSV_INLINE void CSVStat::variance(const long double &x_n, const size_t &i) { - /** Given a record update rolling mean and variance for all columns - * using Welford's Algorithm - * @param[in] x_n Data observation - * @param[out] i The column index that should be updated - */ - long double& current_rolling_mean = this->rolling_means[i]; - long double& current_rolling_var = this->rolling_vars[i]; - long double& current_n = this->n[i]; - long double delta; - long double delta2; + // Catches CRLF (or LFLF) + if (this->data_pos < in.size() && parse_flag(in[this->data_pos]) == ParseFlags::NEWLINE) + this->data_pos++; - current_n++; - - if (current_n == 1) { - current_rolling_mean = x_n; - } else { - delta = x_n - current_rolling_mean; - current_rolling_mean += delta/current_n; - delta2 = x_n - current_rolling_mean; - current_rolling_var += delta*delta2; - } - } + // End of record -> Write record + this->push_field(); + this->push_row(); - /** Useful for uploading CSV files to SQL databases. - * - * Return a data type for each column such that every value in a column can be - * converted to the corresponding data type without data loss. - * @param[in] filename The CSV file - * - * \return A mapping of column names to csv::DataType enums - */ - CSV_INLINE std::unordered_map csv_data_types(const std::string& filename) { - CSVStat stat(filename); - std::unordered_map csv_dtypes; + // Reset + this->current_row = CSVRow(data_ptr, this->data_pos, fields->size()); + break; - auto col_names = stat.get_col_names(); - auto temp = stat.get_dtypes(); + case ParseFlags::NOT_SPECIAL: + this->parse_field(); + break; - for (size_t i = 0; i < stat.get_col_names().size(); i++) { - auto& col = temp[i]; - auto& col_name = col_names[i]; + case ParseFlags::QUOTE_ESCAPE_QUOTE: + if (data_pos + 1 == in.size()) return this->current_row_start(); + else if (data_pos + 1 < in.size()) { + auto next_ch = parse_flag(in[data_pos + 1]); + if (next_ch >= ParseFlags::DELIMITER) { + quote_escape = false; + data_pos++; + break; + } + else if (next_ch == ParseFlags::QUOTE) { + // Case: Escaped quote + data_pos += 2; + this->field_length += 2; + this->field_has_double_quote = true; + break; + } + } + + // Case: Unescaped single quote => not strictly valid but we'll keep it + this->field_length++; + data_pos++; - if (col[DataType::CSV_STRING]) - csv_dtypes[col_name] = DataType::CSV_STRING; - else if (col[DataType::CSV_INT64]) - csv_dtypes[col_name] = DataType::CSV_INT64; - else if (col[DataType::CSV_INT32]) - csv_dtypes[col_name] = DataType::CSV_INT32; - else if (col[DataType::CSV_INT16]) - csv_dtypes[col_name] = DataType::CSV_INT16; - else if (col[DataType::CSV_INT8]) - csv_dtypes[col_name] = DataType::CSV_INT8; - else - csv_dtypes[col_name] = DataType::CSV_DOUBLE; + break; + + default: // Quote (currently not quote escaped) + if (this->field_length == 0) { + quote_escape = true; + data_pos++; + break; + } + + // Case: Unescaped quote + this->field_length++; + data_pos++; + + break; + } + } + + return this->current_row_start(); } - return csv_dtypes; - } -} -#include -#include + CSV_INLINE void IBasicCSVParser::push_row() { + current_row.row_length = fields->size() - current_row.fields_start; + this->_records->push_back(std::move(current_row)); + } + CSV_INLINE void IBasicCSVParser::reset_data_ptr() { + this->data_ptr = std::make_shared(); + this->data_ptr->parse_flags = this->_parse_flags; + this->data_ptr->col_names = this->_col_names; + this->fields = &(this->data_ptr->fields); + } -namespace csv { - /** Shorthand function for parsing an in-memory CSV string - * - * @return A collection of CSVRow objects - * - * @par Example - * @snippet tests/test_read_csv.cpp Parse Example - */ - CSV_INLINE CSVReader parse(csv::string_view in, CSVFormat format) { - std::stringstream stream(in.data()); - return CSVReader(stream, format); - } + CSV_INLINE void IBasicCSVParser::trim_utf8_bom() { + auto& data = this->data_ptr->data; - /** Parses a CSV string with no headers - * - * @return A collection of CSVRow objects - */ - CSV_INLINE CSVReader parse_no_header(csv::string_view in) { - CSVFormat format; - format.header_row(-1); + if (!this->unicode_bom_scan && data.size() >= 3) { + if (data[0] == '\xEF' && data[1] == '\xBB' && data[2] == '\xBF') { + this->data_pos += 3; // Remove BOM from input string + this->_utf8_bom = true; + } - return parse(in, format); - } + this->unicode_bom_scan = true; + } + } +#ifdef _MSC_VER +#pragma endregion +#endif - /** Parse a RFC 4180 CSV string, returning a collection - * of CSVRow objects - * - * @par Example - * @snippet tests/test_read_csv.cpp Escaped Comma - * - */ - CSV_INLINE CSVReader operator ""_csv(const char* in, size_t n) { - return parse(csv::string_view(in, n)); - } +#ifdef _MSC_VER +#pragma region Specializations +#endif + CSV_INLINE void MmapParser::next(size_t bytes = ITERATION_CHUNK_SIZE) { + // Reset parser state + this->field_start = UNINITIALIZED_FIELD; + this->field_length = 0; + this->reset_data_ptr(); - /** A shorthand for csv::parse_no_header() */ - CSV_INLINE CSVReader operator ""_csv_no_header(const char* in, size_t n) { - return parse_no_header(csv::string_view(in, n)); - } + // Create memory map + size_t length = std::min(this->source_size - this->mmap_pos, bytes); + std::error_code error; + this->data_ptr->_data = std::make_shared>(mio::make_mmap_source(this->_filename, this->mmap_pos, length, error)); + this->mmap_pos += length; + if (error) throw error; - /** - * Find the position of a column in a CSV file or CSV_NOT_FOUND otherwise - * - * @param[in] filename Path to CSV file - * @param[in] col_name Column whose position we should resolve - * @param[in] format Format of the CSV file - */ - CSV_INLINE int get_col_pos( - csv::string_view filename, - csv::string_view col_name, - const CSVFormat& format) { - CSVReader reader(filename, format); - return reader.index_of(col_name); - } + auto mmap_ptr = (mio::basic_mmap_source*)(this->data_ptr->_data.get()); - /** Get basic information about a CSV file - * @include programs/csv_info.cpp - */ - CSV_INLINE CSVFileInfo get_file_info(const std::string& filename) { - CSVReader reader(filename); - CSVFormat format = reader.get_format(); - for (auto it = reader.begin(); it != reader.end(); ++it); + // Create string view + this->data_ptr->data = csv::string_view(mmap_ptr->data(), mmap_ptr->length()); - CSVFileInfo info = { - filename, - reader.get_col_names(), - format.get_delim(), - reader.n_rows(), - reader.get_col_names().size() - }; + // Parse + this->current_row = CSVRow(this->data_ptr); + size_t remainder = this->parse(); - return info; + if (this->mmap_pos == this->source_size || no_chunk()) { + this->_eof = true; + this->end_feed(); + } + + this->mmap_pos -= (length - remainder); + } +#ifdef _MSC_VER +#pragma endregion +#endif } } diff --git a/thirdparty/cxxopts/.gitignore b/thirdparty/cxxopts/.gitignore deleted file mode 100644 index 0be95f2f..00000000 --- a/thirdparty/cxxopts/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.swp -build* -CMakeCache.txt -Makefile -CMakeFiles/ -Testing/ -CTestTestfile.cmake -cmake_install.cmake -bazel-* \ No newline at end of file diff --git a/thirdparty/cxxopts/.travis.yml b/thirdparty/cxxopts/.travis.yml deleted file mode 100644 index 87c78a7e..00000000 --- a/thirdparty/cxxopts/.travis.yml +++ /dev/null @@ -1,70 +0,0 @@ -sudo: required -dist: trusty -language: cpp -os: - - linux -matrix: - include: - - os: linux - env: COMPILER=g++-4.9 - addons: - apt: - packages: - - g++-4.9 - sources: &sources - - llvm-toolchain-trusty-3.8 - - llvm-toolchain-trusty-5.0 - - ubuntu-toolchain-r-test - - os: linux - env: COMPILER=g++-4.9 UNICODE_OPTIONS=-DCXXOPTS_USE_UNICODE_HELP=Yes - addons: - apt: - packages: - - g++-4.9 - sources: *sources - - os: linux - env: COMPILER=g++-5 - addons: - apt: - packages: - - g++-5 - sources: *sources - - os: linux - env: COMPILER=g++-5 UNICODE_OPTIONS=-DCXXOPTS_USE_UNICODE_HELP=Yes - addons: - apt: - packages: - - g++-5 - sources: *sources - - os: linux - env: COMPILER=clang++-3.8 CXXFLAGS=-stdlib=libc++ - addons: - apt: - packages: - - clang-3.8 - - libc++-dev - sources: *sources - - os: linux - env: COMPILER=clang++-3.8 CXXFLAGS=-stdlib=libc++ UNICODE_OPTIONS=-DCXXOPTS_USE_UNICODE_HELP=Yes - addons: - apt: - packages: - - clang-3.8 - - libc++-dev - sources: *sources - - os: linux - env: COMPILER=clang++-5.0 CMAKE_OPTIONS=-DCXXOPTS_CXX_STANDARD=17 - addons: - apt: - packages: - - clang-5.0 - - g++-5 - sources: *sources -script: > - cmake -DCXXOPTS_BUILD_TESTS=ON -DCMAKE_CXX_COMPILER=$COMPILER - -DCMAKE_CXX_FLAGS=$CXXFLAGS $UNICODE_OPTIONS $CMAKE_OPTIONS . - && make && make ARGS=--output-on-failure test - -before_install: - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install icu4c; fi diff --git a/thirdparty/cxxopts/BUILD b/thirdparty/cxxopts/BUILD deleted file mode 100644 index a66fae7d..00000000 --- a/thirdparty/cxxopts/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") - -cc_library( - name = "cxxopts", - hdrs = ["include/cxxopts.hpp"], - strip_include_prefix = "include", - visibility = ["//visibility:public"], -) diff --git a/thirdparty/cxxopts/CHANGELOG.md b/thirdparty/cxxopts/CHANGELOG.md deleted file mode 100644 index 16b5a992..00000000 --- a/thirdparty/cxxopts/CHANGELOG.md +++ /dev/null @@ -1,98 +0,0 @@ -# Changelog - -This is the changelog for `cxxopts`, a C++11 library for parsing command line -options. The project adheres to semantic versioning. - -## Next version - -### Changed - -* Only search for a C++ compiler in CMakeLists.txt. -* Allow for exceptions to be disabled. -* Fix duplicate default options when there is a short and long option. -* Add `CXXOPTS_NO_EXCEPTIONS` to disable exceptions. -* Fix char parsing for space and check for length. -* Change argument type in `Options::parse` from `char**` to `const char**`. - -## 2.2 - -### Changed - -* Allow integers to have leading zeroes. -* Build the tests by default. -* Don't check for container when showing positional help. - -### Added - -* Iterator inputs to `parse_positional`. -* Throw an exception if the option in `parse_positional` doesn't exist. -* Parse a delimited list in a single argument for vector options. -* Add an option to disable implicit value on booleans. - -### Bug Fixes - -* Fix a warning about possible loss of data. -* Fix version numbering in CMakeLists.txt -* Remove unused declaration of the undefined `ParseResult::get_option`. -* Throw on invalid option syntax when beginning with a `-`. -* Throw in `as` when option wasn't present. -* Fix catching exceptions by reference. -* Fix out of bounds errors parsing integers. - -## 2.1.1 - -### Bug Fixes - -* Revert the change adding `const` type for `argv`, because most users expect - to pass a non-const `argv` from `main`. - -## 2.1 - -### Changed - -* Options with implicit arguments now require the `--option=value` form if - they are to be specified with an option. This is to remove the ambiguity - when a positional argument could follow an option with an implicit value. - For example, `--foo value`, where `foo` has an implicit value, will be - parsed as `--foo=implicit` and a positional argument `value`. -* Boolean values are no longer special, but are just an option with a default - and implicit value. - -### Added - -* Added support for `std::optional` as a storage type. -* Allow the help string to be customised. -* Use `const` for the type in the `argv` parameter, since the contents of the - arguments is never modified. - -### Bug Fixes - -* Building against GCC 4.9 was broken due to overly strict shadow warnings. -* Fixed an ambiguous overload in the `parse_positional` function when an - `initializer_list` was directly passed. -* Fixed precedence in the Boolean value regex. - -## 2.0 - -### Changed - -* `Options::parse` returns a ParseResult rather than storing the parse - result internally. -* Options with default values now get counted as appearing once if they - were not specified by the user. - -### Added - -* A new `ParseResult` object that is the immutable result of parsing. It - responds to the same `count` and `operator[]` as `Options` of 1.x did. -* The function `ParseResult::arguments` returns a vector of the parsed - arguments to iterate through in the order they were provided. -* The symbol `cxxopts::version` for the version of the library. -* Booleans can be specified with various strings and explicitly set false. - -## 1.x - -The 1.x series was the first major version of the library, with release numbers -starting to follow semantic versioning, after 0.x being unstable. It never had -a changelog maintained for it. Releases mostly contained bug fixes, with the -occasional feature added. diff --git a/thirdparty/cxxopts/CMakeLists.txt b/thirdparty/cxxopts/CMakeLists.txt deleted file mode 100644 index 3e9b8628..00000000 --- a/thirdparty/cxxopts/CMakeLists.txt +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (c) 2014 Jarryd Beck -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -cmake_minimum_required(VERSION 3.1) - -# parse the current version from the cxxopts header -file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/include/cxxopts.hpp" cxxopts_version_defines - REGEX "#define CXXOPTS__VERSION_(MAJOR|MINOR|PATCH)") -foreach(ver ${cxxopts_version_defines}) - if(ver MATCHES "#define CXXOPTS__VERSION_(MAJOR|MINOR|PATCH) +([^ ]+)$") - set(CXXOPTS__VERSION_${CMAKE_MATCH_1} "${CMAKE_MATCH_2}" CACHE INTERNAL "") - endif() -endforeach() -set(VERSION ${CXXOPTS__VERSION_MAJOR}.${CXXOPTS__VERSION_MINOR}.${CXXOPTS__VERSION_PATCH}) -message(STATUS "cxxopts version ${VERSION}") - -project(cxxopts VERSION "${VERSION}" LANGUAGES CXX) - -enable_testing() - -option(CXXOPTS_BUILD_EXAMPLES "Set to ON to build examples" ON) -option(CXXOPTS_BUILD_TESTS "Set to ON to build tests" ON) -option(CXXOPTS_ENABLE_INSTALL "Generate the install target" ON) - -# request c++11 without gnu extension for the whole project and enable more warnings -if (CXXOPTS_CXX_STANDARD) - set(CMAKE_CXX_STANDARD ${CXXOPTS_CXX_STANDARD}) -else() - set(CMAKE_CXX_STANDARD 11) -endif() - -set(CMAKE_CXX_EXTENSIONS OFF) - -if(MSVC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W2") -elseif(CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror -Wextra -Wshadow") -endif() - -add_library(cxxopts INTERFACE) -add_library(cxxopts::cxxopts ALIAS cxxopts) - -# optionally, enable unicode support using the ICU library -set(CXXOPTS_USE_UNICODE_HELP FALSE CACHE BOOL "Use ICU Unicode library") -if(CXXOPTS_USE_UNICODE_HELP) - find_package(PkgConfig) - pkg_check_modules(ICU REQUIRED icu-uc) - - target_link_libraries(cxxopts INTERFACE ${ICU_LDFLAGS}) - target_compile_options(cxxopts INTERFACE ${ICU_CFLAGS}) - target_compile_definitions(cxxopts INTERFACE CXXOPTS_USE_UNICODE) -endif() - -target_include_directories(cxxopts INTERFACE - $ - $ - ) - -if(CXXOPTS_ENABLE_INSTALL) - include(GNUInstallDirs) - include(CMakePackageConfigHelpers) - set(CXXOPTS_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/cxxopts" CACHE STRING - "Installation directory for cmake files, relative to ${CMAKE_INSTALL_PREFIX}.") - set(version_config "${PROJECT_BINARY_DIR}/cxxopts-config-version.cmake") - set(project_config "${PROJECT_BINARY_DIR}/cxxopts-config.cmake") - set(targets_export_name cxxopts-targets) - - # Generate the version, config and target files into the build directory. - write_basic_package_version_file( - ${version_config} - VERSION ${VERSION} - COMPATIBILITY AnyNewerVersion) - configure_package_config_file( - ${PROJECT_SOURCE_DIR}/cxxopts-config.cmake.in - ${project_config} - INSTALL_DESTINATION ${CXXOPTS_CMAKE_DIR}) - export(TARGETS cxxopts NAMESPACE cxxopts:: - FILE ${PROJECT_BINARY_DIR}/${targets_export_name}.cmake) - - # Install version, config and target files. - install( - FILES ${project_config} ${version_config} - DESTINATION ${CXXOPTS_CMAKE_DIR}) - install(EXPORT ${targets_export_name} DESTINATION ${CXXOPTS_CMAKE_DIR} - NAMESPACE cxxopts::) - - # Install the header file and export the target - install(TARGETS cxxopts EXPORT ${targets_export_name} DESTINATION ${CMAKE_INSTALL_LIBDIR}) - install(FILES ${PROJECT_SOURCE_DIR}/include/cxxopts.hpp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) -endif() - -add_subdirectory(src) -add_subdirectory(test) diff --git a/thirdparty/cxxopts/INSTALL b/thirdparty/cxxopts/INSTALL deleted file mode 100644 index 3e1c6f3d..00000000 --- a/thirdparty/cxxopts/INSTALL +++ /dev/null @@ -1,23 +0,0 @@ -== System installation == - -This library is header only. So you can either copy `include/cxxopts.hpp` to `/usr/include` or `/usr/local/include`, or add `include` to your search path. - -== Building the examples and tests == - -It is preferable to build out of source. Make a build directory somewhere, and then -do the following, where `${CXXOPTS_DIR}` is the path that you checked out `cxxopts` -to: - - cmake ${CXXOPTS_DIR} - make - -You can use another build tool, such as ninja. - - cmake -G Ninja ${CXXOPTS_DIR} - ninja - - -To run the tests, you have to configure `cxxopts` with another flag: - cmake -D CXXOPTS_BUILD_TESTS=On ${CXXOPTS_DIR} - make - make test diff --git a/thirdparty/cxxopts/README.md b/thirdparty/cxxopts/README.md deleted file mode 100644 index 88b92d42..00000000 --- a/thirdparty/cxxopts/README.md +++ /dev/null @@ -1,224 +0,0 @@ -[![Build Status](https://travis-ci.org/jarro2783/cxxopts.svg?branch=master)](https://travis-ci.org/jarro2783/cxxopts) - -# Release versions - -Note that `master` is generally a work in progress, and you probably want to use a -tagged release version. - -# Quick start - -This is a lightweight C++ option parser library, supporting the standard GNU -style syntax for options. - -Options can be given as: - - --long - --long=argument - --long argument - -a - -ab - -abc argument - -where c takes an argument, but a and b do not. - -Additionally, anything after `--` will be parsed as a positional argument. - -## Basics - -```cpp -#include -``` - -Create a `cxxopts::Options` instance. - -```cpp -cxxopts::Options options("MyProgram", "One line description of MyProgram"); -``` - -Then use `add_options`. - -```cpp -options.add_options() - ("d,debug", "Enable debugging") // a bool parameter - ("i,integer", "Int param", cxxopts::value()) - ("f,file", "File name", cxxopts::value()) - ("v,verbose", "Verbose output", cxxopts::value()->default_value("false")) - ; -``` - -Options are declared with a long and an optional short option. A description -must be provided. The third argument is the value, if omitted it is boolean. -Any type can be given as long as it can be parsed, with operator>>. - -To parse the command line do: - -```cpp -auto result = options.parse(argc, argv); -``` - -To retrieve an option use `result.count("option")` to get the number of times -it appeared, and - -```cpp -result["opt"].as() -``` - -to get its value. If "opt" doesn't exist, or isn't of the right type, then an -exception will be thrown. - -Note that the result of `options.parse` should only be used as long as the -`options` object that created it is in scope. - -## Exceptions - -Exceptional situations throw C++ exceptions. There are two types of -exceptions: errors defining the options, and errors when parsing a list of -arguments. All exceptions derive from `cxxopts::OptionException`. Errors -defining options derive from `cxxopts::OptionSpecException` and errors -parsing arguments derive from `cxxopts::OptionParseException`. - -All exceptions define a `what()` function to get a printable string -explaining the error. - -## Help groups - -Options can be placed into groups for the purposes of displaying help messages. -To place options in a group, pass the group as a string to `add_options`. Then, -when displaying the help, pass the groups that you would like displayed as a -vector to the `help` function. - -## Positional Arguments - -Positional arguments can be optionally parsed into one or more options. -To set up positional arguments, call - -```cpp -options.parse_positional({"first", "second", "last"}) -``` - -where "last" should be the name of an option with a container type, and the -others should have a single value. - -## Default and implicit values - -An option can be declared with a default or an implicit value, or both. - -A default value is the value that an option takes when it is not specified -on the command line. The following specifies a default value for an option: - -```cpp -cxxopts::value()->default_value("value") -``` - -An implicit value is the value that an option takes when it is given on the -command line without an argument. The following specifies an implicit value: - -```cpp -cxxopts::value()->implicit_value("implicit") -``` - -If an option had both, then not specifying it would give the value `"value"`, -writing it on the command line as `--option` would give the value `"implicit"`, -and writing `--option=another` would give it the value `"another"`. - -Note that the default and implicit value is always stored as a string, -regardless of the type that you want to store it in. It will be parsed as -though it was given on the command line. - -## Boolean values - -Boolean options have a default implicit value of `"true"`, which can be -overridden. The effect is that writing `-o` by itself will set option `o` to -`true`. However, they can also be written with various strings using `=value`. -There is no way to disambiguate positional arguments from the value following -a boolean, so we have chosen that they will be positional arguments, and -therefore, `-o false` does not work. - -## `std::vector` values - -Parsing of list of values in form of an `std::vector` is also supported, as long as `T` -can be parsed. To separate single values in a list the definition `CXXOPTS_VECTOR_DELIMITER` -is used, which is ',' by default. Ensure that you use no whitespaces between values because -those would be interpreted as the next command line option. Example for a command line option -that can be parsed as a `std::vector`: - -~~~ ---my_list=1,-2.1,3,4.5 -~~~ - -## Options specified multiple times - -The same option can be specified several times, with different arguments, which will all -be recorded in order of appearance. An example: - -~~~ ---use train --use bus --use ferry -~~~ - -this is supported through the use of a vector of value for the option: - -~~~ -options.add_options() - ("use", "Usable means of transport", cxxopts::value>()) -~~~ - -## Custom help - -The string after the program name on the first line of the help can be -completely replaced by calling `options.custom_help`. Note that you might -also want to override the positional help by calling `options.positional_help`. - - -## Example - -Putting all together: -```cpp -int main(int argc, char** argv) -{ - cxxopts::Options options("test", "A brief description"); - - options.add_options() - ("b,bar", "Param bar", cxxopts::value()) - ("d,debug", "Enable debugging", cxxopts::value()->default_value("false")) - ("f,foo", "Param foo", cxxopts::value()->default_value("10")) - ("h,help", "Print usage") - ; - - auto result = options.parse(argc, argv); - - if (result.count("help")) - { - std::cout << options.help() << std::endl; - exit(0); - } - bool debug = result["debug"].as(); - std::string bar; - if (result.count("bar")) - bar = result["bar"].as(); - int foo = result["foo"].as(); - - return 0; -} -``` - -# Linking - -This is a header only library. - -# Requirements - -The only build requirement is a C++ compiler that supports C++11 features such as: - -* regex -* constexpr -* default constructors - -GCC >= 4.9 or clang >= 3.1 with libc++ are known to work. - -The following compilers are known not to work: - -* MSVC 2013 - -# TODO list - -* Allow unrecognised options. diff --git a/thirdparty/cxxopts/WORKSPACE b/thirdparty/cxxopts/WORKSPACE deleted file mode 100644 index e69de29b..00000000 diff --git a/thirdparty/cxxopts/cxxopts-config.cmake.in b/thirdparty/cxxopts/cxxopts-config.cmake.in deleted file mode 100644 index c9efaf14..00000000 --- a/thirdparty/cxxopts/cxxopts-config.cmake.in +++ /dev/null @@ -1,4 +0,0 @@ -@PACKAGE_INIT@ - -include(${CMAKE_CURRENT_LIST_DIR}/@targets_export_name@.cmake) -check_required_components(cxxopts) diff --git a/thirdparty/cxxopts/include/cxxopts.hpp b/thirdparty/cxxopts/include/cxxopts.hpp deleted file mode 100644 index e91987d0..00000000 --- a/thirdparty/cxxopts/include/cxxopts.hpp +++ /dev/null @@ -1,2228 +0,0 @@ -/* - -Copyright (c) 2014, 2015, 2016, 2017 Jarryd Beck - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -*/ - -#ifndef CXXOPTS_HPP_INCLUDED -#define CXXOPTS_HPP_INCLUDED - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __cpp_lib_optional -#include -#define CXXOPTS_HAS_OPTIONAL -#endif - -#if __cplusplus >= 201603L -#define CXXOPTS_NODISCARD [[nodiscard]] -#else -#define CXXOPTS_NODISCARD -#endif - -#ifndef CXXOPTS_VECTOR_DELIMITER -#define CXXOPTS_VECTOR_DELIMITER ',' -#endif - -#define CXXOPTS__VERSION_MAJOR 2 -#define CXXOPTS__VERSION_MINOR 2 -#define CXXOPTS__VERSION_PATCH 0 - -namespace cxxopts -{ - static constexpr struct { - uint8_t major, minor, patch; - } version = { - CXXOPTS__VERSION_MAJOR, - CXXOPTS__VERSION_MINOR, - CXXOPTS__VERSION_PATCH - }; -} // namespace cxxopts - -//when we ask cxxopts to use Unicode, help strings are processed using ICU, -//which results in the correct lengths being computed for strings when they -//are formatted for the help output -//it is necessary to make sure that can be found by the -//compiler, and that icu-uc is linked in to the binary. - -#ifdef CXXOPTS_USE_UNICODE -#include - -namespace cxxopts -{ - typedef icu::UnicodeString String; - - inline - String - toLocalString(std::string s) - { - return icu::UnicodeString::fromUTF8(std::move(s)); - } - - class UnicodeStringIterator : public - std::iterator - { - public: - - UnicodeStringIterator(const icu::UnicodeString* string, int32_t pos) - : s(string) - , i(pos) - { - } - - value_type - operator*() const - { - return s->char32At(i); - } - - bool - operator==(const UnicodeStringIterator& rhs) const - { - return s == rhs.s && i == rhs.i; - } - - bool - operator!=(const UnicodeStringIterator& rhs) const - { - return !(*this == rhs); - } - - UnicodeStringIterator& - operator++() - { - ++i; - return *this; - } - - UnicodeStringIterator - operator+(int32_t v) - { - return UnicodeStringIterator(s, i + v); - } - - private: - const icu::UnicodeString* s; - int32_t i; - }; - - inline - String& - stringAppend(String&s, String a) - { - return s.append(std::move(a)); - } - - inline - String& - stringAppend(String& s, int n, UChar32 c) - { - for (int i = 0; i != n; ++i) - { - s.append(c); - } - - return s; - } - - template - String& - stringAppend(String& s, Iterator begin, Iterator end) - { - while (begin != end) - { - s.append(*begin); - ++begin; - } - - return s; - } - - inline - size_t - stringLength(const String& s) - { - return s.length(); - } - - inline - std::string - toUTF8String(const String& s) - { - std::string result; - s.toUTF8String(result); - - return result; - } - - inline - bool - empty(const String& s) - { - return s.isEmpty(); - } -} - -namespace std -{ - inline - cxxopts::UnicodeStringIterator - begin(const icu::UnicodeString& s) - { - return cxxopts::UnicodeStringIterator(&s, 0); - } - - inline - cxxopts::UnicodeStringIterator - end(const icu::UnicodeString& s) - { - return cxxopts::UnicodeStringIterator(&s, s.length()); - } -} - -//ifdef CXXOPTS_USE_UNICODE -#else - -namespace cxxopts -{ - typedef std::string String; - - template - T - toLocalString(T&& t) - { - return std::forward(t); - } - - inline - size_t - stringLength(const String& s) - { - return s.length(); - } - - inline - String& - stringAppend(String&s, const String& a) - { - return s.append(a); - } - - inline - String& - stringAppend(String& s, size_t n, char c) - { - return s.append(n, c); - } - - template - String& - stringAppend(String& s, Iterator begin, Iterator end) - { - return s.append(begin, end); - } - - template - std::string - toUTF8String(T&& t) - { - return std::forward(t); - } - - inline - bool - empty(const std::string& s) - { - return s.empty(); - } -} // namespace cxxopts - -//ifdef CXXOPTS_USE_UNICODE -#endif - -namespace cxxopts -{ - namespace - { -#ifdef _WIN32 - const std::string LQUOTE("\'"); - const std::string RQUOTE("\'"); -#else - const std::string LQUOTE("‘"); - const std::string RQUOTE("’"); -#endif - } // namespace - - class Value : public std::enable_shared_from_this - { - public: - - virtual ~Value() = default; - - virtual - std::shared_ptr - clone() const = 0; - - virtual void - parse(const std::string& text) const = 0; - - virtual void - parse() const = 0; - - virtual bool - has_default() const = 0; - - virtual bool - is_container() const = 0; - - virtual bool - has_implicit() const = 0; - - virtual std::string - get_default_value() const = 0; - - virtual std::string - get_implicit_value() const = 0; - - virtual std::shared_ptr - default_value(const std::string& value) = 0; - - virtual std::shared_ptr - implicit_value(const std::string& value) = 0; - - virtual std::shared_ptr - no_implicit_value() = 0; - - virtual bool - is_boolean() const = 0; - }; - - class OptionException : public std::exception - { - public: - explicit OptionException(std::string message) - : m_message(std::move(message)) - { - } - - CXXOPTS_NODISCARD - const char* - what() const noexcept override - { - return m_message.c_str(); - } - - private: - std::string m_message; - }; - - class OptionSpecException : public OptionException - { - public: - - explicit OptionSpecException(const std::string& message) - : OptionException(message) - { - } - }; - - class OptionParseException : public OptionException - { - public: - explicit OptionParseException(const std::string& message) - : OptionException(message) - { - } - }; - - class option_exists_error : public OptionSpecException - { - public: - explicit option_exists_error(const std::string& option) - : OptionSpecException("Option " + LQUOTE + option + RQUOTE + " already exists") - { - } - }; - - class invalid_option_format_error : public OptionSpecException - { - public: - explicit invalid_option_format_error(const std::string& format) - : OptionSpecException("Invalid option format " + LQUOTE + format + RQUOTE) - { - } - }; - - class option_syntax_exception : public OptionParseException { - public: - explicit option_syntax_exception(const std::string& text) - : OptionParseException("Argument " + LQUOTE + text + RQUOTE + - " starts with a - but has incorrect syntax") - { - } - }; - - class option_not_exists_exception : public OptionParseException - { - public: - explicit option_not_exists_exception(const std::string& option) - : OptionParseException("Option " + LQUOTE + option + RQUOTE + " does not exist") - { - } - }; - - class missing_argument_exception : public OptionParseException - { - public: - explicit missing_argument_exception(const std::string& option) - : OptionParseException( - "Option " + LQUOTE + option + RQUOTE + " is missing an argument" - ) - { - } - }; - - class option_requires_argument_exception : public OptionParseException - { - public: - explicit option_requires_argument_exception(const std::string& option) - : OptionParseException( - "Option " + LQUOTE + option + RQUOTE + " requires an argument" - ) - { - } - }; - - class option_not_has_argument_exception : public OptionParseException - { - public: - option_not_has_argument_exception - ( - const std::string& option, - const std::string& arg - ) - : OptionParseException( - "Option " + LQUOTE + option + RQUOTE + - " does not take an argument, but argument " + - LQUOTE + arg + RQUOTE + " given" - ) - { - } - }; - - class option_not_present_exception : public OptionParseException - { - public: - explicit option_not_present_exception(const std::string& option) - : OptionParseException("Option " + LQUOTE + option + RQUOTE + " not present") - { - } - }; - - class option_has_no_value_exception : public OptionException - { - public: - explicit option_has_no_value_exception(const std::string& option) - : OptionException( - option.empty() ? - ("Option " + LQUOTE + option + RQUOTE + " has no value") : - "Option has no value") - { - } - }; - - class argument_incorrect_type : public OptionParseException - { - public: - explicit argument_incorrect_type - ( - const std::string& arg - ) - : OptionParseException( - "Argument " + LQUOTE + arg + RQUOTE + " failed to parse" - ) - { - } - }; - - class option_required_exception : public OptionParseException - { - public: - explicit option_required_exception(const std::string& option) - : OptionParseException( - "Option " + LQUOTE + option + RQUOTE + " is required but not present" - ) - { - } - }; - - template - void throw_or_mimic(const std::string& text) - { - static_assert(std::is_base_of::value, - "throw_or_mimic only works on std::exception and " - "deriving classes"); - -#ifndef CXXOPTS_NO_EXCEPTIONS - // If CXXOPTS_NO_EXCEPTIONS is not defined, just throw - throw T{text}; -#else - // Otherwise manually instantiate the exception, print what() to stderr, - // and exit - T exception{text}; - std::cerr << exception.what() << std::endl; - std::exit(EXIT_FAILURE); -#endif - } - - namespace values - { - namespace - { - std::basic_regex integer_pattern - ("(-)?(0x)?([0-9a-zA-Z]+)|((0x)?0)"); - std::basic_regex truthy_pattern - ("(t|T)(rue)?|1"); - std::basic_regex falsy_pattern - ("(f|F)(alse)?|0"); - } // namespace - - namespace detail - { - template - struct SignedCheck; - - template - struct SignedCheck - { - template - void - operator()(bool negative, U u, const std::string& text) - { - if (negative) - { - if (u > static_cast((std::numeric_limits::min)())) - { - throw_or_mimic(text); - } - } - else - { - if (u > static_cast((std::numeric_limits::max)())) - { - throw_or_mimic(text); - } - } - } - }; - - template - struct SignedCheck - { - template - void - operator()(bool, U, const std::string&) {} - }; - - template - void - check_signed_range(bool negative, U value, const std::string& text) - { - SignedCheck::is_signed>()(negative, value, text); - } - } // namespace detail - - template - void - checked_negate(R& r, T&& t, const std::string&, std::true_type) - { - // if we got to here, then `t` is a positive number that fits into - // `R`. So to avoid MSVC C4146, we first cast it to `R`. - // See https://github.com/jarro2783/cxxopts/issues/62 for more details. - r = static_cast(-static_cast(t-1)-1); - } - - template - void - checked_negate(R&, T&&, const std::string& text, std::false_type) - { - throw_or_mimic(text); - } - - template - void - integer_parser(const std::string& text, T& value) - { - std::smatch match; - std::regex_match(text, match, integer_pattern); - - if (match.length() == 0) - { - throw_or_mimic(text); - } - - if (match.length(4) > 0) - { - value = 0; - return; - } - - using US = typename std::make_unsigned::type; - - constexpr bool is_signed = std::numeric_limits::is_signed; - const bool negative = match.length(1) > 0; - const uint8_t base = match.length(2) > 0 ? 16 : 10; - - auto value_match = match[3]; - - US result = 0; - - for (auto iter = value_match.first; iter != value_match.second; ++iter) - { - US digit = 0; - - if (*iter >= '0' && *iter <= '9') - { - digit = static_cast(*iter - '0'); - } - else if (base == 16 && *iter >= 'a' && *iter <= 'f') - { - digit = static_cast(*iter - 'a' + 10); - } - else if (base == 16 && *iter >= 'A' && *iter <= 'F') - { - digit = static_cast(*iter - 'A' + 10); - } - else - { - throw_or_mimic(text); - } - - const US next = static_cast(result * base + digit); - if (result > next) - { - throw_or_mimic(text); - } - - result = next; - } - - detail::check_signed_range(negative, result, text); - - if (negative) - { - checked_negate(value, result, text, std::integral_constant()); - } - else - { - value = static_cast(result); - } - } - - template - void stringstream_parser(const std::string& text, T& value) - { - std::stringstream in(text); - in >> value; - if (!in) { - throw_or_mimic(text); - } - } - - inline - void - parse_value(const std::string& text, uint8_t& value) - { - integer_parser(text, value); - } - - inline - void - parse_value(const std::string& text, int8_t& value) - { - integer_parser(text, value); - } - - inline - void - parse_value(const std::string& text, uint16_t& value) - { - integer_parser(text, value); - } - - inline - void - parse_value(const std::string& text, int16_t& value) - { - integer_parser(text, value); - } - - inline - void - parse_value(const std::string& text, uint32_t& value) - { - integer_parser(text, value); - } - - inline - void - parse_value(const std::string& text, int32_t& value) - { - integer_parser(text, value); - } - - inline - void - parse_value(const std::string& text, uint64_t& value) - { - integer_parser(text, value); - } - - inline - void - parse_value(const std::string& text, int64_t& value) - { - integer_parser(text, value); - } - - inline - void - parse_value(const std::string& text, bool& value) - { - std::smatch result; - std::regex_match(text, result, truthy_pattern); - - if (!result.empty()) - { - value = true; - return; - } - - std::regex_match(text, result, falsy_pattern); - if (!result.empty()) - { - value = false; - return; - } - - throw_or_mimic(text); - } - - inline - void - parse_value(const std::string& text, std::string& value) - { - value = text; - } - - // The fallback parser. It uses the stringstream parser to parse all types - // that have not been overloaded explicitly. It has to be placed in the - // source code before all other more specialized templates. - template - void - parse_value(const std::string& text, T& value) { - stringstream_parser(text, value); - } - - template - void - parse_value(const std::string& text, std::vector& value) - { - std::stringstream in(text); - std::string token; - while(!in.eof() && std::getline(in, token, CXXOPTS_VECTOR_DELIMITER)) { - T v; - parse_value(token, v); - value.emplace_back(std::move(v)); - } - } - -#ifdef CXXOPTS_HAS_OPTIONAL - template - void - parse_value(const std::string& text, std::optional& value) - { - T result; - parse_value(text, result); - value = std::move(result); - } -#endif - - inline - void parse_value(const std::string& text, char& c) - { - if (text.length() != 1) - { - throw_or_mimic(text); - } - - c = text[0]; - } - - template - struct type_is_container - { - static constexpr bool value = false; - }; - - template - struct type_is_container> - { - static constexpr bool value = true; - }; - - template - class abstract_value : public Value - { - using Self = abstract_value; - - public: - abstract_value() - : m_result(std::make_shared()) - , m_store(m_result.get()) - { - } - - explicit abstract_value(T* t) - : m_store(t) - { - } - - ~abstract_value() override = default; - - abstract_value(const abstract_value& rhs) - { - if (rhs.m_result) - { - m_result = std::make_shared(); - m_store = m_result.get(); - } - else - { - m_store = rhs.m_store; - } - - m_default = rhs.m_default; - m_implicit = rhs.m_implicit; - m_default_value = rhs.m_default_value; - m_implicit_value = rhs.m_implicit_value; - } - - void - parse(const std::string& text) const override - { - parse_value(text, *m_store); - } - - bool - is_container() const override - { - return type_is_container::value; - } - - void - parse() const override - { - parse_value(m_default_value, *m_store); - } - - bool - has_default() const override - { - return m_default; - } - - bool - has_implicit() const override - { - return m_implicit; - } - - std::shared_ptr - default_value(const std::string& value) override - { - m_default = true; - m_default_value = value; - return shared_from_this(); - } - - std::shared_ptr - implicit_value(const std::string& value) override - { - m_implicit = true; - m_implicit_value = value; - return shared_from_this(); - } - - std::shared_ptr - no_implicit_value() override - { - m_implicit = false; - return shared_from_this(); - } - - std::string - get_default_value() const override - { - return m_default_value; - } - - std::string - get_implicit_value() const override - { - return m_implicit_value; - } - - bool - is_boolean() const override - { - return std::is_same::value; - } - - const T& - get() const - { - if (m_store == nullptr) - { - return *m_result; - } - return *m_store; - } - - protected: - std::shared_ptr m_result; - T* m_store; - - bool m_default = false; - bool m_implicit = false; - - std::string m_default_value; - std::string m_implicit_value; - }; - - template - class standard_value : public abstract_value - { - public: - using abstract_value::abstract_value; - - CXXOPTS_NODISCARD - std::shared_ptr - clone() const - { - return std::make_shared>(*this); - } - }; - - template <> - class standard_value : public abstract_value - { - public: - ~standard_value() override = default; - - standard_value() - { - set_default_and_implicit(); - } - - explicit standard_value(bool* b) - : abstract_value(b) - { - set_default_and_implicit(); - } - - std::shared_ptr - clone() const override - { - return std::make_shared>(*this); - } - - private: - - void - set_default_and_implicit() - { - m_default = true; - m_default_value = "false"; - m_implicit = true; - m_implicit_value = "true"; - } - }; - } // namespace values - - template - std::shared_ptr - value() - { - return std::make_shared>(); - } - - template - std::shared_ptr - value(T& t) - { - return std::make_shared>(&t); - } - - class OptionAdder; - - class OptionDetails - { - public: - OptionDetails - ( - std::string short_, - std::string long_, - String desc, - std::shared_ptr val - ) - : m_short(std::move(short_)) - , m_long(std::move(long_)) - , m_desc(std::move(desc)) - , m_value(std::move(val)) - , m_count(0) - { - } - - OptionDetails(const OptionDetails& rhs) - : m_desc(rhs.m_desc) - , m_count(rhs.m_count) - { - m_value = rhs.m_value->clone(); - } - - OptionDetails(OptionDetails&& rhs) = default; - - CXXOPTS_NODISCARD - const String& - description() const - { - return m_desc; - } - - CXXOPTS_NODISCARD - const Value& - value() const { - return *m_value; - } - - CXXOPTS_NODISCARD - std::shared_ptr - make_storage() const - { - return m_value->clone(); - } - - CXXOPTS_NODISCARD - const std::string& - short_name() const - { - return m_short; - } - - CXXOPTS_NODISCARD - const std::string& - long_name() const - { - return m_long; - } - - private: - std::string m_short; - std::string m_long; - String m_desc; - std::shared_ptr m_value; - int m_count; - }; - - struct HelpOptionDetails - { - std::string s; - std::string l; - String desc; - bool has_default; - std::string default_value; - bool has_implicit; - std::string implicit_value; - std::string arg_help; - bool is_container; - bool is_boolean; - }; - - struct HelpGroupDetails - { - std::string name; - std::string description; - std::vector options; - }; - - class OptionValue - { - public: - void - parse - ( - const std::shared_ptr& details, - const std::string& text - ) - { - ensure_value(details); - ++m_count; - m_value->parse(text); - m_long_name = &details->long_name(); - } - - void - parse_default(const std::shared_ptr& details) - { - ensure_value(details); - m_default = true; - m_long_name = &details->long_name(); - m_value->parse(); - } - - CXXOPTS_NODISCARD - size_t - count() const noexcept - { - return m_count; - } - - // TODO: maybe default options should count towards the number of arguments - CXXOPTS_NODISCARD - bool - has_default() const noexcept - { - return m_default; - } - - template - const T& - as() const - { - if (m_value == nullptr) { - throw_or_mimic( - m_long_name == nullptr ? "" : *m_long_name); - } - -#ifdef CXXOPTS_NO_RTTI - return static_cast&>(*m_value).get(); -#else - return dynamic_cast&>(*m_value).get(); -#endif - } - - private: - void - ensure_value(const std::shared_ptr& details) - { - if (m_value == nullptr) - { - m_value = details->make_storage(); - } - } - - const std::string* m_long_name = nullptr; - // Holding this pointer is safe, since OptionValue's only exist in key-value pairs, - // where the key has the string we point to. - std::shared_ptr m_value; - size_t m_count = 0; - bool m_default = false; - }; - - class KeyValue - { - public: - KeyValue(std::string key_, std::string value_) - : m_key(std::move(key_)) - , m_value(std::move(value_)) - { - } - - CXXOPTS_NODISCARD - const std::string& - key() const - { - return m_key; - } - - CXXOPTS_NODISCARD - const std::string& - value() const - { - return m_value; - } - - template - T - as() const - { - T result; - values::parse_value(m_value, result); - return result; - } - - private: - std::string m_key; - std::string m_value; - }; - - class ParseResult - { - public: - - ParseResult( - std::shared_ptr< - std::unordered_map> - >, - std::vector, - bool allow_unrecognised, - int&, const char**&); - - size_t - count(const std::string& o) const - { - auto iter = m_options->find(o); - if (iter == m_options->end()) - { - return 0; - } - - auto riter = m_results.find(iter->second); - - return riter->second.count(); - } - - const OptionValue& - operator[](const std::string& option) const - { - auto iter = m_options->find(option); - - if (iter == m_options->end()) - { - throw_or_mimic(option); - } - - auto riter = m_results.find(iter->second); - - return riter->second; - } - - const std::vector& - arguments() const - { - return m_sequential; - } - - private: - - void - parse(int& argc, const char**& argv); - - void - add_to_option(const std::string& option, const std::string& arg); - - bool - consume_positional(const std::string& a); - - void - parse_option - ( - const std::shared_ptr& value, - const std::string& name, - const std::string& arg = "" - ); - - void - parse_default(const std::shared_ptr& details); - - void - checked_parse_arg - ( - int argc, - const char* argv[], - int& current, - const std::shared_ptr& value, - const std::string& name - ); - - const std::shared_ptr< - std::unordered_map> - > m_options; - std::vector m_positional; - std::vector::iterator m_next_positional; - std::unordered_set m_positional_set; - std::unordered_map, OptionValue> m_results; - - bool m_allow_unrecognised; - - std::vector m_sequential; - }; - - struct Option - { - Option - ( - std::string opts, - std::string desc, - std::shared_ptr value = ::cxxopts::value(), - std::string arg_help = "" - ) - : opts_(std::move(opts)) - , desc_(std::move(desc)) - , value_(std::move(value)) - , arg_help_(std::move(arg_help)) - { - } - - std::string opts_; - std::string desc_; - std::shared_ptr value_; - std::string arg_help_; - }; - - class Options - { - using OptionMap = std::unordered_map>; - public: - - explicit Options(std::string program, std::string help_string = "") - : m_program(std::move(program)) - , m_help_string(toLocalString(std::move(help_string))) - , m_custom_help("[OPTION...]") - , m_positional_help("positional parameters") - , m_show_positional(false) - , m_allow_unrecognised(false) - , m_options(std::make_shared()) - , m_next_positional(m_positional.end()) - { - } - - Options& - positional_help(std::string help_text) - { - m_positional_help = std::move(help_text); - return *this; - } - - Options& - custom_help(std::string help_text) - { - m_custom_help = std::move(help_text); - return *this; - } - - Options& - show_positional_help() - { - m_show_positional = true; - return *this; - } - - Options& - allow_unrecognised_options() - { - m_allow_unrecognised = true; - return *this; - } - - ParseResult - parse(int& argc, const char**& argv); - - OptionAdder - add_options(std::string group = ""); - - void - add_options - ( - const std::string& group, - std::initializer_list