From b310d44ebf9cdde3495102e4868e8493fd7e0812 Mon Sep 17 00:00:00 2001 From: Tristan Date: Thu, 31 Jul 2025 20:38:26 -0700 Subject: [PATCH 01/51] Added the skeleton of a C++ module to directly evaluate FTAs using llama.cpp --- .dockerignore | 54 ++++++++ .gitignore | 2 + .gitmodules | 3 + Dockerfile | 26 ++++ README.md | 15 +++ libs/autocog/llama/CMakeLists.txt | 61 +++++++++ libs/autocog/llama/evaluator.cxx | 9 ++ libs/autocog/llama/evaluator.hxx | 12 ++ libs/autocog/llama/fta.cxx | 16 +++ libs/autocog/llama/fta.hxx | 53 ++++++++ libs/autocog/llama/ftt.cxx | 7 ++ libs/autocog/llama/ftt.hxx | 26 ++++ libs/autocog/llama/python_bindings.cxx | 65 ++++++++++ libs/autocog/llama/tokenizer.cxx | 9 ++ libs/autocog/llama/tokenizer.hxx | 12 ++ {autocog => modules/autocog}/__init__.py | 0 {autocog => modules/autocog}/__main__.py | 0 {autocog => modules/autocog}/arch/__init__.py | 0 .../autocog}/arch/architecture.py | 0 {autocog => modules/autocog}/arch/cogs.py | 0 .../autocog}/arch/orchestrator.py | 0 {autocog => modules/autocog}/arch/utility.py | 0 {autocog => modules/autocog}/config.py | 0 {autocog => modules/autocog}/fta/__init__.py | 0 {autocog => modules/autocog}/fta/actions.py | 0 {autocog => modules/autocog}/fta/automaton.py | 0 {autocog => modules/autocog}/fta/beam.py | 0 {autocog => modules/autocog}/fta/ftt.py | 2 +- {autocog => modules/autocog}/fta/tct.py | 0 {autocog => modules/autocog}/fta/utils.py | 0 {autocog => modules/autocog}/fta/vocab.py | 0 {autocog => modules/autocog}/lm/__init__.py | 0 {autocog => modules/autocog}/lm/llama.py | 0 {autocog => modules/autocog}/lm/lm.py | 0 {autocog => modules/autocog}/lm/random.py | 0 .../autocog}/lm/transformers.py | 0 {autocog => modules/autocog}/sta/__init__.py | 0 {autocog => modules/autocog}/sta/ast.py | 0 {autocog => modules/autocog}/sta/automaton.py | 0 {autocog => modules/autocog}/sta/compile.py | 0 {autocog => modules/autocog}/sta/devel.py | 0 {autocog => modules/autocog}/sta/frontend.py | 0 {autocog => modules/autocog}/sta/grammar.py | 2 +- {autocog => modules/autocog}/sta/ir.py | 0 .../autocog}/sta/parse_tree.py | 0 {autocog => modules/autocog}/sta/runtime.py | 0 {autocog => modules/autocog}/sta/syntax.py | 0 .../autocog}/utility/__init__.py | 0 .../autocog}/utility/args2arch.py | 0 .../autocog}/utility/dashboard.py | 0 {autocog => modules/autocog}/utility/enums.py | 0 .../autocog}/utility/gv2html.py | 0 .../autocog}/utility/models.py | 0 {autocog => modules/autocog}/utility/pynb.py | 0 .../autocog}/utility/server.py | 0 pyproject.toml | 59 +++++++++ requirements.txt | 4 - setup.py | 116 ++++++++++++++---- vendors/llama | 1 + 59 files changed, 523 insertions(+), 31 deletions(-) create mode 100644 .dockerignore create mode 100644 .gitmodules create mode 100644 Dockerfile create mode 100644 libs/autocog/llama/CMakeLists.txt create mode 100644 libs/autocog/llama/evaluator.cxx create mode 100644 libs/autocog/llama/evaluator.hxx create mode 100644 libs/autocog/llama/fta.cxx create mode 100644 libs/autocog/llama/fta.hxx create mode 100644 libs/autocog/llama/ftt.cxx create mode 100644 libs/autocog/llama/ftt.hxx create mode 100644 libs/autocog/llama/python_bindings.cxx create mode 100644 libs/autocog/llama/tokenizer.cxx create mode 100644 libs/autocog/llama/tokenizer.hxx rename {autocog => modules/autocog}/__init__.py (100%) rename {autocog => modules/autocog}/__main__.py (100%) rename {autocog => modules/autocog}/arch/__init__.py (100%) rename {autocog => modules/autocog}/arch/architecture.py (100%) rename {autocog => modules/autocog}/arch/cogs.py (100%) rename {autocog => modules/autocog}/arch/orchestrator.py (100%) rename {autocog => modules/autocog}/arch/utility.py (100%) rename {autocog => modules/autocog}/config.py (100%) rename {autocog => modules/autocog}/fta/__init__.py (100%) rename {autocog => modules/autocog}/fta/actions.py (100%) rename {autocog => modules/autocog}/fta/automaton.py (100%) rename {autocog => modules/autocog}/fta/beam.py (100%) rename {autocog => modules/autocog}/fta/ftt.py (98%) rename {autocog => modules/autocog}/fta/tct.py (100%) rename {autocog => modules/autocog}/fta/utils.py (100%) rename {autocog => modules/autocog}/fta/vocab.py (100%) rename {autocog => modules/autocog}/lm/__init__.py (100%) rename {autocog => modules/autocog}/lm/llama.py (100%) rename {autocog => modules/autocog}/lm/lm.py (100%) rename {autocog => modules/autocog}/lm/random.py (100%) rename {autocog => modules/autocog}/lm/transformers.py (100%) rename {autocog => modules/autocog}/sta/__init__.py (100%) rename {autocog => modules/autocog}/sta/ast.py (100%) rename {autocog => modules/autocog}/sta/automaton.py (100%) rename {autocog => modules/autocog}/sta/compile.py (100%) rename {autocog => modules/autocog}/sta/devel.py (100%) rename {autocog => modules/autocog}/sta/frontend.py (100%) rename {autocog => modules/autocog}/sta/grammar.py (99%) rename {autocog => modules/autocog}/sta/ir.py (100%) rename {autocog => modules/autocog}/sta/parse_tree.py (100%) rename {autocog => modules/autocog}/sta/runtime.py (100%) rename {autocog => modules/autocog}/sta/syntax.py (100%) rename {autocog => modules/autocog}/utility/__init__.py (100%) rename {autocog => modules/autocog}/utility/args2arch.py (100%) rename {autocog => modules/autocog}/utility/dashboard.py (100%) rename {autocog => modules/autocog}/utility/enums.py (100%) rename {autocog => modules/autocog}/utility/gv2html.py (100%) rename {autocog => modules/autocog}/utility/models.py (100%) rename {autocog => modules/autocog}/utility/pynb.py (100%) rename {autocog => modules/autocog}/utility/server.py (100%) create mode 100644 pyproject.toml delete mode 100644 requirements.txt mode change 100755 => 100644 setup.py create mode 160000 vendors/llama diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e26ced7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,54 @@ +# Git +.git +.gitignore + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual environments +venv/ +env/ +ENV/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +# Logs +*.log + +# Documentation +docs/_build/ + +# Test coverage +.coverage +htmlcov/ + +# Temporary files +*.tmp +*.temp diff --git a/.gitignore b/.gitignore index 90ab3e6..a9ff4c4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ **/__pycache__/ /models /Pipfile +/modules/autocog/*.so +/build/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ae8ec1f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vendors/llama"] + path = vendors/llama + url = https://github.com/ggerganov/llama.cpp.git diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fc352af --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 + +RUN apt-get update && \ + apt-get install -y \ + build-essential \ + cmake \ + pkg-config \ + python3 \ + python3-pip \ + python3-dev \ + python3-venv && \ + rm -rf /var/lib/apt/lists/* + +RUN python3 -m venv /venv +ENV PATH="/venv/bin:$PATH" + +RUN pip install --upgrade pip setuptools wheel pybind11 numpy + +COPY . /workspace/autocog +WORKDIR /workspace/autocog + +RUN pip install -e . + diff --git a/README.md b/README.md index a0bff45..95b6f2f 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,21 @@ We broke down the documentation into a few files: The libraries have [their own documentation](./share/library/README.md). +## Develop Command Cheat Sheet + +### C++ module: autocog.llama + +Build image and "test": +``` +docker build -t autocog:latest . +docker run --rm -it autocog:latest python3 -c "import autocog.llama ; print(autocog.llama)" +``` + +Fast rebuild (in mounted: +``` +docker run --rm -v $(pwd):/workspace/autocog -w /workspace/autocog -it autocog:latest python setup.py build_ext --inplace +``` + ## Contributing Contributions are welcome! diff --git a/libs/autocog/llama/CMakeLists.txt b/libs/autocog/llama/CMakeLists.txt new file mode 100644 index 0000000..e4dc7d6 --- /dev/null +++ b/libs/autocog/llama/CMakeLists.txt @@ -0,0 +1,61 @@ +cmake_minimum_required(VERSION 3.18) +project(autocog_llama) + +# Set C++ standard +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# Find required packages +find_package(pybind11 REQUIRED) + +# Add llama.cpp as subdirectory +add_subdirectory(../../../vendors/llama llama_cpp EXCLUDE_FROM_ALL) + +# Source files +set(SOURCES + fta.cxx + ftt.cxx + evaluator.cxx + tokenizer.cxx + python_bindings.cxx +) + +# Create the Python module +pybind11_add_module(autocog_llama ${SOURCES}) + +set_target_properties(autocog_llama PROPERTIES OUTPUT_NAME "llama") +# Include directories +target_include_directories(autocog_llama PRIVATE + ../.. + ../../../vendors/llama +) + +# Link against llama.cpp +target_link_libraries(autocog_llama PRIVATE llama) + +# Compiler flags +target_compile_features(autocog_llama PRIVATE cxx_std_17) + +# Optimization flags +if(CMAKE_BUILD_TYPE STREQUAL "Release") + target_compile_options(autocog_llama PRIVATE + $<$:-O3 -march=native> + $<$:/O2> + ) +endif() + +# Enable warnings +target_compile_options(autocog_llama PRIVATE + $<$:-Wall -Wextra -Wpedantic> + $<$:/W4> +) + +# Python extension properties +set_target_properties(autocog_llama PROPERTIES + PREFIX "${PYTHON_MODULE_PREFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}" +) + +# Installation +install(TARGETS autocog_llama DESTINATION autocog) + diff --git a/libs/autocog/llama/evaluator.cxx b/libs/autocog/llama/evaluator.cxx new file mode 100644 index 0000000..b96e183 --- /dev/null +++ b/libs/autocog/llama/evaluator.cxx @@ -0,0 +1,9 @@ + +#include "autocog/llama/evaluator.hxx" + +namespace autocog { namespace llama { + +// TODO + +} } + diff --git a/libs/autocog/llama/evaluator.hxx b/libs/autocog/llama/evaluator.hxx new file mode 100644 index 0000000..44c0fa2 --- /dev/null +++ b/libs/autocog/llama/evaluator.hxx @@ -0,0 +1,12 @@ +#ifndef __AUTOCOG_LLAMA_EVALUATOR__HXX_ +#define __AUTOCOG_LLAMA_EVALUATOR__HXX_ + +namespace autocog { +namespace llama { + +class Evaluator {}; + +} } + +#endif /* __AUTOCOG_LLAMA_EVALUATOR__HXX_ */ + diff --git a/libs/autocog/llama/fta.cxx b/libs/autocog/llama/fta.cxx new file mode 100644 index 0000000..7a1d699 --- /dev/null +++ b/libs/autocog/llama/fta.cxx @@ -0,0 +1,16 @@ + +#include "autocog/llama/fta.hxx" + +namespace autocog { namespace llama { + +Action::Action(ActionKind const kind_, NodeID const id_) : kind(kind_), id(id_) {} + +Text::Text(NodeID const id_) : Action(ActionKind::Text, id_) {} + +Completion::Completion(NodeID const id_) : Action(ActionKind::Completion, id_) {} + +Choice::Choice(NodeID const id_) : Action(ActionKind::Choice, id_) {} + + +} } + diff --git a/libs/autocog/llama/fta.hxx b/libs/autocog/llama/fta.hxx new file mode 100644 index 0000000..5a20d67 --- /dev/null +++ b/libs/autocog/llama/fta.hxx @@ -0,0 +1,53 @@ +#ifndef __AUTOCOG_LLAMA_FTA__HXX_ +#define __AUTOCOG_LLAMA_FTA__HXX_ + +#include + +namespace autocog { +namespace llama { + +using NodeID = unsigned; + +enum class ActionKind { + Text, + Completion, + Choice +}; + +class Action { + public: + ActionKind const kind; + NodeID const id; + + Action(ActionKind const kind_, NodeID const id_); + + protected: + std::vector next; +}; + +class Text : public Action { + public: + Text(NodeID const id_); +}; + +class Completion : public Action { + public: + Completion(NodeID const id_); +}; + +class Choice : public Action { + public: + Choice(NodeID const id_); +}; + +class FTA { + public: + Action const & action(NodeID const & id) const; + private: + std::vector nodes; +}; + +} } + +#endif /* __AUTOCOG_LLAMA_FTA__HXX_ */ + diff --git a/libs/autocog/llama/ftt.cxx b/libs/autocog/llama/ftt.cxx new file mode 100644 index 0000000..6befe3d --- /dev/null +++ b/libs/autocog/llama/ftt.cxx @@ -0,0 +1,7 @@ + +#include "autocog/llama/ftt.hxx" + +namespace autocog { namespace llama { + +} } + diff --git a/libs/autocog/llama/ftt.hxx b/libs/autocog/llama/ftt.hxx new file mode 100644 index 0000000..8b25ba7 --- /dev/null +++ b/libs/autocog/llama/ftt.hxx @@ -0,0 +1,26 @@ +#ifndef __AUTOCOG_LLAMA_FTT__HXX_ +#define __AUTOCOG_LLAMA_FTT__HXX_ + +#include +#include + +namespace autocog { +namespace llama { + +using TokenID = unsigned; +using probability_t = float; + +class FTT { + public: + FTT() = default; + + private: + std::vector tokens; + std::vector probas; + std::vector children; +}; + +} } + +#endif /* __AUTOCOG_LLAMA_FTT__HXX_ */ + diff --git a/libs/autocog/llama/python_bindings.cxx b/libs/autocog/llama/python_bindings.cxx new file mode 100644 index 0000000..bdff94d --- /dev/null +++ b/libs/autocog/llama/python_bindings.cxx @@ -0,0 +1,65 @@ + +#include +#include +#include + +#include "fta.hxx" +#include "ftt.hxx" +#include "evaluator.hxx" +#include "tokenizer.hxx" + +static void populate_fta_from_python_dict(autocog::llama::FTA & fta, pybind11::dict const & fta_dict) { + // TODO: Implementation +} + +static pybind11::dict ftt_to_python_dict(autocog::llama::FTT const & ftt) { + pybind11::dict result; + // TODO: Implementation + return result; +} + +// Wrapper functions for Python integration +pybind11::dict evaluate_fta_python(uintptr_t ctx_ptr, const pybind11::dict& fta_dict, const pybind11::dict& config_dict = pybind11::dict()) { + autocog::llama::FTA fta; + populate_fta_from_python_dict(fta, fta_dict); + autocog::llama::FTT ftt; + + // TODO: Implementation + + return ftt_to_python_dict(ftt); +} + +// Tokenizer wrapper functions +std::vector tokenize_python(uintptr_t model_ptr, const std::string& text, bool add_special = true) { + // TODO: Implementation + return {}; +} + +std::string detokenize_python(uintptr_t model_ptr, const std::vector& tokens) { + // TODO: Implementation + return ""; +} + +PYBIND11_MODULE(llama, m) { + m.doc() = "AutoCog llama.cpp integration module"; + + m.def("get_version", []() { return "0.1.0"; }); + + m.def("evaluate_fta", &evaluate_fta_python, + "Evaluate Finite Thoughts Automata", + pybind11::arg("ctx_ptr"), pybind11::arg("fta_dict"), pybind11::arg("config_dict") = pybind11::dict()); + + m.def("tokenize", &tokenize_python, + "Tokenize text using llama.cpp tokenizer", + pybind11::arg("model_ptr"), pybind11::arg("text"), pybind11::arg("add_special") = true); + + m.def("detokenize", &detokenize_python, + "Detokenize tokens using llama.cpp tokenizer", + pybind11::arg("model_ptr"), pybind11::arg("tokens")); + + pybind11::enum_(m, "ActionKind") + .value("Text", autocog::llama::ActionKind::Text ) + .value("Completion", autocog::llama::ActionKind::Completion ) + .value("Choice", autocog::llama::ActionKind::Choice ); +} + diff --git a/libs/autocog/llama/tokenizer.cxx b/libs/autocog/llama/tokenizer.cxx new file mode 100644 index 0000000..b67d7c0 --- /dev/null +++ b/libs/autocog/llama/tokenizer.cxx @@ -0,0 +1,9 @@ + +#include "autocog/llama/tokenizer.hxx" + +namespace autocog { namespace llama { + +// TODO + +} } + diff --git a/libs/autocog/llama/tokenizer.hxx b/libs/autocog/llama/tokenizer.hxx new file mode 100644 index 0000000..00944d2 --- /dev/null +++ b/libs/autocog/llama/tokenizer.hxx @@ -0,0 +1,12 @@ +#ifndef __AUTOCOG_LLAMA_TOKENIZER__HXX_ +#define __AUTOCOG_LLAMA_TOKENIZER__HXX_ + +namespace autocog { +namespace llama { + +class Tokenizer {}; + +} } + +#endif /* __AUTOCOG_LLAMA_TOKENIZER__HXX_ */ + diff --git a/autocog/__init__.py b/modules/autocog/__init__.py similarity index 100% rename from autocog/__init__.py rename to modules/autocog/__init__.py diff --git a/autocog/__main__.py b/modules/autocog/__main__.py similarity index 100% rename from autocog/__main__.py rename to modules/autocog/__main__.py diff --git a/autocog/arch/__init__.py b/modules/autocog/arch/__init__.py similarity index 100% rename from autocog/arch/__init__.py rename to modules/autocog/arch/__init__.py diff --git a/autocog/arch/architecture.py b/modules/autocog/arch/architecture.py similarity index 100% rename from autocog/arch/architecture.py rename to modules/autocog/arch/architecture.py diff --git a/autocog/arch/cogs.py b/modules/autocog/arch/cogs.py similarity index 100% rename from autocog/arch/cogs.py rename to modules/autocog/arch/cogs.py diff --git a/autocog/arch/orchestrator.py b/modules/autocog/arch/orchestrator.py similarity index 100% rename from autocog/arch/orchestrator.py rename to modules/autocog/arch/orchestrator.py diff --git a/autocog/arch/utility.py b/modules/autocog/arch/utility.py similarity index 100% rename from autocog/arch/utility.py rename to modules/autocog/arch/utility.py diff --git a/autocog/config.py b/modules/autocog/config.py similarity index 100% rename from autocog/config.py rename to modules/autocog/config.py diff --git a/autocog/fta/__init__.py b/modules/autocog/fta/__init__.py similarity index 100% rename from autocog/fta/__init__.py rename to modules/autocog/fta/__init__.py diff --git a/autocog/fta/actions.py b/modules/autocog/fta/actions.py similarity index 100% rename from autocog/fta/actions.py rename to modules/autocog/fta/actions.py diff --git a/autocog/fta/automaton.py b/modules/autocog/fta/automaton.py similarity index 100% rename from autocog/fta/automaton.py rename to modules/autocog/fta/automaton.py diff --git a/autocog/fta/beam.py b/modules/autocog/fta/beam.py similarity index 100% rename from autocog/fta/beam.py rename to modules/autocog/fta/beam.py diff --git a/autocog/fta/ftt.py b/modules/autocog/fta/ftt.py similarity index 98% rename from autocog/fta/ftt.py rename to modules/autocog/fta/ftt.py index e273094..245860f 100644 --- a/autocog/fta/ftt.py +++ b/modules/autocog/fta/ftt.py @@ -217,7 +217,7 @@ def toGraphViz(self, lm): label = label.replace('\n',r'\l') # label = json.dumps(label.replace(r'\n',r'\l'))[1:-1] # label = 'text' - dotstr += f'n_{tree.id}' + '[shape=record, label="{' + label + '\l|finalized=' + str(tree.finalized) + '\l' + ( "" if tree.probas is None else tree.probas.toGraphVizRecord() ) + '}"];\n' + dotstr += f'n_{tree.id}' + '[shape=record, label="{' + label + '\\l|finalized=' + str(tree.finalized) + '\\l' + ( "" if tree.probas is None else tree.probas.toGraphVizRecord() ) + '}"];\n' if tree.parent is not None: dotstr += f'n_{tree.parent.id} -> n_{tree.id};\n' return dotstr diff --git a/autocog/fta/tct.py b/modules/autocog/fta/tct.py similarity index 100% rename from autocog/fta/tct.py rename to modules/autocog/fta/tct.py diff --git a/autocog/fta/utils.py b/modules/autocog/fta/utils.py similarity index 100% rename from autocog/fta/utils.py rename to modules/autocog/fta/utils.py diff --git a/autocog/fta/vocab.py b/modules/autocog/fta/vocab.py similarity index 100% rename from autocog/fta/vocab.py rename to modules/autocog/fta/vocab.py diff --git a/autocog/lm/__init__.py b/modules/autocog/lm/__init__.py similarity index 100% rename from autocog/lm/__init__.py rename to modules/autocog/lm/__init__.py diff --git a/autocog/lm/llama.py b/modules/autocog/lm/llama.py similarity index 100% rename from autocog/lm/llama.py rename to modules/autocog/lm/llama.py diff --git a/autocog/lm/lm.py b/modules/autocog/lm/lm.py similarity index 100% rename from autocog/lm/lm.py rename to modules/autocog/lm/lm.py diff --git a/autocog/lm/random.py b/modules/autocog/lm/random.py similarity index 100% rename from autocog/lm/random.py rename to modules/autocog/lm/random.py diff --git a/autocog/lm/transformers.py b/modules/autocog/lm/transformers.py similarity index 100% rename from autocog/lm/transformers.py rename to modules/autocog/lm/transformers.py diff --git a/autocog/sta/__init__.py b/modules/autocog/sta/__init__.py similarity index 100% rename from autocog/sta/__init__.py rename to modules/autocog/sta/__init__.py diff --git a/autocog/sta/ast.py b/modules/autocog/sta/ast.py similarity index 100% rename from autocog/sta/ast.py rename to modules/autocog/sta/ast.py diff --git a/autocog/sta/automaton.py b/modules/autocog/sta/automaton.py similarity index 100% rename from autocog/sta/automaton.py rename to modules/autocog/sta/automaton.py diff --git a/autocog/sta/compile.py b/modules/autocog/sta/compile.py similarity index 100% rename from autocog/sta/compile.py rename to modules/autocog/sta/compile.py diff --git a/autocog/sta/devel.py b/modules/autocog/sta/devel.py similarity index 100% rename from autocog/sta/devel.py rename to modules/autocog/sta/devel.py diff --git a/autocog/sta/frontend.py b/modules/autocog/sta/frontend.py similarity index 100% rename from autocog/sta/frontend.py rename to modules/autocog/sta/frontend.py diff --git a/autocog/sta/grammar.py b/modules/autocog/sta/grammar.py similarity index 99% rename from autocog/sta/grammar.py rename to modules/autocog/sta/grammar.py index ad1eb44..16aeb37 100644 --- a/autocog/sta/grammar.py +++ b/modules/autocog/sta/grammar.py @@ -126,7 +126,7 @@ int_literal = ~r'\d+' int_infinty = "INF" -WS = ~"\s*" +WS = ~r'\s*' # Keywords diff --git a/autocog/sta/ir.py b/modules/autocog/sta/ir.py similarity index 100% rename from autocog/sta/ir.py rename to modules/autocog/sta/ir.py diff --git a/autocog/sta/parse_tree.py b/modules/autocog/sta/parse_tree.py similarity index 100% rename from autocog/sta/parse_tree.py rename to modules/autocog/sta/parse_tree.py diff --git a/autocog/sta/runtime.py b/modules/autocog/sta/runtime.py similarity index 100% rename from autocog/sta/runtime.py rename to modules/autocog/sta/runtime.py diff --git a/autocog/sta/syntax.py b/modules/autocog/sta/syntax.py similarity index 100% rename from autocog/sta/syntax.py rename to modules/autocog/sta/syntax.py diff --git a/autocog/utility/__init__.py b/modules/autocog/utility/__init__.py similarity index 100% rename from autocog/utility/__init__.py rename to modules/autocog/utility/__init__.py diff --git a/autocog/utility/args2arch.py b/modules/autocog/utility/args2arch.py similarity index 100% rename from autocog/utility/args2arch.py rename to modules/autocog/utility/args2arch.py diff --git a/autocog/utility/dashboard.py b/modules/autocog/utility/dashboard.py similarity index 100% rename from autocog/utility/dashboard.py rename to modules/autocog/utility/dashboard.py diff --git a/autocog/utility/enums.py b/modules/autocog/utility/enums.py similarity index 100% rename from autocog/utility/enums.py rename to modules/autocog/utility/enums.py diff --git a/autocog/utility/gv2html.py b/modules/autocog/utility/gv2html.py similarity index 100% rename from autocog/utility/gv2html.py rename to modules/autocog/utility/gv2html.py diff --git a/autocog/utility/models.py b/modules/autocog/utility/models.py similarity index 100% rename from autocog/utility/models.py rename to modules/autocog/utility/models.py diff --git a/autocog/utility/pynb.py b/modules/autocog/utility/pynb.py similarity index 100% rename from autocog/utility/pynb.py rename to modules/autocog/utility/pynb.py diff --git a/autocog/utility/server.py b/modules/autocog/utility/server.py similarity index 100% rename from autocog/utility/server.py rename to modules/autocog/utility/server.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e090a06 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,59 @@ +[build-system] +requires = [ + "setuptools>=64", + "wheel", + "pybind11>=2.6.0", + "cmake>=3.18", +] +build-backend = "setuptools.build_meta" + +[project] +name = "autocog" +dynamic = ["version"] +description = "Automaton & Cognition: programming models for language models" +readme = "README.md" +license = {text = "Apache 2.0"} +authors = [ + {name = "Tristan Vanderbruggen", email = "vanderbrugge1@llnl.gov"} +] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: C++", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development :: Libraries :: Python Modules", +] +requires-python = ">=3.8" +dependencies = [ + "numpy", + "pydantic>=2", + "typing_extensions", + "parsimonious", + "pybind11>=2.6.0", +] + +[project.urls] +Homepage = "https://github.com/LLNL/autocog/" + +[tool.setuptools.dynamic] +version = {file = "VERSION"} + +[tool.setuptools.packages.find] +where = ["modules"] +include = ["autocog", "autocog.*"] + +[tool.setuptools.package-dir] +"" = "modules" + +[tool.setuptools.package-data] +autocog = ["py.typed"] + +[tool.setuptools.data-files] +"share/autocog/library/mcq" = ["share/library/mcq/*"] +"share/autocog/library/dfl" = ["share/library/dfl/*"] +"share/autocog/library/elementary" = ["share/library/elementary/*"] +"share/autocog/library/tools" = ["share/library/tools/*"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 0d4a237..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -numpy -pydantic>=2 -typing_extensions -parsimonious diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 211bfb7..784a877 --- a/setup.py +++ b/setup.py @@ -1,25 +1,91 @@ -import io, os, glob -from setuptools import find_packages, setup - -def read(path): - return io.open(os.path.join(os.path.dirname(__file__), path), encoding="utf8").read().strip() - -def read_requirements(path): - return list(map(lambda l: l.strip(), filter(lambda l: not l.startswith(('"', "#", "-", "git+")), read(path).split("\n")))) - -setup( - name="AutoCog", - version=read("VERSION"), - description="Automaton & Cognition: programming models for language models", - url="https://github.com/LLNL/autocog/", - long_description=read("README.md"), - long_description_content_type="text/markdown", - packages=find_packages(exclude=["share", "tests"]), - install_requires=read_requirements("requirements.txt"), - data_files=[ - ( 'share/autocog/library/mcq', glob.glob("share/library/mcq/*") ), - ( 'share/autocog/library/dfl', glob.glob("share/library/dfl/*") ), - ( 'share/autocog/library/elementary', glob.glob("share/library/elementary/*") ), - ( 'share/autocog/library/tools', glob.glob("share/library/tools/*") ) - ], -) +#!/usr/bin/env python3 + +import os +import sys +import subprocess +from pathlib import Path +from pybind11.setup_helpers import Pybind11Extension, build_ext +from pybind11 import get_cmake_dir +import pybind11 + +from setuptools import setup, Extension + +class CMakeExtension(Extension): + def __init__(self, name, sourcedir=""): + Extension.__init__(self, name, sources=[]) + self.sourcedir = os.path.abspath(sourcedir) + +class CMakeBuild(build_ext): + def build_extension(self, ext): + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + + if not extdir.endswith(os.path.sep): + extdir += os.path.sep + + debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug + cfg = "Debug" if debug else "Release" + + cmake_args = [ + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", + f"-DPYTHON_EXECUTABLE={sys.executable}", + f"-DCMAKE_BUILD_TYPE={cfg}", + f"-Dpybind11_DIR={get_cmake_dir()}", + ] + + build_args = [] + if "CMAKE_ARGS" in os.environ: + cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item] + + cmake_generator = os.environ.get("CMAKE_GENERATOR", "") + if self.compiler.compiler_type != "msvc": + if not cmake_generator or cmake_generator == "Ninja": + try: + import ninja + ninja_executable_path = os.path.join(ninja.BIN_DIR, "ninja") + cmake_args += [ + "-GNinja", + f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}", + ] + except ImportError: + pass + + if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: + if hasattr(self, "parallel") and self.parallel: + build_args += [f"-j{self.parallel}"] + + build_temp = Path(self.build_temp) / ext.name + build_temp.mkdir(parents=True, exist_ok=True) + + subprocess.check_call( + ["cmake", ext.sourcedir] + cmake_args, cwd=build_temp + ) + subprocess.check_call( + ["cmake", "--build", "."] + build_args, cwd=build_temp + ) + +def check_submodules(): + """Check if git submodules are initialized""" + llama_cpp_path = Path(__file__).parent / "vendors/llama" + if not llama_cpp_path.exists() or not any(llama_cpp_path.iterdir()): + print("Error: llama.cpp submodule not found.") + print("Please run: git submodule update --init --recursive") + sys.exit(1) + +def main(): + # Check submodules + check_submodules() + + # Define the extension + ext_modules = [ + CMakeExtension("autocog.llama", sourcedir="libs/autocog/llama"), + ] + + # Minimal setup - most configuration is in pyproject.toml + setup( + ext_modules=ext_modules, + cmdclass={"build_ext": CMakeBuild}, + zip_safe=False, + ) + +if __name__ == "__main__": + main() diff --git a/vendors/llama b/vendors/llama new file mode 160000 index 0000000..484b209 --- /dev/null +++ b/vendors/llama @@ -0,0 +1 @@ +Subproject commit 484b2091ce5017901483b5204c07878f171d1441 From 8eecae80cf67315ed34fa9f24d655bdde07537aa Mon Sep 17 00:00:00 2001 From: Tristan Date: Fri, 1 Aug 2025 07:48:18 -0700 Subject: [PATCH 02/51] Started implementing the base types for autocog::llama C++ module --- README.md | 2 +- libs/autocog/llama/fta.cxx | 52 ++++++++++++++--- libs/autocog/llama/fta.hxx | 78 +++++++++++++++++++------- libs/autocog/llama/ftt.cxx | 4 ++ libs/autocog/llama/ftt.hxx | 11 ++-- libs/autocog/llama/python_bindings.cxx | 17 +----- libs/autocog/llama/types.hxx | 17 ++++++ 7 files changed, 132 insertions(+), 49 deletions(-) create mode 100644 libs/autocog/llama/types.hxx diff --git a/README.md b/README.md index 95b6f2f..1e84d76 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The libraries have [their own documentation](./share/library/README.md). Build image and "test": ``` docker build -t autocog:latest . -docker run --rm -it autocog:latest python3 -c "import autocog.llama ; print(autocog.llama)" + docker run --rm -it autocog:latest python3 -c "import autocog.llama; print(autocog.llama.__doc__)" ``` Fast rebuild (in mounted: diff --git a/libs/autocog/llama/fta.cxx b/libs/autocog/llama/fta.cxx index 7a1d699..cda6e00 100644 --- a/libs/autocog/llama/fta.cxx +++ b/libs/autocog/llama/fta.cxx @@ -3,14 +3,50 @@ namespace autocog { namespace llama { -Action::Action(ActionKind const kind_, NodeID const id_) : kind(kind_), id(id_) {} - -Text::Text(NodeID const id_) : Action(ActionKind::Text, id_) {} - -Completion::Completion(NodeID const id_) : Action(ActionKind::Completion, id_) {} - -Choice::Choice(NodeID const id_) : Action(ActionKind::Choice, id_) {} - +Action::Action( + ActionKind const kind_, NodeID const id_, float threshold_ +) : kind(kind_), id(id_), threshold(threshold_), successors() {} + +Text::Text(NodeID const id_, float threshold_) : Action(ActionKind::Text, id_, threshold_) {} + +Completion::Completion( + NodeID const id_, float threshold_, unsigned length_, unsigned beams_, unsigned ahead_ +) : Action(ActionKind::Completion, id_, threshold_), length(length_), beams(beams_), ahead(ahead_) {} + +Choice::Choice(NodeID const id_, float threshold_, unsigned width_) : Action(ActionKind::Choice, id_, threshold_), width(width_) {} + +Action const & FTA::action(NodeID const & id) const { + Action const & action = *(this->actions.at(id)); + if (action.id != id) { + throw std::runtime_error("Action's ID does not match position in FTA::actions!"); + } + return action; +} + +Text & FTA::insert(float threshold_) { + NodeID id = this->actions.size(); + Text * action = new Text(id, threshold_); + this->actions.push_back(std::unique_ptr(action)); + return *action; +} + +Completion & FTA::insert(float threshold_, unsigned length_, unsigned beams_, unsigned ahead_) { + NodeID id = this->actions.size(); + Completion * action = new Completion(id, threshold_, length_, beams_, ahead_); + this->actions.push_back(std::unique_ptr(action)); + return *action; +} + +Choice & FTA::insert(float threshold_, unsigned width_) { + NodeID id = this->actions.size(); + Choice * action = new Choice(id, threshold_, width_); + this->actions.push_back(std::unique_ptr(action)); + return *action; +} + +FTA::FTA(pybind11::dict const & pydata) { + throw std::runtime_error("Loading FTA from python dictionary is not implemented yet!"); +} } } diff --git a/libs/autocog/llama/fta.hxx b/libs/autocog/llama/fta.hxx index 5a20d67..97e1831 100644 --- a/libs/autocog/llama/fta.hxx +++ b/libs/autocog/llama/fta.hxx @@ -1,12 +1,20 @@ #ifndef __AUTOCOG_LLAMA_FTA__HXX_ #define __AUTOCOG_LLAMA_FTA__HXX_ -#include +#include "autocog/llama/types.hxx" + +#include + +#include namespace autocog { namespace llama { -using NodeID = unsigned; +struct Vocab { + std::vector mask; + + void select(unsigned num, std::vector const & input, std::vector> const & output) const; +}; enum class ActionKind { Text, @@ -14,37 +22,69 @@ enum class ActionKind { Choice }; -class Action { - public: - ActionKind const kind; - NodeID const id; +struct Action { + ActionKind const kind; + NodeID const id; - Action(ActionKind const kind_, NodeID const id_); + float const threshold; //< Probability threshold for pruning - protected: - std::vector next; + std::vector successors; + + Action(ActionKind const kind_, NodeID const id_, float threshold_); + + template + T const & as() const { + if (T::Kind != kind) { + throw std::runtime_error("Calling Action::as() with uncompatible ActionKind."); + } + return *(T const *)this; + } }; -class Text : public Action { - public: - Text(NodeID const id_); +struct Text : public Action { + static constexpr ActionKind Kind = ActionKind::Text; + + TokenSequence tokens; + + Text(NodeID const id_, float threshold_); }; -class Completion : public Action { - public: - Completion(NodeID const id_); +struct Completion : public Action { + static constexpr ActionKind Kind = ActionKind::Completion; + + unsigned const length; + unsigned const beams; + unsigned const ahead; + + Vocab vocab; + TokenSequence stop; + + Completion(NodeID const id_, float threshold_, unsigned length_, unsigned beams_, unsigned ahead_); }; -class Choice : public Action { - public: - Choice(NodeID const id_); +struct Choice : public Action { + static constexpr ActionKind Kind = ActionKind::Choice; + + unsigned const width; // Maximum number of choices to explore + + std::vector choices; // Each choice is a token sequence + + Choice(NodeID const id_, float threshold_, unsigned width_); }; class FTA { public: Action const & action(NodeID const & id) const; + + Text & insert(float threshold_); + Completion & insert(float threshold_, unsigned length_, unsigned beams_, unsigned ahead_); + Choice & insert(float threshold_, unsigned width_); + + FTA() = default; + FTA(pybind11::dict const & pydata); + private: - std::vector nodes; + std::vector> actions; }; } } diff --git a/libs/autocog/llama/ftt.cxx b/libs/autocog/llama/ftt.cxx index 6befe3d..1ee1f9c 100644 --- a/libs/autocog/llama/ftt.cxx +++ b/libs/autocog/llama/ftt.cxx @@ -3,5 +3,9 @@ namespace autocog { namespace llama { +pybind11::dict FTT::pydict() const { + throw std::runtime_error("Exporting FTT to python dictionary is not implemented yet!"); +} + } } diff --git a/libs/autocog/llama/ftt.hxx b/libs/autocog/llama/ftt.hxx index 8b25ba7..117a931 100644 --- a/libs/autocog/llama/ftt.hxx +++ b/libs/autocog/llama/ftt.hxx @@ -1,22 +1,21 @@ #ifndef __AUTOCOG_LLAMA_FTT__HXX_ #define __AUTOCOG_LLAMA_FTT__HXX_ -#include -#include +#include "autocog/llama/types.hxx" + +#include namespace autocog { namespace llama { -using TokenID = unsigned; -using probability_t = float; - class FTT { public: FTT() = default; + pybind11::dict pydict() const; private: std::vector tokens; - std::vector probas; + std::vector probas; std::vector children; }; diff --git a/libs/autocog/llama/python_bindings.cxx b/libs/autocog/llama/python_bindings.cxx index bdff94d..d727fc3 100644 --- a/libs/autocog/llama/python_bindings.cxx +++ b/libs/autocog/llama/python_bindings.cxx @@ -8,25 +8,14 @@ #include "evaluator.hxx" #include "tokenizer.hxx" -static void populate_fta_from_python_dict(autocog::llama::FTA & fta, pybind11::dict const & fta_dict) { - // TODO: Implementation -} - -static pybind11::dict ftt_to_python_dict(autocog::llama::FTT const & ftt) { - pybind11::dict result; - // TODO: Implementation - return result; -} - // Wrapper functions for Python integration pybind11::dict evaluate_fta_python(uintptr_t ctx_ptr, const pybind11::dict& fta_dict, const pybind11::dict& config_dict = pybind11::dict()) { - autocog::llama::FTA fta; - populate_fta_from_python_dict(fta, fta_dict); + autocog::llama::FTA fta(fta_dict); autocog::llama::FTT ftt; // TODO: Implementation - return ftt_to_python_dict(ftt); + return ftt.pydict(); } // Tokenizer wrapper functions @@ -42,8 +31,6 @@ std::string detokenize_python(uintptr_t model_ptr, const std::vector& token PYBIND11_MODULE(llama, m) { m.doc() = "AutoCog llama.cpp integration module"; - - m.def("get_version", []() { return "0.1.0"; }); m.def("evaluate_fta", &evaluate_fta_python, "Evaluate Finite Thoughts Automata", diff --git a/libs/autocog/llama/types.hxx b/libs/autocog/llama/types.hxx new file mode 100644 index 0000000..0f61296 --- /dev/null +++ b/libs/autocog/llama/types.hxx @@ -0,0 +1,17 @@ +#ifndef __AUTOCOG_LLAMA_TYPES__HXX_ +#define __AUTOCOG_LLAMA_TYPES__HXX_ + +#include + +namespace autocog { +namespace llama { + +using NodeID = unsigned; +using TokenID = unsigned; + +using TokenSequence = std::vector; + +} } + +#endif /* __AUTOCOG_LLAMA_TYPES__HXX_ */ + From 2b10b36306814f9ce2e377506d84a024d2300777 Mon Sep 17 00:00:00 2001 From: Tristan Date: Fri, 1 Aug 2025 09:22:13 -0700 Subject: [PATCH 03/51] Added model with tokenization to autocog::llama C++ module --- .dockerignore | 2 + Dockerfile | 27 +++- README.md | 3 +- libs/autocog/llama/CMakeLists.txt | 11 +- libs/autocog/llama/fta.hxx | 2 +- libs/autocog/llama/model.cxx | 136 ++++++++++++++++++ libs/autocog/llama/model.hxx | 39 +++++ libs/autocog/llama/python_bindings.cxx | 50 +++---- libs/autocog/llama/tokenizer.cxx | 9 -- libs/autocog/llama/tokenizer.hxx | 12 -- libs/autocog/llama/types.hxx | 4 +- setup.py | 12 +- tests/autocog/llama/roundtrip_tokenization.py | 16 +++ 13 files changed, 246 insertions(+), 77 deletions(-) create mode 100644 libs/autocog/llama/model.cxx create mode 100644 libs/autocog/llama/model.hxx delete mode 100644 libs/autocog/llama/tokenizer.cxx delete mode 100644 libs/autocog/llama/tokenizer.hxx create mode 100644 tests/autocog/llama/roundtrip_tokenization.py diff --git a/.dockerignore b/.dockerignore index e26ced7..a1b9dec 100644 --- a/.dockerignore +++ b/.dockerignore @@ -52,3 +52,5 @@ htmlcov/ # Temporary files *.tmp *.temp + +models/ diff --git a/Dockerfile b/Dockerfile index fc352af..7bd412c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,13 +14,30 @@ RUN apt-get update && \ python3-venv && \ rm -rf /var/lib/apt/lists/* -RUN python3 -m venv /venv -ENV PATH="/venv/bin:$PATH" +# LLama.cpp -RUN pip install --upgrade pip setuptools wheel pybind11 numpy +COPY vendors/llama /tmp/llama_cpp + +RUN cd /tmp/llama_cpp && \ + cmake -DLLAMA_BUILD_COMMON=OFF -B build && \ + make -C build -j$(nproc) install + +ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" + +# Venv + +RUN python3 -m venv /opt +ENV PATH="/opt/bin:$PATH" +ENV PYTHONPATH="/opt/bin:$PATH" +RUN pip install --upgrade pip + +# Autocog COPY . /workspace/autocog -WORKDIR /workspace/autocog -RUN pip install -e . +RUN rm -rf /workspace/autocog/vendors && \ + pip install /workspace/autocog && \ + rm -rf /workspace/autocog + +WORKDIR /workspace diff --git a/README.md b/README.md index 1e84d76..3d0f540 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ The libraries have [their own documentation](./share/library/README.md). Build image and "test": ``` docker build -t autocog:latest . - docker run --rm -it autocog:latest python3 -c "import autocog.llama; print(autocog.llama.__doc__)" +docker run --rm -it autocog:latest python3 -c "import autocog.llama; print(autocog.llama.__doc__)" +docker run --rm -v $(pwd)/models:/workspace/autocog/models -v $(pwd)/tests:/workspace/autocog/tests -it autocog:latest python3 /workspace/autocog/tests/autocog/llama/roundtrip_tokenization.py /workspace/autocog/models/stories260K.gguf ``` Fast rebuild (in mounted: diff --git a/libs/autocog/llama/CMakeLists.txt b/libs/autocog/llama/CMakeLists.txt index e4dc7d6..56f420a 100644 --- a/libs/autocog/llama/CMakeLists.txt +++ b/libs/autocog/llama/CMakeLists.txt @@ -8,15 +8,12 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) # Find required packages find_package(pybind11 REQUIRED) -# Add llama.cpp as subdirectory -add_subdirectory(../../../vendors/llama llama_cpp EXCLUDE_FROM_ALL) - # Source files set(SOURCES fta.cxx ftt.cxx + model.cxx evaluator.cxx - tokenizer.cxx python_bindings.cxx ) @@ -24,11 +21,9 @@ set(SOURCES pybind11_add_module(autocog_llama ${SOURCES}) set_target_properties(autocog_llama PROPERTIES OUTPUT_NAME "llama") + # Include directories -target_include_directories(autocog_llama PRIVATE - ../.. - ../../../vendors/llama -) +target_include_directories(autocog_llama PRIVATE ../.. ) # Link against llama.cpp target_link_libraries(autocog_llama PRIVATE llama) diff --git a/libs/autocog/llama/fta.hxx b/libs/autocog/llama/fta.hxx index 97e1831..53f9830 100644 --- a/libs/autocog/llama/fta.hxx +++ b/libs/autocog/llama/fta.hxx @@ -13,7 +13,7 @@ namespace llama { struct Vocab { std::vector mask; - void select(unsigned num, std::vector const & input, std::vector> const & output) const; + void topk(unsigned k, std::vector const & input, std::vector> const & output) const; }; enum class ActionKind { diff --git a/libs/autocog/llama/model.cxx b/libs/autocog/llama/model.cxx new file mode 100644 index 0000000..b18229e --- /dev/null +++ b/libs/autocog/llama/model.cxx @@ -0,0 +1,136 @@ +#include "autocog/llama/model.hxx" + +#include + +#include + +namespace autocog { namespace llama { + +Model::Model(std::string const & model_path, int n_ctx) : model(nullptr), contexts() { + llama_backend_init(); + + // Load model + llama_model_params model_params = llama_model_default_params(); + this->model = llama_model_load_from_file(model_path.c_str(), model_params); + if (!this->model) { + throw std::runtime_error("Failed to load model from: " + model_path); + } + + // Create context parameters + llama_context_params ctx_params = llama_context_default_params(); + ctx_params.n_ctx = n_ctx; + + // Create single context with ID=0 + llama_context * ctx = llama_init_from_model(this->model, ctx_params); + if (!ctx) { + llama_model_free(this->model); + throw std::runtime_error("Failed to create llama context"); + } + this->contexts.push_back(ctx); +} + +Model::~Model() { + for (auto* ctx : this->contexts) if (ctx) llama_free(ctx); + contexts.clear(); + + if (this->model) llama_free_model(model); + + llama_backend_free(); +} + +llama_context * Model::get_context(ContextID const id) const { + if (id >= this->contexts.size()) { + throw std::runtime_error("Invalid context ID: " + std::to_string(id)); + } + return this->contexts[id]; +} + +ContextID Model::fork_context(ContextID const id) { + throw std::runtime_error("Context forking is not implemented yet (Phase 3 feature)"); +} + +TokenSequence Model::tokenize(std::string const & text, bool add_bos, bool special) { + std::vector tokens; + tokens.resize(text.length() + (add_bos ? 1 : 0) + 1); // Rough upper bound + + int n_tokens = llama_tokenize( + this->get_vocab(), + text.c_str(), + text.length(), + tokens.data(), + tokens.size(), + add_bos, + special + ); + + if (n_tokens < 0) { + throw std::runtime_error("Tokenization failed for text: " + text); + } + + TokenSequence result(tokens.begin(), tokens.begin() + n_tokens); + return result; +} + +std::string Model::detokenize(TokenSequence const & tokens, bool spec_rm, bool spec_unp) { + if (tokens.empty()) { + return ""; + } + + // Detokenize + std::string result; + result.resize(tokens.size() * 8); // Rough estimate for buffer size + + int n_chars = llama_detokenize( + this->get_vocab(), + tokens.data(), + tokens.size(), + &result[0], + result.size(), + spec_rm, + spec_unp + ); + + if (n_chars < 0) { + throw std::runtime_error("Detokenization failed"); + } + + result.resize(n_chars); + return result; +} + +const llama_vocab * Model::get_vocab() const { + return llama_model_get_vocab(this->model); +} + +size_t Model::vocab_size() const { + return llama_vocab_n_tokens(this->get_vocab()); +} + +TokenID Model::bos_token() const { + return llama_vocab_bos(this->get_vocab()); +} + +TokenID Model::eos_token() const { + return llama_vocab_eos(this->get_vocab()); +} + +std::vector Model::get_logits(ContextID const id) { + float * logits = llama_get_logits(this->get_context(id)); + if (!logits) { + throw std::runtime_error("Failed to get logits - context may not have been evaluated"); + } + return std::vector(logits, logits + this->vocab_size()); +} + +bool Model::eval_tokens(TokenSequence const & tokens, ContextID const id) { + if (tokens.empty()) { + return true; + } + + llama_batch batch = llama_batch_get_one(const_cast(tokens.data()), tokens.size()); + int result = llama_decode(this->get_context(id), batch); + + return result == 0; +} + +} } diff --git a/libs/autocog/llama/model.hxx b/libs/autocog/llama/model.hxx new file mode 100644 index 0000000..4a63731 --- /dev/null +++ b/libs/autocog/llama/model.hxx @@ -0,0 +1,39 @@ +#ifndef __AUTOCOG_LLAMA_MODEL__HXX_ +#define __AUTOCOG_LLAMA_MODEL__HXX_ + +#include "autocog/llama/types.hxx" + +#include + +namespace autocog { +namespace llama { + +class Model { + private: + llama_model * model; + std::vector contexts; + + llama_context * get_context(ContextID const id = 0) const; + const llama_vocab * get_vocab() const; + + public: + Model(std::string const & model_path, int n_ctx); + ~Model(); + + ContextID fork_context(ContextID const id = 0); + + TokenSequence tokenize(std::string const & text, bool add_bos, bool special); + std::string detokenize(TokenSequence const & tokens, bool spec_rm, bool spec_unp); + + size_t vocab_size() const; + TokenID bos_token() const; + TokenID eos_token() const; + + std::vector get_logits(ContextID const id = 0); + bool eval_tokens(const TokenSequence& tokens, ContextID const id = 0); +}; + +} } + +#endif /* __AUTOCOG_LLAMA_MODEL__HXX_ */ + diff --git a/libs/autocog/llama/python_bindings.cxx b/libs/autocog/llama/python_bindings.cxx index d727fc3..9c7487a 100644 --- a/libs/autocog/llama/python_bindings.cxx +++ b/libs/autocog/llama/python_bindings.cxx @@ -5,8 +5,8 @@ #include "fta.hxx" #include "ftt.hxx" +#include "model.hxx" #include "evaluator.hxx" -#include "tokenizer.hxx" // Wrapper functions for Python integration pybind11::dict evaluate_fta_python(uintptr_t ctx_ptr, const pybind11::dict& fta_dict, const pybind11::dict& config_dict = pybind11::dict()) { @@ -18,35 +18,27 @@ pybind11::dict evaluate_fta_python(uintptr_t ctx_ptr, const pybind11::dict& fta_ return ftt.pydict(); } -// Tokenizer wrapper functions -std::vector tokenize_python(uintptr_t model_ptr, const std::string& text, bool add_special = true) { - // TODO: Implementation - return {}; -} - -std::string detokenize_python(uintptr_t model_ptr, const std::vector& tokens) { - // TODO: Implementation - return ""; -} - -PYBIND11_MODULE(llama, m) { - m.doc() = "AutoCog llama.cpp integration module"; +PYBIND11_MODULE(llama, module) { + module.doc() = "AutoCog's llama.cpp integration module"; - m.def("evaluate_fta", &evaluate_fta_python, - "Evaluate Finite Thoughts Automata", - pybind11::arg("ctx_ptr"), pybind11::arg("fta_dict"), pybind11::arg("config_dict") = pybind11::dict()); - - m.def("tokenize", &tokenize_python, - "Tokenize text using llama.cpp tokenizer", - pybind11::arg("model_ptr"), pybind11::arg("text"), pybind11::arg("add_special") = true); + module.def("create", [](const std::string& model_path, int n_ctx=4096) { + auto model = new autocog::llama::Model(model_path, n_ctx); + return reinterpret_cast(model); + }); + + module.def("tokenize", [](uintptr_t ptr, const std::string & text, bool add_bos = false, bool special = false) { + auto model = reinterpret_cast(ptr); + auto tokens = model->tokenize(text, add_bos, special); + pybind11::list result; + for (auto token : tokens) result.append(token); + return result; + }); - m.def("detokenize", &detokenize_python, - "Detokenize tokens using llama.cpp tokenizer", - pybind11::arg("model_ptr"), pybind11::arg("tokens")); - - pybind11::enum_(m, "ActionKind") - .value("Text", autocog::llama::ActionKind::Text ) - .value("Completion", autocog::llama::ActionKind::Completion ) - .value("Choice", autocog::llama::ActionKind::Choice ); + module.def("detokenize", [](uintptr_t ptr, const pybind11::list & py_tokens, bool spec_rm = false, bool spec_unp = false) { + auto model = reinterpret_cast(ptr); + autocog::llama::TokenSequence tokens; + for (auto item : py_tokens) tokens.push_back(item.cast()); + return model->detokenize(tokens, spec_rm, spec_unp); + }); } diff --git a/libs/autocog/llama/tokenizer.cxx b/libs/autocog/llama/tokenizer.cxx deleted file mode 100644 index b67d7c0..0000000 --- a/libs/autocog/llama/tokenizer.cxx +++ /dev/null @@ -1,9 +0,0 @@ - -#include "autocog/llama/tokenizer.hxx" - -namespace autocog { namespace llama { - -// TODO - -} } - diff --git a/libs/autocog/llama/tokenizer.hxx b/libs/autocog/llama/tokenizer.hxx deleted file mode 100644 index 00944d2..0000000 --- a/libs/autocog/llama/tokenizer.hxx +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef __AUTOCOG_LLAMA_TOKENIZER__HXX_ -#define __AUTOCOG_LLAMA_TOKENIZER__HXX_ - -namespace autocog { -namespace llama { - -class Tokenizer {}; - -} } - -#endif /* __AUTOCOG_LLAMA_TOKENIZER__HXX_ */ - diff --git a/libs/autocog/llama/types.hxx b/libs/autocog/llama/types.hxx index 0f61296..efba002 100644 --- a/libs/autocog/llama/types.hxx +++ b/libs/autocog/llama/types.hxx @@ -1,13 +1,15 @@ #ifndef __AUTOCOG_LLAMA_TYPES__HXX_ #define __AUTOCOG_LLAMA_TYPES__HXX_ +#include #include namespace autocog { namespace llama { +using ContextID = unsigned; using NodeID = unsigned; -using TokenID = unsigned; +using TokenID = llama_token; using TokenSequence = std::vector; diff --git a/setup.py b/setup.py index 784a877..7bcb80a 100644 --- a/setup.py +++ b/setup.py @@ -63,18 +63,7 @@ def build_extension(self, ext): ["cmake", "--build", "."] + build_args, cwd=build_temp ) -def check_submodules(): - """Check if git submodules are initialized""" - llama_cpp_path = Path(__file__).parent / "vendors/llama" - if not llama_cpp_path.exists() or not any(llama_cpp_path.iterdir()): - print("Error: llama.cpp submodule not found.") - print("Please run: git submodule update --init --recursive") - sys.exit(1) - def main(): - # Check submodules - check_submodules() - # Define the extension ext_modules = [ CMakeExtension("autocog.llama", sourcedir="libs/autocog/llama"), @@ -89,3 +78,4 @@ def main(): if __name__ == "__main__": main() + diff --git a/tests/autocog/llama/roundtrip_tokenization.py b/tests/autocog/llama/roundtrip_tokenization.py new file mode 100644 index 0000000..244802e --- /dev/null +++ b/tests/autocog/llama/roundtrip_tokenization.py @@ -0,0 +1,16 @@ + +import sys +import autocog.llama + +# Create both implementations +model = autocog.llama.create(sys.argv[1], 4096) + +# Test tokenization round-trip +test_texts = ["Hello world", "This is a test", "System prompt with\nmultiple lines"] + +for text in test_texts: + cpp_tokens = autocog.llama.tokenize(model, text, False, False) + cpp_reconstructed = autocog.llama.detokenize(model, cpp_tokens, False, False) + assert text == cpp_reconstructed, f"Round-trip failed: {text} vs {cpp_reconstructed}" + +print("All tokenization tests passed!") From 3e52461f7e429d0bb35769f03af6703e6f94bbc9 Mon Sep 17 00:00:00 2001 From: Tristan Date: Fri, 1 Aug 2025 13:43:42 -0700 Subject: [PATCH 04/51] Added tests for sta->fta and executing STA with LLama.cpp --- README.md | 4 +- libs/autocog/llama/fta.cxx | 122 +++++++++++++++++- libs/autocog/llama/ftt.cxx | 30 ++++- libs/autocog/llama/ftt.hxx | 2 + libs/autocog/llama/python_bindings.cxx | 5 +- modules/autocog/sta/automaton.py | 2 +- modules/autocog/sta/compile.py | 7 +- .../compilation/compile_sta_to_ftas.py | 28 ++++ .../llama/execute_sta_with_llama_cpp.py | 57 ++++++++ 9 files changed, 251 insertions(+), 6 deletions(-) create mode 100644 tests/autocog/compilation/compile_sta_to_ftas.py create mode 100644 tests/autocog/llama/execute_sta_with_llama_cpp.py diff --git a/README.md b/README.md index 3d0f540..9b4c406 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,9 @@ Build image and "test": ``` docker build -t autocog:latest . docker run --rm -it autocog:latest python3 -c "import autocog.llama; print(autocog.llama.__doc__)" -docker run --rm -v $(pwd)/models:/workspace/autocog/models -v $(pwd)/tests:/workspace/autocog/tests -it autocog:latest python3 /workspace/autocog/tests/autocog/llama/roundtrip_tokenization.py /workspace/autocog/models/stories260K.gguf +docker run --rm -v $(pwd):/workspace/autocog -it autocog:latest python3 /workspace/autocog/tests/autocog/llama/roundtrip_tokenization.py /workspace/autocog/models/stories260K.gguf +docker run --rm -v $(pwd):/workspace/autocog -it autocog:latest python3 /workspace/autocog/tests/autocog/compilation/compile_sta_to_ftas.py /workspace/autocog/share/library/mcq/select.sta '{"topic":"a topic", "question":"my question", "choices" : [ "a", "b", "c", "d" ] }' +docker run --rm -v $(pwd):/workspace/autocog -it autocog:latest python3 /workspace/autocog/tests/autocog/llama/execute_sta_with_llama_cpp.py /workspace/autocog/share/library/mcq/select.sta '{"topic":"a topic", "question":"my question", "choices" : [ "a", "b", "c", "d" ] }' /workspace/autocog/models/stories260K.gguf ``` Fast rebuild (in mounted: diff --git a/libs/autocog/llama/fta.cxx b/libs/autocog/llama/fta.cxx index cda6e00..28e89d9 100644 --- a/libs/autocog/llama/fta.cxx +++ b/libs/autocog/llama/fta.cxx @@ -1,6 +1,9 @@ #include "autocog/llama/fta.hxx" +#include +#include + namespace autocog { namespace llama { Action::Action( @@ -44,8 +47,125 @@ Choice & FTA::insert(float threshold_, unsigned width_) { return *action; } +// TODO review + FTA::FTA(pybind11::dict const & pydata) { - throw std::runtime_error("Loading FTA from python dictionary is not implemented yet!"); + // Get the actions dictionary from Python FTA + if (!pydata.contains("actions")) { + throw std::runtime_error("FTA dictionary missing 'actions' field"); + } + + auto py_actions = pydata["actions"].cast(); + + // First pass: create all actions and build ID mapping + std::map uid_to_id; + + for (auto item : py_actions) { + std::string uid = item.first.cast(); + auto action_dict = item.second.cast(); + + if (!action_dict.contains("__type__")) { + throw std::runtime_error("Action missing '__type__' field: " + uid); + } + + std::string action_type = action_dict["__type__"].cast(); + float threshold = action_dict.contains("threshold") ? action_dict["threshold"].cast() : 0.0f; + + NodeID node_id = actions.size(); // Assign sequential IDs + uid_to_id[uid] = node_id; + + std::unique_ptr action; + + if (action_type == "Text") { + action = std::make_unique(node_id, threshold); + Text* text_action = static_cast(action.get()); + + // Set tokens + if (action_dict.contains("tokens")) { + auto py_tokens = action_dict["tokens"].cast(); + text_action->tokens.clear(); + for (auto token : py_tokens) { + text_action->tokens.push_back(token.cast()); + } + } + + } else if (action_type == "Completion") { + // Extract parameters with defaults + unsigned length = action_dict.contains("length") ? action_dict["length"].cast() : 1; + unsigned beams = action_dict.contains("beams") ? action_dict["beams"].cast() : 1; + unsigned ahead = action_dict.contains("ahead") ? action_dict["ahead"].cast() : 1; + + action = std::make_unique(node_id, threshold, length, beams, ahead); + Completion* completion_action = static_cast(action.get()); + + // Set stop tokens + if (action_dict.contains("stop_tokens")) { + auto py_stop = action_dict["stop_tokens"].cast(); + completion_action->stop.clear(); + for (auto token : py_stop) { + completion_action->stop.push_back(token.cast()); + } + } + + // Set vocabulary mask + if (action_dict.contains("vocab_mask")) { + auto py_mask = action_dict["vocab_mask"].cast(); + completion_action->vocab.mask.clear(); + completion_action->vocab.mask.reserve(py_mask.size()); + for (auto val : py_mask) { + completion_action->vocab.mask.push_back(val.cast()); + } + } + + } else if (action_type == "Choice") { + unsigned width = action_dict.contains("width") ? action_dict["width"].cast() : 1; + action = std::make_unique(node_id, threshold, width); + Choice* choice_action = static_cast(action.get()); + + // Set choices + if (action_dict.contains("choices")) { + auto py_choices = action_dict["choices"].cast(); + choice_action->choices.clear(); + choice_action->choices.reserve(py_choices.size()); + + for (auto py_choice : py_choices) { + auto py_tokens = py_choice.cast(); + TokenSequence choice_tokens; + choice_tokens.reserve(py_tokens.size()); + for (auto token : py_tokens) { + choice_tokens.push_back(token.cast()); + } + choice_action->choices.push_back(std::move(choice_tokens)); + } + } + + } else { + throw std::runtime_error("Unknown action type: " + action_type); + } + + actions.push_back(std::move(action)); + } + + // Second pass: set up successors using the UID mapping + for (auto item : py_actions) { + std::string uid = item.first.cast(); + auto action_dict = item.second.cast(); + + NodeID node_id = uid_to_id[uid]; + Action* action = actions[node_id].get(); + + // Add successors + if (action_dict.contains("successors")) { + auto py_successors = action_dict["successors"].cast(); + for (auto successor : py_successors) { + std::string successor_uid = successor.cast(); + if (uid_to_id.find(successor_uid) == uid_to_id.end()) { + throw std::runtime_error("Unknown successor UID: " + successor_uid); + } + action->successors.push_back(uid_to_id[successor_uid]); + } + } + } } } } diff --git a/libs/autocog/llama/ftt.cxx b/libs/autocog/llama/ftt.cxx index 1ee1f9c..47b0644 100644 --- a/libs/autocog/llama/ftt.cxx +++ b/libs/autocog/llama/ftt.cxx @@ -3,8 +3,36 @@ namespace autocog { namespace llama { +// TODO review code below + pybind11::dict FTT::pydict() const { - throw std::runtime_error("Exporting FTT to python dictionary is not implemented yet!"); + pybind11::dict result; + + // Convert tokens + pybind11::list token_list; + for (TokenID token : tokens) { + token_list.append(token); + } + result["tokens"] = token_list; + + // Convert probabilities + pybind11::list proba_list; + for (float prob : probas) { + proba_list.append(prob); + } + result["probabilities"] = proba_list; + + // Convert children recursively + pybind11::list children_list; + for (const FTT& child : children) { + children_list.append(child.pydict()); + } + result["children"] = children_list; + + // Add metadata + result["pruned"] = pruned; + + return result; } } } diff --git a/libs/autocog/llama/ftt.hxx b/libs/autocog/llama/ftt.hxx index 117a931..7bd89e8 100644 --- a/libs/autocog/llama/ftt.hxx +++ b/libs/autocog/llama/ftt.hxx @@ -16,6 +16,8 @@ class FTT { private: std::vector tokens; std::vector probas; + bool pruned = false; + std::vector children; }; diff --git a/libs/autocog/llama/python_bindings.cxx b/libs/autocog/llama/python_bindings.cxx index 9c7487a..efcb8ae 100644 --- a/libs/autocog/llama/python_bindings.cxx +++ b/libs/autocog/llama/python_bindings.cxx @@ -9,7 +9,8 @@ #include "evaluator.hxx" // Wrapper functions for Python integration -pybind11::dict evaluate_fta_python(uintptr_t ctx_ptr, const pybind11::dict& fta_dict, const pybind11::dict& config_dict = pybind11::dict()) { +pybind11::dict evaluate_fta_python(uintptr_t ptr, const pybind11::dict& fta_dict) { + auto model = reinterpret_cast(ptr); autocog::llama::FTA fta(fta_dict); autocog::llama::FTT ftt; @@ -40,5 +41,7 @@ PYBIND11_MODULE(llama, module) { for (auto item : py_tokens) tokens.push_back(item.cast()); return model->detokenize(tokens, spec_rm, spec_unp); }); + + module.def("evaluate", &evaluate_fta_python); } diff --git a/modules/autocog/sta/automaton.py b/modules/autocog/sta/automaton.py index c04a581..75e181a 100644 --- a/modules/autocog/sta/automaton.py +++ b/modules/autocog/sta/automaton.py @@ -119,7 +119,7 @@ def prompt(self, syntax): prompt += fmt if syntax.prompt_with_index: idx = self.indices[-1] - if not prompt_zero_index: + if not syntax.prompt_zero_index: idx += 1 idx = f'[{idx}]' if field.is_list() else '' prompt += idx diff --git a/modules/autocog/sta/compile.py b/modules/autocog/sta/compile.py index e2520af..0e8b35f 100644 --- a/modules/autocog/sta/compile.py +++ b/modules/autocog/sta/compile.py @@ -305,7 +305,7 @@ def compile_prompt(ast: AstPrompt, program: IrProgram, ctx:Context): return prompt -def compile(arch:"CogArch", tag:str, source:str): +def compile_source_to_program_and_stas(source:str): program = IrProgram() ctx = Context() @@ -323,4 +323,9 @@ def compile(arch:"CogArch", tag:str, source:str): sta.build_concrete() stas.update({ptag:sta}) + return (program, stas) + +def compile(arch:"CogArch", tag:str, source:str): + (program, stas) = compile_source_to_program_and_stas(source) return CogAutomaton(tag=tag, arch=arch, program=program, prompts=stas) + diff --git a/tests/autocog/compilation/compile_sta_to_ftas.py b/tests/autocog/compilation/compile_sta_to_ftas.py new file mode 100644 index 0000000..492a54c --- /dev/null +++ b/tests/autocog/compilation/compile_sta_to_ftas.py @@ -0,0 +1,28 @@ + +import sys, json + +from autocog.sta.compile import compile_source_to_program_and_stas +from autocog.sta.syntax import Syntax +from autocog.sta.runtime import Frame +from autocog.fta.automaton import FiniteThoughtAutomaton as FTA + +sta_file = sys.argv[1] +json_data = sys.argv[2] + +sta = compile_source_to_program_and_stas( + open(sta_file, 'r').read() +)[1]['main'] + +fta = sta.instantiate( + syntax=Syntax(), + frame=Frame( + state={ st.label() : None for st in sta.concretes.values() if st.abstract.field is not None }, + data=json.loads(json_data) + ), + branches={}, + inputs=None +) + +for action in fta.actions.values(): + print(action) + diff --git a/tests/autocog/llama/execute_sta_with_llama_cpp.py b/tests/autocog/llama/execute_sta_with_llama_cpp.py new file mode 100644 index 0000000..d7cbd85 --- /dev/null +++ b/tests/autocog/llama/execute_sta_with_llama_cpp.py @@ -0,0 +1,57 @@ + +import sys, json + +from autocog.sta.compile import compile_source_to_program_and_stas +from autocog.sta.syntax import Syntax +from autocog.sta.runtime import Frame +from autocog.fta.automaton import FiniteThoughtAutomaton as FTA + +import autocog.llama +from autocog.llama import tokenize, detokenize + +sta_file = sys.argv[1] +json_data = sys.argv[2] +model_path = sys.argv[3] + +sta = compile_source_to_program_and_stas( + open(sta_file, 'r').read() +)[1]['main'] + +fta = sta.instantiate( + syntax=Syntax(), + frame=Frame( + state={ st.label() : None for st in sta.concretes.values() if st.abstract.field is not None }, + data=json.loads(json_data) + ), + branches={}, + inputs=None +) + +model = autocog.llama.create(model_path, 4096) + +actions = { +## "root": { +## "__type__": "Text", +## "threshold": 0.0, +## "tokens": [1, 2, 3], +## "successors": ["choice1"] +## }, +## "choice1": { +## "__type__": "Choice", +## "threshold": 0.1, +## "width": 2, +## "choices": [[4, 5], [6, 7]], +## "successors": [] +## } +} + +for action in fta.actions.values(): + ## tokenize(model, text, False, False) +# print(action) + actions.update({}) + +ftt = autocog.llama.evaluate(model, { 'actions' : actions }) + +print(ftt) + +## cpp_reconstructed = detokenize(model, cpp_tokens, False, False) From 10be985b14b3a5969ac93c3095508624fafc14f2 Mon Sep 17 00:00:00 2001 From: Tristan Date: Sat, 2 Aug 2025 08:34:00 -0700 Subject: [PATCH 05/51] Prototype export of FTA to dict for serialization --- libs/autocog/llama/fta.cxx | 28 +++++++-------- libs/autocog/llama/fta.hxx | 4 ++- libs/autocog/llama/python_bindings.cxx | 2 +- .../llama/execute_sta_with_llama_cpp.py | 36 ++++++++++++++++--- 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/libs/autocog/llama/fta.cxx b/libs/autocog/llama/fta.cxx index 28e89d9..57462e4 100644 --- a/libs/autocog/llama/fta.cxx +++ b/libs/autocog/llama/fta.cxx @@ -1,5 +1,6 @@ #include "autocog/llama/fta.hxx" +#include "autocog/llama/model.hxx" #include #include @@ -49,7 +50,7 @@ Choice & FTA::insert(float threshold_, unsigned width_) { // TODO review -FTA::FTA(pybind11::dict const & pydata) { +FTA::FTA(Model * model, pybind11::dict const & pydata) { // Get the actions dictionary from Python FTA if (!pydata.contains("actions")) { throw std::runtime_error("FTA dictionary missing 'actions' field"); @@ -99,22 +100,21 @@ FTA::FTA(pybind11::dict const & pydata) { Completion* completion_action = static_cast(action.get()); // Set stop tokens - if (action_dict.contains("stop_tokens")) { - auto py_stop = action_dict["stop_tokens"].cast(); - completion_action->stop.clear(); - for (auto token : py_stop) { - completion_action->stop.push_back(token.cast()); - } + if (!action_dict.contains("stops")) { + throw std::runtime_error("Completion missing 'stops' field: " + uid); + } else { + auto py_stop = action_dict["stop"].cast(); + completion_action->stop.clear(); + for (auto token : py_stop) { + completion_action->stop.push_back(token.cast()); + } } // Set vocabulary mask - if (action_dict.contains("vocab_mask")) { - auto py_mask = action_dict["vocab_mask"].cast(); - completion_action->vocab.mask.clear(); - completion_action->vocab.mask.reserve(py_mask.size()); - for (auto val : py_mask) { - completion_action->vocab.mask.push_back(val.cast()); - } + completion_action->vocab.mask.reserve(model->vocab_size()); + completion_action->vocab.mask.assign(model->vocab_size(), true); + if (action_dict.contains("vocab")) { + throw std::runtime_error("Setting the vocabulary from PY desc is not implemented yet!"); } } else if (action_type == "Choice") { diff --git a/libs/autocog/llama/fta.hxx b/libs/autocog/llama/fta.hxx index 53f9830..afaf464 100644 --- a/libs/autocog/llama/fta.hxx +++ b/libs/autocog/llama/fta.hxx @@ -10,6 +10,8 @@ namespace autocog { namespace llama { +class Model; + struct Vocab { std::vector mask; @@ -81,7 +83,7 @@ class FTA { Choice & insert(float threshold_, unsigned width_); FTA() = default; - FTA(pybind11::dict const & pydata); + FTA(Model * model, pybind11::dict const & pydata); private: std::vector> actions; diff --git a/libs/autocog/llama/python_bindings.cxx b/libs/autocog/llama/python_bindings.cxx index efcb8ae..77013a6 100644 --- a/libs/autocog/llama/python_bindings.cxx +++ b/libs/autocog/llama/python_bindings.cxx @@ -11,7 +11,7 @@ // Wrapper functions for Python integration pybind11::dict evaluate_fta_python(uintptr_t ptr, const pybind11::dict& fta_dict) { auto model = reinterpret_cast(ptr); - autocog::llama::FTA fta(fta_dict); + autocog::llama::FTA fta(model, fta_dict); autocog::llama::FTT ftt; // TODO: Implementation diff --git a/tests/autocog/llama/execute_sta_with_llama_cpp.py b/tests/autocog/llama/execute_sta_with_llama_cpp.py index d7cbd85..64a6530 100644 --- a/tests/autocog/llama/execute_sta_with_llama_cpp.py +++ b/tests/autocog/llama/execute_sta_with_llama_cpp.py @@ -4,7 +4,9 @@ from autocog.sta.compile import compile_source_to_program_and_stas from autocog.sta.syntax import Syntax from autocog.sta.runtime import Frame + from autocog.fta.automaton import FiniteThoughtAutomaton as FTA +from autocog.fta.actions import Choose, Text, Complete import autocog.llama from autocog.llama import tokenize, detokenize @@ -45,10 +47,36 @@ ## } } -for action in fta.actions.values(): - ## tokenize(model, text, False, False) -# print(action) - actions.update({}) +for act in fta.actions.values(): + action = { "successors" : act.successors } + if isinstance(act, Text): + action.update({ + "__type__" : "Text", + "threshold" : 1. if act.threshold is None else act.threshold, + "tokens" : tokenize(model, act.text, False, False) + }) + elif isinstance(act, Choose): + action.update({ + "__type__" : "Text", + "threshold" : 1. if act.threshold is None else act.threshold, + "width" : 1 if act.width is None else act.width, + "choices" : [ + tokenize(model, choice[0], False, False) for choice in act.choices + ] + }) + elif isinstance(act, Complete): + action.update({ + "__type__" : "Text", + "threshold" : 1. if act.threshold is None else act.threshold, + "length" : 1 if act.length is None else act.length, + "beams" : 1 if act.beams is None else act.beams, + "ahead" : 1 if act.ahead is None else act.ahead, + "stop" : tokenize(model, act.stop, False, False), +# "vocab" : "TODO", + }) + else: + raise Exception() + actions.update({ act.uid : action }) ftt = autocog.llama.evaluate(model, { 'actions' : actions }) From 5f336b2301a6a9254fbe2e65f8a21f4ad47b5423 Mon Sep 17 00:00:00 2001 From: Tristan Date: Tue, 5 Aug 2025 19:34:24 -0700 Subject: [PATCH 06/51] Working on simplest evaluation of FTA --- libs/autocog/llama/CMakeLists.txt | 6 +- libs/autocog/llama/evaluation-choice.cxx | 19 +++ libs/autocog/llama/evaluation-completion.cxx | 19 +++ libs/autocog/llama/evaluation-text.cxx | 29 ++++ libs/autocog/llama/evaluation.cxx | 113 ++++++++++++++ libs/autocog/llama/evaluation.hxx | 60 +++++++ libs/autocog/llama/evaluator.cxx | 9 -- libs/autocog/llama/evaluator.hxx | 12 -- libs/autocog/llama/fta.cxx | 64 +++++--- libs/autocog/llama/fta.hxx | 18 +-- libs/autocog/llama/ftt.cxx | 19 +++ libs/autocog/llama/ftt.hxx | 23 +-- libs/autocog/llama/manager.cxx | 71 +++++++++ libs/autocog/llama/manager.hxx | 43 +++++ libs/autocog/llama/model.cxx | 131 +++++++++++++--- libs/autocog/llama/model.hxx | 13 +- libs/autocog/llama/python_bindings.cxx | 147 ++++++++++++++---- libs/autocog/llama/types.hxx | 5 +- .../llama/execute_sta_with_llama_cpp.py | 6 +- 19 files changed, 693 insertions(+), 114 deletions(-) create mode 100644 libs/autocog/llama/evaluation-choice.cxx create mode 100644 libs/autocog/llama/evaluation-completion.cxx create mode 100644 libs/autocog/llama/evaluation-text.cxx create mode 100644 libs/autocog/llama/evaluation.cxx create mode 100644 libs/autocog/llama/evaluation.hxx delete mode 100644 libs/autocog/llama/evaluator.cxx delete mode 100644 libs/autocog/llama/evaluator.hxx create mode 100644 libs/autocog/llama/manager.cxx create mode 100644 libs/autocog/llama/manager.hxx diff --git a/libs/autocog/llama/CMakeLists.txt b/libs/autocog/llama/CMakeLists.txt index 56f420a..2b695c9 100644 --- a/libs/autocog/llama/CMakeLists.txt +++ b/libs/autocog/llama/CMakeLists.txt @@ -13,7 +13,11 @@ set(SOURCES fta.cxx ftt.cxx model.cxx - evaluator.cxx + manager.cxx + evaluation.cxx + evaluation-choice.cxx + evaluation-completion.cxx + evaluation-text.cxx python_bindings.cxx ) diff --git a/libs/autocog/llama/evaluation-choice.cxx b/libs/autocog/llama/evaluation-choice.cxx new file mode 100644 index 0000000..fb484e8 --- /dev/null +++ b/libs/autocog/llama/evaluation-choice.cxx @@ -0,0 +1,19 @@ + +#include "autocog/llama/evaluation.hxx" +#include "autocog/llama/model.hxx" +#include "autocog/llama/fta.hxx" +#include "autocog/llama/ftt.hxx" + +#include + +namespace autocog { namespace llama { + +unsigned Evaluation::evaluate_choice(PathState & state) { + auto [model,ctx] = this->restore_context(state); + Choice const & action = this->fta.action(state.action).as(); + throw std::runtime_error("Not implemented yet"); + return 0; +} + +} } + diff --git a/libs/autocog/llama/evaluation-completion.cxx b/libs/autocog/llama/evaluation-completion.cxx new file mode 100644 index 0000000..2eb7788 --- /dev/null +++ b/libs/autocog/llama/evaluation-completion.cxx @@ -0,0 +1,19 @@ + +#include "autocog/llama/evaluation.hxx" +#include "autocog/llama/model.hxx" +#include "autocog/llama/fta.hxx" +#include "autocog/llama/ftt.hxx" + +#include + +namespace autocog { namespace llama { + +unsigned Evaluation::evaluate_completion(PathState & state) { + auto [model,ctx] = this->restore_context(state); + Completion const & action = this->fta.action(state.action).as(); + throw std::runtime_error("Not implemented yet"); + return 0; +} + +} } + diff --git a/libs/autocog/llama/evaluation-text.cxx b/libs/autocog/llama/evaluation-text.cxx new file mode 100644 index 0000000..9c9c1c0 --- /dev/null +++ b/libs/autocog/llama/evaluation-text.cxx @@ -0,0 +1,29 @@ + +#include "autocog/llama/manager.hxx" +#include "autocog/llama/evaluation.hxx" +#include "autocog/llama/model.hxx" +#include "autocog/llama/fta.hxx" +#include "autocog/llama/ftt.hxx" + +#include + +namespace autocog { namespace llama { + +unsigned Evaluation::evaluate_text(PathState & state) { + auto [model,ctx] = this->restore_context(state); + Text const & action = this->fta.action(state.action).as(); + + ProbaSequence probas(action.tokens.size(), 1.); + unsigned num_token_eval = model.eval_sequences(action.tokens, probas, ctx); + + float probability = 1.; + for (auto proba: probas) probability *= proba; + state.parent.add(action.id, action.tokens, probas, probability); + + this->enqueue(action.successors[0], state.parent.children.back(), model.get_tokens_const(), state); + + return num_token_eval; +} + +} } + diff --git a/libs/autocog/llama/evaluation.cxx b/libs/autocog/llama/evaluation.cxx new file mode 100644 index 0000000..d50d2ab --- /dev/null +++ b/libs/autocog/llama/evaluation.cxx @@ -0,0 +1,113 @@ + +#include "autocog/llama/manager.hxx" +#include "autocog/llama/evaluation.hxx" +#include "autocog/llama/model.hxx" +#include "autocog/llama/ftt.hxx" +#include "autocog/llama/fta.hxx" + +#include + +namespace autocog { namespace llama { + +PathState::PathState(ActionID const action_, FTT & parent_, TokenSequence const & tokens_, std::optional context_) : + action(action_), + parent(parent_), + tokens(tokens_), + context(context_) +{} + +Evaluation::Evaluation(ModelID const model_, FTA const & fta_) : + model(model_), + fta(fta_), + queue(), + root(nullptr) +{} + +Evaluation::~Evaluation() { + if (this->root) + delete this->root; +} + +unsigned Evaluation::advance(std::optional max_token_eval) { + if (this->root == nullptr) this->initial(); + + unsigned num_token_eval = 0; + while (!queue.empty() && (max_token_eval == std::nullopt || num_token_eval < max_token_eval)) { + PathState & state = queue.front(); + Action const & action = this->fta.action(state.action); + switch (action.kind) { + case ActionKind::Text: + std::cerr << "Executing Text #" << action.id << std::endl; + num_token_eval += this->evaluate_text(state); + break; + case ActionKind::Completion: + std::cerr << "Executing Completion #" << action.id << std::endl; + num_token_eval += this->evaluate_completion(state); + break; + case ActionKind::Choice: + std::cerr << "Executing Choice #" << action.id << std::endl; + num_token_eval += this->evaluate_choice(state); + break; + } + queue.pop(); + } + + return num_token_eval; +} + +FTT const & Evaluation::get() const { + return *(this->root); +} + +void Evaluation::initial() { + Text const & init = this->fta.action(0).as(); + TokenSequence tokens = init.tokens; + TokenID bos = Manager::get_model(this->model).bos_token(); + if (tokens[0] != bos) { + tokens.insert(tokens.begin(), bos); + } + ProbaSequence probas(init.tokens.size(), 1.); + float probability = 1.; + for (auto proba: probas) probability *= proba; + + this->root = new FTT(0, init.tokens, probas, probability); + this->queue.emplace(init.successors[0], *(this->root), init.tokens, std::nullopt); +} + +std::pair Evaluation::restore_context(PathState & state) const { + Model & model = Manager::get_model(this->model); + if (state.context) { + throw std::runtime_error("Context saving is not implemented yet, this should not happen"); + } else { + // for now we always use context 0 and reset it before evaluation any action + model.set_tokens(state.tokens, 0); + state.context = 0; + } + return std::pair(model, state.context.value()); +} + +void Evaluation::enqueue(ActionID const action, FTT & parent, std::vector const & tokens, PathState const & state) { + std::optional ctx = state.context; + ctx.reset(); // TODO context saving logic + this->queue.emplace(action, parent, tokens, ctx); +} + +std::vector softmax(float * logits, int vocab_size) { + std::vector result(logits, logits + vocab_size); + + // Apply softmax + float max_logit = *std::max_element(result.begin(), result.end()); + float sum = 0.0f; + for (float& logit : result) { + logit = std::exp(logit - max_logit); + sum += logit; + } + for (float& prob : result) { + prob /= sum; + } + + return result; +} + +} } + diff --git a/libs/autocog/llama/evaluation.hxx b/libs/autocog/llama/evaluation.hxx new file mode 100644 index 0000000..c0373f6 --- /dev/null +++ b/libs/autocog/llama/evaluation.hxx @@ -0,0 +1,60 @@ +#ifndef __AUTOCOG_LLAMA_Evaluation__HXX_ +#define __AUTOCOG_LLAMA_Evaluation__HXX_ + +#include "autocog/llama/types.hxx" + +#include +#include + +namespace autocog { +namespace llama { + +class Model; +class FTA; +class FTT; +class Text; +class Completion; +class Choice; + +struct PathState { + ActionID const action; //< Action to be evaluated next + FTT & parent; //< Previous FTT in the path, reslts of exploring this state will be added to that tree + TokenSequence const tokens; //< Tokens that lead to this state + + std::optional context; //< Context used to evaluate this path + + PathState(ActionID const action_, FTT & parent, std::vector const & tokens_, std::optional context); +}; + +class Evaluation { + public: + using Queue = std::queue; + + private: + ModelID const model; + FTA const & fta; + + Queue queue; + FTT * root; + + protected: + std::pair restore_context(PathState & state) const; + + void initial(); + void enqueue(ActionID const action, FTT & parent, std::vector const & tokens, PathState const & current); + + unsigned evaluate_text (PathState & state); + unsigned evaluate_completion (PathState & state); + unsigned evaluate_choice (PathState & state); + + public: + Evaluation(ModelID const model_, FTA const & fta_); + ~Evaluation(); + unsigned advance(std::optional max_token_eval); + FTT const & get() const; +}; + +} } + +#endif /* __AUTOCOG_LLAMA_Evaluation__HXX_ */ + diff --git a/libs/autocog/llama/evaluator.cxx b/libs/autocog/llama/evaluator.cxx deleted file mode 100644 index b96e183..0000000 --- a/libs/autocog/llama/evaluator.cxx +++ /dev/null @@ -1,9 +0,0 @@ - -#include "autocog/llama/evaluator.hxx" - -namespace autocog { namespace llama { - -// TODO - -} } - diff --git a/libs/autocog/llama/evaluator.hxx b/libs/autocog/llama/evaluator.hxx deleted file mode 100644 index 44c0fa2..0000000 --- a/libs/autocog/llama/evaluator.hxx +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef __AUTOCOG_LLAMA_EVALUATOR__HXX_ -#define __AUTOCOG_LLAMA_EVALUATOR__HXX_ - -namespace autocog { -namespace llama { - -class Evaluator {}; - -} } - -#endif /* __AUTOCOG_LLAMA_EVALUATOR__HXX_ */ - diff --git a/libs/autocog/llama/fta.cxx b/libs/autocog/llama/fta.cxx index 57462e4..95235dc 100644 --- a/libs/autocog/llama/fta.cxx +++ b/libs/autocog/llama/fta.cxx @@ -8,18 +8,46 @@ namespace autocog { namespace llama { Action::Action( - ActionKind const kind_, NodeID const id_, float threshold_ -) : kind(kind_), id(id_), threshold(threshold_), successors() {} + ActionKind const kind_, + ActionID const id_, + float threshold_ +) : + kind(kind_), + id(id_), + threshold(threshold_), + successors() +{} -Text::Text(NodeID const id_, float threshold_) : Action(ActionKind::Text, id_, threshold_) {} +Text::Text( + ActionID const id_, + float threshold_ +) : + Action(ActionKind::Text, id_, threshold_) +{} Completion::Completion( - NodeID const id_, float threshold_, unsigned length_, unsigned beams_, unsigned ahead_ -) : Action(ActionKind::Completion, id_, threshold_), length(length_), beams(beams_), ahead(ahead_) {} + ActionID const id_, + float threshold_, + unsigned length_, + unsigned beams_, + unsigned ahead_ +) : + Action(ActionKind::Completion, id_, threshold_), + length(length_), + beams(beams_), + ahead(ahead_) +{} -Choice::Choice(NodeID const id_, float threshold_, unsigned width_) : Action(ActionKind::Choice, id_, threshold_), width(width_) {} +Choice::Choice( + ActionID const id_, + float threshold_, + unsigned width_ +) : + Action(ActionKind::Choice, id_, threshold_), + width(width_) +{} -Action const & FTA::action(NodeID const & id) const { +Action const & FTA::action(ActionID const id) const { Action const & action = *(this->actions.at(id)); if (action.id != id) { throw std::runtime_error("Action's ID does not match position in FTA::actions!"); @@ -28,21 +56,21 @@ Action const & FTA::action(NodeID const & id) const { } Text & FTA::insert(float threshold_) { - NodeID id = this->actions.size(); + ActionID id = this->actions.size(); Text * action = new Text(id, threshold_); this->actions.push_back(std::unique_ptr(action)); return *action; } Completion & FTA::insert(float threshold_, unsigned length_, unsigned beams_, unsigned ahead_) { - NodeID id = this->actions.size(); + ActionID id = this->actions.size(); Completion * action = new Completion(id, threshold_, length_, beams_, ahead_); this->actions.push_back(std::unique_ptr(action)); return *action; } Choice & FTA::insert(float threshold_, unsigned width_) { - NodeID id = this->actions.size(); + ActionID id = this->actions.size(); Choice * action = new Choice(id, threshold_, width_); this->actions.push_back(std::unique_ptr(action)); return *action; @@ -50,7 +78,7 @@ Choice & FTA::insert(float threshold_, unsigned width_) { // TODO review -FTA::FTA(Model * model, pybind11::dict const & pydata) { +FTA::FTA(Model const & model, pybind11::dict const & pydata) { // Get the actions dictionary from Python FTA if (!pydata.contains("actions")) { throw std::runtime_error("FTA dictionary missing 'actions' field"); @@ -59,7 +87,7 @@ FTA::FTA(Model * model, pybind11::dict const & pydata) { auto py_actions = pydata["actions"].cast(); // First pass: create all actions and build ID mapping - std::map uid_to_id; + std::map uid_to_id; for (auto item : py_actions) { std::string uid = item.first.cast(); @@ -72,7 +100,7 @@ FTA::FTA(Model * model, pybind11::dict const & pydata) { std::string action_type = action_dict["__type__"].cast(); float threshold = action_dict.contains("threshold") ? action_dict["threshold"].cast() : 0.0f; - NodeID node_id = actions.size(); // Assign sequential IDs + ActionID node_id = actions.size(); // Assign sequential IDs uid_to_id[uid] = node_id; std::unique_ptr action; @@ -100,8 +128,8 @@ FTA::FTA(Model * model, pybind11::dict const & pydata) { Completion* completion_action = static_cast(action.get()); // Set stop tokens - if (!action_dict.contains("stops")) { - throw std::runtime_error("Completion missing 'stops' field: " + uid); + if (!action_dict.contains("stop")) { + throw std::runtime_error("Completion missing 'stop' field: " + uid); } else { auto py_stop = action_dict["stop"].cast(); completion_action->stop.clear(); @@ -111,8 +139,8 @@ FTA::FTA(Model * model, pybind11::dict const & pydata) { } // Set vocabulary mask - completion_action->vocab.mask.reserve(model->vocab_size()); - completion_action->vocab.mask.assign(model->vocab_size(), true); + completion_action->vocab.mask.reserve(model.vocab_size()); + completion_action->vocab.mask.assign(model.vocab_size(), true); if (action_dict.contains("vocab")) { throw std::runtime_error("Setting the vocabulary from PY desc is not implemented yet!"); } @@ -151,7 +179,7 @@ FTA::FTA(Model * model, pybind11::dict const & pydata) { std::string uid = item.first.cast(); auto action_dict = item.second.cast(); - NodeID node_id = uid_to_id[uid]; + ActionID node_id = uid_to_id[uid]; Action* action = actions[node_id].get(); // Add successors diff --git a/libs/autocog/llama/fta.hxx b/libs/autocog/llama/fta.hxx index afaf464..25ac8d7 100644 --- a/libs/autocog/llama/fta.hxx +++ b/libs/autocog/llama/fta.hxx @@ -26,20 +26,20 @@ enum class ActionKind { struct Action { ActionKind const kind; - NodeID const id; + ActionID const id; float const threshold; //< Probability threshold for pruning - std::vector successors; + std::vector successors; - Action(ActionKind const kind_, NodeID const id_, float threshold_); + Action(ActionKind const kind_, ActionID const id_, float threshold_); template T const & as() const { if (T::Kind != kind) { throw std::runtime_error("Calling Action::as() with uncompatible ActionKind."); } - return *(T const *)this; + return static_cast(*this); } }; @@ -48,7 +48,7 @@ struct Text : public Action { TokenSequence tokens; - Text(NodeID const id_, float threshold_); + Text(ActionID const id_, float threshold_); }; struct Completion : public Action { @@ -61,7 +61,7 @@ struct Completion : public Action { Vocab vocab; TokenSequence stop; - Completion(NodeID const id_, float threshold_, unsigned length_, unsigned beams_, unsigned ahead_); + Completion(ActionID const id_, float threshold_, unsigned length_, unsigned beams_, unsigned ahead_); }; struct Choice : public Action { @@ -71,19 +71,19 @@ struct Choice : public Action { std::vector choices; // Each choice is a token sequence - Choice(NodeID const id_, float threshold_, unsigned width_); + Choice(ActionID const id_, float threshold_, unsigned width_); }; class FTA { public: - Action const & action(NodeID const & id) const; + Action const & action(ActionID const id) const; Text & insert(float threshold_); Completion & insert(float threshold_, unsigned length_, unsigned beams_, unsigned ahead_); Choice & insert(float threshold_, unsigned width_); FTA() = default; - FTA(Model * model, pybind11::dict const & pydata); + FTA(Model const & model, pybind11::dict const & pydata); private: std::vector> actions; diff --git a/libs/autocog/llama/ftt.cxx b/libs/autocog/llama/ftt.cxx index 47b0644..0dda4c9 100644 --- a/libs/autocog/llama/ftt.cxx +++ b/libs/autocog/llama/ftt.cxx @@ -3,6 +3,25 @@ namespace autocog { namespace llama { +FTT::FTT( + ActionID const action_, + TokenSequence const & tokens_, + ProbaSequence const & probas_, + float probability_ +) : + action(action_), + tokens(tokens_), + probas(probas_), + probability(probability_), + children(), + pruned(false) +{} + +FTT & FTT::add(ActionID const action_, TokenSequence const & tokens_, ProbaSequence const & probas_, float probability_) { + this->children.emplace_back(action_, tokens_, probas_, probability_); + return this->children.back(); +} + // TODO review code below pybind11::dict FTT::pydict() const { diff --git a/libs/autocog/llama/ftt.hxx b/libs/autocog/llama/ftt.hxx index 7bd89e8..48be603 100644 --- a/libs/autocog/llama/ftt.hxx +++ b/libs/autocog/llama/ftt.hxx @@ -8,17 +8,22 @@ namespace autocog { namespace llama { -class FTT { - public: - FTT() = default; - pybind11::dict pydict() const; +struct FTT { + ActionID const action; + TokenSequence const tokens; + ProbaSequence const probas; + float const probability; - private: - std::vector tokens; - std::vector probas; - bool pruned = false; + std::vector children; + bool pruned{false}; - std::vector children; + FTT(ActionID const action_, TokenSequence const & tokens_, ProbaSequence const & probas_, float probability_); + + FTT & add(ActionID const action_, TokenSequence const & tokens_, ProbaSequence const & probas_, float probability_); + + pybind11::dict pydict() const; + + }; } } diff --git a/libs/autocog/llama/manager.cxx b/libs/autocog/llama/manager.cxx new file mode 100644 index 0000000..5ca1675 --- /dev/null +++ b/libs/autocog/llama/manager.cxx @@ -0,0 +1,71 @@ + +#include "autocog/llama/manager.hxx" +#include "autocog/llama/evaluation.hxx" + +#include +#include +#include + +namespace autocog { +namespace llama { + +Manager & Manager::instance() { + static Manager __instance; + return __instance; +} + +Manager::~Manager() { + cleanup(); +} + +void Manager::cleanup() { + models.clear(); + llama_backend_free(); +} + +void Manager::initialize() { + llama_backend_init(); + + auto & manager = instance(); + manager.models.emplace_back(); // adding Model #0 which is a simple character level random number generator (for ultra-fast testing) + + std::atexit([]() { + instance().cleanup(); + }); +} + +ModelID Manager::add_model(std::string const & path, int n_ctx) { + auto & manager = instance(); + ModelID id = manager.models.size(); + manager.models.emplace_back(id, path, n_ctx); + return id; +} + +Model & Manager::get_model(ModelID id) { + auto & manager = instance(); + return manager.models[id]; +} + +EvalID Manager::add_eval(ModelID const model, FTA const & fta) { + auto & manager = instance(); + EvalID id = manager.next_eval_id++; + manager.evaluations.try_emplace(id, model, fta); + return id; +} + +Evaluation & Manager::get_eval(EvalID id) { + auto & manager = instance(); + auto it = manager.evaluations.find(id); + if (it == manager.evaluations.end()) { + throw std::runtime_error("Invalid Evaluation ID: " + std::to_string(id)); + } + return it->second; +} + +void Manager::rm_eval(EvalID id) { + auto & manager = instance(); + manager.evaluations.erase(id); +} + +} } + diff --git a/libs/autocog/llama/manager.hxx b/libs/autocog/llama/manager.hxx new file mode 100644 index 0000000..65bcb5c --- /dev/null +++ b/libs/autocog/llama/manager.hxx @@ -0,0 +1,43 @@ +#ifndef __AUTOCOG_LLAMA_MANAGER__HXX_ +#define __AUTOCOG_LLAMA_MANAGER__HXX_ + +#include "autocog/llama/types.hxx" +#include "autocog/llama/model.hxx" + +#include +#include + +namespace autocog { +namespace llama { + +class Evaluation; +class FTA; + +class Manager { + private: + std::vector models; + + EvalID next_eval_id = 0; + std::unordered_map evaluations; + + Manager() = default; + void cleanup(); + static Manager & instance(); + + public: + ~Manager(); + + static void initialize(); + + static ModelID add_model(std::string const & path, int n_ctx); + static Model & get_model(ModelID id); + + static EvalID add_eval(ModelID const model_, FTA const & fta); + static Evaluation & get_eval(EvalID id); + static void rm_eval(EvalID id); +}; + +} } + +#endif /* __AUTOCOG_LLAMA_MANAGER__HXX_ */ + diff --git a/libs/autocog/llama/model.cxx b/libs/autocog/llama/model.cxx index b18229e..f47fcdb 100644 --- a/libs/autocog/llama/model.cxx +++ b/libs/autocog/llama/model.cxx @@ -6,9 +6,17 @@ namespace autocog { namespace llama { -Model::Model(std::string const & model_path, int n_ctx) : model(nullptr), contexts() { - llama_backend_init(); - +Model::Model() : + id(0), + model(nullptr), + contexts() +{} + +Model::Model(ModelID const id_, std::string const & model_path, int n_ctx) : + id(id_), + model(nullptr), + contexts() +{ // Load model llama_model_params model_params = llama_model_default_params(); this->model = llama_model_load_from_file(model_path.c_str(), model_params); @@ -20,36 +28,58 @@ Model::Model(std::string const & model_path, int n_ctx) : model(nullptr), contex llama_context_params ctx_params = llama_context_default_params(); ctx_params.n_ctx = n_ctx; - // Create single context with ID=0 + // Create single context with ID=0 (and associated token sequence) llama_context * ctx = llama_init_from_model(this->model, ctx_params); if (!ctx) { llama_model_free(this->model); throw std::runtime_error("Failed to create llama context"); } this->contexts.push_back(ctx); + this->tokens.emplace_back(); } Model::~Model() { for (auto* ctx : this->contexts) if (ctx) llama_free(ctx); contexts.clear(); - if (this->model) llama_free_model(model); - - llama_backend_free(); + if (this->model) llama_model_free(model); } -llama_context * Model::get_context(ContextID const id) const { +void Model::check_context_id(ContextID const id) const { + if (this->contexts.size() != this->tokens.size()) { + throw std::runtime_error("Discrepency between contexts and tokens vector size."); + } if (id >= this->contexts.size()) { throw std::runtime_error("Invalid context ID: " + std::to_string(id)); } + if (this->contexts[id] == nullptr) { + throw std::runtime_error("Missing context ID: " + std::to_string(id)); + } +} + +llama_context * Model::get_context(ContextID const id) const { + check_context_id(id); return this->contexts[id]; } -ContextID Model::fork_context(ContextID const id) { +TokenSequence const & Model::get_tokens_const(ContextID const id) const { + check_context_id(id); + return this->tokens[id]; +} + +TokenSequence & Model::get_tokens(ContextID const id) { + check_context_id(id); + return this->tokens[id]; +} + +ContextID Model::fork_context(ContextID const) { throw std::runtime_error("Context forking is not implemented yet (Phase 3 feature)"); } -TokenSequence Model::tokenize(std::string const & text, bool add_bos, bool special) { +TokenSequence Model::tokenize(std::string const & text, bool add_bos, bool special) { + if (this->id == 0) { + throw std::runtime_error("Using model #0 (RNG) is not implemented yet!"); + } std::vector tokens; tokens.resize(text.length() + (add_bos ? 1 : 0) + 1); // Rough upper bound @@ -72,6 +102,10 @@ TokenSequence Model::tokenize(std::string const & text, bool add_bos, bool speci } std::string Model::detokenize(TokenSequence const & tokens, bool spec_rm, bool spec_unp) { + if (this->id == 0) { + throw std::runtime_error("Using model #0 (RNG) is not implemented yet!"); + } + if (tokens.empty()) { return ""; } @@ -99,38 +133,87 @@ std::string Model::detokenize(TokenSequence const & tokens, bool spec_rm, bool s } const llama_vocab * Model::get_vocab() const { + if (this->id == 0) { + throw std::runtime_error("Using model #0 (RNG) is not implemented yet!"); + } return llama_model_get_vocab(this->model); } size_t Model::vocab_size() const { - return llama_vocab_n_tokens(this->get_vocab()); + if (this->id == 0) { + throw std::runtime_error("Using model #0 (RNG) is not implemented yet!"); + } + return llama_vocab_n_tokens(this->get_vocab()); } TokenID Model::bos_token() const { - return llama_vocab_bos(this->get_vocab()); + if (this->id == 0) { + throw std::runtime_error("Using model #0 (RNG) is not implemented yet!"); + } + return llama_vocab_bos(this->get_vocab()); } TokenID Model::eos_token() const { - return llama_vocab_eos(this->get_vocab()); + if (this->id == 0) { + throw std::runtime_error("Using model #0 (RNG) is not implemented yet!"); + } + return llama_vocab_eos(this->get_vocab()); } -std::vector Model::get_logits(ContextID const id) { - float * logits = llama_get_logits(this->get_context(id)); - if (!logits) { - throw std::runtime_error("Failed to get logits - context may not have been evaluated"); +static unsigned run_batch_full(llama_context * ctx, TokenSequence const & tokens) { + llama_memory_t mem = llama_get_memory(ctx); + llama_memory_clear(mem, false); + + unsigned n_ctx = llama_n_ctx(ctx); + if (tokens.size() > n_ctx) { + throw std::runtime_error("Token sequence too long: " + std::to_string(tokens.size()) + " > " + std::to_string(n_ctx)); + } + llama_batch batch = llama_batch_get_one(const_cast(tokens.data()), tokens.size()); + if (llama_decode(ctx, batch) != 0) { + throw std::runtime_error("Failed to set the token sequence."); } - return std::vector(logits, logits + this->vocab_size()); + return tokens.size(); } -bool Model::eval_tokens(TokenSequence const & tokens, ContextID const id) { - if (tokens.empty()) { - return true; +unsigned Model::set_tokens(TokenSequence const & tokens_, ContextID const id) { + if (this->id == 0) { + return tokens_.size(); } + check_context_id(id); - llama_batch batch = llama_batch_get_one(const_cast(tokens.data()), tokens.size()); - int result = llama_decode(this->get_context(id), batch); + TokenSequence & loc_tokens = this->get_tokens(id); + loc_tokens = tokens_; + return run_batch_full(this->get_context(id), loc_tokens); +} - return result == 0; +unsigned Model::eval_sequences(TokenSequence const & new_tokens, ProbaSequence & probas, ContextID const id) { + if (this->id == 0) { + throw std::runtime_error("Using model #0 (RNG) is not implemented yet!"); + // TODO fill `probas` with random float in [0,1] + return new_tokens.size(); + } + + TokenSequence & loc_tokens = this->get_tokens(id); + llama_pos token_pos = loc_tokens.size(); + probas.clear(); + + for (auto token: new_tokens) { + + llama_batch batch = llama_batch_get_one(&token, 1); + batch.pos = &token_pos; + + if (llama_decode(this->get_context(id), batch) != 0) { + throw std::runtime_error("Failed to decode token"); + } + + float tok_probas = llama_get_logits(this->get_context(id))[token]; + probas.push_back(tok_probas); + + token_pos++; + } + loc_tokens.insert(loc_tokens.end(), new_tokens.begin(), new_tokens.end()); + return new_tokens.size(); } } } + diff --git a/libs/autocog/llama/model.hxx b/libs/autocog/llama/model.hxx index 4a63731..58e60a9 100644 --- a/libs/autocog/llama/model.hxx +++ b/libs/autocog/llama/model.hxx @@ -10,14 +10,20 @@ namespace llama { class Model { private: + ModelID const id; llama_model * model; std::vector contexts; + std::vector tokens; llama_context * get_context(ContextID const id = 0) const; + TokenSequence & get_tokens(ContextID const id = 0); + void check_context_id(ContextID const id = 0) const; + const llama_vocab * get_vocab() const; public: - Model(std::string const & model_path, int n_ctx); + Model(); + Model(ModelID const id, std::string const & model_path, int n_ctx); ~Model(); ContextID fork_context(ContextID const id = 0); @@ -29,8 +35,9 @@ class Model { TokenID bos_token() const; TokenID eos_token() const; - std::vector get_logits(ContextID const id = 0); - bool eval_tokens(const TokenSequence& tokens, ContextID const id = 0); + TokenSequence const & get_tokens_const(ContextID const id = 0) const; + unsigned set_tokens(TokenSequence const & tokens, ContextID const id = 0); + unsigned eval_sequences(TokenSequence const & tokens, ProbaSequence & probas, ContextID const id = 0); }; } } diff --git a/libs/autocog/llama/python_bindings.cxx b/libs/autocog/llama/python_bindings.cxx index 77013a6..4e5626c 100644 --- a/libs/autocog/llama/python_bindings.cxx +++ b/libs/autocog/llama/python_bindings.cxx @@ -1,47 +1,142 @@ +#include "fta.hxx" +#include "ftt.hxx" +#include "model.hxx" +#include "evaluation.hxx" +#include "manager.hxx" + #include #include #include -#include "fta.hxx" -#include "ftt.hxx" -#include "model.hxx" -#include "evaluator.hxx" +#include + +namespace autocog { +namespace llama { + +static FTA convert_pydict_to_fta(ModelID const id, pybind11::dict const & fta_dict) { + Model & model = Manager::get_model(id); + + FTA fta(model, fta_dict); // TODO move from class -// Wrapper functions for Python integration -pybind11::dict evaluate_fta_python(uintptr_t ptr, const pybind11::dict& fta_dict) { - auto model = reinterpret_cast(ptr); - autocog::llama::FTA fta(model, fta_dict); - autocog::llama::FTT ftt; + return fta; +} - // TODO: Implementation +static pybind11::dict convert_ftt_to_pydict(ModelID const id, FTT const & ftt) { + Model & model = Manager::get_model(id); - return ftt.pydict(); + return ftt.pydict(); // TODO move from class } +} } + PYBIND11_MODULE(llama, module) { + using namespace autocog::llama; + + Manager::initialize(); + module.doc() = "AutoCog's llama.cpp integration module"; - module.def("create", [](const std::string& model_path, int n_ctx=4096) { - auto model = new autocog::llama::Model(model_path, n_ctx); - return reinterpret_cast(model); - }); + module.def("create", + [](std::string const & model_path, int n_ctx) { + return Manager::add_model(model_path, n_ctx); + }, + "Instantiate a GGML model with llama.cpp", + pybind11::arg("model_path"), + pybind11::arg("n_ctx") = 4096 + ); - module.def("tokenize", [](uintptr_t ptr, const std::string & text, bool add_bos = false, bool special = false) { - auto model = reinterpret_cast(ptr); - auto tokens = model->tokenize(text, add_bos, special); + module.def("tokenize", + [](ModelID model, const std::string & text, bool add_bos, bool special) { + auto tokens = Manager::get_model(model).tokenize(text, add_bos, special); pybind11::list result; for (auto token : tokens) result.append(token); return result; - }); + }, + "Tokenize text using llama.cpp", + pybind11::arg("model"), + pybind11::arg("text"), + pybind11::arg("add_bos") = false, + pybind11::arg("special") = false + ); - module.def("detokenize", [](uintptr_t ptr, const pybind11::list & py_tokens, bool spec_rm = false, bool spec_unp = false) { - auto model = reinterpret_cast(ptr); - autocog::llama::TokenSequence tokens; - for (auto item : py_tokens) tokens.push_back(item.cast()); - return model->detokenize(tokens, spec_rm, spec_unp); - }); + module.def("detokenize", + [](ModelID model, const pybind11::list & py_tokens, bool spec_rm, bool spec_unp) { + TokenSequence tokens; + for (auto item : py_tokens) tokens.push_back(item.cast()); + return Manager::get_model(model).detokenize(tokens, spec_rm, spec_unp); + }, + "Detokenize tokens to text", + pybind11::arg("model"), + pybind11::arg("tokens"), + pybind11::arg("spec_rm") = false, + pybind11::arg("spec_unp") = false + ); - module.def("evaluate", &evaluate_fta_python); + module.def("evaluate", + [](ModelID model, pybind11::dict const & fta_dict) { + FTA fta = convert_pydict_to_fta(model, fta_dict); + EvalID eval = Manager::add_eval(model, fta); + Manager::get_eval(eval).advance(std::nullopt); + pybind11::dict ftt = convert_ftt_to_pydict(model, Manager::get_eval(eval).get()); + Manager::rm_eval(eval); + return ftt; + }, + "Evaluate a FTA using a model and return the FTT." + ); + +#ifdef ASYNC_EXEC + module.def("instantiate", + [](ModelID model, pybind11::dict const & fta) { + FTA fta_ = convert_pydict_to_fta(model, fta); + // TODO + }, + "Instantiate a FTA using a model and return the EvalID.", + pybind11::arg("model"), + pybind11::arg("fta") + ); + + module.def("advance", + [](EvalID eval, std::optional max_token_eval) { + // TODO + }, + "Advance a FTA evaluation with an optional (soft) limit on the number of token evaluation, return the number of evaluated tokens.", + pybind11::arg("eval"), + pybind11::arg("max_token_eval") = std::nullopt + ); + + module.def("advance_bg", + [](EvalID eval, std::optional max_token_eval) { + // TODO + }, + "Signal an evaluation to run in the background, return nothing immediately.", + pybind11::arg("eval"), + pybind11::arg("max_token_eval") = std::nullopt + ); + + module.def("finished", + [](EvalID eval) { + // TODO + }, + "Check if a FTA evaluation is finished.", + pybind11::arg("eval") + ); + + module.def("retrieve", + [](EvalID eval) { + // TODO + }, + "Retrieve the FTT being generated by an evaluation.", + pybind11::arg("eval") + ); + + module.def("release", + [](EvalID eval) { + // TODO + }, + "Wait for the evaluation to finish, retrieve the generated FTT, then remove the evaluation.", + pybind11::arg("eval") + ); +#endif /* ASYNC_EXEC */ } diff --git a/libs/autocog/llama/types.hxx b/libs/autocog/llama/types.hxx index efba002..29082f3 100644 --- a/libs/autocog/llama/types.hxx +++ b/libs/autocog/llama/types.hxx @@ -7,11 +7,14 @@ namespace autocog { namespace llama { +using EvalID = unsigned; +using ModelID = unsigned; using ContextID = unsigned; -using NodeID = unsigned; +using ActionID = unsigned; using TokenID = llama_token; using TokenSequence = std::vector; +using ProbaSequence = std::vector; } } diff --git a/tests/autocog/llama/execute_sta_with_llama_cpp.py b/tests/autocog/llama/execute_sta_with_llama_cpp.py index 64a6530..e1b203a 100644 --- a/tests/autocog/llama/execute_sta_with_llama_cpp.py +++ b/tests/autocog/llama/execute_sta_with_llama_cpp.py @@ -57,7 +57,7 @@ }) elif isinstance(act, Choose): action.update({ - "__type__" : "Text", + "__type__" : "Choice", "threshold" : 1. if act.threshold is None else act.threshold, "width" : 1 if act.width is None else act.width, "choices" : [ @@ -66,7 +66,7 @@ }) elif isinstance(act, Complete): action.update({ - "__type__" : "Text", + "__type__" : "Completion", "threshold" : 1. if act.threshold is None else act.threshold, "length" : 1 if act.length is None else act.length, "beams" : 1 if act.beams is None else act.beams, @@ -78,6 +78,8 @@ raise Exception() actions.update({ act.uid : action }) +# print(json.dumps(actions,indent=4)) + ftt = autocog.llama.evaluate(model, { 'actions' : actions }) print(ftt) From 769d4b89c366a43ba8c4d6c808663167f4696f37 Mon Sep 17 00:00:00 2001 From: Tristan Date: Thu, 7 Aug 2025 00:13:12 -0700 Subject: [PATCH 07/51] LLama.cpp went al the way through FTA eval but segfault in FFT to pydict --- libs/autocog/llama/evaluation-choice.cxx | 115 +++++++++- libs/autocog/llama/evaluation-completion.cxx | 200 +++++++++++++++++- libs/autocog/llama/evaluation-text.cxx | 6 +- libs/autocog/llama/evaluation.cxx | 3 - libs/autocog/llama/evaluation.hxx | 9 + libs/autocog/llama/model.cxx | 65 ++++++ libs/autocog/llama/model.hxx | 21 +- .../llama/execute_sta_with_llama_cpp.py | 3 + tests/samples/mini.sta | 22 ++ 9 files changed, 432 insertions(+), 12 deletions(-) create mode 100644 tests/samples/mini.sta diff --git a/libs/autocog/llama/evaluation-choice.cxx b/libs/autocog/llama/evaluation-choice.cxx index fb484e8..0d61691 100644 --- a/libs/autocog/llama/evaluation-choice.cxx +++ b/libs/autocog/llama/evaluation-choice.cxx @@ -4,15 +4,124 @@ #include "autocog/llama/fta.hxx" #include "autocog/llama/ftt.hxx" +#include + +#include #include +#include +#include namespace autocog { namespace llama { unsigned Evaluation::evaluate_choice(PathState & state) { - auto [model,ctx] = this->restore_context(state); + auto [model, ctx] = this->restore_context(state); Choice const & action = this->fta.action(state.action).as(); - throw std::runtime_error("Not implemented yet"); - return 0; + std::cerr << "Executing Choice #" << action.id << std::endl; + std::cerr << " - width: " << action.width << std::endl; + std::cerr << " - number of choices: " << action.choices.size() << std::endl; + + if (action.choices.empty()) { + throw std::runtime_error("Choice action has no choices"); + } + + if (action.successors.size() != action.choices.size()) { + throw std::runtime_error("Choice action must have as many successors as choices"); + } + + unsigned num_token_eval = 0; + + // Structure to hold choice evaluation results + struct ChoiceResult { + size_t index; + TokenSequence tokens; + ProbaSequence probas; + float total_probability; + }; + std::vector results; + + // Evaluate ALL choices in full + for (size_t i = 0; i < action.choices.size(); ++i) { + TokenSequence const & choice_tokens = action.choices[i]; + std::cerr << " - choice[" << i << "]:" << std::endl; + std::cerr << " - number of tokens: " << choice_tokens.size() << std::endl; + + // Save current state to restore after evaluation + TokenSequence saved_tokens = model.get_tokens_const(ctx); + + // Evaluate this choice + ProbaSequence probas; + num_token_eval += model.eval_sequences(choice_tokens, probas, ctx); + + // Calculate total probability for this choice + // TODO: In the future, support different probability strategies: + // - product (current): p_total = p1 * p2 * ... * pn + // - average: p_total = (p1 + p2 + ... + pn) / n + // - min: p_total = min(p1, p2, ..., pn) + // - weighted: custom weighting function + // This could be specified as a parameter in the Choice action + float total_prob = 1.0f; + for (float p : probas) { + total_prob *= p; // Product strategy (default) + } + std::cerr << " - probability: " << total_prob << std::endl; + + results.push_back({i, choice_tokens, probas, total_prob}); + + // Restore context state for next choice evaluation + model.set_tokens(saved_tokens, ctx); + } + + // Sort by probability (descending) + std::sort(results.begin(), results.end(), + [](const ChoiceResult& a, const ChoiceResult& b) { + return a.total_probability > b.total_probability; + }); + + // Select top 'width' choices that are above threshold + std::vector selected_indices; + + for (const auto& result : results) { + if (result.total_probability >= action.threshold) { + selected_indices.push_back(result.index); + if (selected_indices.size() >= action.width) { + break; + } + } + } + + // If none above threshold, pick the best one + if (selected_indices.empty() && !results.empty()) { + selected_indices.push_back(results[0].index); + } + + // Add selected choices to FTT and enqueue their corresponding successors + for (size_t idx : selected_indices) { + // Find the result for this index + auto it = std::find_if(results.begin(), results.end(), + [idx](const ChoiceResult& r) { return r.index == idx; }); + + if (it != results.end()) { + // Add this choice to the FTT + FTT& child = state.parent.add(action.id, it->tokens, it->probas, it->total_probability); + + // Mark as pruned if below threshold (for tracking purposes) + if (it->total_probability < action.threshold) { + child.pruned = true; + // Note: We still enqueue it since it was the best available + } + + // Enqueue the corresponding successor action for this specific choice + ActionID next_action = action.successors[idx]; + + // Prepare tokens for next action (current context + chosen tokens) + TokenSequence next_tokens = model.get_tokens_const(ctx); + next_tokens.insert(next_tokens.end(), it->tokens.begin(), it->tokens.end()); + + this->enqueue(next_action, child, next_tokens, state); + } + } + + return num_token_eval; } } } diff --git a/libs/autocog/llama/evaluation-completion.cxx b/libs/autocog/llama/evaluation-completion.cxx index 2eb7788..d38d7f3 100644 --- a/libs/autocog/llama/evaluation-completion.cxx +++ b/libs/autocog/llama/evaluation-completion.cxx @@ -4,15 +4,209 @@ #include "autocog/llama/fta.hxx" #include "autocog/llama/ftt.hxx" +#include + +#include #include +#include +#include +#include +#include namespace autocog { namespace llama { +struct Beam { + TokenSequence tokens; + ProbaSequence probas; + float cumulative_log_prob; + bool stopped; + + float get_probability() const { + return std::exp(cumulative_log_prob); + } +}; + +// Expand a single beam by evaluating next token +static void expand_beam( + Model& model, + ContextID ctx, + const Beam& beam, + const Completion& action, + const TokenSequence& base_tokens, + std::vector& next_beams, + unsigned& num_token_eval +) { + TokenSequence context_tokens = base_tokens; + context_tokens.insert(context_tokens.end(), beam.tokens.begin(), beam.tokens.end()); + model.set_tokens(context_tokens, ctx); + + // Get top candidates + std::vector topk_tokens; + std::vector topk_logits; + size_t n_candidates = static_cast(action.beams * action.ahead); + num_token_eval += model.eval_topk_tokens( + action.vocab.mask, + n_candidates, + topk_tokens, + topk_logits, + ctx + ); + + // Create new beams from candidates + for (size_t i = 0; i < topk_tokens.size(); ++i) { + TokenID token = topk_tokens[i]; + float logit = topk_logits[i]; + + // Check if this is a stop token + bool is_stop = std::find(action.stop.begin(), action.stop.end(), token) != action.stop.end(); + + if (is_stop) { + Beam new_beam = beam; + new_beam.stopped = true; + next_beams.push_back(new_beam); + } else { + Beam new_beam = beam; + new_beam.tokens.push_back(token); + new_beam.probas.push_back(std::exp(logit)); // Simplified probability + new_beam.cumulative_log_prob += logit; + next_beams.push_back(new_beam); + } + } +} + +// Prune beams to keep only top k +static void prune_beams( + std::vector& beams, + unsigned beam_width +) { + // Sort by score + std::sort(beams.begin(), beams.end(), [](const Beam& a, const Beam& b) { + if (a.stopped != b.stopped) { + return a.stopped > b.stopped; // Prioritize stopped beams + } + return a.cumulative_log_prob > b.cumulative_log_prob; + }); + + // Keep top beams + std::vector pruned; + size_t kept_active = 0; + for (const auto& beam : beams) { + if (beam.stopped) { + pruned.push_back(beam); + } else if (kept_active < beam_width) { + pruned.push_back(beam); + kept_active++; + } + } + + beams = std::move(pruned); +} + +// Run beam search for one position +static bool beam_search_step( + Model& model, + ContextID ctx, + const Completion& action, + const TokenSequence& base_tokens, + std::vector& current_beams, + unsigned& num_token_eval +) { + std::vector next_beams; + + for (const Beam& beam : current_beams) { + if (beam.stopped) { + next_beams.push_back(beam); + continue; + } + expand_beam(model, ctx, beam, action, base_tokens, next_beams, num_token_eval); + } + + // Check for early termination + bool all_stopped = std::all_of(next_beams.begin(), next_beams.end(), [](const Beam& b) { return b.stopped; }); + + if (all_stopped) { + current_beams = std::move(next_beams); + return true; // Signal early termination + } + + // Prune and update beams + prune_beams(next_beams, action.beams); + + if (next_beams.empty()) { + throw std::runtime_error("No valid beams remaining in completion"); + } + + current_beams = std::move(next_beams); + return false; // Continue beam search +} + +// Add selected beams to FTT +void add_beams_to_ftt( + std::vector const & beams, + Completion const & action, + PathState & state, + Evaluation * eval +) { + // Sort by score + std::vector sorted_beams = beams; + std::sort(sorted_beams.begin(), sorted_beams.end(), [](const Beam& a, const Beam& b) { + return a.cumulative_log_prob > b.cumulative_log_prob; + }); + + // Add top beams to FTT + size_t n_keep = std::min(static_cast(action.ahead), sorted_beams.size()); + + for (size_t i = 0; i < n_keep; ++i) { + const Beam& beam = sorted_beams[i]; + float probability = beam.get_probability(); + + FTT & child = state.parent.add(action.id, beam.tokens, beam.probas, probability); + + // Apply threshold pruning + if (probability < action.threshold) { + child.pruned = true; + continue; + } + + // Enqueue next action + if (!action.successors.empty()) { + TokenSequence next_tokens = state.tokens; + next_tokens.insert(next_tokens.end(), beam.tokens.begin(), beam.tokens.end()); + eval->enqueue(action.successors[0], child, next_tokens, state); + } + } +} + unsigned Evaluation::evaluate_completion(PathState & state) { - auto [model,ctx] = this->restore_context(state); + auto [model, ctx] = this->restore_context(state); Completion const & action = this->fta.action(state.action).as(); - throw std::runtime_error("Not implemented yet"); - return 0; + std::cerr << "Executing Completion #" << action.id << std::endl; + std::cerr << " - beams: " << action.beams << std::endl; + std::cerr << " - length: " << action.length << std::endl; + std::cerr << " - ahead: " << action.ahead << std::endl; + std::cerr << " - threshold: " << action.threshold << std::endl; + + unsigned num_token_eval = 0; + + // Initialize beam search + std::vector current_beams; + current_beams.push_back({TokenSequence(), ProbaSequence(), 0.0f, false}); + + // Run beam search + for (unsigned pos = 0; pos < action.length; ++pos) { + std::cerr << " - pos[" << pos << "]..." << std::endl; + bool should_stop = beam_search_step( + model, ctx, action, state.tokens, current_beams, num_token_eval + ); + + if (should_stop) { + break; + } + } + + add_beams_to_ftt(current_beams, action, state, this); + + return num_token_eval; } } } diff --git a/libs/autocog/llama/evaluation-text.cxx b/libs/autocog/llama/evaluation-text.cxx index 9c9c1c0..889fdb9 100644 --- a/libs/autocog/llama/evaluation-text.cxx +++ b/libs/autocog/llama/evaluation-text.cxx @@ -5,6 +5,7 @@ #include "autocog/llama/fta.hxx" #include "autocog/llama/ftt.hxx" +#include #include namespace autocog { namespace llama { @@ -12,13 +13,16 @@ namespace autocog { namespace llama { unsigned Evaluation::evaluate_text(PathState & state) { auto [model,ctx] = this->restore_context(state); Text const & action = this->fta.action(state.action).as(); + std::cerr << "Executing Text #" << action.id << std::endl; + std::cerr << " - number of tokens: " << action.tokens.size() << std::endl; ProbaSequence probas(action.tokens.size(), 1.); unsigned num_token_eval = model.eval_sequences(action.tokens, probas, ctx); float probability = 1.; for (auto proba: probas) probability *= proba; - state.parent.add(action.id, action.tokens, probas, probability); + std::cerr << " - probability: " << probability << std::endl; + state.parent.add(action.id, action.tokens, probas, probability); this->enqueue(action.successors[0], state.parent.children.back(), model.get_tokens_const(), state); diff --git a/libs/autocog/llama/evaluation.cxx b/libs/autocog/llama/evaluation.cxx index d50d2ab..ae44378 100644 --- a/libs/autocog/llama/evaluation.cxx +++ b/libs/autocog/llama/evaluation.cxx @@ -37,15 +37,12 @@ unsigned Evaluation::advance(std::optional max_token_eval) { Action const & action = this->fta.action(state.action); switch (action.kind) { case ActionKind::Text: - std::cerr << "Executing Text #" << action.id << std::endl; num_token_eval += this->evaluate_text(state); break; case ActionKind::Completion: - std::cerr << "Executing Completion #" << action.id << std::endl; num_token_eval += this->evaluate_completion(state); break; case ActionKind::Choice: - std::cerr << "Executing Choice #" << action.id << std::endl; num_token_eval += this->evaluate_choice(state); break; } diff --git a/libs/autocog/llama/evaluation.hxx b/libs/autocog/llama/evaluation.hxx index c0373f6..5ef3016 100644 --- a/libs/autocog/llama/evaluation.hxx +++ b/libs/autocog/llama/evaluation.hxx @@ -16,6 +16,8 @@ class Text; class Completion; class Choice; +class Beam; + struct PathState { ActionID const action; //< Action to be evaluated next FTT & parent; //< Previous FTT in the path, reslts of exploring this state will be added to that tree @@ -52,6 +54,13 @@ class Evaluation { ~Evaluation(); unsigned advance(std::optional max_token_eval); FTT const & get() const; + + friend void add_beams_to_ftt( + std::vector const & beams, + Completion const & action, + PathState & state, + Evaluation * eval + ); }; } } diff --git a/libs/autocog/llama/model.cxx b/libs/autocog/llama/model.cxx index f47fcdb..f1714bf 100644 --- a/libs/autocog/llama/model.cxx +++ b/libs/autocog/llama/model.cxx @@ -2,6 +2,7 @@ #include +#include #include namespace autocog { namespace llama { @@ -215,5 +216,69 @@ unsigned Model::eval_sequences(TokenSequence const & new_tokens, ProbaSequence & return new_tokens.size(); } +unsigned Model::eval_topk_tokens( + std::vector const & vocab_mask, + size_t max_candidates, + std::vector & topk_tokens, + std::vector & topk_probas, + ContextID const id +) { + + check_context_id(id); + if (this->id == 0) { + throw std::runtime_error("Using model #0 (RNG) is not implemented yet!"); + } + + size_t vocab_size = this->vocab_size(); + if (vocab_mask.size() != vocab_size) { + throw std::runtime_error("vocab_mask size (" + std::to_string(vocab_mask.size()) + ") does not match vocabulary size (" + std::to_string(vocab_size) + ")"); + } + + llama_context * ctx = this->get_context(id); + TokenSequence const & current_tokens = this->get_tokens_const(id); + + topk_tokens.clear(); + topk_probas.clear(); + + llama_batch batch = llama_batch_get_one( + const_cast(current_tokens.data()), + current_tokens.size() + ); + + if (llama_decode(ctx, batch) != 0) { + throw std::runtime_error("Failed to decode for token evaluation"); + } + + float * logits = llama_get_logits(ctx); + + // Collect valid candidates based on mask + std::vector> candidates; + for (TokenID tid = 0; tid < vocab_size; ++tid) { + if (vocab_mask[tid]) { + candidates.emplace_back(tid, logits[tid]); + } + } + + // Handle edge case: no valid candidates + if (candidates.empty()) { + throw std::runtime_error("Failed to find candidate token. Cannot have an empty vocabularity mask (all false)."); + } + + std::sort(candidates.begin(), candidates.end(), [](const auto& a, const auto& b) { + return a.second > b.second; + }); + + size_t k = std::min(max_candidates, candidates.size()); + topk_tokens.reserve(k); + topk_probas.reserve(k); + + for (size_t i = 0; i < k; ++i) { + topk_tokens.push_back(candidates[i].first); + topk_probas.push_back(candidates[i].second); // Raw logits from llama.cpp + } + + return 1; +} + } } diff --git a/libs/autocog/llama/model.hxx b/libs/autocog/llama/model.hxx index 58e60a9..5b3cd24 100644 --- a/libs/autocog/llama/model.hxx +++ b/libs/autocog/llama/model.hxx @@ -36,8 +36,25 @@ class Model { TokenID eos_token() const; TokenSequence const & get_tokens_const(ContextID const id = 0) const; - unsigned set_tokens(TokenSequence const & tokens, ContextID const id = 0); - unsigned eval_sequences(TokenSequence const & tokens, ProbaSequence & probas, ContextID const id = 0); + + unsigned set_tokens( + TokenSequence const & tokens, + ContextID const id = 0 + ); + + unsigned eval_sequences( + TokenSequence const & tokens, + ProbaSequence & probas, + ContextID const id = 0 + ); + + unsigned eval_topk_tokens( + std::vector const & vocab_mask, + size_t max_candidates, + std::vector & topk_tokens, + std::vector & topk_probas, + ContextID const id + ); }; } } diff --git a/tests/autocog/llama/execute_sta_with_llama_cpp.py b/tests/autocog/llama/execute_sta_with_llama_cpp.py index e1b203a..5c5be7a 100644 --- a/tests/autocog/llama/execute_sta_with_llama_cpp.py +++ b/tests/autocog/llama/execute_sta_with_llama_cpp.py @@ -56,6 +56,9 @@ "tokens" : tokenize(model, act.text, False, False) }) elif isinstance(act, Choose): + if len(act.successors) == 1: + action.update({ "successors" : [ act.successors[0] ] * len(act.choices) }) + assert len(act.successors) == 0 or len(action['successors']) == len(act.choices) action.update({ "__type__" : "Choice", "threshold" : 1. if act.threshold is None else act.threshold, diff --git a/tests/samples/mini.sta b/tests/samples/mini.sta new file mode 100644 index 0000000..cb9c2c9 --- /dev/null +++ b/tests/samples/mini.sta @@ -0,0 +1,22 @@ +format sentence { + is text<5>; + annotate f"a well formed sentence in english"; +} + +format boolean { + is enum("true","false"); + annotate f"a boolean"; +} + +prompt main { + is { + statement is sentence; + correct is boolean; + } + channel { + to .statement from ?statement; + } + return { + from .correct; + } +} From aa9539d1f9b1c473b261a2e322a712d834c442ad Mon Sep 17 00:00:00 2001 From: Tristan Date: Fri, 8 Aug 2025 14:42:53 -0700 Subject: [PATCH 08/51] Tons of fixes to get to a working implementation --- README.md | 12 +- libs/autocog/llama/CMakeLists.txt | 15 +- libs/autocog/llama/evaluation-choice.cxx | 143 ++++----- libs/autocog/llama/evaluation-completion.cxx | 195 ++++++------ libs/autocog/llama/evaluation-text.cxx | 46 ++- libs/autocog/llama/evaluation.cxx | 77 +++-- libs/autocog/llama/evaluation.hxx | 24 +- libs/autocog/llama/fta.cxx | 102 +++---- libs/autocog/llama/fta.hxx | 26 +- libs/autocog/llama/ftt.cxx | 59 +++- libs/autocog/llama/ftt.hxx | 42 ++- libs/autocog/llama/manager.cxx | 3 +- libs/autocog/llama/model.cxx | 68 +++-- libs/autocog/llama/model.hxx | 8 +- libs/autocog/llama/python_bindings.cxx | 13 +- libs/autocog/llama/types.hxx | 4 + modules/autocog/fta/actions.py | 15 +- modules/autocog/fta/automaton.py | 1 + modules/autocog/sta/automaton.py | 2 +- modules/autocog/sta/compile.py | 26 +- modules/autocog/sta/ir.py | 22 +- .../llama/execute_sta_with_llama_cpp.py | 284 +++++++++++++----- tests/samples/micro.sta | 27 ++ tests/samples/mini.sta | 7 +- tests/samples/test.sta | 34 +++ 25 files changed, 779 insertions(+), 476 deletions(-) create mode 100644 tests/samples/micro.sta create mode 100644 tests/samples/test.sta diff --git a/README.md b/README.md index 9b4c406..42ff003 100644 --- a/README.md +++ b/README.md @@ -25,15 +25,15 @@ The libraries have [their own documentation](./share/library/README.md). Build image and "test": ``` docker build -t autocog:latest . -docker run --rm -it autocog:latest python3 -c "import autocog.llama; print(autocog.llama.__doc__)" -docker run --rm -v $(pwd):/workspace/autocog -it autocog:latest python3 /workspace/autocog/tests/autocog/llama/roundtrip_tokenization.py /workspace/autocog/models/stories260K.gguf -docker run --rm -v $(pwd):/workspace/autocog -it autocog:latest python3 /workspace/autocog/tests/autocog/compilation/compile_sta_to_ftas.py /workspace/autocog/share/library/mcq/select.sta '{"topic":"a topic", "question":"my question", "choices" : [ "a", "b", "c", "d" ] }' -docker run --rm -v $(pwd):/workspace/autocog -it autocog:latest python3 /workspace/autocog/tests/autocog/llama/execute_sta_with_llama_cpp.py /workspace/autocog/share/library/mcq/select.sta '{"topic":"a topic", "question":"my question", "choices" : [ "a", "b", "c", "d" ] }' /workspace/autocog/models/stories260K.gguf +docker run --rm -v $(pwd):/workspace/autocog -it autocog:latest python3 /workspace/autocog/tests/autocog/llama/execute_sta_with_llama_cpp.py /workspace/autocog/tests/samples/mini.sta '{}' /workspace/autocog/models/SmolLM3-Q4_K_M.gguf ``` -Fast rebuild (in mounted: +In container: ``` -docker run --rm -v $(pwd):/workspace/autocog -w /workspace/autocog -it autocog:latest python setup.py build_ext --inplace +docker run --rm -v $(pwd):/workspace/autocog -w /workspace/autocog -it autocog:latest bash +apt update && apt install -y gdb vim +pip install -e . +python3 tests/autocog/llama/execute_sta_with_llama_cpp.py tests/samples/mini.sta '{}' models/SmolLM3-Q4_K_M.gguf ``` ## Contributing diff --git a/libs/autocog/llama/CMakeLists.txt b/libs/autocog/llama/CMakeLists.txt index 2b695c9..dcb50b2 100644 --- a/libs/autocog/llama/CMakeLists.txt +++ b/libs/autocog/llama/CMakeLists.txt @@ -35,15 +35,12 @@ target_link_libraries(autocog_llama PRIVATE llama) # Compiler flags target_compile_features(autocog_llama PRIVATE cxx_std_17) -# Optimization flags -if(CMAKE_BUILD_TYPE STREQUAL "Release") - target_compile_options(autocog_llama PRIVATE - $<$:-O3 -march=native> - $<$:/O2> - ) -endif() - -# Enable warnings +#target_compile_options(autocog_llama PRIVATE +# $<$:-O3 -march=native -g> +# $<$:/O2> +#) +target_compile_options(autocog_llama PRIVATE -O0 -g3 -ggdb -fno-omit-frame-pointer -DVERBOSE=1) + target_compile_options(autocog_llama PRIVATE $<$:-Wall -Wextra -Wpedantic> $<$:/W4> diff --git a/libs/autocog/llama/evaluation-choice.cxx b/libs/autocog/llama/evaluation-choice.cxx index 0d61691..ff459fc 100644 --- a/libs/autocog/llama/evaluation-choice.cxx +++ b/libs/autocog/llama/evaluation-choice.cxx @@ -6,19 +6,40 @@ #include -#include #include #include #include +#if VERBOSE +# include +#endif + +#define DEBUG_Evaluation_evaluate_choice VERBOSE && 1 + namespace autocog { namespace llama { +struct ChoiceResult { + size_t index; + ProbaSequence logprobs; + float proba; + + ChoiceResult( + size_t index_, ProbaSequence logprobs_, float proba_ + ) : + index(index_), logprobs(logprobs_), proba(proba_) + {} +}; + unsigned Evaluation::evaluate_choice(PathState & state) { - auto [model, ctx] = this->restore_context(state); +#if DEBUG_Evaluation_evaluate_choice + std::cerr << "Executing Choice #" << state.action << std::endl; +#endif Choice const & action = this->fta.action(state.action).as(); - std::cerr << "Executing Choice #" << action.id << std::endl; +#if DEBUG_Evaluation_evaluate_choice + std::cerr << " - name: " << action.name << std::endl; std::cerr << " - width: " << action.width << std::endl; std::cerr << " - number of choices: " << action.choices.size() << std::endl; +#endif if (action.choices.empty()) { throw std::runtime_error("Choice action has no choices"); @@ -29,97 +50,55 @@ unsigned Evaluation::evaluate_choice(PathState & state) { } unsigned num_token_eval = 0; - - // Structure to hold choice evaluation results - struct ChoiceResult { - size_t index; - TokenSequence tokens; - ProbaSequence probas; - float total_probability; - }; std::vector results; // Evaluate ALL choices in full - for (size_t i = 0; i < action.choices.size(); ++i) { - TokenSequence const & choice_tokens = action.choices[i]; - std::cerr << " - choice[" << i << "]:" << std::endl; - std::cerr << " - number of tokens: " << choice_tokens.size() << std::endl; + for (size_t idx = 0; idx < action.choices.size(); ++idx) { + auto [model, ctx] = this->restore(state); +#if DEBUG_Evaluation_evaluate_choice + std::cerr << " - Model #" << model.id << std::endl; + std::cerr << " - Context #" << ctx << std::endl; + std::cerr << " - choice[" << idx << "]:" << std::endl; + std::cerr << " - number of tokens: " << action.choices[idx].size() << std::endl; +#endif // Save current state to restore after evaluation TokenSequence saved_tokens = model.get_tokens_const(ctx); - - // Evaluate this choice - ProbaSequence probas; - num_token_eval += model.eval_sequences(choice_tokens, probas, ctx); - - // Calculate total probability for this choice - // TODO: In the future, support different probability strategies: - // - product (current): p_total = p1 * p2 * ... * pn - // - average: p_total = (p1 + p2 + ... + pn) / n - // - min: p_total = min(p1, p2, ..., pn) - // - weighted: custom weighting function - // This could be specified as a parameter in the Choice action - float total_prob = 1.0f; - for (float p : probas) { - total_prob *= p; // Product strategy (default) - } - std::cerr << " - probability: " << total_prob << std::endl; - - results.push_back({i, choice_tokens, probas, total_prob}); - - // Restore context state for next choice evaluation - model.set_tokens(saved_tokens, ctx); + + ProbaSequence logprobs; + num_token_eval += model.eval_sequences(action.choices[idx], logprobs, ctx); + + float proba = 0.; + for (float lpb : logprobs) proba += lpb; + proba = std::exp(-proba/logprobs.size()); +#if DEBUG_Evaluation_evaluate_choice + std::cerr << " - proba: " << proba << std::endl; +#endif + + results.emplace_back(idx, logprobs, proba); + + state.context.reset(); // TODO remove once context saving/restore/rewind is implemented } - - // Sort by probability (descending) + std::sort(results.begin(), results.end(), [](const ChoiceResult& a, const ChoiceResult& b) { - return a.total_probability > b.total_probability; + return a.proba > b.proba; }); - - // Select top 'width' choices that are above threshold - std::vector selected_indices; - - for (const auto& result : results) { - if (result.total_probability >= action.threshold) { - selected_indices.push_back(result.index); - if (selected_indices.size() >= action.width) { - break; - } - } - } - - // If none above threshold, pick the best one - if (selected_indices.empty() && !results.empty()) { - selected_indices.push_back(results[0].index); - } - - // Add selected choices to FTT and enqueue their corresponding successors - for (size_t idx : selected_indices) { - // Find the result for this index - auto it = std::find_if(results.begin(), results.end(), - [idx](const ChoiceResult& r) { return r.index == idx; }); - - if (it != results.end()) { - // Add this choice to the FTT - FTT& child = state.parent.add(action.id, it->tokens, it->probas, it->total_probability); - - // Mark as pruned if below threshold (for tracking purposes) - if (it->total_probability < action.threshold) { - child.pruned = true; - // Note: We still enqueue it since it was the best available - } - - // Enqueue the corresponding successor action for this specific choice - ActionID next_action = action.successors[idx]; - - // Prepare tokens for next action (current context + chosen tokens) - TokenSequence next_tokens = model.get_tokens_const(ctx); - next_tokens.insert(next_tokens.end(), it->tokens.begin(), it->tokens.end()); - - this->enqueue(next_action, child, next_tokens, state); + + unsigned count = 0; + for (const auto & result : results) { + auto & choice_tokens = action.choices[result.index]; + FTT & child = state.parent.add(action.id, choice_tokens, result.logprobs); + child.pruned = ( count > action.width ) || (count > 0 && result.proba < action.threshold); + if (!child.pruned) { + this->enqueue(action.successors[result.index], child, state); } + count++; } + +#if DEBUG_Evaluation_evaluate_choice + std::cerr << " > evaluated: " << num_token_eval << std::endl; +#endif return num_token_eval; } diff --git a/libs/autocog/llama/evaluation-completion.cxx b/libs/autocog/llama/evaluation-completion.cxx index d38d7f3..f25b59b 100644 --- a/libs/autocog/llama/evaluation-completion.cxx +++ b/libs/autocog/llama/evaluation-completion.cxx @@ -6,91 +6,88 @@ #include -#include #include #include #include #include #include +#if VERBOSE +# include +#endif + +#define DEBUG_Evaluation_evaluate_completion VERBOSE && 1 + namespace autocog { namespace llama { -struct Beam { +struct BeamState { TokenSequence tokens; - ProbaSequence probas; - float cumulative_log_prob; - bool stopped; - - float get_probability() const { - return std::exp(cumulative_log_prob); + ProbaSequence logprobs; + float logprob{0.}; + bool stopped{false}; + + float proba() const { + return std::exp(-logprob/logprobs.size()); } }; // Expand a single beam by evaluating next token -static void expand_beam( - Model& model, +static unsigned expand_beam( + Model & model, ContextID ctx, - const Beam& beam, - const Completion& action, - const TokenSequence& base_tokens, - std::vector& next_beams, - unsigned& num_token_eval + BeamState const & beam, + Completion const & action, + TokenSequence const & base_tokens, + std::vector & beams ) { TokenSequence context_tokens = base_tokens; context_tokens.insert(context_tokens.end(), beam.tokens.begin(), beam.tokens.end()); model.set_tokens(context_tokens, ctx); - // Get top candidates std::vector topk_tokens; std::vector topk_logits; - size_t n_candidates = static_cast(action.beams * action.ahead); - num_token_eval += model.eval_topk_tokens( + unsigned num_token_eval = model.eval_topk_tokens( action.vocab.mask, - n_candidates, + action.beams, topk_tokens, topk_logits, ctx ); - - // Create new beams from candidates + for (size_t i = 0; i < topk_tokens.size(); ++i) { - TokenID token = topk_tokens[i]; - float logit = topk_logits[i]; - - // Check if this is a stop token - bool is_stop = std::find(action.stop.begin(), action.stop.end(), token) != action.stop.end(); - - if (is_stop) { - Beam new_beam = beam; - new_beam.stopped = true; - next_beams.push_back(new_beam); - } else { - Beam new_beam = beam; - new_beam.tokens.push_back(token); - new_beam.probas.push_back(std::exp(logit)); // Simplified probability - new_beam.cumulative_log_prob += logit; - next_beams.push_back(new_beam); + BeamState & new_beam = beams.emplace_back(beam); + new_beam.tokens.push_back(topk_tokens[i]); + new_beam.logprobs.push_back(topk_logits[i]); + new_beam.logprob += topk_logits[i]; + new_beam.stopped = ( + action.stop.size() <= new_beam.tokens.size() + ) && std::equal( + action.stop.begin(), + action.stop.end(), + new_beam.tokens.end() - action.stop.size() + ); + if (new_beam.stopped) { + new_beam.tokens.erase(new_beam.tokens.end() - action.stop.size(), new_beam.tokens.end()); } } + return num_token_eval; } // Prune beams to keep only top k static void prune_beams( - std::vector& beams, + std::vector & beams, unsigned beam_width ) { // Sort by score - std::sort(beams.begin(), beams.end(), [](const Beam& a, const Beam& b) { - if (a.stopped != b.stopped) { - return a.stopped > b.stopped; // Prioritize stopped beams - } - return a.cumulative_log_prob > b.cumulative_log_prob; + std::sort(beams.begin(), beams.end(), [](BeamState const & a, BeamState const & b) { + if (a.stopped != b.stopped) return a.stopped; + return a.proba() < b.proba(); }); // Keep top beams - std::vector pruned; + std::vector pruned; size_t kept_active = 0; - for (const auto& beam : beams) { + for (BeamState const & beam : beams) { if (beam.stopped) { pruned.push_back(beam); } else if (kept_active < beam_width) { @@ -104,25 +101,25 @@ static void prune_beams( // Run beam search for one position static bool beam_search_step( - Model& model, + Model & model, ContextID ctx, const Completion& action, const TokenSequence& base_tokens, - std::vector& current_beams, - unsigned& num_token_eval + std::vector & current_beams, + unsigned & num_token_eval ) { - std::vector next_beams; + std::vector next_beams; - for (const Beam& beam : current_beams) { + for (BeamState const & beam : current_beams) { if (beam.stopped) { next_beams.push_back(beam); - continue; + } else { + num_token_eval += expand_beam(model, ctx, beam, action, base_tokens, next_beams); } - expand_beam(model, ctx, beam, action, base_tokens, next_beams, num_token_eval); } // Check for early termination - bool all_stopped = std::all_of(next_beams.begin(), next_beams.end(), [](const Beam& b) { return b.stopped; }); + bool all_stopped = std::all_of(next_beams.begin(), next_beams.end(), [](BeamState const & b) { return b.stopped; }); if (all_stopped) { current_beams = std::move(next_beams); @@ -140,71 +137,65 @@ static bool beam_search_step( return false; // Continue beam search } -// Add selected beams to FTT -void add_beams_to_ftt( - std::vector const & beams, - Completion const & action, - PathState & state, - Evaluation * eval -) { - // Sort by score - std::vector sorted_beams = beams; - std::sort(sorted_beams.begin(), sorted_beams.end(), [](const Beam& a, const Beam& b) { - return a.cumulative_log_prob > b.cumulative_log_prob; - }); - - // Add top beams to FTT - size_t n_keep = std::min(static_cast(action.ahead), sorted_beams.size()); - - for (size_t i = 0; i < n_keep; ++i) { - const Beam& beam = sorted_beams[i]; - float probability = beam.get_probability(); - - FTT & child = state.parent.add(action.id, beam.tokens, beam.probas, probability); - - // Apply threshold pruning - if (probability < action.threshold) { - child.pruned = true; - continue; - } - - // Enqueue next action - if (!action.successors.empty()) { - TokenSequence next_tokens = state.tokens; - next_tokens.insert(next_tokens.end(), beam.tokens.begin(), beam.tokens.end()); - eval->enqueue(action.successors[0], child, next_tokens, state); - } - } -} - unsigned Evaluation::evaluate_completion(PathState & state) { - auto [model, ctx] = this->restore_context(state); +#if DEBUG_Evaluation_evaluate_completion + std::cerr << "Executing Completion #" << state.action << std::endl; +#endif Completion const & action = this->fta.action(state.action).as(); - std::cerr << "Executing Completion #" << action.id << std::endl; +#if DEBUG_Evaluation_evaluate_completion + std::cerr << " - name: " << action.name << std::endl; std::cerr << " - beams: " << action.beams << std::endl; std::cerr << " - length: " << action.length << std::endl; std::cerr << " - ahead: " << action.ahead << std::endl; + std::cerr << " - width: " << action.width << std::endl; std::cerr << " - threshold: " << action.threshold << std::endl; +#endif + auto [model, ctx] = this->restore(state); +#if DEBUG_Evaluation_evaluate_completion + std::cerr << " - Model #" << model.id << std::endl; + std::cerr << " - Context #" << ctx << std::endl; +#endif + + std::vector beams; + beams.emplace_back(); - unsigned num_token_eval = 0; - - // Initialize beam search - std::vector current_beams; - current_beams.push_back({TokenSequence(), ProbaSequence(), 0.0f, false}); - - // Run beam search + unsigned num_token_eval = 0; for (unsigned pos = 0; pos < action.length; ++pos) { +#if DEBUG_Evaluation_evaluate_completion std::cerr << " - pos[" << pos << "]..." << std::endl; +#endif bool should_stop = beam_search_step( - model, ctx, action, state.tokens, current_beams, num_token_eval + model, ctx, action, state.tokens, beams, num_token_eval ); if (should_stop) { break; } } - - add_beams_to_ftt(current_beams, action, state, this); + + std::sort(beams.begin(), beams.end(), [](BeamState const & a, BeamState const & b) { + return a.proba() > b.proba(); + }); + + unsigned count = 0; + for (auto & beam: beams) { +#if DEBUG_Evaluation_evaluate_completion + std::cerr << " - beam[" << count << "]:" << std::endl; + std::cerr << " length: " << beam.tokens.size() << std::endl; + std::cerr << " logprob: " << beam.logprob << std::endl; + std::cerr << " proba: " << beam.proba() << std::endl; +#endif + FTT & child = state.parent.add(action.id, beam.tokens, beam.logprobs); + child.pruned = ( count > action.width ) || (count > 0 && beam.proba() < action.threshold); + if (!child.pruned) { + this->enqueue(action.successors[0], child, state); + } + count++; + } + +#if DEBUG_Evaluation_evaluate_completion + std::cerr << " > evaluated: " << num_token_eval << std::endl; +#endif return num_token_eval; } diff --git a/libs/autocog/llama/evaluation-text.cxx b/libs/autocog/llama/evaluation-text.cxx index 889fdb9..4de33fa 100644 --- a/libs/autocog/llama/evaluation-text.cxx +++ b/libs/autocog/llama/evaluation-text.cxx @@ -5,26 +5,46 @@ #include "autocog/llama/fta.hxx" #include "autocog/llama/ftt.hxx" -#include #include +#if VERBOSE +# include +#endif + +#define DEBUG_Evaluation_evaluate_text VERBOSE && 1 + namespace autocog { namespace llama { unsigned Evaluation::evaluate_text(PathState & state) { - auto [model,ctx] = this->restore_context(state); +#if DEBUG_Evaluation_evaluate_text + std::cerr << "Executing Text #" << state.action << std::endl; +#endif Text const & action = this->fta.action(state.action).as(); - std::cerr << "Executing Text #" << action.id << std::endl; +#if DEBUG_Evaluation_evaluate_text + std::cerr << " - name: " << action.name << std::endl; std::cerr << " - number of tokens: " << action.tokens.size() << std::endl; - - ProbaSequence probas(action.tokens.size(), 1.); - unsigned num_token_eval = model.eval_sequences(action.tokens, probas, ctx); - - float probability = 1.; - for (auto proba: probas) probability *= proba; - std::cerr << " - probability: " << probability << std::endl; - state.parent.add(action.id, action.tokens, probas, probability); - - this->enqueue(action.successors[0], state.parent.children.back(), model.get_tokens_const(), state); +#endif + + unsigned num_token_eval = 0; + ProbaSequence logprobs(action.tokens.size(), 0.); + if (this->config.evaluate_text) { + auto [model,ctx] = this->restore(state); +#if DEBUG_Evaluation_evaluate_text + std::cerr << " - Model #" << model.id << std::endl; + std::cerr << " - Context #" << ctx << std::endl; +#endif + num_token_eval += model.eval_sequences(action.tokens, logprobs, ctx); + } +#if DEBUG_Evaluation_evaluate_text + std::cerr << " > evaluated: " << num_token_eval << std::endl; +#endif + + auto & child = state.parent.add(action.id, action.tokens, logprobs); + if (action.successors.size() == 1) { + this->enqueue(action.successors[0], child, state); + } else if (action.successors.size() > 1) { + throw std::runtime_error("Text action should never have more than 1 successor."); + } return num_token_eval; } diff --git a/libs/autocog/llama/evaluation.cxx b/libs/autocog/llama/evaluation.cxx index ae44378..4134aae 100644 --- a/libs/autocog/llama/evaluation.cxx +++ b/libs/autocog/llama/evaluation.cxx @@ -5,7 +5,12 @@ #include "autocog/llama/ftt.hxx" #include "autocog/llama/fta.hxx" -#include +#if VERBOSE +# include +#endif + +#define DEBUG_Evaluation_enqueue VERBOSE && 0 +#define DEBUG_Evaluation_advance VERBOSE && 1 namespace autocog { namespace llama { @@ -16,7 +21,12 @@ PathState::PathState(ActionID const action_, FTT & parent_, TokenSequence const context(context_) {} -Evaluation::Evaluation(ModelID const model_, FTA const & fta_) : +float PathState::proba() const { + return this->parent.proba(); +} + +Evaluation::Evaluation(EvaluationConfig const & config_, ModelID const model_, FTA const & fta_) : + config(config_), model(model_), fta(fta_), queue(), @@ -31,10 +41,22 @@ Evaluation::~Evaluation() { unsigned Evaluation::advance(std::optional max_token_eval) { if (this->root == nullptr) this->initial(); + unsigned num_action_eval = 0; unsigned num_token_eval = 0; while (!queue.empty() && (max_token_eval == std::nullopt || num_token_eval < max_token_eval)) { PathState & state = queue.front(); - Action const & action = this->fta.action(state.action); +#if DEBUG_Evaluation_advance + std::cerr << "Evaluation::advance" << std::endl; + std::cerr << " queue.size() = " << queue.size() << std::endl; + std::cerr << " num_action_eval = " << num_action_eval << std::endl; + std::cerr << " num_token_eval = " << num_token_eval << std::endl; + std::cerr << " state.action = " << state.action << std::endl; + std::cerr << " state.tokens.size() = " << state.tokens.size() << std::endl; + std::cerr << " state.proba() = " << state.proba() << std::endl; + std::cerr << " state.parent.length = " << state.parent.length << std::endl; + std::cerr << " state.parent.logprob = " << state.parent.logprob << std::endl; +#endif + Action const & action = this->fta.action(state.action); switch (action.kind) { case ActionKind::Text: num_token_eval += this->evaluate_text(state); @@ -47,6 +69,7 @@ unsigned Evaluation::advance(std::optional max_token_eval) { break; } queue.pop(); + num_action_eval++; } return num_token_eval; @@ -63,15 +86,28 @@ void Evaluation::initial() { if (tokens[0] != bos) { tokens.insert(tokens.begin(), bos); } - ProbaSequence probas(init.tokens.size(), 1.); - float probability = 1.; - for (auto proba: probas) probability *= proba; - - this->root = new FTT(0, init.tokens, probas, probability); + this->root = FTT::make_root(init.tokens); this->queue.emplace(init.successors[0], *(this->root), init.tokens, std::nullopt); } -std::pair Evaluation::restore_context(PathState & state) const { +void Evaluation::enqueue( + ActionID const action, + FTT & parent, + PathState const & state +) { +#if DEBUG_Evaluation_enqueue + std::cerr << ">> Evaluation::enqueue <<" << std::endl; +#endif + std::optional ctx = state.context; + ctx.reset(); // TODO context saving logic + + std::vector tokens(state.tokens.begin(), state.tokens.end()); + tokens.insert(tokens.end(), parent.tokens.begin(), parent.tokens.end()); + + this->queue.emplace(action, parent, tokens, ctx); +} + +std::pair Evaluation::restore(PathState & state) const { Model & model = Manager::get_model(this->model); if (state.context) { throw std::runtime_error("Context saving is not implemented yet, this should not happen"); @@ -83,28 +119,5 @@ std::pair Evaluation::restore_context(PathState & state) con return std::pair(model, state.context.value()); } -void Evaluation::enqueue(ActionID const action, FTT & parent, std::vector const & tokens, PathState const & state) { - std::optional ctx = state.context; - ctx.reset(); // TODO context saving logic - this->queue.emplace(action, parent, tokens, ctx); -} - -std::vector softmax(float * logits, int vocab_size) { - std::vector result(logits, logits + vocab_size); - - // Apply softmax - float max_logit = *std::max_element(result.begin(), result.end()); - float sum = 0.0f; - for (float& logit : result) { - logit = std::exp(logit - max_logit); - sum += logit; - } - for (float& prob : result) { - prob /= sum; - } - - return result; -} - } } diff --git a/libs/autocog/llama/evaluation.hxx b/libs/autocog/llama/evaluation.hxx index 5ef3016..441572d 100644 --- a/libs/autocog/llama/evaluation.hxx +++ b/libs/autocog/llama/evaluation.hxx @@ -16,21 +16,24 @@ class Text; class Completion; class Choice; -class Beam; - struct PathState { ActionID const action; //< Action to be evaluated next - FTT & parent; //< Previous FTT in the path, reslts of exploring this state will be added to that tree + FTT & parent; //< Previous FTT in the path, results of exploring this state will be added to that tree TokenSequence const tokens; //< Tokens that lead to this state - std::optional context; //< Context used to evaluate this path PathState(ActionID const action_, FTT & parent, std::vector const & tokens_, std::optional context); + float proba() const; +}; + +struct EvaluationConfig { + bool evaluate_text{false}; }; class Evaluation { public: using Queue = std::queue; + EvaluationConfig const config; private: ModelID const model; @@ -40,27 +43,20 @@ class Evaluation { FTT * root; protected: - std::pair restore_context(PathState & state) const; + std::pair restore(PathState & state) const; void initial(); - void enqueue(ActionID const action, FTT & parent, std::vector const & tokens, PathState const & current); + void enqueue(ActionID const action, FTT & parent, PathState const & current); unsigned evaluate_text (PathState & state); unsigned evaluate_completion (PathState & state); unsigned evaluate_choice (PathState & state); public: - Evaluation(ModelID const model_, FTA const & fta_); + Evaluation(EvaluationConfig const & config_, ModelID const model_, FTA const & fta_); ~Evaluation(); unsigned advance(std::optional max_token_eval); FTT const & get() const; - - friend void add_beams_to_ftt( - std::vector const & beams, - Completion const & action, - PathState & state, - Evaluation * eval - ); }; } } diff --git a/libs/autocog/llama/fta.cxx b/libs/autocog/llama/fta.cxx index 95235dc..da28bff 100644 --- a/libs/autocog/llama/fta.cxx +++ b/libs/autocog/llama/fta.cxx @@ -10,40 +10,46 @@ namespace autocog { namespace llama { Action::Action( ActionKind const kind_, ActionID const id_, - float threshold_ + std::string const & name_ ) : kind(kind_), id(id_), - threshold(threshold_), + name(name_), successors() {} Text::Text( ActionID const id_, - float threshold_ + std::string const & name_ ) : - Action(ActionKind::Text, id_, threshold_) + Action(ActionKind::Text, id_, name_) {} Completion::Completion( ActionID const id_, + std::string const & name_, float threshold_, unsigned length_, unsigned beams_, - unsigned ahead_ + unsigned ahead_, + unsigned width_ ) : - Action(ActionKind::Completion, id_, threshold_), + Action(ActionKind::Completion, id_, name_), + threshold(threshold_), length(length_), beams(beams_), - ahead(ahead_) + ahead(ahead_), + width(width_) {} Choice::Choice( ActionID const id_, + std::string const & name_, float threshold_, unsigned width_ ) : - Action(ActionKind::Choice, id_, threshold_), + Action(ActionKind::Choice, id_, name_), + threshold(threshold_), width(width_) {} @@ -55,27 +61,6 @@ Action const & FTA::action(ActionID const id) const { return action; } -Text & FTA::insert(float threshold_) { - ActionID id = this->actions.size(); - Text * action = new Text(id, threshold_); - this->actions.push_back(std::unique_ptr(action)); - return *action; -} - -Completion & FTA::insert(float threshold_, unsigned length_, unsigned beams_, unsigned ahead_) { - ActionID id = this->actions.size(); - Completion * action = new Completion(id, threshold_, length_, beams_, ahead_); - this->actions.push_back(std::unique_ptr(action)); - return *action; -} - -Choice & FTA::insert(float threshold_, unsigned width_) { - ActionID id = this->actions.size(); - Choice * action = new Choice(id, threshold_, width_); - this->actions.push_back(std::unique_ptr(action)); - return *action; -} - // TODO review FTA::FTA(Model const & model, pybind11::dict const & pydata) { @@ -84,21 +69,23 @@ FTA::FTA(Model const & model, pybind11::dict const & pydata) { throw std::runtime_error("FTA dictionary missing 'actions' field"); } - auto py_actions = pydata["actions"].cast(); + auto py_actions = pydata["actions"].cast(); // First pass: create all actions and build ID mapping std::map uid_to_id; - + for (auto item : py_actions) { - std::string uid = item.first.cast(); - auto action_dict = item.second.cast(); - + auto action_dict = item.cast(); + + if (!action_dict.contains("uid")) { + throw std::runtime_error("Action missing 'uid' field."); + } + auto uid = action_dict["uid"].cast(); if (!action_dict.contains("__type__")) { - throw std::runtime_error("Action missing '__type__' field: " + uid); + throw std::runtime_error("Action missing '__type__' field: " + uid); } std::string action_type = action_dict["__type__"].cast(); - float threshold = action_dict.contains("threshold") ? action_dict["threshold"].cast() : 0.0f; ActionID node_id = actions.size(); // Assign sequential IDs uid_to_id[uid] = node_id; @@ -106,25 +93,23 @@ FTA::FTA(Model const & model, pybind11::dict const & pydata) { std::unique_ptr action; if (action_type == "Text") { - action = std::make_unique(node_id, threshold); - Text* text_action = static_cast(action.get()); - - // Set tokens - if (action_dict.contains("tokens")) { - auto py_tokens = action_dict["tokens"].cast(); - text_action->tokens.clear(); - for (auto token : py_tokens) { - text_action->tokens.push_back(token.cast()); - } - } + action = std::make_unique(node_id, uid); + Text * text_action = static_cast(action.get()); + + auto py_tokens = action_dict["tokens"].cast(); + text_action->tokens.clear(); + for (auto token : py_tokens) { + text_action->tokens.push_back(token.cast()); + } - } else if (action_type == "Completion") { - // Extract parameters with defaults - unsigned length = action_dict.contains("length") ? action_dict["length"].cast() : 1; - unsigned beams = action_dict.contains("beams") ? action_dict["beams"].cast() : 1; - unsigned ahead = action_dict.contains("ahead") ? action_dict["ahead"].cast() : 1; + } else if (action_type == "Complete") { + float threshold = action_dict["threshold"].cast(); + unsigned length = action_dict["length"].cast(); + unsigned beams = action_dict["beams"].cast(); + unsigned ahead = action_dict["ahead"].cast(); + unsigned width = action_dict["width"].cast(); - action = std::make_unique(node_id, threshold, length, beams, ahead); + action = std::make_unique(node_id, uid, threshold, length, beams, ahead, width); Completion* completion_action = static_cast(action.get()); // Set stop tokens @@ -145,9 +130,10 @@ FTA::FTA(Model const & model, pybind11::dict const & pydata) { throw std::runtime_error("Setting the vocabulary from PY desc is not implemented yet!"); } - } else if (action_type == "Choice") { - unsigned width = action_dict.contains("width") ? action_dict["width"].cast() : 1; - action = std::make_unique(node_id, threshold, width); + } else if (action_type == "Choose") { + float threshold = action_dict["threshold"].cast(); + unsigned width = action_dict["width"].cast(); + action = std::make_unique(node_id, uid, threshold, width); Choice* choice_action = static_cast(action.get()); // Set choices @@ -176,8 +162,8 @@ FTA::FTA(Model const & model, pybind11::dict const & pydata) { // Second pass: set up successors using the UID mapping for (auto item : py_actions) { - std::string uid = item.first.cast(); - auto action_dict = item.second.cast(); + auto action_dict = item.cast(); + auto uid = action_dict["uid"].cast(); ActionID node_id = uid_to_id[uid]; Action* action = actions[node_id].get(); diff --git a/libs/autocog/llama/fta.hxx b/libs/autocog/llama/fta.hxx index 25ac8d7..914ebe9 100644 --- a/libs/autocog/llama/fta.hxx +++ b/libs/autocog/llama/fta.hxx @@ -27,12 +27,11 @@ enum class ActionKind { struct Action { ActionKind const kind; ActionID const id; - - float const threshold; //< Probability threshold for pruning + std::string const name; std::vector successors; - Action(ActionKind const kind_, ActionID const id_, float threshold_); + Action(ActionKind const kind_, ActionID const id_, std::string const & name_); template T const & as() const { @@ -48,40 +47,39 @@ struct Text : public Action { TokenSequence tokens; - Text(ActionID const id_, float threshold_); + Text(ActionID const id_, std::string const & name_); }; struct Completion : public Action { static constexpr ActionKind Kind = ActionKind::Completion; - unsigned const length; - unsigned const beams; - unsigned const ahead; + float const threshold; //< Probability threshold for pruning + unsigned const length; // Maximum length of the completion + unsigned const beams; // Number of concurrent exploration beams + unsigned const ahead; // Look ahead parameter for beam search + unsigned const width; // Maximum number of beams to select Vocab vocab; TokenSequence stop; - Completion(ActionID const id_, float threshold_, unsigned length_, unsigned beams_, unsigned ahead_); + Completion(ActionID const id_, std::string const & name_, float threshold_, unsigned length_, unsigned beams_, unsigned ahead_, unsigned width_); }; struct Choice : public Action { static constexpr ActionKind Kind = ActionKind::Choice; - unsigned const width; // Maximum number of choices to explore + float const threshold; //< Probability threshold for pruning + unsigned const width; // Maximum number of choices to select std::vector choices; // Each choice is a token sequence - Choice(ActionID const id_, float threshold_, unsigned width_); + Choice(ActionID const id_, std::string const & name_, float threshold_, unsigned width_); }; class FTA { public: Action const & action(ActionID const id) const; - Text & insert(float threshold_); - Completion & insert(float threshold_, unsigned length_, unsigned beams_, unsigned ahead_); - Choice & insert(float threshold_, unsigned width_); - FTA() = default; FTA(Model const & model, pybind11::dict const & pydata); diff --git a/libs/autocog/llama/ftt.cxx b/libs/autocog/llama/ftt.cxx index 0dda4c9..fcb5c89 100644 --- a/libs/autocog/llama/ftt.cxx +++ b/libs/autocog/llama/ftt.cxx @@ -1,58 +1,87 @@ #include "autocog/llama/ftt.hxx" +#if VERBOSE +# include +#endif + +#define DEBUG_FTT_add VERBOSE && 0 + namespace autocog { namespace llama { FTT::FTT( ActionID const action_, TokenSequence const & tokens_, - ProbaSequence const & probas_, - float probability_ + ProbaSequence const & logprobs_, + float logprob_, + unsigned length_ ) : action(action_), tokens(tokens_), - probas(probas_), - probability(probability_), - children(), - pruned(false) + logprobs(logprobs_), + logprob(logprob_), + length(length_), + pruned(false), + children() {} -FTT & FTT::add(ActionID const action_, TokenSequence const & tokens_, ProbaSequence const & probas_, float probability_) { - this->children.emplace_back(action_, tokens_, probas_, probability_); +FTT & FTT::add( + ActionID const action_, + TokenSequence const & tokens_, + ProbaSequence const & logprobs_ +) { +#if DEBUG_FTT_add + std::cerr << ">> FTT::add <<" << std::endl; +#endif + float logprob_ = this->logprob; + for (auto lpb: logprobs_) logprob_ += lpb; + this->children.emplace_back(action_, tokens_, logprobs_, logprob_, this->length + tokens_.size()); return this->children.back(); } +FTT * FTT::make_root(TokenSequence const & tokens) { + ProbaSequence logprobs(tokens.size(), 0.); + return new FTT(0, tokens, logprobs, 0, tokens.size()); +} + // TODO review code below pybind11::dict FTT::pydict() const { pybind11::dict result; + result["action"] = this->action; // Convert tokens pybind11::list token_list; - for (TokenID token : tokens) { + for (TokenID token : this->tokens) { token_list.append(token); } result["tokens"] = token_list; // Convert probabilities - pybind11::list proba_list; - for (float prob : probas) { - proba_list.append(prob); + pybind11::list logprobs_list; + for (float lpb : this->logprobs) { + logprobs_list.append(lpb); } - result["probabilities"] = proba_list; + result["logprobs"] = logprobs_list; + result["logprob"] = this->logprob; + result["length"] = this->length; // Convert children recursively pybind11::list children_list; - for (const FTT& child : children) { + for (const FTT& child : this->children) { children_list.append(child.pydict()); } result["children"] = children_list; // Add metadata - result["pruned"] = pruned; + result["pruned"] = this->pruned; return result; } +float FTT::proba() const { + return std::exp(-this->logprob / this->length); +} + } } diff --git a/libs/autocog/llama/ftt.hxx b/libs/autocog/llama/ftt.hxx index 48be603..efaac39 100644 --- a/libs/autocog/llama/ftt.hxx +++ b/libs/autocog/llama/ftt.hxx @@ -5,25 +5,35 @@ #include +#include + namespace autocog { namespace llama { -struct FTT { - ActionID const action; - TokenSequence const tokens; - ProbaSequence const probas; - float const probability; - - std::vector children; - bool pruned{false}; - - FTT(ActionID const action_, TokenSequence const & tokens_, ProbaSequence const & probas_, float probability_); - - FTT & add(ActionID const action_, TokenSequence const & tokens_, ProbaSequence const & probas_, float probability_); - - pybind11::dict pydict() const; - - +class FTT { + public: + ActionID const action; + TokenSequence const tokens; + ProbaSequence const logprobs; + float const logprob; + unsigned const length; + + bool pruned{false}; + private: + std::list children; + + public: + /// I'd like that constructor to be private but it prevents the use of `emplace_back` in `add`. + /// Adding `friend class std::list;` does not solve the issue... + FTT(ActionID const action_, TokenSequence const & tokens_, ProbaSequence const & logprobs_, float logprob_, unsigned length_); + + public: + static FTT * make_root(TokenSequence const & tokens_); + FTT & add(ActionID const action_, TokenSequence const & tokens_, ProbaSequence const & logprobs_); + + float proba() const; + + pybind11::dict pydict() const; }; } } diff --git a/libs/autocog/llama/manager.cxx b/libs/autocog/llama/manager.cxx index 5ca1675..052e581 100644 --- a/libs/autocog/llama/manager.cxx +++ b/libs/autocog/llama/manager.cxx @@ -49,7 +49,8 @@ Model & Manager::get_model(ModelID id) { EvalID Manager::add_eval(ModelID const model, FTA const & fta) { auto & manager = instance(); EvalID id = manager.next_eval_id++; - manager.evaluations.try_emplace(id, model, fta); + EvaluationConfig config; + manager.evaluations.try_emplace(id, config, model, fta); return id; } diff --git a/libs/autocog/llama/model.cxx b/libs/autocog/llama/model.cxx index f1714bf..e5d15b0 100644 --- a/libs/autocog/llama/model.cxx +++ b/libs/autocog/llama/model.cxx @@ -2,6 +2,7 @@ #include +#include #include #include @@ -176,6 +177,43 @@ static unsigned run_batch_full(llama_context * ctx, TokenSequence const & tokens return tokens.size(); } +static float logit_to_log_sum_exp(float * logit, unsigned vocab_size) { + // Find max logit for numerical stability + float max_logit = *std::max_element(logit, logit + vocab_size); + + // Compute log-sum-exp for normalization + float log_sum_exp = 0.0f; + for (unsigned i = 0; i < vocab_size; ++i) { + log_sum_exp += std::exp(logit[i] - max_logit); + } + return max_logit + std::log(log_sum_exp); +} + +static void retrieve_logprobs(llama_context * ctx, unsigned vocab_size, std::vector & logprobs) { + float * logits = llama_get_logits(ctx); + float log_sum_exp = logit_to_log_sum_exp(logits, vocab_size); + logprobs.resize(vocab_size); + for (unsigned tok = 0; tok < vocab_size; tok++) { + logprobs[tok] = log_sum_exp - logits[tok]; + } +} + +static void sample_logprobs(llama_context * ctx, std::vector const & mask, std::vector> & candidates) { + float * logits = llama_get_logits(ctx); + float log_sum_exp = logit_to_log_sum_exp(logits, mask.size()); + + for (unsigned tok = 0; tok < mask.size(); tok++) { + if (mask[tok]) { + candidates.emplace_back(tok, log_sum_exp - logits[tok]); + } + } +} + +static float retrieve_logprob(llama_context * ctx, unsigned vocab_size, TokenID token) { + float * logits = llama_get_logits(ctx); + return logit_to_log_sum_exp(logits, vocab_size) - logits[token]; +} + unsigned Model::set_tokens(TokenSequence const & tokens_, ContextID const id) { if (this->id == 0) { return tokens_.size(); @@ -187,16 +225,16 @@ unsigned Model::set_tokens(TokenSequence const & tokens_, ContextID const id) { return run_batch_full(this->get_context(id), loc_tokens); } -unsigned Model::eval_sequences(TokenSequence const & new_tokens, ProbaSequence & probas, ContextID const id) { +unsigned Model::eval_sequences(TokenSequence const & new_tokens, ProbaSequence & logprobs, ContextID const id) { if (this->id == 0) { throw std::runtime_error("Using model #0 (RNG) is not implemented yet!"); - // TODO fill `probas` with random float in [0,1] + // TODO fill `logprobs` with random vales return new_tokens.size(); } TokenSequence & loc_tokens = this->get_tokens(id); llama_pos token_pos = loc_tokens.size(); - probas.clear(); + logprobs.clear(); for (auto token: new_tokens) { @@ -207,8 +245,7 @@ unsigned Model::eval_sequences(TokenSequence const & new_tokens, ProbaSequence & throw std::runtime_error("Failed to decode token"); } - float tok_probas = llama_get_logits(this->get_context(id))[token]; - probas.push_back(tok_probas); + logprobs.push_back(retrieve_logprob(this->get_context(id), this->vocab_size(), token)); token_pos++; } @@ -220,7 +257,7 @@ unsigned Model::eval_topk_tokens( std::vector const & vocab_mask, size_t max_candidates, std::vector & topk_tokens, - std::vector & topk_probas, + std::vector & topk_lobprobs, ContextID const id ) { @@ -238,7 +275,7 @@ unsigned Model::eval_topk_tokens( TokenSequence const & current_tokens = this->get_tokens_const(id); topk_tokens.clear(); - topk_probas.clear(); + topk_lobprobs.clear(); llama_batch batch = llama_batch_get_one( const_cast(current_tokens.data()), @@ -249,32 +286,25 @@ unsigned Model::eval_topk_tokens( throw std::runtime_error("Failed to decode for token evaluation"); } - float * logits = llama_get_logits(ctx); - - // Collect valid candidates based on mask std::vector> candidates; - for (TokenID tid = 0; tid < vocab_size; ++tid) { - if (vocab_mask[tid]) { - candidates.emplace_back(tid, logits[tid]); - } - } - + sample_logprobs(ctx, vocab_mask, candidates); + // Handle edge case: no valid candidates if (candidates.empty()) { throw std::runtime_error("Failed to find candidate token. Cannot have an empty vocabularity mask (all false)."); } std::sort(candidates.begin(), candidates.end(), [](const auto& a, const auto& b) { - return a.second > b.second; + return a.second < b.second; }); size_t k = std::min(max_candidates, candidates.size()); topk_tokens.reserve(k); - topk_probas.reserve(k); + topk_lobprobs.reserve(k); for (size_t i = 0; i < k; ++i) { topk_tokens.push_back(candidates[i].first); - topk_probas.push_back(candidates[i].second); // Raw logits from llama.cpp + topk_lobprobs.push_back(candidates[i].second); } return 1; diff --git a/libs/autocog/llama/model.hxx b/libs/autocog/llama/model.hxx index 5b3cd24..c95575d 100644 --- a/libs/autocog/llama/model.hxx +++ b/libs/autocog/llama/model.hxx @@ -9,8 +9,10 @@ namespace autocog { namespace llama { class Model { - private: + public: ModelID const id; + + private: llama_model * model; std::vector contexts; std::vector tokens; @@ -44,7 +46,7 @@ class Model { unsigned eval_sequences( TokenSequence const & tokens, - ProbaSequence & probas, + ProbaSequence & logprobs, ContextID const id = 0 ); @@ -52,7 +54,7 @@ class Model { std::vector const & vocab_mask, size_t max_candidates, std::vector & topk_tokens, - std::vector & topk_probas, + std::vector & topk_logprobs, ContextID const id ); }; diff --git a/libs/autocog/llama/python_bindings.cxx b/libs/autocog/llama/python_bindings.cxx index 4e5626c..77eab01 100644 --- a/libs/autocog/llama/python_bindings.cxx +++ b/libs/autocog/llama/python_bindings.cxx @@ -9,6 +9,7 @@ #include #include +#include #include namespace autocog { @@ -75,12 +76,20 @@ PYBIND11_MODULE(llama, module) { module.def("evaluate", [](ModelID model, pybind11::dict const & fta_dict) { + std::cerr << "IN evaluate (pybind): START" << std::endl; FTA fta = convert_pydict_to_fta(model, fta_dict); + std::cerr << "IN evaluate (pybind): FTA" << std::endl; EvalID eval = Manager::add_eval(model, fta); + std::cerr << "IN evaluate (pybind): EVAL" << std::endl; Manager::get_eval(eval).advance(std::nullopt); - pybind11::dict ftt = convert_ftt_to_pydict(model, Manager::get_eval(eval).get()); + std::cerr << "IN evaluate (pybind): FTT" << std::endl; + FTT const & ftt = Manager::get_eval(eval).get(); + std::cerr << "IN evaluate (pybind): RES" << std::endl; + pybind11::dict res = convert_ftt_to_pydict(model, ftt); + std::cerr << "IN evaluate (pybind): CLEAN" << std::endl; Manager::rm_eval(eval); - return ftt; + std::cerr << "IN evaluate (pybind): DONE" << std::endl; + return res; }, "Evaluate a FTA using a model and return the FTT." ); diff --git a/libs/autocog/llama/types.hxx b/libs/autocog/llama/types.hxx index 29082f3..c6d5a00 100644 --- a/libs/autocog/llama/types.hxx +++ b/libs/autocog/llama/types.hxx @@ -4,6 +4,10 @@ #include #include +#ifndef VERBOSE +# define VERBOSE 0 +#endif + namespace autocog { namespace llama { diff --git a/modules/autocog/fta/actions.py b/modules/autocog/fta/actions.py index 3923f3c..4b93012 100644 --- a/modules/autocog/fta/actions.py +++ b/modules/autocog/fta/actions.py @@ -84,15 +84,22 @@ def toGraphVizLabel(self): class Complete(Action): length: int = 1 - beams: int = 1 - ahead: int = 1 stop: str = '' + threshold: Optional[float] + beams: Optional[int] + ahead: Optional[int] + width: Optional[int] seeds: Optional[List[str]] vocab: Vocab - def __init__(self, uid:str, length:int, stop: str='', seeds: Optional[List[str]] = None, successors: List[str]=[]): - super().__init__(uid=uid, successors=successors, length=length, stop=stop, seeds=seeds, vocab=Vocab()) + def __init__(self, uid:str, + length:int, stop: str='', + threshold=None, beams=None, ahead=None, width=None, + seeds: Optional[List[str]] = None, + successors: List[str]=[] + ): + super().__init__(uid=uid, successors=successors, length=length, stop=stop, threshold=threshold, beams=beams, ahead=ahead, width=width, seeds=seeds, vocab=Vocab()) def prepare(self, lm): if self.seeds is not None: diff --git a/modules/autocog/fta/automaton.py b/modules/autocog/fta/automaton.py index 61b2a95..b5a3327 100644 --- a/modules/autocog/fta/automaton.py +++ b/modules/autocog/fta/automaton.py @@ -53,6 +53,7 @@ def simplify(self): pred.successors.clear() pred.successors.extend(curr.successors) del self.actions[cuid] + return self def greedy_rec(self, ptree:FiniteTokenTree, lm:LM, tokens:List[Token], action:Action): todos = [] diff --git a/modules/autocog/sta/automaton.py b/modules/autocog/sta/automaton.py index 75e181a..a1d9ec2 100644 --- a/modules/autocog/sta/automaton.py +++ b/modules/autocog/sta/automaton.py @@ -385,7 +385,7 @@ def instantiate_rec(self, syntax: Syntax, frame: Frame, fta: FTA, concrete: Conc path = [ ( p.name, i if p.is_list() else None ) for (p,i) in zip(successor.abstract.parents(), successor.indices) ] fta.create(uid=uid, cls=Text, text=frame.read(path)) elif isinstance(fmt, IrCompletion): - fta.create(uid=uid, cls=Complete, length=fmt.length, stop='\n') + fta.create(uid=uid, cls=Complete, length=fmt.length, threshold=fmt.threshold, beams=fmt.beams, ahead=fmt.ahead, width=fmt.width, stop='\n') elif isinstance(fmt, IrEnum): fta.create(uid=uid, cls=Choose, choices=fmt.values, width=fmt.width) elif isinstance(fmt, IrChoice): diff --git a/modules/autocog/sta/compile.py b/modules/autocog/sta/compile.py index 0e8b35f..c6d7220 100644 --- a/modules/autocog/sta/compile.py +++ b/modules/autocog/sta/compile.py @@ -66,14 +66,24 @@ def resolve_type(type: Union[AstRecord,AstTypeRef,AstEnumType], path:List[str], if type.name == 'text': length = None - if len(type.arguments) == 1: - arg = type.arguments[0] - if arg.name is not None and arg.name != 'length': - raise Exception(f"Builtin format `text` expect only `length` arguments (got: {args})") - length = arg.value.eval(values=values) - elif len(type.arguments) > 1: - raise Exception(f"Builtin format `text` expect single `length` arguments (got: {args})") - fmt = IrCompletion(name=pathname, length=length) + threshold = None + beams = None + ahead = None + width = None + for arg in type.arguments: + if arg.name is None or arg.name == 'length': + length = arg.value.eval(values=values) + elif arg.name == 'beams': + beams = arg.value.eval(values=values) + elif arg.name == 'ahead': + ahead = arg.value.eval(values=values) + elif arg.name == 'width': + width = arg.value.eval(values=values) + elif arg.name == 'threshold': + threshold = arg.value.eval(values=values) + else: + raise Exception(f"Builtin format `text` does not expect `{arg.name}` arguments (got: {type.arguments})") + fmt = IrCompletion(name=pathname, length=length, threshold=threshold, beams=beams, ahead=ahead, width=width) program.formats.update({ pathname : fmt }) return fmt diff --git a/modules/autocog/sta/ir.py b/modules/autocog/sta/ir.py index 384b446..c02bc49 100644 --- a/modules/autocog/sta/ir.py +++ b/modules/autocog/sta/ir.py @@ -128,17 +128,29 @@ def toGraphViz(self): class Completion(Format): length: Optional[int] + threshold: Optional[float] + beams: Optional[int] + ahead: Optional[int] + width: Optional[int] within: Optional[List[str]] = None def str(self): - res = 'text' + res = [] if self.length is not None: - res += f'({self.length})' - return res + res.append(f'length={self.length}') + if self.threshold is not None: + res.append(f'threshold={self.threshold}') + if self.beams is not None: + res.append(f'beams={self.beams}') + if self.ahead is not None: + res.append(f'ahead={self.ahead}') + if self.width is not None: + res.append(f'width={self.width}') + return 'text<' + ','.join(res) + '>' class Enum(Format): values: List[str] - width: int = 0 + width: Optional[int] = None def str(self): str = '","'.join(self.values) @@ -147,7 +159,7 @@ def str(self): class Choice(Format): path: Path mode: str - width: int = 0 + width: Optional[int] = None def str(self): return f'{self.mode}({self.path.str()})' diff --git a/tests/autocog/llama/execute_sta_with_llama_cpp.py b/tests/autocog/llama/execute_sta_with_llama_cpp.py index 5c5be7a..18b19b2 100644 --- a/tests/autocog/llama/execute_sta_with_llama_cpp.py +++ b/tests/autocog/llama/execute_sta_with_llama_cpp.py @@ -1,5 +1,5 @@ -import sys, json +import sys, json, html, math, graphviz from autocog.sta.compile import compile_source_to_program_and_stas from autocog.sta.syntax import Syntax @@ -11,80 +11,222 @@ import autocog.llama from autocog.llama import tokenize, detokenize -sta_file = sys.argv[1] -json_data = sys.argv[2] -model_path = sys.argv[3] - -sta = compile_source_to_program_and_stas( - open(sta_file, 'r').read() -)[1]['main'] - -fta = sta.instantiate( - syntax=Syntax(), - frame=Frame( - state={ st.label() : None for st in sta.concretes.values() if st.abstract.field is not None }, - data=json.loads(json_data) - ), - branches={}, - inputs=None -) - -model = autocog.llama.create(model_path, 4096) - -actions = { -## "root": { -## "__type__": "Text", -## "threshold": 0.0, -## "tokens": [1, 2, 3], -## "successors": ["choice1"] -## }, -## "choice1": { -## "__type__": "Choice", -## "threshold": 0.1, -## "width": 2, -## "choices": [[4, 5], [6, 7]], -## "successors": [] -## } +fta_defaults = { + "Text" : {}, + "Choose" : { + "threshold" : 0e-10, + "width" : 3 + }, + "Complete" : { + "threshold" : 0e-8, + "width" : 3, + "beams" : 2, + "ahead" : 1 + } } -for act in fta.actions.values(): - action = { "successors" : act.successors } - if isinstance(act, Text): - action.update({ - "__type__" : "Text", - "threshold" : 1. if act.threshold is None else act.threshold, - "tokens" : tokenize(model, act.text, False, False) - }) - elif isinstance(act, Choose): - if len(act.successors) == 1: - action.update({ "successors" : [ act.successors[0] ] * len(act.choices) }) - assert len(act.successors) == 0 or len(action['successors']) == len(act.choices) - action.update({ - "__type__" : "Choice", - "threshold" : 1. if act.threshold is None else act.threshold, - "width" : 1 if act.width is None else act.width, - "choices" : [ - tokenize(model, choice[0], False, False) for choice in act.choices - ] - }) - elif isinstance(act, Complete): - action.update({ - "__type__" : "Completion", - "threshold" : 1. if act.threshold is None else act.threshold, - "length" : 1 if act.length is None else act.length, - "beams" : 1 if act.beams is None else act.beams, - "ahead" : 1 if act.ahead is None else act.ahead, - "stop" : tokenize(model, act.stop, False, False), -# "vocab" : "TODO", - }) +def main(argv): + sta_file = argv[1] + json_data = argv[2] + model_path = argv[3] + + sta = compile_source_to_program_and_stas( + open(sta_file, 'r').read() + )[1]['main'] + + fta = sta.instantiate( + syntax=Syntax(), + frame=Frame( + state={ st.label() : None for st in sta.concretes.values() if st.abstract.field is not None }, + data=json.loads(json_data) + ), + branches={}, + inputs=None + ).simplify() + + model = autocog.llama.create(model_path, 4096) if len(model_path) > 0 else 0 + + ftt = cxx_to_ftt(model, autocog.llama.evaluate(model, fta_to_cxx(model, fta, fta_defaults))) + +# print(json.dumps(ftt, indent=4)) + paths = extract_paths_from_ftt(ftt) + paths = sorted(paths, key=lambda x: x[1]) + for (text, proba) in paths: + print("========================================") + print(f"[[ {proba} ]]") + print("> " + "\n> ".join(text.split('\n'))) + + ftt_to_graphviz_detailed( + ftt, + output_filename='ftt', + format='svg', + max_text_length=40, + show_text_preview=True + ) + +def fta_to_cxx(model, fta, defaults): + actions = [] + for act in fta.actions.values(): + action = { "uid" : act.uid, "successors" : act.successors } + if isinstance(act, Text): + action.update({ + "__type__" : "Text", + "tokens" : tokenize(model, act.text, False, False) + }) + elif isinstance(act, Choose): + if len(act.successors) == 1: + action.update({ "successors" : [ act.successors[0] ] * len(act.choices) }) + assert len(act.successors) == 0 or len(action['successors']) == len(act.choices) + action.update({ + "__type__" : "Choose", + "choices" : [ + tokenize(model, choice[0], False, False) for choice in act.choices + ], + "threshold" : defaults["Choose"]["threshold"] if act.threshold is None else act.threshold, + "width" : defaults["Choose"]["width"] if act.width is None else act.width + }) + elif isinstance(act, Complete): + assert act.length is not None + action.update({ + "__type__" : "Complete", + "length" : act.length, + "stop" : tokenize(model, act.stop, False, False), +# "vocab" : "TODO", + "threshold" : defaults["Complete"]["threshold"] if act.threshold is None else act.threshold, + "beams" : defaults["Complete"]["beams"] if act.beams is None else act.beams, + "ahead" : defaults["Complete"]["ahead"] if act.ahead is None else act.ahead, + "width" : defaults["Complete"]["width"] if act.width is None else act.width, + }) + else: + raise Exception() + actions.append( action ) + return { 'actions' : actions } + +def cxx_to_ftt(model, ftt): +# print(f'#{ftt["action"]}: {ftt["tokens"]}') + return { + "action" : ftt["action"], + "text" : detokenize(model, ftt["tokens"], False, False), + "length" : ftt["length"], + "logprobs" : ftt["logprobs"], + "logprob" : ftt["logprob"], + "probability" : math.exp(-ftt["logprob"]), + "normprob" : math.exp(-ftt["logprob"]/ftt["length"]), + "probabilities" : [ math.exp(-lpb) for lpb in ftt["logprobs"] ], + "children" : [ cxx_to_ftt(model, child) for child in ftt["children"] ], + "pruned" : ftt["pruned"], + } + +def extract_paths_from_ftt(ftt, current_path=""): + paths = [] + new_path = current_path + ftt["text"] + + if not "children" in ftt or len(ftt["children"]) == 0: + if not ftt["pruned"]: + paths.append((new_path, ftt["probability"])) else: - raise Exception() - actions.update({ act.uid : action }) + for child in ftt["children"]: + child_paths = extract_paths_from_ftt(child, new_path) + paths.extend(child_paths) + return paths + -# print(json.dumps(actions,indent=4)) +def ftt_to_graphviz_detailed(ftt, output_filename='ftt_tree_detailed', format='png', + max_text_length=30, show_text_preview=True): + """ + Detailed version with HTML-like labels for better formatting. + """ + dot = graphviz.Digraph(comment='FTT Tree Detailed', format=format) + dot.attr(rankdir='TB') + # Important: use 'plaintext' shape for HTML labels + dot.attr('node', shape='plaintext') + + node_counter = [0] + + def format_text_for_display(text, max_len): + """Format text for display in node, handling newlines and long text.""" + if not text: + return "[empty]" + + # Replace newlines with spaces for display + text = text.replace('\n', ' ').replace('\r', '') + + # Escape HTML characters + text = html.escape(text) + + # Truncate if needed + if len(text) > max_len: + text = text[:max_len] + "..." + + return text + + def add_node_recursive(node, parent_id=None, depth=0, is_best=True): + node_id = f"node_{node_counter[0]}" + node_counter[0] += 1 + + # Extract node properties + probability = node["probability"] + text = node.get("text", "") + is_pruned = node["pruned"] + + # Format probability + if probability < 0.001: + prob_str = f"{probability:.2e}" + elif probability < 0.01: + prob_str = f"{probability:.4f}" + else: + prob_str = f"{probability:.3f}" + + # Determine colors + if is_pruned: + header_color = "#FFB6C1" # lightpink + border_color = "#FF0000" # red + elif probability > 0.01: + header_color = "#90EE90" # lightgreen + border_color = "#008000" # green + elif probability > 0.0001: + header_color = "#ADD8E6" # lightblue + border_color = "#0000FF" # blue + else: + header_color = "#FFFFE0" # lightyellow + border_color = "#FFA500" # orange + + # Format text for display + display_text = format_text_for_display(text, max_text_length) + + label = f'''< + + +
P = {prob_str}
{display_text}
>''' + + # Add node with HTML label + dot.node(node_id, label) + + # Add edge from parent + if parent_id is not None: + edge_style = 'dashed' if is_pruned else 'solid' + edge_color = 'red' if is_pruned else 'black' -ftt = autocog.llama.evaluate(model, { 'actions' : actions }) + edge_width = "4.0" if is_best else "1.0" + + dot.edge(parent_id, node_id, style=edge_style, color=edge_color, penwidth=edge_width) + + # Recursively add children + if "children" in node: + for child in sorted(node["children"], key=lambda x: x["probability"], reverse=True): + add_node_recursive(child, node_id, depth + 1, is_best) + is_best = False + + return node_id + + # Build the tree + add_node_recursive(ftt) + + # Render + dot.render(output_filename, view=False, cleanup=True) + + return dot -print(ftt) +if __name__ == '__main__': + main(sys.argv) -## cpp_reconstructed = detokenize(model, cpp_tokens, False, False) diff --git a/tests/samples/micro.sta b/tests/samples/micro.sta new file mode 100644 index 0000000..ed2e997 --- /dev/null +++ b/tests/samples/micro.sta @@ -0,0 +1,27 @@ +format sentence { + is text<5, beams=2, width=2>; + annotate f"a well formed sentence in english"; +} + +format boolean { + is enum("true","false"); + annotate f"a boolean"; +} + +prompt main { + is { + statement is sentence; + correct is boolean; + } + channel { + to .statement from ?statement; + } + return { + from .correct; + } + annotate { + _ as "You are verifying whether statements are true or false."; + .statement as "the statement that you must verify"; + .correct as "your decision whether it is true or false"; + } +} diff --git a/tests/samples/mini.sta b/tests/samples/mini.sta index cb9c2c9..9ebe8fd 100644 --- a/tests/samples/mini.sta +++ b/tests/samples/mini.sta @@ -1,5 +1,5 @@ format sentence { - is text<5>; + is text<10, beams=5, width=3>; annotate f"a well formed sentence in english"; } @@ -19,4 +19,9 @@ prompt main { return { from .correct; } + annotate { + _ as "You are verifying whether statements are true or false."; + .statement as "the statement that you must verify"; + .correct as "your decision whether it is true or false"; + } } diff --git a/tests/samples/test.sta b/tests/samples/test.sta new file mode 100644 index 0000000..b0f4094 --- /dev/null +++ b/tests/samples/test.sta @@ -0,0 +1,34 @@ +format sentence { + is text<20, beams=5, width=5>; + annotate f"a well formed sentence in english"; +} + +format thought { + is text<10, beams=2, width=2>; + annotate f"show your thinking"; +} + +format boolean { + is enum("true","false"); + annotate f"a boolean"; +} + +prompt main { + is { + statement is sentence; + work[1:5] is thought; + correct is boolean; + } + channel { + to .statement from ?statement; + } + return { + from .correct; + } + annotate { + _ as "You are verifying whether statements are true or false."; + .statement as "the statement that you must verify"; + .thought as "up to five items that show what you consider to make that decision"; + .correct as "your decision whether it is true or false"; + } +} From 84486c1e82badc10180ea76895d6bd4ae887e5ca Mon Sep 17 00:00:00 2001 From: Tristan Date: Sat, 9 Aug 2025 15:55:03 -0700 Subject: [PATCH 09/51] Context reuse based on prefixes. Repetition penalty and diversity bonus. --- libs/autocog/llama/evaluation-choice.cxx | 2 +- libs/autocog/llama/evaluation-completion.cxx | 118 ++++++++++++++- libs/autocog/llama/evaluation-text.cxx | 4 +- libs/autocog/llama/evaluation.cxx | 11 +- libs/autocog/llama/evaluation.hxx | 2 +- libs/autocog/llama/fta.cxx | 35 +++-- libs/autocog/llama/fta.hxx | 23 ++- libs/autocog/llama/model.cxx | 120 +++++++++++---- libs/autocog/llama/python_bindings.cxx | 4 + modules/autocog/fta/actions.py | 18 ++- .../llama/execute_sta_with_llama_cpp.py | 141 ++++++++++++++---- tests/samples/micro.sta | 6 +- tests/samples/mini.sta | 8 +- tests/samples/small.sta | 27 ++++ tests/samples/test.sta | 16 +- 15 files changed, 421 insertions(+), 114 deletions(-) create mode 100644 tests/samples/small.sta diff --git a/libs/autocog/llama/evaluation-choice.cxx b/libs/autocog/llama/evaluation-choice.cxx index ff459fc..f8208b4 100644 --- a/libs/autocog/llama/evaluation-choice.cxx +++ b/libs/autocog/llama/evaluation-choice.cxx @@ -14,7 +14,7 @@ # include #endif -#define DEBUG_Evaluation_evaluate_choice VERBOSE && 1 +#define DEBUG_Evaluation_evaluate_choice VERBOSE && 0 namespace autocog { namespace llama { diff --git a/libs/autocog/llama/evaluation-completion.cxx b/libs/autocog/llama/evaluation-completion.cxx index f25b59b..31bea1d 100644 --- a/libs/autocog/llama/evaluation-completion.cxx +++ b/libs/autocog/llama/evaluation-completion.cxx @@ -11,12 +11,15 @@ #include #include #include +#include #if VERBOSE # include #endif -#define DEBUG_Evaluation_evaluate_completion VERBOSE && 1 +#define DEBUG_Evaluation_evaluate_completion VERBOSE && 0 +#define DEBUG_expand_beam VERBOSE && 0 +#define DEBUG_beam_search_step VERBOSE && 0 namespace autocog { namespace llama { @@ -24,14 +27,92 @@ struct BeamState { TokenSequence tokens; ProbaSequence logprobs; float logprob{0.}; + float repetition_penalty = 1.0f; + float diversity_bonus = 0.0f; + float lookahead_bonus = 0.0f; bool stopped{false}; float proba() const { return std::exp(-logprob/logprobs.size()); } + + float score() const { + return (this->proba() + lookahead_bonus + diversity_bonus) / repetition_penalty; + } }; -// Expand a single beam by evaluating next token +static float calculate_repetition_penalty(TokenSequence const & tokens, float & penalty, float const penalty_weight) { + size_t const min_length = 3; + size_t const max_window_size = 256; + + size_t window_size = std::min(tokens.size(), max_window_size); + for (size_t i = min_length; i < tokens.size(); ++i) { + size_t best_length = 0; + size_t best_distance = 0; + + size_t search_start = (i >= window_size) ? i - window_size : 0; + + for (size_t j = search_start; j < i; ++j) { + size_t match_length = 0; + + while (j + match_length < i && + i + match_length < tokens.size() && + tokens[j + match_length] == tokens[i + match_length]) { + match_length++; + } + + if (match_length >= min_length && match_length > best_length) { + best_length = match_length; + best_distance = i - j; + } + } + + if (best_length >= min_length) { + // Stronger penalty for: + // - Longer repetitions + // - Recent repetitions (smaller distance) + float length_factor = std::log(best_length + 1); + float recency_factor = 1.0f / std::log(best_distance + 2); + penalty *= (1.0f + penalty_weight * length_factor * recency_factor); + } + } + return penalty; +} + +static float token_sequence_diversity(TokenSequence const & a, TokenSequence const & b) { + // Jaccard distance or edit distance + std::set set_a(a.begin(), a.end()); + std::set set_b(b.begin(), b.end()); + + std::set intersection; + std::set_intersection( + set_a.begin(), set_a.end(), + set_b.begin(), set_b.end(), + std::inserter(intersection, intersection.begin()) + ); + + std::set union_set; + std::set_union( + set_a.begin(), set_a.end(), + set_b.begin(), set_b.end(), + std::inserter(union_set, union_set.begin()) + ); + + return 1.0f - (float)intersection.size() / union_set.size(); +} + +static void calculate_diversity_bonuses(std::vector & beams, float const weight) { + for (size_t i = 0; i < beams.size(); ++i) { + float diversity = 0.0f; + for (size_t j = 0; j < beams.size(); ++j) { + if (i != j) { + diversity += token_sequence_diversity(beams[i].tokens, beams[j].tokens); + } + } + beams[i].diversity_bonus = weight * diversity / (beams.size() - 1); + } +} + static unsigned expand_beam( Model & model, ContextID ctx, @@ -40,6 +121,11 @@ static unsigned expand_beam( TokenSequence const & base_tokens, std::vector & beams ) { +#if DEBUG_expand_beam + std::cerr << "expand_beam(...):" << std::endl; + std::cerr << " - base_tokens.size() = " << base_tokens.size() << std::endl; + std::cerr << " - beam.tokens.size() = " << beam.tokens.size() << std::endl; +#endif TokenSequence context_tokens = base_tokens; context_tokens.insert(context_tokens.end(), beam.tokens.begin(), beam.tokens.end()); model.set_tokens(context_tokens, ctx); @@ -56,9 +142,18 @@ static unsigned expand_beam( for (size_t i = 0; i < topk_tokens.size(); ++i) { BeamState & new_beam = beams.emplace_back(beam); + new_beam.tokens.push_back(topk_tokens[i]); new_beam.logprobs.push_back(topk_logits[i]); new_beam.logprob += topk_logits[i]; + + if (action.repetition) { + TokenSequence beam_tokens; + beam_tokens.insert(beam_tokens.end(), base_tokens.begin(), base_tokens.end()); + beam_tokens.insert(beam_tokens.end(), new_beam.tokens.begin(), new_beam.tokens.end()); + calculate_repetition_penalty(beam_tokens, new_beam.repetition_penalty, action.repetition.value()); + } + new_beam.stopped = ( action.stop.size() <= new_beam.tokens.size() ) && std::equal( @@ -81,7 +176,7 @@ static void prune_beams( // Sort by score std::sort(beams.begin(), beams.end(), [](BeamState const & a, BeamState const & b) { if (a.stopped != b.stopped) return a.stopped; - return a.proba() < b.proba(); + return a.score() > b.score(); }); // Keep top beams @@ -103,11 +198,15 @@ static void prune_beams( static bool beam_search_step( Model & model, ContextID ctx, - const Completion& action, - const TokenSequence& base_tokens, + Completion const & action, + TokenSequence const & base_tokens, std::vector & current_beams, unsigned & num_token_eval ) { +#if DEBUG_beam_search_step + std::cerr << "beam_search_step(...):" << std::endl; + std::cerr << " - current_beams.size() = " << current_beams.size() << std::endl; +#endif std::vector next_beams; for (BeamState const & beam : current_beams) { @@ -117,15 +216,18 @@ static bool beam_search_step( num_token_eval += expand_beam(model, ctx, beam, action, base_tokens, next_beams); } } + if (action.diversity) { + calculate_diversity_bonuses(next_beams, action.diversity.value()); + } // Check for early termination bool all_stopped = std::all_of(next_beams.begin(), next_beams.end(), [](BeamState const & b) { return b.stopped; }); - + if (all_stopped) { current_beams = std::move(next_beams); return true; // Signal early termination } - + // Prune and update beams prune_beams(next_beams, action.beams); @@ -174,7 +276,7 @@ unsigned Evaluation::evaluate_completion(PathState & state) { } std::sort(beams.begin(), beams.end(), [](BeamState const & a, BeamState const & b) { - return a.proba() > b.proba(); + return a.score() > b.score(); }); unsigned count = 0; diff --git a/libs/autocog/llama/evaluation-text.cxx b/libs/autocog/llama/evaluation-text.cxx index 4de33fa..5e3b704 100644 --- a/libs/autocog/llama/evaluation-text.cxx +++ b/libs/autocog/llama/evaluation-text.cxx @@ -11,7 +11,7 @@ # include #endif -#define DEBUG_Evaluation_evaluate_text VERBOSE && 1 +#define DEBUG_Evaluation_evaluate_text VERBOSE && 0 namespace autocog { namespace llama { @@ -27,7 +27,7 @@ unsigned Evaluation::evaluate_text(PathState & state) { unsigned num_token_eval = 0; ProbaSequence logprobs(action.tokens.size(), 0.); - if (this->config.evaluate_text) { + if (action.evaluate) { auto [model,ctx] = this->restore(state); #if DEBUG_Evaluation_evaluate_text std::cerr << " - Model #" << model.id << std::endl; diff --git a/libs/autocog/llama/evaluation.cxx b/libs/autocog/llama/evaluation.cxx index 4134aae..19e835d 100644 --- a/libs/autocog/llama/evaluation.cxx +++ b/libs/autocog/llama/evaluation.cxx @@ -109,13 +109,12 @@ void Evaluation::enqueue( std::pair Evaluation::restore(PathState & state) const { Model & model = Manager::get_model(this->model); - if (state.context) { - throw std::runtime_error("Context saving is not implemented yet, this should not happen"); - } else { - // for now we always use context 0 and reset it before evaluation any action - model.set_tokens(state.tokens, 0); - state.context = 0; + + if (!state.context) { + state.context = 0; // TODO look at existing context for the largest prefix? } + model.set_tokens(state.tokens, state.context.value()); + return std::pair(model, state.context.value()); } diff --git a/libs/autocog/llama/evaluation.hxx b/libs/autocog/llama/evaluation.hxx index 441572d..10367ed 100644 --- a/libs/autocog/llama/evaluation.hxx +++ b/libs/autocog/llama/evaluation.hxx @@ -27,7 +27,7 @@ struct PathState { }; struct EvaluationConfig { - bool evaluate_text{false}; + bool evaluate_text{true}; }; class Evaluation { diff --git a/libs/autocog/llama/fta.cxx b/libs/autocog/llama/fta.cxx index da28bff..6ae86c3 100644 --- a/libs/autocog/llama/fta.cxx +++ b/libs/autocog/llama/fta.cxx @@ -20,9 +20,11 @@ Action::Action( Text::Text( ActionID const id_, - std::string const & name_ + std::string const & name_, + bool const eval ) : - Action(ActionKind::Text, id_, name_) + Action(ActionKind::Text, id_, name_), + evaluate(eval) {} Completion::Completion( @@ -32,14 +34,18 @@ Completion::Completion( unsigned length_, unsigned beams_, unsigned ahead_, - unsigned width_ + unsigned width_, + std::optional repetition_, + std::optional diversity_ ) : Action(ActionKind::Completion, id_, name_), threshold(threshold_), length(length_), beams(beams_), ahead(ahead_), - width(width_) + width(width_), + repetition(repetition_), + diversity(diversity_) {} Choice::Choice( @@ -93,7 +99,10 @@ FTA::FTA(Model const & model, pybind11::dict const & pydata) { std::unique_ptr action; if (action_type == "Text") { - action = std::make_unique(node_id, uid); + bool evaluate = false; + if (action_dict.contains("evaluate")) evaluate = action_dict["evaluate"].cast(); + + action = std::make_unique(node_id, uid, evaluate); Text * text_action = static_cast(action.get()); auto py_tokens = action_dict["tokens"].cast(); @@ -108,8 +117,12 @@ FTA::FTA(Model const & model, pybind11::dict const & pydata) { unsigned beams = action_dict["beams"].cast(); unsigned ahead = action_dict["ahead"].cast(); unsigned width = action_dict["width"].cast(); - - action = std::make_unique(node_id, uid, threshold, length, beams, ahead, width); + std::optional repetition = std::nullopt; + if (action_dict.contains("repetition") && !action_dict["repetition"].is_none()) repetition = action_dict["repetition"].cast(); + std::optional diversity = std::nullopt; + if (action_dict.contains("diversity") && !action_dict["diversity"].is_none()) repetition = action_dict["diversity"].cast(); + + action = std::make_unique(node_id, uid, threshold, length, beams, ahead, width, repetition, diversity); Completion* completion_action = static_cast(action.get()); // Set stop tokens @@ -124,12 +137,12 @@ FTA::FTA(Model const & model, pybind11::dict const & pydata) { } // Set vocabulary mask + completion_action->vocab.mask.clear(); completion_action->vocab.mask.reserve(model.vocab_size()); - completion_action->vocab.mask.assign(model.vocab_size(), true); - if (action_dict.contains("vocab")) { - throw std::runtime_error("Setting the vocabulary from PY desc is not implemented yet!"); + for (auto tok_msk : action_dict["vocab"]) { + completion_action->vocab.mask.push_back(tok_msk.cast()); } - + } else if (action_type == "Choose") { float threshold = action_dict["threshold"].cast(); unsigned width = action_dict["width"].cast(); diff --git a/libs/autocog/llama/fta.hxx b/libs/autocog/llama/fta.hxx index 914ebe9..f522f08 100644 --- a/libs/autocog/llama/fta.hxx +++ b/libs/autocog/llama/fta.hxx @@ -45,24 +45,35 @@ struct Action { struct Text : public Action { static constexpr ActionKind Kind = ActionKind::Text; + bool const evaluate; //< whether to evaluate the probability using the model (else p=1.) + TokenSequence tokens; - Text(ActionID const id_, std::string const & name_); + Text(ActionID const id_, std::string const & name_, bool const eval); }; struct Completion : public Action { static constexpr ActionKind Kind = ActionKind::Completion; float const threshold; //< Probability threshold for pruning - unsigned const length; // Maximum length of the completion - unsigned const beams; // Number of concurrent exploration beams - unsigned const ahead; // Look ahead parameter for beam search - unsigned const width; // Maximum number of beams to select + unsigned const length; //< Maximum length of the completion + unsigned const beams; //< Number of concurrent exploration beams + unsigned const ahead; //< Look ahead parameter for beam search + unsigned const width; //< Maximum number of beams to select + + std::optional const repetition; //< Penalize repeting pattern + std::optional const diversity; //< Encourage diversity across beams Vocab vocab; TokenSequence stop; - Completion(ActionID const id_, std::string const & name_, float threshold_, unsigned length_, unsigned beams_, unsigned ahead_, unsigned width_); + Completion( + ActionID const id_, std::string const & name_, + float threshold_, unsigned length_, + unsigned beams_, unsigned ahead_, unsigned width_, + std::optional repetition_, + std::optional diversity_ + ); }; struct Choice : public Action { diff --git a/libs/autocog/llama/model.cxx b/libs/autocog/llama/model.cxx index e5d15b0..f4dfa19 100644 --- a/libs/autocog/llama/model.cxx +++ b/libs/autocog/llama/model.cxx @@ -6,6 +6,14 @@ #include #include +#if VERBOSE +# include +#endif + +#define DEBUG_Model_set_tokens VERBOSE && 0 +#define DEBUG_Model_eval_sequences VERBOSE && 0 +#define DEBUG_Model_eval_topk_tokens VERBOSE && 0 + namespace autocog { namespace llama { Model::Model() : @@ -162,21 +170,6 @@ TokenID Model::eos_token() const { return llama_vocab_eos(this->get_vocab()); } -static unsigned run_batch_full(llama_context * ctx, TokenSequence const & tokens) { - llama_memory_t mem = llama_get_memory(ctx); - llama_memory_clear(mem, false); - - unsigned n_ctx = llama_n_ctx(ctx); - if (tokens.size() > n_ctx) { - throw std::runtime_error("Token sequence too long: " + std::to_string(tokens.size()) + " > " + std::to_string(n_ctx)); - } - llama_batch batch = llama_batch_get_one(const_cast(tokens.data()), tokens.size()); - if (llama_decode(ctx, batch) != 0) { - throw std::runtime_error("Failed to set the token sequence."); - } - return tokens.size(); -} - static float logit_to_log_sum_exp(float * logit, unsigned vocab_size) { // Find max logit for numerical stability float max_logit = *std::max_element(logit, logit + vocab_size); @@ -214,18 +207,88 @@ static float retrieve_logprob(llama_context * ctx, unsigned vocab_size, TokenID return logit_to_log_sum_exp(logits, vocab_size) - logits[token]; } -unsigned Model::set_tokens(TokenSequence const & tokens_, ContextID const id) { +static llama_pos find_common_prefix(const TokenSequence& a, const TokenSequence& b) { + llama_pos common = 0; + size_t min_size = std::min(a.size(), b.size()); + while (common < min_size && a[common] == b[common]) { + common++; + } + return common; +} + +unsigned Model::set_tokens(TokenSequence const & target_tokens, ContextID const id) { +#if DEBUG_Model_set_tokens + std::cerr << "Model::set_tokens(...):" << std::endl; + std::cerr << " > target_tokens.size() = " << target_tokens.size() << std::endl; +#endif if (this->id == 0) { - return tokens_.size(); + return target_tokens.size(); } check_context_id(id); - TokenSequence & loc_tokens = this->get_tokens(id); - loc_tokens = tokens_; - return run_batch_full(this->get_context(id), loc_tokens); + TokenSequence & current_tokens = this->get_tokens(id); +#if DEBUG_Model_set_tokens + std::cerr << " > current_tokens.size() = " << current_tokens.size() << std::endl; +#endif + llama_context * ctx = this->get_context(id); + unsigned n_ctx = llama_n_ctx(ctx); + if (current_tokens.size() > n_ctx) { + throw std::runtime_error("Token sequence too long: " + std::to_string(current_tokens.size()) + " > " + std::to_string(n_ctx)); + } + + llama_memory_t mem = llama_get_memory(ctx); +#if DEBUG_Model_set_tokens + llama_pos kv_pos_max = llama_memory_seq_pos_max(mem, 0); + llama_pos kv_pos_min = llama_memory_seq_pos_min(mem, 0); + std::cerr << " > KV cache pos_min = " << kv_pos_min << std::endl; + std::cerr << " > KV cache pos_max = " << kv_pos_max << std::endl; +#endif + + llama_pos common_prefix = find_common_prefix(current_tokens, target_tokens); +#if DEBUG_Model_set_tokens + std::cerr << " > common_prefix = " << common_prefix << std::endl; +#endif + + unsigned num_token_eval = 0; + if (common_prefix == 0) { + llama_memory_seq_rm(mem, 0, 0, -1); + + llama_batch batch = llama_batch_get_one(const_cast(target_tokens.data()), target_tokens.size()); + if (llama_decode(ctx, batch) != 0) { + throw std::runtime_error("Failed to set the token sequence."); + } + num_token_eval += target_tokens.size(); + } else { + if (common_prefix < current_tokens.size()) { + llama_memory_seq_rm(mem, 0, common_prefix, -1); + } + + if (common_prefix < target_tokens.size()) { + TokenSequence extension(target_tokens.begin() + common_prefix, target_tokens.end()); + + llama_batch batch = llama_batch_get_one(const_cast(extension.data()), extension.size()); + + std::vector positions(extension.size()); + for (size_t i = 0; i < extension.size(); ++i) { + positions[i] = common_prefix + i; + } + batch.pos = positions.data(); + + if (llama_decode(ctx, batch) != 0) { + throw std::runtime_error("Failed to decode token"); + } + num_token_eval += extension.size(); + } + } + current_tokens = target_tokens; + return num_token_eval; } unsigned Model::eval_sequences(TokenSequence const & new_tokens, ProbaSequence & logprobs, ContextID const id) { +#if DEBUG_Model_eval_sequences + std::cerr << "Model::eval_sequences(...):" << std::endl; + std::cerr << " > new_tokens.size() = " << new_tokens.size() << std::endl; +#endif if (this->id == 0) { throw std::runtime_error("Using model #0 (RNG) is not implemented yet!"); // TODO fill `logprobs` with random vales @@ -237,6 +300,9 @@ unsigned Model::eval_sequences(TokenSequence const & new_tokens, ProbaSequence & logprobs.clear(); for (auto token: new_tokens) { +#if DEBUG_Model_set_tokens + std::cerr << " > token_pos = " << token_pos << std::endl; +#endif llama_batch batch = llama_batch_get_one(&token, 1); batch.pos = &token_pos; @@ -253,6 +319,7 @@ unsigned Model::eval_sequences(TokenSequence const & new_tokens, ProbaSequence & return new_tokens.size(); } + unsigned Model::eval_topk_tokens( std::vector const & vocab_mask, size_t max_candidates, @@ -260,6 +327,10 @@ unsigned Model::eval_topk_tokens( std::vector & topk_lobprobs, ContextID const id ) { +#if DEBUG_Model_eval_topk_tokens + std::cerr << "Model::eval_topk_tokens(...):" << std::endl; + std::cerr << " > max_candidates = " << max_candidates << std::endl; +#endif check_context_id(id); if (this->id == 0) { @@ -277,15 +348,6 @@ unsigned Model::eval_topk_tokens( topk_tokens.clear(); topk_lobprobs.clear(); - llama_batch batch = llama_batch_get_one( - const_cast(current_tokens.data()), - current_tokens.size() - ); - - if (llama_decode(ctx, batch) != 0) { - throw std::runtime_error("Failed to decode for token evaluation"); - } - std::vector> candidates; sample_logprobs(ctx, vocab_mask, candidates); diff --git a/libs/autocog/llama/python_bindings.cxx b/libs/autocog/llama/python_bindings.cxx index 77eab01..757e37f 100644 --- a/libs/autocog/llama/python_bindings.cxx +++ b/libs/autocog/llama/python_bindings.cxx @@ -47,6 +47,10 @@ PYBIND11_MODULE(llama, module) { pybind11::arg("n_ctx") = 4096 ); + module.def("vocab_size", [](ModelID model) { + return Manager::get_model(model).vocab_size(); + }, "Get vocabulary size"); + module.def("tokenize", [](ModelID model, const std::string & text, bool add_bos, bool special) { auto tokens = Manager::get_model(model).tokenize(text, add_bos, special); diff --git a/modules/autocog/fta/actions.py b/modules/autocog/fta/actions.py index 4b93012..cbdfb7b 100644 --- a/modules/autocog/fta/actions.py +++ b/modules/autocog/fta/actions.py @@ -11,8 +11,6 @@ class Action(BaseModel): uid: str successors: List[str] = [] - width: Optional[int] = None - threshold: Optional[float] = None def __init__(self, uid:Optional[str]=None, **kwargs): if uid is None: @@ -41,6 +39,7 @@ def toGraphVizNode(self, label_with_uid:bool=False): class Text(Action): text: str tokens: List[Token] = [] + evaluate: Optional[bool] = None def __init__(self, uid:str, text:str, successors: List[str]=[]): super().__init__(uid=uid, successors=successors, text=text) @@ -63,8 +62,11 @@ def toGraphVizLabel(self): class Choose(Action): choices: List[Tuple[str,List[Token]]] - def __init__(self, uid:str, choices:List[str], successors: List[str]=[], width:Optional[int]=None): - super().__init__(uid=uid, successors=successors, choices=[ ( c, [] ) for c in choices ], width=None if width == 0 else width) + width: Optional[int] = None + threshold: Optional[float] = None + + def __init__(self, uid:str, choices:List[str], successors: List[str]=[], width:Optional[int]=None, threshold: Optional[float]=None): + super().__init__(uid=uid, successors=successors, choices=[ ( c, [] ) for c in choices ], width=None if width == 0 else width, threshold=threshold) def prepare(self, lm): for choice in self.choices: @@ -85,10 +87,11 @@ def toGraphVizLabel(self): class Complete(Action): length: int = 1 stop: str = '' + threshold: Optional[float] - beams: Optional[int] - ahead: Optional[int] - width: Optional[int] + beams: Optional[int] + ahead: Optional[int] + width: Optional[int] seeds: Optional[List[str]] vocab: Vocab @@ -116,3 +119,4 @@ def toGraphVizShape(self): def toGraphVizLabel(self): return f"length={self.length}\nvocab={self.vocab.toGraphVizLabel() if self.vocab is not None else ''}\nstop={self.stop}\n" + diff --git a/tests/autocog/llama/execute_sta_with_llama_cpp.py b/tests/autocog/llama/execute_sta_with_llama_cpp.py index 18b19b2..3f91351 100644 --- a/tests/autocog/llama/execute_sta_with_llama_cpp.py +++ b/tests/autocog/llama/execute_sta_with_llama_cpp.py @@ -1,5 +1,5 @@ -import sys, json, html, math, graphviz +import sys, json, html, math, string, graphviz from autocog.sta.compile import compile_source_to_program_and_stas from autocog.sta.syntax import Syntax @@ -12,16 +12,20 @@ from autocog.llama import tokenize, detokenize fta_defaults = { - "Text" : {}, + "Text" : { + "evaluate" : True + }, "Choose" : { - "threshold" : 0e-10, - "width" : 3 + "threshold" : 0e-3, + "width" : 1 }, "Complete" : { - "threshold" : 0e-8, - "width" : 3, + "threshold" : 0e-3, + "width" : 2, "beams" : 2, - "ahead" : 1 + "ahead" : 1, + "diversity" : 1., + "repetition" : .2 } } @@ -46,7 +50,10 @@ def main(argv): model = autocog.llama.create(model_path, 4096) if len(model_path) > 0 else 0 - ftt = cxx_to_ftt(model, autocog.llama.evaluate(model, fta_to_cxx(model, fta, fta_defaults))) + safe_mask = create_safe_vocab_mask(model) + char_mask = create_single_char_vocab(model) + + ftt = cxx_to_ftt(model, autocog.llama.evaluate(model, fta_to_cxx(model, fta, fta_defaults, safe_mask, char_mask))) # print(json.dumps(ftt, indent=4)) paths = extract_paths_from_ftt(ftt) @@ -60,17 +67,87 @@ def main(argv): ftt, output_filename='ftt', format='svg', - max_text_length=40, + max_text_length=100, show_text_preview=True ) -def fta_to_cxx(model, fta, defaults): +def create_safe_vocab_mask(model): + """Create vocab mask that excludes problematic tokens""" + import autocog.llama + + vocab_size = autocog.llama.vocab_size(model) + mask = [True] * vocab_size + + problematic_count = 0 + + for token_id in range(vocab_size): + try: + # Test detokenize single token + text = autocog.llama.detokenize(model, [token_id], False, False) + + # Flag problematic tokens + is_problematic = ( + '\n' in text or # Newlines + '\r' in text or # Carriage returns + '\t' in text or # Tabs + len(text) == 0 or # Empty tokens + any(ord(c) < 32 for c in text if c not in [' ']) or # Control characters + any(ord(c) > 126 for c in text) # Non-ASCII + ) + + if is_problematic: + mask[token_id] = False + problematic_count += 1 + + except Exception: + # If detokenization fails, exclude the token + mask[token_id] = False + problematic_count += 1 + + print(f"Excluded {problematic_count}/{vocab_size} problematic tokens") + return mask + +def create_single_char_vocab(model): + vocab_size = autocog.llama.vocab_size(model) + mask = [False] * vocab_size + + # Define character sets to include + chars_to_test = ( + string.ascii_letters + # a-z, A-Z + string.digits + # 0-9 + string.punctuation + # .,!? etc. + ' ' + '\n' # spaces + ) + + single_char_count = 0 + + for char in chars_to_test: + try: + # Tokenize the single character + tokens = autocog.llama.tokenize(model, char, False, False) + + # Check if it tokenizes to exactly one token + if len(tokens) == 1: + token_id = tokens[0] + if 0 <= token_id < vocab_size: # Validate range + mask[token_id] = True + single_char_count += 1 + + except Exception: + # Skip characters that can't be tokenized + pass + + print(f"Found {single_char_count} single-character tokens from {len(chars_to_test)} tested characters") + return mask + +def fta_to_cxx(model, fta, defaults, safe_mask, char_mask): actions = [] for act in fta.actions.values(): action = { "uid" : act.uid, "successors" : act.successors } if isinstance(act, Text): action.update({ "__type__" : "Text", + "evaluate" : defaults["Text"]["evaluate"] if act.evaluate is None else act.evaluate, "tokens" : tokenize(model, act.text, False, False) }) elif isinstance(act, Choose): @@ -91,7 +168,7 @@ def fta_to_cxx(model, fta, defaults): "__type__" : "Complete", "length" : act.length, "stop" : tokenize(model, act.stop, False, False), -# "vocab" : "TODO", + "vocab" : safe_mask, "threshold" : defaults["Complete"]["threshold"] if act.threshold is None else act.threshold, "beams" : defaults["Complete"]["beams"] if act.beams is None else act.beams, "ahead" : defaults["Complete"]["ahead"] if act.ahead is None else act.ahead, @@ -103,16 +180,12 @@ def fta_to_cxx(model, fta, defaults): return { 'actions' : actions } def cxx_to_ftt(model, ftt): -# print(f'#{ftt["action"]}: {ftt["tokens"]}') return { "action" : ftt["action"], "text" : detokenize(model, ftt["tokens"], False, False), "length" : ftt["length"], "logprobs" : ftt["logprobs"], - "logprob" : ftt["logprob"], - "probability" : math.exp(-ftt["logprob"]), - "normprob" : math.exp(-ftt["logprob"]/ftt["length"]), - "probabilities" : [ math.exp(-lpb) for lpb in ftt["logprobs"] ], + "probability" : math.exp(-ftt["logprob"]/ftt["length"]), "children" : [ cxx_to_ftt(model, child) for child in ftt["children"] ], "pruned" : ftt["pruned"], } @@ -130,9 +203,7 @@ def extract_paths_from_ftt(ftt, current_path=""): paths.extend(child_paths) return paths - -def ftt_to_graphviz_detailed(ftt, output_filename='ftt_tree_detailed', format='png', - max_text_length=30, show_text_preview=True): +def ftt_to_graphviz_detailed(ftt, output_filename='ftt_tree_detailed', format='png', max_text_length=30, show_text_preview=True): """ Detailed version with HTML-like labels for better formatting. """ @@ -147,17 +218,31 @@ def format_text_for_display(text, max_len): """Format text for display in node, handling newlines and long text.""" if not text: return "[empty]" - - # Replace newlines with spaces for display - text = text.replace('\n', ' ').replace('\r', '') - - # Escape HTML characters - text = html.escape(text) - - # Truncate if needed + if len(text) > max_len: text = text[:max_len] + "..." - + + # HTML entity escaping + text = text.replace('&', '＆') # Must be first + text = text.replace('<', '‹') + text = text.replace('>', '›') + text = text.replace('"', '"') + text = text.replace("'", ''') + + # GraphViz special characters + text = text.replace('\\', '\\\\') + text = text.replace('{', '\\{') + text = text.replace('}', '\\}') + text = text.replace('|', '\\|') + + # Handle newlines and control characters + text = text.replace('\n', '↵') + text = text.replace('\r', '⏎') + text = text.replace('\t', '→') + + # Remove remaining control characters + text = ''.join(c if ord(c) >= 32 or c in ['\n', '\r', '\t'] else f'\\x{ord(c):02x}' for c in text) + return text def add_node_recursive(node, parent_id=None, depth=0, is_best=True): diff --git a/tests/samples/micro.sta b/tests/samples/micro.sta index ed2e997..d1e6c8b 100644 --- a/tests/samples/micro.sta +++ b/tests/samples/micro.sta @@ -1,11 +1,11 @@ format sentence { is text<5, beams=2, width=2>; - annotate f"a well formed sentence in english"; + annotate f"Single well formed sentence in correct english"; } format boolean { - is enum("true","false"); - annotate f"a boolean"; + is enum("True","False"); + annotate f"A boolean value to specify True or False"; } prompt main { diff --git a/tests/samples/mini.sta b/tests/samples/mini.sta index 9ebe8fd..ebb37aa 100644 --- a/tests/samples/mini.sta +++ b/tests/samples/mini.sta @@ -1,11 +1,11 @@ format sentence { - is text<10, beams=5, width=3>; - annotate f"a well formed sentence in english"; + is text<10, beams=3, width=3>; + annotate f"Single well formed sentence in correct english"; } format boolean { - is enum("true","false"); - annotate f"a boolean"; + is enum("True","False"); + annotate f"A boolean value to specify True or False"; } prompt main { diff --git a/tests/samples/small.sta b/tests/samples/small.sta new file mode 100644 index 0000000..ded90e8 --- /dev/null +++ b/tests/samples/small.sta @@ -0,0 +1,27 @@ +format sentence { + is text<20, beams=5, width=3>; + annotate f"Single well formed sentence in correct english"; +} + +format boolean { + is enum("True","False"); + annotate f"A boolean value to specify True or False"; +} + +prompt main { + is { + statement is sentence; + correct is boolean; + } + channel { + to .statement from ?statement; + } + return { + from .correct; + } + annotate { + _ as "You are verifying whether statements are true or false."; + .statement as "the statement that you must verify"; + .correct as "your decision whether it is true or false"; + } +} diff --git a/tests/samples/test.sta b/tests/samples/test.sta index b0f4094..cf36ca2 100644 --- a/tests/samples/test.sta +++ b/tests/samples/test.sta @@ -1,16 +1,16 @@ format sentence { - is text<20, beams=5, width=5>; - annotate f"a well formed sentence in english"; + is text<50, beams=5, width=3>; + annotate f"Single well formed sentence in correct english"; } format thought { - is text<10, beams=2, width=2>; - annotate f"show your thinking"; + is text<20, beams=5, width=2>; + annotate f"Working through figuring out an answer. No need to use proper sentences."; } format boolean { - is enum("true","false"); - annotate f"a boolean"; + is enum("True","False"); + annotate f"A boolean value to specify True or False"; } prompt main { @@ -26,9 +26,9 @@ prompt main { from .correct; } annotate { - _ as "You are verifying whether statements are true or false."; + _ as "You are verifying whether statements are true or false. Each prompt ends with a new line."; .statement as "the statement that you must verify"; - .thought as "up to five items that show what you consider to make that decision"; + .work as "up to five items that show what you consider to make that decision"; .correct as "your decision whether it is true or false"; } } From 95a7dbbc82311f013a92ad742918bc32556d2204 Mon Sep 17 00:00:00 2001 From: Tristan Date: Sat, 9 Aug 2025 19:14:37 -0700 Subject: [PATCH 10/51] Getting started on integrating C++ work with full Autocog flow --- modules/autocog/arch/__init__.py | 7 ++++++ modules/autocog/arch/architecture.py | 1 + modules/autocog/arch/cogs.py | 6 ++++- modules/autocog/arch/orchestrator.py | 3 ++- modules/autocog/fta/automaton.py | 3 +++ modules/autocog/lm/__init__.py | 3 ++- modules/autocog/lm/llama_cxx.py | 26 ++++++++++++++++++++ modules/autocog/lm/{llama.py => llama_py.py} | 3 ++- modules/autocog/lm/lm.py | 1 + modules/autocog/utility/args2arch.py | 2 +- modules/autocog/utility/models.py | 6 ++--- tests/cli/mcq/run.sh | 4 ++- 12 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 modules/autocog/lm/llama_cxx.py rename modules/autocog/lm/{llama.py => llama_py.py} (98%) diff --git a/modules/autocog/arch/__init__.py b/modules/autocog/arch/__init__.py index e69de29..b33994c 100644 --- a/modules/autocog/arch/__init__.py +++ b/modules/autocog/arch/__init__.py @@ -0,0 +1,7 @@ + +from .architecture import CognitiveArchitecture as CogArch +from .orchestrator import Serial, Async + +Serial.model_rebuild() +Async.model_rebuild() +CogArch.model_rebuild() diff --git a/modules/autocog/arch/architecture.py b/modules/autocog/arch/architecture.py index 3a82c0b..586e089 100644 --- a/modules/autocog/arch/architecture.py +++ b/modules/autocog/arch/architecture.py @@ -109,3 +109,4 @@ def toGraphViz(self): dotstr += cog.toGraphViz() + "\n" dotstr += "}\n" return dotstr + diff --git a/modules/autocog/arch/cogs.py b/modules/autocog/arch/cogs.py index 0ac6506..95069a0 100644 --- a/modules/autocog/arch/cogs.py +++ b/modules/autocog/arch/cogs.py @@ -82,7 +82,10 @@ async def __call__(self, __page: Optional[Page]=None, **inputs) -> Any: fta = sta.instantiate(syntax=self.arch.syntax, frame=frame, branches=__page.branches[ptag], inputs=inputs) __page.ftas[ptag].append(fta) fta.simplify() - ftt = fta.greedy(lm=self.arch.lm) + if hasattr(self.arch.lm, 'evaluate'): + ftt = fta.execute(lm=self.arch.lm) + else: + ftt = fta.greedy(lm=self.arch.lm) __page.ftts[ptag].append(ftt) next = sta.parse(lm=self.arch.lm, syntax=self.arch.syntax, stacks=__page.stacks, ftt=ftt) if isinstance(next, Return): @@ -106,3 +109,4 @@ def toGraphViz(self): for (tag,prompt) in self.prompts: dotstr += prompt.toGraphViz_abstract() return dotstr + diff --git a/modules/autocog/arch/orchestrator.py b/modules/autocog/arch/orchestrator.py index 7612809..d388496 100644 --- a/modules/autocog/arch/orchestrator.py +++ b/modules/autocog/arch/orchestrator.py @@ -15,7 +15,7 @@ class Orchestrator(BaseModel): pages: List[Page] - cogs: Dict[str,Cog] = {} + cogs: Dict[str,"Cog"] = {} def __init__(self): super().__init__(pages=[ Page.root() ]) @@ -55,3 +55,4 @@ async def execute(self, jobs:List[Tuple[str,str,Any]], parent:int, progress:bool gather = asyncio.gather return await gather(*super().coroutines(jobs, parent)) + diff --git a/modules/autocog/fta/automaton.py b/modules/autocog/fta/automaton.py index b5a3327..dea17a0 100644 --- a/modules/autocog/fta/automaton.py +++ b/modules/autocog/fta/automaton.py @@ -141,6 +141,9 @@ def greedy(self, lm: LM): assert root.finalized return root + def execute(self, lm): + raise NotImplementedError() + def toGraphViz(self, label_with_uid:bool=False): dotstr = "" for act in self.actions.values(): diff --git a/modules/autocog/lm/__init__.py b/modules/autocog/lm/__init__.py index 389d323..1075eb1 100644 --- a/modules/autocog/lm/__init__.py +++ b/modules/autocog/lm/__init__.py @@ -1,4 +1,5 @@ from .random import RLM -from .llama import Llama +from .llama_cxx import LlamaCXX +from .llama_py import LlamaPY from .transformers import TfLM diff --git a/modules/autocog/lm/llama_cxx.py b/modules/autocog/lm/llama_cxx.py new file mode 100644 index 0000000..e38d44f --- /dev/null +++ b/modules/autocog/lm/llama_cxx.py @@ -0,0 +1,26 @@ + +from typing import Any, Dict, List, Tuple, Union, Optional, Callable +from .lm import LM + +from ..llama import create as autocog_llama_create + +class LlamaCXX(LM): + model: Any + + def __init__(self, model_path:str, n_ctx=2048, **kwargs): + super().__init__( + model=autocog_llama_create(model_path, n_ctx) if len(model_path) > 0 else 0 + ) + + def tokenize(self, text:str, whole:bool=True) -> List[int]: + raise NotImplementedError("LlamaCXX.tokenize") + + def detokenize(self, tokens:List[int], whole:bool=True) -> str: + raise NotImplementedError("LlamaCXX.detokenize") + + def evaluate(self, sta): + raise NotImplementedError("LlamaCXX.evaluate") + + def impl_greedy(self, **kwargs): + raise Exception("LlamaCXX does not implement `impl_greedy`") + diff --git a/modules/autocog/lm/llama.py b/modules/autocog/lm/llama_py.py similarity index 98% rename from modules/autocog/lm/llama.py rename to modules/autocog/lm/llama_py.py index afef34d..f9ffe08 100644 --- a/modules/autocog/lm/llama.py +++ b/modules/autocog/lm/llama_py.py @@ -8,7 +8,7 @@ llama_cpp = "Package `llama_cpp` needed for LLaMa wrapper (pip install git+https://github.com/tristanvdb/llama-cpp-python@choice-dev)" print(f"Warning: {llama_cpp}") -class Llama(LM): +class LlamaPY(LM): model: Any def __init__(self, model_path:str, logits_all=True, verbose=False, n_ctx=2048, **kwargs): @@ -51,3 +51,4 @@ def detokenize(self, tokens:List[int], whole:bool=True) -> str: def impl_greedy(self, prompt: Union[str,List[int]]) -> List[float]: output = self.model.create_completion(prompt, max_tokens=1, logprobs=-1, full_logprobs=True) return output['choices'][0]['logprobs'][0] + diff --git a/modules/autocog/lm/lm.py b/modules/autocog/lm/lm.py index f1e10c1..3b722e5 100644 --- a/modules/autocog/lm/lm.py +++ b/modules/autocog/lm/lm.py @@ -50,3 +50,4 @@ def greedy(self, prompt: Union[str,List[int]]): params = f"retries={self.retries}, delta={self.delta}s, growth={self.growth}x" errors = '\n - '.join(list(set(map(str,errors)))) raise Exception(f"Persisting exception when calling {self.__class__.__name__}.greedy()\n => {params}\n - {errors}") + diff --git a/modules/autocog/utility/args2arch.py b/modules/autocog/utility/args2arch.py index a660e8a..94407fd 100644 --- a/modules/autocog/utility/args2arch.py +++ b/modules/autocog/utility/args2arch.py @@ -26,7 +26,7 @@ def argparser(): parser.add_argument('--command', help="""Command to be executed by the architecture as a dictionary. `__tag` identify the cog while `__entry` identify the entry point in this cog (defaults to `main`). All other field will be forwarded as keyworded args. Example: `{ "__tag" : "writer", "__entry" : "main", **kwarg }` (inlined JSON or path to a file). Any command argument can be a list of dictionary.""", action='append') - parser.add_argument('--libdir', help="""Directory where results are stored.""", action='append', default=[]) + parser.add_argument('--libdir', help="""Directory where libraries are stored.""", action='append', default=[]) parser.add_argument('--output', help="""Directory where results are stored.""", default=os.getcwd()) parser.add_argument('--prefix', help="""String to identify this instance of AutoCog""", default='autocog') diff --git a/modules/autocog/utility/models.py b/modules/autocog/utility/models.py index a181a3a..6804217 100644 --- a/modules/autocog/utility/models.py +++ b/modules/autocog/utility/models.py @@ -1,14 +1,14 @@ from ..sta.syntax import Syntax, syntax_kwargs as SyntaxKwargs from ..lm import RLM -from ..lm import Llama +from ..lm import LlamaCXX def loader(models_path=None, syntax=None, n_ctx=4096, **syntax_kwargs): if models_path is None or models_path == '': models_path = '' lm = RLM() elif models_path.endswith('.gguf'): - lm = Llama(model_path=models_path, n_ctx=n_ctx) + lm = LlamaCXX(model_path=models_path, n_ctx=n_ctx) else: raise Exception(f'Unrecognized model file extension: {models_path.split(".")[-1]}') @@ -34,4 +34,4 @@ def loader(models_path=None, syntax=None, n_ctx=4096, **syntax_kwargs): syntax = Syntax(**syntax) - return (lm,syntax) \ No newline at end of file + return (lm,syntax) diff --git a/tests/cli/mcq/run.sh b/tests/cli/mcq/run.sh index 2596ae4..54d76ff 100755 --- a/tests/cli/mcq/run.sh +++ b/tests/cli/mcq/run.sh @@ -1,5 +1,7 @@ #!/bin/bash -e +bdir=$(realpath $(dirname "${BASH_SOURCE[0]}")/../../..) + model=$1 [ -z $model ] && model=none @@ -25,6 +27,6 @@ for cog in $COGS; do kind=$(basename $model | cut -d. -f1) quant=$(basename $model | cut -d. -f2) fi - python3 -m autocog --model "$model_path" --prefix $tag-$kind-$quant --cog $cog --command data.json \ + python3 -m autocog --model "$model_path" --prefix $tag-$kind-$quant --cog $cog --command $bdir/tests/cli/mcq/data.json --libdir $bdir/share/library \ --syntax '{ "prompt_with_format" : false, "prompt_with_index" : false, "prompt_indent" : " " }' done From 164d74990c10c7e48e5b26e3d433739e1f923358 Mon Sep 17 00:00:00 2001 From: Tristan Date: Sat, 9 Aug 2025 22:16:44 -0700 Subject: [PATCH 11/51] Finished integrating C++ evaluation of FTA into complete AutoCog flow. Restore optimization levels and disable verbosity --- Dockerfile | 6 +- libs/autocog/llama/CMakeLists.txt | 4 +- libs/autocog/llama/evaluation.cxx | 9 +- libs/autocog/llama/manager.cxx | 9 + libs/autocog/llama/python_bindings.cxx | 20 +- modules/autocog/arch/cogs.py | 13 +- modules/autocog/fta/automaton.py | 3 - modules/autocog/lm/__init__.py | 4 +- modules/autocog/lm/llama_cxx.py | 173 +++++++++++++++++- modules/autocog/sta/automaton.py | 8 +- .../llama/execute_sta_with_llama_cpp.py | 166 +---------------- 11 files changed, 226 insertions(+), 189 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7bd412c..2f1f653 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,8 @@ RUN apt-get update && \ python3 \ python3-pip \ python3-dev \ - python3-venv && \ + python3-venv \ + graphviz && \ rm -rf /var/lib/apt/lists/* # LLama.cpp @@ -29,7 +30,8 @@ ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" RUN python3 -m venv /opt ENV PATH="/opt/bin:$PATH" ENV PYTHONPATH="/opt/bin:$PATH" -RUN pip install --upgrade pip +RUN pip install --upgrade pip && \ + pip install graphviz # Autocog diff --git a/libs/autocog/llama/CMakeLists.txt b/libs/autocog/llama/CMakeLists.txt index dcb50b2..ce24565 100644 --- a/libs/autocog/llama/CMakeLists.txt +++ b/libs/autocog/llama/CMakeLists.txt @@ -39,7 +39,9 @@ target_compile_features(autocog_llama PRIVATE cxx_std_17) # $<$:-O3 -march=native -g> # $<$:/O2> #) -target_compile_options(autocog_llama PRIVATE -O0 -g3 -ggdb -fno-omit-frame-pointer -DVERBOSE=1) +#target_compile_options(autocog_llama PRIVATE -O0 -g3 -ggdb -fno-omit-frame-pointer -DVERBOSE=1) +#target_compile_options(autocog_llama PRIVATE -O3 -DVERBOSE=1) +target_compile_options(autocog_llama PRIVATE -O3) target_compile_options(autocog_llama PRIVATE $<$:-Wall -Wextra -Wpedantic> diff --git a/libs/autocog/llama/evaluation.cxx b/libs/autocog/llama/evaluation.cxx index 19e835d..7f301a4 100644 --- a/libs/autocog/llama/evaluation.cxx +++ b/libs/autocog/llama/evaluation.cxx @@ -10,7 +10,7 @@ #endif #define DEBUG_Evaluation_enqueue VERBOSE && 0 -#define DEBUG_Evaluation_advance VERBOSE && 1 +#define DEBUG_Evaluation_advance VERBOSE && 0 namespace autocog { namespace llama { @@ -45,11 +45,10 @@ unsigned Evaluation::advance(std::optional max_token_eval) { unsigned num_token_eval = 0; while (!queue.empty() && (max_token_eval == std::nullopt || num_token_eval < max_token_eval)) { PathState & state = queue.front(); +#if VERBOSE + std::cerr << "Evaluation::advance [ Q=" << queue.size() << ", A=" << num_action_eval << ", T=" << num_token_eval << " ]" << std::endl; +#endif #if DEBUG_Evaluation_advance - std::cerr << "Evaluation::advance" << std::endl; - std::cerr << " queue.size() = " << queue.size() << std::endl; - std::cerr << " num_action_eval = " << num_action_eval << std::endl; - std::cerr << " num_token_eval = " << num_token_eval << std::endl; std::cerr << " state.action = " << state.action << std::endl; std::cerr << " state.tokens.size() = " << state.tokens.size() << std::endl; std::cerr << " state.proba() = " << state.proba() << std::endl; diff --git a/libs/autocog/llama/manager.cxx b/libs/autocog/llama/manager.cxx index 052e581..7cfa02b 100644 --- a/libs/autocog/llama/manager.cxx +++ b/libs/autocog/llama/manager.cxx @@ -9,6 +9,12 @@ namespace autocog { namespace llama { +void quiet_log_callback(enum ggml_log_level level, const char * text, void * user_data) { + if (level == GGML_LOG_LEVEL_ERROR) { + fprintf(stderr, "%s", text); + } +} + Manager & Manager::instance() { static Manager __instance; return __instance; @@ -24,6 +30,9 @@ void Manager::cleanup() { } void Manager::initialize() { +#if VERBOSE == 0 + llama_log_set(quiet_log_callback, nullptr); +#endif llama_backend_init(); auto & manager = instance(); diff --git a/libs/autocog/llama/python_bindings.cxx b/libs/autocog/llama/python_bindings.cxx index 757e37f..8c21cb0 100644 --- a/libs/autocog/llama/python_bindings.cxx +++ b/libs/autocog/llama/python_bindings.cxx @@ -9,9 +9,13 @@ #include #include -#include #include +#if VERBOSE +# include +#endif +#define DEBUG_pybind_evaluate VERBOSE && 0 + namespace autocog { namespace llama { @@ -80,19 +84,33 @@ PYBIND11_MODULE(llama, module) { module.def("evaluate", [](ModelID model, pybind11::dict const & fta_dict) { +#if DEBUG_pybind_evaluate std::cerr << "IN evaluate (pybind): START" << std::endl; +#endif FTA fta = convert_pydict_to_fta(model, fta_dict); +#if DEBUG_pybind_evaluate std::cerr << "IN evaluate (pybind): FTA" << std::endl; +#endif EvalID eval = Manager::add_eval(model, fta); +#if DEBUG_pybind_evaluate std::cerr << "IN evaluate (pybind): EVAL" << std::endl; +#endif Manager::get_eval(eval).advance(std::nullopt); +#if DEBUG_pybind_evaluate std::cerr << "IN evaluate (pybind): FTT" << std::endl; +#endif FTT const & ftt = Manager::get_eval(eval).get(); +#if DEBUG_pybind_evaluate std::cerr << "IN evaluate (pybind): RES" << std::endl; +#endif pybind11::dict res = convert_ftt_to_pydict(model, ftt); +#if DEBUG_pybind_evaluate std::cerr << "IN evaluate (pybind): CLEAN" << std::endl; +#endif Manager::rm_eval(eval); +#if DEBUG_pybind_evaluate std::cerr << "IN evaluate (pybind): DONE" << std::endl; +#endif return res; }, "Evaluate a FTA using a model and return the FTT." diff --git a/modules/autocog/arch/cogs.py b/modules/autocog/arch/cogs.py index 95069a0..41b17ae 100644 --- a/modules/autocog/arch/cogs.py +++ b/modules/autocog/arch/cogs.py @@ -83,11 +83,18 @@ async def __call__(self, __page: Optional[Page]=None, **inputs) -> Any: __page.ftas[ptag].append(fta) fta.simplify() if hasattr(self.arch.lm, 'evaluate'): - ftt = fta.execute(lm=self.arch.lm) + (ftt, paths) = self.arch.lm.evaluate(fta) + (text, proba) = paths[0] + __page.ftts[ptag].append(ftt) else: ftt = fta.greedy(lm=self.arch.lm) - __page.ftts[ptag].append(ftt) - next = sta.parse(lm=self.arch.lm, syntax=self.arch.syntax, stacks=__page.stacks, ftt=ftt) + __page.ftts[ptag].append(ftt) + results = ftt.results(lm=lm, normalized=True) + # for r,res in enumerate(results): + # lines = res[0].split('\nstart:\n')[2].split('\n') + # print(f"[{r}]\n> " + "\n> ".join(lines) + f"\n[/{r}]") + text = results[-1][0] + next = sta.parse(lm=self.arch.lm, syntax=self.arch.syntax, stacks=__page.stacks, text=text) if isinstance(next, Return): if len(next.fields) == 1 and '_' in next.fields: return frame.read(next.fields['_']) diff --git a/modules/autocog/fta/automaton.py b/modules/autocog/fta/automaton.py index dea17a0..b5a3327 100644 --- a/modules/autocog/fta/automaton.py +++ b/modules/autocog/fta/automaton.py @@ -141,9 +141,6 @@ def greedy(self, lm: LM): assert root.finalized return root - def execute(self, lm): - raise NotImplementedError() - def toGraphViz(self, label_with_uid:bool=False): dotstr = "" for act in self.actions.values(): diff --git a/modules/autocog/lm/__init__.py b/modules/autocog/lm/__init__.py index 1075eb1..c310564 100644 --- a/modules/autocog/lm/__init__.py +++ b/modules/autocog/lm/__init__.py @@ -1,5 +1,5 @@ from .random import RLM from .llama_cxx import LlamaCXX -from .llama_py import LlamaPY -from .transformers import TfLM +# from .llama_py import LlamaPY +# from .transformers import TfLM diff --git a/modules/autocog/lm/llama_cxx.py b/modules/autocog/lm/llama_cxx.py index e38d44f..1e5f165 100644 --- a/modules/autocog/lm/llama_cxx.py +++ b/modules/autocog/lm/llama_cxx.py @@ -1,15 +1,176 @@ +import math +import string + from typing import Any, Dict, List, Tuple, Union, Optional, Callable from .lm import LM +from ..fta.actions import Choose, Text, Complete + from ..llama import create as autocog_llama_create +from ..llama import tokenize as autocog_llama_tokenize +from ..llama import detokenize as autocog_llama_detokenize +from ..llama import vocab_size as autocog_llama_vocab_size +from ..llama import evaluate as autocog_llama_evaluate + +def fta_to_cxx(model, fta, defaults, safe_mask, char_mask): + actions = [] + for act in fta.actions.values(): + action = { "uid" : act.uid, "successors" : act.successors } + if isinstance(act, Text): + action.update({ + "__type__" : "Text", + "evaluate" : defaults["Text"]["evaluate"] if act.evaluate is None else act.evaluate, + "tokens" : autocog_llama_tokenize(model, act.text, False, False) + }) + elif isinstance(act, Choose): + if len(act.successors) == 1: + action.update({ "successors" : [ act.successors[0] ] * len(act.choices) }) + assert len(act.successors) == 0 or len(action['successors']) == len(act.choices) + action.update({ + "__type__" : "Choose", + "choices" : [ + autocog_llama_tokenize(model, choice[0], False, False) for choice in act.choices + ], + "threshold" : defaults["Choose"]["threshold"] if act.threshold is None else act.threshold, + "width" : defaults["Choose"]["width"] if act.width is None else act.width + }) + elif isinstance(act, Complete): + assert act.length is not None + action.update({ + "__type__" : "Complete", + "length" : act.length, + "stop" : autocog_llama_tokenize(model, act.stop, False, False), + "vocab" : safe_mask, + "threshold" : defaults["Complete"]["threshold"] if act.threshold is None else act.threshold, + "beams" : defaults["Complete"]["beams"] if act.beams is None else act.beams, + "ahead" : defaults["Complete"]["ahead"] if act.ahead is None else act.ahead, + "width" : defaults["Complete"]["width"] if act.width is None else act.width, + }) + else: + raise Exception() + actions.append( action ) + return { 'actions' : actions } + +def cxx_to_ftt(model, ftt): + return { + "action" : ftt["action"], + "text" : autocog_llama_detokenize(model, ftt["tokens"], False, False), + "length" : ftt["length"], + "logprobs" : ftt["logprobs"], + "probability" : math.exp(-ftt["logprob"]/ftt["length"]), + "children" : [ cxx_to_ftt(model, child) for child in ftt["children"] ], + "pruned" : ftt["pruned"], + } + +def create_safe_vocab_mask(model): + vocab_size = autocog_llama_vocab_size(model) + mask = [True] * vocab_size + + problematic_count = 0 + + for token_id in range(vocab_size): + try: + # Test detokenize single token + text = autocog_llama_detokenize(model, [token_id], False, False) + + # Flag problematic tokens + is_problematic = ( + '\n' in text or # Newlines + '\r' in text or # Carriage returns + '\t' in text or # Tabs + len(text) == 0 or # Empty tokens + any(ord(c) < 32 for c in text if c not in [' ']) or # Control characters + any(ord(c) > 126 for c in text) # Non-ASCII + ) + + if is_problematic: + mask[token_id] = False + problematic_count += 1 + + except Exception: + # If detokenization fails, exclude the token + mask[token_id] = False + problematic_count += 1 + +# print(f"Excluded {problematic_count}/{vocab_size} problematic tokens") + return mask + +def create_single_char_vocab(model): + vocab_size = autocog_llama_vocab_size(model) + mask = [False] * vocab_size + + # Define character sets to include + chars_to_test = ( + string.ascii_letters + # a-z, A-Z + string.digits + # 0-9 + string.punctuation + # .,!? etc. + ' ' + '\n' # spaces + ) + + single_char_count = 0 + + for char in chars_to_test: + try: + # Tokenize the single character + tokens = autocog_llama_tokenize(model, char, False, False) + + # Check if it tokenizes to exactly one token + if len(tokens) == 1: + token_id = tokens[0] + if 0 <= token_id < vocab_size: # Validate range + mask[token_id] = True + single_char_count += 1 + + except Exception: + # Skip characters that can't be tokenized + pass + +# print(f"Found {single_char_count} single-character tokens from {len(chars_to_test)} tested characters") + return mask + +def extract_paths_from_ftt(ftt, current_path=""): + paths = [] + new_path = current_path + ftt["text"] + + if not "children" in ftt or len(ftt["children"]) == 0: + if not ftt["pruned"]: + paths.append((new_path, ftt["probability"])) + else: + for child in ftt["children"]: + child_paths = extract_paths_from_ftt(child, new_path) + paths.extend(child_paths) + return sorted(paths, key=lambda path: path[1], reverse=True) + +fta_defaults = { + "Text" : { + "evaluate" : True + }, + "Choose" : { + "threshold" : 0e-1, + "width" : 2 + }, + "Complete" : { + "threshold" : 0e-1, + "width" : 2, + "beams" : 3, + "ahead" : 1, + "diversity" : 1., + "repetition" : .2 + } +} class LlamaCXX(LM): model: Any + safe_mask: Any + char_mask: Any - def __init__(self, model_path:str, n_ctx=2048, **kwargs): + def __init__(self, model_path:str, n_ctx=4096, **kwargs): + model = autocog_llama_create(model_path, n_ctx) if len(model_path) > 0 else 0 super().__init__( - model=autocog_llama_create(model_path, n_ctx) if len(model_path) > 0 else 0 + model=model, + safe_mask=create_safe_vocab_mask(model), + char_mask=create_single_char_vocab(model) ) def tokenize(self, text:str, whole:bool=True) -> List[int]: @@ -18,8 +179,12 @@ def tokenize(self, text:str, whole:bool=True) -> List[int]: def detokenize(self, tokens:List[int], whole:bool=True) -> str: raise NotImplementedError("LlamaCXX.detokenize") - def evaluate(self, sta): - raise NotImplementedError("LlamaCXX.evaluate") + def evaluate(self, fta): + fta = fta_to_cxx(self.model, fta, fta_defaults, self.safe_mask, self.char_mask) + ftt = autocog_llama_evaluate(self.model, fta) + ftt = cxx_to_ftt(self.model, ftt) + paths = extract_paths_from_ftt(ftt) + return (ftt, paths) def impl_greedy(self, **kwargs): raise Exception("LlamaCXX does not implement `impl_greedy`") diff --git a/modules/autocog/sta/automaton.py b/modules/autocog/sta/automaton.py index a1d9ec2..5d6f2c4 100644 --- a/modules/autocog/sta/automaton.py +++ b/modules/autocog/sta/automaton.py @@ -538,14 +538,8 @@ def instantiate(self, syntax: Syntax, frame: Any, branches: Any, inputs: Any): fta.connect('next.field', 'next.choice') return fta - def parse(self, lm:LM, ftt:FTT, syntax: Syntax, stacks: Any): + def parse(self, lm:LM, text:str, syntax: Syntax, stacks: Any): result = None - - results = ftt.results(lm=lm, normalized=True) - # for r,res in enumerate(results): - # lines = res[0].split('\nstart:\n')[2].split('\n') - # print(f"[{r}]\n> " + "\n> ".join(lines) + f"\n[/{r}]") - text = results[-1][0] lines = text.split('\nstart:\n')[2].split('\n') # print("[Lines]\n> " + "\n> ".join(lines) + "\n[/Lines]") diff --git a/tests/autocog/llama/execute_sta_with_llama_cpp.py b/tests/autocog/llama/execute_sta_with_llama_cpp.py index 3f91351..9e09831 100644 --- a/tests/autocog/llama/execute_sta_with_llama_cpp.py +++ b/tests/autocog/llama/execute_sta_with_llama_cpp.py @@ -6,37 +6,19 @@ from autocog.sta.runtime import Frame from autocog.fta.automaton import FiniteThoughtAutomaton as FTA -from autocog.fta.actions import Choose, Text, Complete import autocog.llama from autocog.llama import tokenize, detokenize -fta_defaults = { - "Text" : { - "evaluate" : True - }, - "Choose" : { - "threshold" : 0e-3, - "width" : 1 - }, - "Complete" : { - "threshold" : 0e-3, - "width" : 2, - "beams" : 2, - "ahead" : 1, - "diversity" : 1., - "repetition" : .2 - } -} - def main(argv): sta_file = argv[1] json_data = argv[2] - model_path = argv[3] + model_path = argv[3] if len(argv) >= 4 else '' + prompt_name = argv[4] if len(argv) >= 5 else 'main' sta = compile_source_to_program_and_stas( open(sta_file, 'r').read() - )[1]['main'] + )[1][prompt_name] fta = sta.instantiate( syntax=Syntax(), @@ -48,16 +30,10 @@ def main(argv): inputs=None ).simplify() - model = autocog.llama.create(model_path, 4096) if len(model_path) > 0 else 0 - - safe_mask = create_safe_vocab_mask(model) - char_mask = create_single_char_vocab(model) + model = autocog.lm.LlamaCXX(model_path, 4096) - ftt = cxx_to_ftt(model, autocog.llama.evaluate(model, fta_to_cxx(model, fta, fta_defaults, safe_mask, char_mask))) + (ftt, paths) = model.evaluate(fta) -# print(json.dumps(ftt, indent=4)) - paths = extract_paths_from_ftt(ftt) - paths = sorted(paths, key=lambda x: x[1]) for (text, proba) in paths: print("========================================") print(f"[[ {proba} ]]") @@ -71,138 +47,6 @@ def main(argv): show_text_preview=True ) -def create_safe_vocab_mask(model): - """Create vocab mask that excludes problematic tokens""" - import autocog.llama - - vocab_size = autocog.llama.vocab_size(model) - mask = [True] * vocab_size - - problematic_count = 0 - - for token_id in range(vocab_size): - try: - # Test detokenize single token - text = autocog.llama.detokenize(model, [token_id], False, False) - - # Flag problematic tokens - is_problematic = ( - '\n' in text or # Newlines - '\r' in text or # Carriage returns - '\t' in text or # Tabs - len(text) == 0 or # Empty tokens - any(ord(c) < 32 for c in text if c not in [' ']) or # Control characters - any(ord(c) > 126 for c in text) # Non-ASCII - ) - - if is_problematic: - mask[token_id] = False - problematic_count += 1 - - except Exception: - # If detokenization fails, exclude the token - mask[token_id] = False - problematic_count += 1 - - print(f"Excluded {problematic_count}/{vocab_size} problematic tokens") - return mask - -def create_single_char_vocab(model): - vocab_size = autocog.llama.vocab_size(model) - mask = [False] * vocab_size - - # Define character sets to include - chars_to_test = ( - string.ascii_letters + # a-z, A-Z - string.digits + # 0-9 - string.punctuation + # .,!? etc. - ' ' + '\n' # spaces - ) - - single_char_count = 0 - - for char in chars_to_test: - try: - # Tokenize the single character - tokens = autocog.llama.tokenize(model, char, False, False) - - # Check if it tokenizes to exactly one token - if len(tokens) == 1: - token_id = tokens[0] - if 0 <= token_id < vocab_size: # Validate range - mask[token_id] = True - single_char_count += 1 - - except Exception: - # Skip characters that can't be tokenized - pass - - print(f"Found {single_char_count} single-character tokens from {len(chars_to_test)} tested characters") - return mask - -def fta_to_cxx(model, fta, defaults, safe_mask, char_mask): - actions = [] - for act in fta.actions.values(): - action = { "uid" : act.uid, "successors" : act.successors } - if isinstance(act, Text): - action.update({ - "__type__" : "Text", - "evaluate" : defaults["Text"]["evaluate"] if act.evaluate is None else act.evaluate, - "tokens" : tokenize(model, act.text, False, False) - }) - elif isinstance(act, Choose): - if len(act.successors) == 1: - action.update({ "successors" : [ act.successors[0] ] * len(act.choices) }) - assert len(act.successors) == 0 or len(action['successors']) == len(act.choices) - action.update({ - "__type__" : "Choose", - "choices" : [ - tokenize(model, choice[0], False, False) for choice in act.choices - ], - "threshold" : defaults["Choose"]["threshold"] if act.threshold is None else act.threshold, - "width" : defaults["Choose"]["width"] if act.width is None else act.width - }) - elif isinstance(act, Complete): - assert act.length is not None - action.update({ - "__type__" : "Complete", - "length" : act.length, - "stop" : tokenize(model, act.stop, False, False), - "vocab" : safe_mask, - "threshold" : defaults["Complete"]["threshold"] if act.threshold is None else act.threshold, - "beams" : defaults["Complete"]["beams"] if act.beams is None else act.beams, - "ahead" : defaults["Complete"]["ahead"] if act.ahead is None else act.ahead, - "width" : defaults["Complete"]["width"] if act.width is None else act.width, - }) - else: - raise Exception() - actions.append( action ) - return { 'actions' : actions } - -def cxx_to_ftt(model, ftt): - return { - "action" : ftt["action"], - "text" : detokenize(model, ftt["tokens"], False, False), - "length" : ftt["length"], - "logprobs" : ftt["logprobs"], - "probability" : math.exp(-ftt["logprob"]/ftt["length"]), - "children" : [ cxx_to_ftt(model, child) for child in ftt["children"] ], - "pruned" : ftt["pruned"], - } - -def extract_paths_from_ftt(ftt, current_path=""): - paths = [] - new_path = current_path + ftt["text"] - - if not "children" in ftt or len(ftt["children"]) == 0: - if not ftt["pruned"]: - paths.append((new_path, ftt["probability"])) - else: - for child in ftt["children"]: - child_paths = extract_paths_from_ftt(child, new_path) - paths.extend(child_paths) - return paths - def ftt_to_graphviz_detailed(ftt, output_filename='ftt_tree_detailed', format='png', max_text_length=30, show_text_preview=True): """ Detailed version with HTML-like labels for better formatting. From ea1f9f7cb756b72a945ef441d8484476286b50a7 Mon Sep 17 00:00:00 2001 From: vanderbrugge1 Date: Tue, 12 Aug 2025 05:34:06 -0700 Subject: [PATCH 12/51] Stated working on RHEL support (ubi container and setup script) --- .gitignore | 2 ++ Dockerfile.ubi | 49 +++++++++++++++++++++++++++++++++ Dockerfile => Dockerfile.ubuntu | 2 +- libs/autocog/llama/manager.hxx | 1 + setup.sh | 47 +++++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.ubi rename Dockerfile => Dockerfile.ubuntu (94%) create mode 100755 setup.sh diff --git a/.gitignore b/.gitignore index a9ff4c4..89b1d61 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ /Pipfile /modules/autocog/*.so /build/ +ftt.svg +.venv/ diff --git a/Dockerfile.ubi b/Dockerfile.ubi new file mode 100644 index 0000000..fef2a3d --- /dev/null +++ b/Dockerfile.ubi @@ -0,0 +1,49 @@ +FROM registry.access.redhat.com/ubi9/ubi:latest + +ENV PYTHONUNBUFFERED=1 + +# RUN rm -f /etc/yum.repos.d/*.repo && \ +# dnf clean all + +# RUN dnf config-manager --disable \* && \ +# dnf config-manager --enable ubi-9-appstream-rpms && \ +# dnf config-manager --enable ubi-9-baseos-rpms + +RUN cat /etc/yum.repos.d/*.repo && \ + dnf --disablerepo=\* --enablerepo=ubi-\* install -y \ + gcc gcc-c++ \ + cmake \ + pkgconf-pkg-config \ + python3 \ + python3-pip \ + python3-devel \ + graphviz && \ + dnf clean all + +# LLama.cpp + +COPY vendors/llama /tmp/llama_cpp + +RUN cd /tmp/llama_cpp && \ + cmake -DLLAMA_BUILD_COMMON=OFF -B build && \ + make -C build -j$(nproc) install + +ENV LD_LIBRARY_PATH="/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH" + +# Venv + +RUN python3 -m venv /opt +ENV PATH="/opt/bin:$PATH" +ENV PYTHONPATH="/opt/bin:$PATH" +RUN pip install --upgrade pip && \ + pip install graphviz + +# Autocog + +COPY . /workspace/autocog + +RUN rm -rf /workspace/autocog/vendors && \ + pip install /workspace/autocog && \ + rm -rf /workspace/autocog + +WORKDIR /workspace diff --git a/Dockerfile b/Dockerfile.ubuntu similarity index 94% rename from Dockerfile rename to Dockerfile.ubuntu index 2f1f653..2c473d9 100644 --- a/Dockerfile +++ b/Dockerfile.ubuntu @@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND=noninteractive ENV PYTHONUNBUFFERED=1 RUN apt-get update && \ - apt-get install -y \ + apt-get install -y --no-install-recommends \ build-essential \ cmake \ pkg-config \ diff --git a/libs/autocog/llama/manager.hxx b/libs/autocog/llama/manager.hxx index 65bcb5c..a056292 100644 --- a/libs/autocog/llama/manager.hxx +++ b/libs/autocog/llama/manager.hxx @@ -3,6 +3,7 @@ #include "autocog/llama/types.hxx" #include "autocog/llama/model.hxx" +#include "autocog/llama/evaluation.hxx" #include #include diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..f49a745 --- /dev/null +++ b/setup.sh @@ -0,0 +1,47 @@ +#!/bin/bash -ex + +# Simple AutoCog Setup Script +# Creates local .venv with llama.cpp and AutoCog + +echo "Setting up AutoCog in local .venv..." + +# Create and activate virtual environment +python3 -m venv .venv +source .venv/bin/activate + +# Upgrade pip and install build dependencies +pip install --upgrade pip setuptools wheel pybind11 numpy + +exit 0 + +# Build and install llama.cpp to .venv +echo "Building llama.cpp..." +cd vendors/llama +cmake -B build \ + -DCMAKE_INSTALL_PREFIX="$(pwd)/../../.venv" \ + -DLLAMA_BUILD_COMMON=OFF \ + -DLLAMA_LOG_DISABLE=ON \ + -DCMAKE_BUILD_TYPE=Release + +make -C build -j$(nproc) install +cd ../.. + +# Set library path for AutoCog build +export LD_LIBRARY_PATH="$(pwd)/.venv/lib:$LD_LIBRARY_PATH" +export CMAKE_PREFIX_PATH="$(pwd)/.venv" +export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) + +# Build and install AutoCog +echo "Building AutoCog..." +pip install -e . + +# Test installation +echo "Testing installation..." +python3 -c "import autocog.llama; print('Success!')" + +echo "Setup complete!" +echo "" +echo "To use AutoCog:" +echo " source .venv/bin/activate" +echo " python3 tests/autocog/llama/roundtrip_tokenization.py path/to/model.gguf" + From f49904c6c135f807bf0bf80529d88fa4f84d73cc Mon Sep 17 00:00:00 2001 From: vanderbrugge1 Date: Thu, 14 Aug 2025 06:36:29 -0700 Subject: [PATCH 13/51] Added CUDA support --- Dockerfile.ubi | 10 +------- Dockerfile.ubi-cuda | 41 +++++++++++++++++++++++++++++++ libs/autocog/llama/CMakeLists.txt | 4 +-- libs/autocog/llama/manager.cxx | 21 ++++++++++++++-- libs/autocog/llama/manager.hxx | 2 ++ libs/autocog/llama/model.cxx | 14 ++++++++--- modules/autocog/sta/syntax.py | 2 +- tests/samples/small.sta | 6 ++--- 8 files changed, 79 insertions(+), 21 deletions(-) create mode 100644 Dockerfile.ubi-cuda diff --git a/Dockerfile.ubi b/Dockerfile.ubi index fef2a3d..b03f8a4 100644 --- a/Dockerfile.ubi +++ b/Dockerfile.ubi @@ -2,15 +2,7 @@ FROM registry.access.redhat.com/ubi9/ubi:latest ENV PYTHONUNBUFFERED=1 -# RUN rm -f /etc/yum.repos.d/*.repo && \ -# dnf clean all - -# RUN dnf config-manager --disable \* && \ -# dnf config-manager --enable ubi-9-appstream-rpms && \ -# dnf config-manager --enable ubi-9-baseos-rpms - -RUN cat /etc/yum.repos.d/*.repo && \ - dnf --disablerepo=\* --enablerepo=ubi-\* install -y \ +RUN dnf --disablerepo=\* --enablerepo=ubi-\* install -y \ gcc gcc-c++ \ cmake \ pkgconf-pkg-config \ diff --git a/Dockerfile.ubi-cuda b/Dockerfile.ubi-cuda new file mode 100644 index 0000000..f192370 --- /dev/null +++ b/Dockerfile.ubi-cuda @@ -0,0 +1,41 @@ +FROM docker.io/nvidia/cuda:13.0.0-devel-ubi9 + +ENV PYTHONUNBUFFERED=1 + +RUN dnf --disablerepo=\* --enablerepo=ubi-\* install -y \ + gcc gcc-c++ \ + cmake \ + pkgconf-pkg-config \ + python3 \ + python3-pip \ + python3-devel \ + graphviz && \ + dnf clean all + +# LLama.cpp + +COPY vendors/llama /tmp/llama_cpp + +RUN cd /tmp/llama_cpp && \ + cmake -DLLAMA_BUILD_COMMON=OFF -DLLAMA_CUDA=ON -B build && \ + make -C build -j$(nproc) install + +ENV LD_LIBRARY_PATH="/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH" + +# Venv + +RUN python3 -m venv /opt +ENV PATH="/opt/bin:$PATH" +ENV PYTHONPATH="/opt/bin:$PATH" +RUN pip install --upgrade pip && \ + pip install graphviz + +# Autocog + +COPY . /workspace/autocog + +RUN rm -rf /workspace/autocog/vendors && \ + pip install /workspace/autocog && \ + rm -rf /workspace/autocog + +WORKDIR /workspace diff --git a/libs/autocog/llama/CMakeLists.txt b/libs/autocog/llama/CMakeLists.txt index ce24565..161c213 100644 --- a/libs/autocog/llama/CMakeLists.txt +++ b/libs/autocog/llama/CMakeLists.txt @@ -40,8 +40,8 @@ target_compile_features(autocog_llama PRIVATE cxx_std_17) # $<$:/O2> #) #target_compile_options(autocog_llama PRIVATE -O0 -g3 -ggdb -fno-omit-frame-pointer -DVERBOSE=1) -#target_compile_options(autocog_llama PRIVATE -O3 -DVERBOSE=1) -target_compile_options(autocog_llama PRIVATE -O3) +target_compile_options(autocog_llama PRIVATE -O3 -DVERBOSE=1) +#target_compile_options(autocog_llama PRIVATE -O3) target_compile_options(autocog_llama PRIVATE $<$:-Wall -Wextra -Wpedantic> diff --git a/libs/autocog/llama/manager.cxx b/libs/autocog/llama/manager.cxx index 7cfa02b..f6cd4d7 100644 --- a/libs/autocog/llama/manager.cxx +++ b/libs/autocog/llama/manager.cxx @@ -6,6 +6,10 @@ #include #include +#if VERBOSE +# include +#endif + namespace autocog { namespace llama { @@ -15,6 +19,8 @@ void quiet_log_callback(enum ggml_log_level level, const char * text, void * use } } +bool Manager::initialized{false}; + Manager & Manager::instance() { static Manager __instance; return __instance; @@ -25,11 +31,21 @@ Manager::~Manager() { } void Manager::cleanup() { - models.clear(); - llama_backend_free(); +#if VERBOSE + std::cerr << "Manager::cleanup()" << std::endl; +#endif + if (Manager::initialized) { + evaluations.clear(); + models.clear(); + llama_backend_free(); + Manager::initialized = false; + } } void Manager::initialize() { +#if VERBOSE + std::cerr << "Manager::initialize()" << std::endl; +#endif #if VERBOSE == 0 llama_log_set(quiet_log_callback, nullptr); #endif @@ -41,6 +57,7 @@ void Manager::initialize() { std::atexit([]() { instance().cleanup(); }); + Manager::initialized = true; } ModelID Manager::add_model(std::string const & path, int n_ctx) { diff --git a/libs/autocog/llama/manager.hxx b/libs/autocog/llama/manager.hxx index a056292..3e5784f 100644 --- a/libs/autocog/llama/manager.hxx +++ b/libs/autocog/llama/manager.hxx @@ -15,6 +15,8 @@ class Evaluation; class FTA; class Manager { + public: + static bool initialized; private: std::vector models; diff --git a/libs/autocog/llama/model.cxx b/libs/autocog/llama/model.cxx index f4dfa19..3ac7c58 100644 --- a/libs/autocog/llama/model.cxx +++ b/libs/autocog/llama/model.cxx @@ -49,10 +49,16 @@ Model::Model(ModelID const id_, std::string const & model_path, int n_ctx) : } Model::~Model() { - for (auto* ctx : this->contexts) if (ctx) llama_free(ctx); - contexts.clear(); - - if (this->model) llama_model_free(model); +#if VERBOSE + std::cerr << "Model::~Model(" << this->id << ")" << std::endl; +#endif + if (this->id == 0) { + // NOP + } else { + for (auto* ctx : this->contexts) if (ctx) llama_free(ctx); + contexts.clear(); + if (this->model) llama_model_free(model); + } } void Model::check_context_id(ContextID const id) const { diff --git a/modules/autocog/sta/syntax.py b/modules/autocog/sta/syntax.py index 6583dfa..88ed493 100644 --- a/modules/autocog/sta/syntax.py +++ b/modules/autocog/sta/syntax.py @@ -41,7 +41,7 @@ class Syntax(BaseModel): format_listing: str = "- " prompt_indent: str = "> " - system_msg: str = 'You are an AI expert interacting with your environment using a set of interactive questionnaires.' + system_msg: str = 'You are an AI expert interacting with your environment using a set of interactive questionnaires. A newline ends each statement (or prompt).' header_pre: str = '' header_mid: str = '\n' header_post: str = '\n' diff --git a/tests/samples/small.sta b/tests/samples/small.sta index ded90e8..9dbed9d 100644 --- a/tests/samples/small.sta +++ b/tests/samples/small.sta @@ -20,8 +20,8 @@ prompt main { from .correct; } annotate { - _ as "You are verifying whether statements are true or false."; - .statement as "the statement that you must verify"; - .correct as "your decision whether it is true or false"; + _ as "You are verifying whether statements are True or False."; + .statement as "The statement that you must verify. It is a single sentence."; + .correct as "Your decision whether the statement is True or False."; } } From 133842d9f4cd55cf262bd1a3ed57e17bac0d0f92 Mon Sep 17 00:00:00 2001 From: Tristan Date: Thu, 14 Aug 2025 06:42:33 -0700 Subject: [PATCH 14/51] Added parameters to control beam search and some minor changes of STA IR/syntax --- DEVEL.md | 36 ++++++++++++++++++ README.md | 18 --------- libs/autocog/llama/evaluation-completion.cxx | 22 +++++++---- libs/autocog/llama/ftt.hxx | 11 +++--- modules/autocog/lm/llama_cxx.py | 37 ++++++++++--------- modules/autocog/sta/ir.py | 20 +++++++--- modules/autocog/sta/syntax.py | 12 ++++-- modules/autocog/utility/models.py | 9 +++-- .../llama/execute_sta_with_llama_cpp.py | 18 +++++++-- 9 files changed, 119 insertions(+), 64 deletions(-) create mode 100644 DEVEL.md diff --git a/DEVEL.md b/DEVEL.md new file mode 100644 index 0000000..8a6f7dd --- /dev/null +++ b/DEVEL.md @@ -0,0 +1,36 @@ +# Develop Commands Cheat Sheet + +## C++ module: autocog.llama + +Build image and "test": +``` +docker build -t autocog:latest . +docker run --rm -v $(pwd):/workspace/autocog -it autocog:latest python3 /workspace/autocog/tests/autocog/llama/execute_sta_with_llama_cpp.py /workspace/autocog/tests/samples/mini.sta '{}' /workspace/autocog/models/SmolLM3-Q4_K_M.gguf +``` + +In container: +``` +docker run --rm -v $(pwd):/workspace/autocog -w /workspace/autocog -it autocog:latest bash +apt update && apt install -y gdb vim +pip install -e . +python3 tests/autocog/llama/execute_sta_with_llama_cpp.py tests/samples/mini.sta '{}' models/SmolLM3-Q4_K_M.gguf +``` + +## CUDA on RHEL with Podman + +First setup CUDA for container use (3rd command is for FIPS enable machines): +``` +curl -s -L https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo | sudo tee /etc/yum.repos.d/nvidia-container-toolkit.repo +sudo dnf --disablerepo=\* --enablerepo=nvidia-container-toolkit-experimental install -y nvidia-container-toolkit +sudo rpm -ivh --nodigest --nofiledigest /var/cache/dnf/nvidia-container-toolkit-experimental-*/packages/*.rpm +sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml +sudo chmod o+r /etc/cdi/nvidia.yaml +sudo chmod o+rx /etc/cdi +podman run --rm --device nvidia.com/gpu=all docker.io/nvidia/cuda:12.4.0-runtime-ubuntu22.04 nvidia-smi +``` +Then building and running the container: +``` +podman build --device nvidia.com/gpu=all -f Dockerfile.ubi-cuda -t autocog:ubi-cuda .\ +podman run --name autocog --rm -d --device nvidia.com/gpu=all -v $(pwd):/workspace/autocog -w /workspace/autocog -it autocog:ubi-cuda sleep infinity +podman exec -ti autocog python3 tests/autocog/llama/execute_sta_with_llama_cpp.py tests/samples/mini.sta '{}' models/SmolLM3-Q4_K_M.gguf +``` diff --git a/README.md b/README.md index 42ff003..a0bff45 100644 --- a/README.md +++ b/README.md @@ -18,24 +18,6 @@ We broke down the documentation into a few files: The libraries have [their own documentation](./share/library/README.md). -## Develop Command Cheat Sheet - -### C++ module: autocog.llama - -Build image and "test": -``` -docker build -t autocog:latest . -docker run --rm -v $(pwd):/workspace/autocog -it autocog:latest python3 /workspace/autocog/tests/autocog/llama/execute_sta_with_llama_cpp.py /workspace/autocog/tests/samples/mini.sta '{}' /workspace/autocog/models/SmolLM3-Q4_K_M.gguf -``` - -In container: -``` -docker run --rm -v $(pwd):/workspace/autocog -w /workspace/autocog -it autocog:latest bash -apt update && apt install -y gdb vim -pip install -e . -python3 tests/autocog/llama/execute_sta_with_llama_cpp.py tests/samples/mini.sta '{}' models/SmolLM3-Q4_K_M.gguf -``` - ## Contributing Contributions are welcome! diff --git a/libs/autocog/llama/evaluation-completion.cxx b/libs/autocog/llama/evaluation-completion.cxx index 31bea1d..47fa06a 100644 --- a/libs/autocog/llama/evaluation-completion.cxx +++ b/libs/autocog/llama/evaluation-completion.cxx @@ -41,9 +41,14 @@ struct BeamState { } }; -static float calculate_repetition_penalty(TokenSequence const & tokens, float & penalty, float const penalty_weight) { - size_t const min_length = 3; - size_t const max_window_size = 256; +static float calculate_repetition_penalty( + TokenSequence const & tokens, float & penalty, + float const penalty_weight, + size_t const min_length = 3, + size_t const max_window_size = 256, + float const length_weight = 1.0, + float const recency_weight = 1.0 +) { size_t window_size = std::min(tokens.size(), max_window_size); for (size_t i = min_length; i < tokens.size(); ++i) { @@ -71,9 +76,9 @@ static float calculate_repetition_penalty(TokenSequence const & tokens, float & // Stronger penalty for: // - Longer repetitions // - Recent repetitions (smaller distance) - float length_factor = std::log(best_length + 1); - float recency_factor = 1.0f / std::log(best_distance + 2); - penalty *= (1.0f + penalty_weight * length_factor * recency_factor); + float length_factor = std::log(1. + length_weight * best_length ); + float recency_factor = std::log(1. + recency_weight * best_distance); + penalty *= (1.0f + penalty_weight * length_factor / recency_factor); } } return penalty; @@ -151,7 +156,10 @@ static unsigned expand_beam( TokenSequence beam_tokens; beam_tokens.insert(beam_tokens.end(), base_tokens.begin(), base_tokens.end()); beam_tokens.insert(beam_tokens.end(), new_beam.tokens.begin(), new_beam.tokens.end()); - calculate_repetition_penalty(beam_tokens, new_beam.repetition_penalty, action.repetition.value()); + calculate_repetition_penalty( + beam_tokens, new_beam.repetition_penalty, + action.repetition.value() + ); } new_beam.stopped = ( diff --git a/libs/autocog/llama/ftt.hxx b/libs/autocog/llama/ftt.hxx index efaac39..d7614ef 100644 --- a/libs/autocog/llama/ftt.hxx +++ b/libs/autocog/llama/ftt.hxx @@ -12,11 +12,12 @@ namespace llama { class FTT { public: - ActionID const action; - TokenSequence const tokens; - ProbaSequence const logprobs; - float const logprob; - unsigned const length; + ActionID const action; //< Action evaluated for this node + TokenSequence const tokens; //< Tokens generated at this node + ProbaSequence const logprobs; //< Logprob for each token + + float const logprob; //< Cumulative logprob from the root + unsigned const length; //< Total length from the root bool pruned{false}; private: diff --git a/modules/autocog/lm/llama_cxx.py b/modules/autocog/lm/llama_cxx.py index 1e5f165..2a3e4be 100644 --- a/modules/autocog/lm/llama_cxx.py +++ b/modules/autocog/lm/llama_cxx.py @@ -13,6 +13,24 @@ from ..llama import vocab_size as autocog_llama_vocab_size from ..llama import evaluate as autocog_llama_evaluate +fta_defaults = { + "Text" : { + "evaluate" : True + }, + "Choose" : { + "threshold" : 0.4, + "width" : 2 + }, + "Complete" : { + "threshold" : 0.3, + "width" : 2, + "beams" : 3, + "ahead" : 1, + "diversity" : 1., + "repetition" : .5 + } +} + def fta_to_cxx(model, fta, defaults, safe_mask, char_mask): actions = [] for act in fta.actions.values(): @@ -59,6 +77,7 @@ def cxx_to_ftt(model, ftt): "length" : ftt["length"], "logprobs" : ftt["logprobs"], "probability" : math.exp(-ftt["logprob"]/ftt["length"]), + "locproba" : math.exp(-sum(ftt["logprobs"])/len(ftt["logprobs"])), "children" : [ cxx_to_ftt(model, child) for child in ftt["children"] ], "pruned" : ftt["pruned"], } @@ -142,24 +161,6 @@ def extract_paths_from_ftt(ftt, current_path=""): paths.extend(child_paths) return sorted(paths, key=lambda path: path[1], reverse=True) -fta_defaults = { - "Text" : { - "evaluate" : True - }, - "Choose" : { - "threshold" : 0e-1, - "width" : 2 - }, - "Complete" : { - "threshold" : 0e-1, - "width" : 2, - "beams" : 3, - "ahead" : 1, - "diversity" : 1., - "repetition" : .2 - } -} - class LlamaCXX(LM): model: Any safe_mask: Any diff --git a/modules/autocog/sta/ir.py b/modules/autocog/sta/ir.py index c02bc49..3b3cc01 100644 --- a/modules/autocog/sta/ir.py +++ b/modules/autocog/sta/ir.py @@ -97,21 +97,31 @@ def mechanics(self, mech, indent): mechs = '\n'.join(mechs) return mech + '\n```\n' + mechs + '\n```' - def formats(self, fmt, lst): + def formats(self, fmt, lst, detailed_formats): # TODO add enum, repeat, select, and text description as needed # TODO next if len(self.flows) > 0 formats = [ fld.format for fld in self.fields if fld.format is not None and fld.format.refname is not None ] if len(formats) > 0: fmtstrs = [] for f in formats: - fmtstrs.append(f"{lst}{f.label()}: {f.str()}") - fmtstrs += [ f" {lst}{desc}" for desc in f.desc ] + if detailed_formats: + fmtstrs.append(f"{lst}{f.label()}: {f.str()}") + fmtstrs += [ f" {lst}{desc}" for desc in f.desc ] + else: + fmtstr = f"{lst}{f.label()}: " + if len(f.desc) > 1: + fmtstrs.append(fmtstr) + fmtstrs += [ f" {lst}{desc}" for desc in f.desc ] + elif len(f.desc) == 1: + fmtstrs.append(f"{fmtstr}{f.desc[0]}") + else: + fmtstrs.append(fmtstr) return '\n' + fmt + '\n' + '\n'.join(fmtstrs) else: return '' - def header(self, mech:str, indent:str, fmt:str, lst:str): - return ' '.join(self.desc) + '\n' + self.mechanics(mech=mech, indent=indent) + self.formats(fmt=fmt, lst=lst) + def header(self, mech:str, indent:str, fmt:str, lst:str, detailed_formats:bool): + return ' '.join(self.desc) + '\n' + self.mechanics(mech=mech, indent=indent) + self.formats(fmt=fmt, lst=lst, detailed_formats=detailed_formats) class Program(BaseModel): desc: Optional[str] = None diff --git a/modules/autocog/sta/syntax.py b/modules/autocog/sta/syntax.py index 88ed493..e381b99 100644 --- a/modules/autocog/sta/syntax.py +++ b/modules/autocog/sta/syntax.py @@ -39,16 +39,18 @@ class Syntax(BaseModel): header_mechanic: str = "You are using the following syntax:" header_formats: str = "It includes the folowing named formats:" format_listing: str = "- " - prompt_indent: str = "> " + prompt_indent: str = "\t" system_msg: str = 'You are an AI expert interacting with your environment using a set of interactive questionnaires. A newline ends each statement (or prompt).' header_pre: str = '' header_mid: str = '\n' header_post: str = '\n' - prompt_with_format: bool = True - prompt_with_index: bool = True + prompt_with_format: bool = False + prompt_with_index: bool = False prompt_zero_index: bool = False + + detailed_formats: bool = False @staticmethod def Llama2Chat(**kwargs): @@ -70,6 +72,8 @@ def header(self, prompt: Prompt): mech=self.header_mechanic, indent=self.prompt_indent, fmt=self.header_formats, - lst=self.format_listing + lst=self.format_listing, + detailed_formats=self.detailed_formats ) return self.header_pre + self.system_msg + self.header_mid + header + self.header_post + 'start:\n' + diff --git a/modules/autocog/utility/models.py b/modules/autocog/utility/models.py index 6804217..8fc91e5 100644 --- a/modules/autocog/utility/models.py +++ b/modules/autocog/utility/models.py @@ -1,14 +1,17 @@ from ..sta.syntax import Syntax, syntax_kwargs as SyntaxKwargs from ..lm import RLM -from ..lm import LlamaCXX -def loader(models_path=None, syntax=None, n_ctx=4096, **syntax_kwargs): +def loader(models_path=None, syntax=None, n_ctx=4096, use_cxx=True, **syntax_kwargs): if models_path is None or models_path == '': models_path = '' lm = RLM() - elif models_path.endswith('.gguf'): + elif models_path.endswith('.gguf') and use_cxx: + from ..lm import LlamaCXX lm = LlamaCXX(model_path=models_path, n_ctx=n_ctx) + elif models_path.endswith('.gguf') and not use_cxx: + from ..lm import LlamaPY + lm = LlamaPY(model_path=models_path, n_ctx=n_ctx) else: raise Exception(f'Unrecognized model file extension: {models_path.split(".")[-1]}') diff --git a/tests/autocog/llama/execute_sta_with_llama_cpp.py b/tests/autocog/llama/execute_sta_with_llama_cpp.py index 9e09831..b363e2a 100644 --- a/tests/autocog/llama/execute_sta_with_llama_cpp.py +++ b/tests/autocog/llama/execute_sta_with_llama_cpp.py @@ -7,6 +7,8 @@ from autocog.fta.automaton import FiniteThoughtAutomaton as FTA +from autocog.utility.models import loader + import autocog.llama from autocog.llama import tokenize, detokenize @@ -20,8 +22,10 @@ def main(argv): open(sta_file, 'r').read() )[1][prompt_name] + (model, syntax) = loader(models_path=model_path, n_ctx=4096, use_cxx=True) + fta = sta.instantiate( - syntax=Syntax(), + syntax=syntax, frame=Frame( state={ st.label() : None for st in sta.concretes.values() if st.abstract.field is not None }, data=json.loads(json_data) @@ -30,8 +34,6 @@ def main(argv): inputs=None ).simplify() - model = autocog.lm.LlamaCXX(model_path, 4096) - (ftt, paths) = model.evaluate(fta) for (text, proba) in paths: @@ -95,6 +97,7 @@ def add_node_recursive(node, parent_id=None, depth=0, is_best=True): # Extract node properties probability = node["probability"] + locproba = node["locproba"] text = node.get("text", "") is_pruned = node["pruned"] @@ -105,6 +108,13 @@ def add_node_recursive(node, parent_id=None, depth=0, is_best=True): prob_str = f"{probability:.4f}" else: prob_str = f"{probability:.3f}" + + if locproba < 0.001: + locprob_str = f"{locproba:.2e}" + elif locproba < 0.01: + locprob_str = f"{locproba:.4f}" + else: + locprob_str = f"{locproba:.3f}" # Determine colors if is_pruned: @@ -124,7 +134,7 @@ def add_node_recursive(node, parent_id=None, depth=0, is_best=True): display_text = format_text_for_display(text, max_text_length) label = f'''< - +
P = {prob_str}
P = {prob_str}(p = {locprob_str})
{display_text}
>''' From c11cd7e1c817005b9697f07b8e4095c2393bbcda Mon Sep 17 00:00:00 2001 From: Tristan Vanderbruggen Date: Mon, 25 Aug 2025 05:26:37 -0700 Subject: [PATCH 15/51] Added modules/autocog.egg-info/ to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 89b1d61..73c612c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ /build/ ftt.svg .venv/ +modules/autocog.egg-info/ From e6ce75fd6ddef25775bbaecce02906d7e1b7d15b Mon Sep 17 00:00:00 2001 From: Tristan Vanderbruggen Date: Tue, 26 Aug 2025 05:52:23 -0700 Subject: [PATCH 16/51] Temporary disabled deleting llama contexts as it causes a CUDA error --- libs/autocog/llama/CMakeLists.txt | 4 ++-- libs/autocog/llama/model.cxx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/autocog/llama/CMakeLists.txt b/libs/autocog/llama/CMakeLists.txt index 161c213..ce24565 100644 --- a/libs/autocog/llama/CMakeLists.txt +++ b/libs/autocog/llama/CMakeLists.txt @@ -40,8 +40,8 @@ target_compile_features(autocog_llama PRIVATE cxx_std_17) # $<$:/O2> #) #target_compile_options(autocog_llama PRIVATE -O0 -g3 -ggdb -fno-omit-frame-pointer -DVERBOSE=1) -target_compile_options(autocog_llama PRIVATE -O3 -DVERBOSE=1) -#target_compile_options(autocog_llama PRIVATE -O3) +#target_compile_options(autocog_llama PRIVATE -O3 -DVERBOSE=1) +target_compile_options(autocog_llama PRIVATE -O3) target_compile_options(autocog_llama PRIVATE $<$:-Wall -Wextra -Wpedantic> diff --git a/libs/autocog/llama/model.cxx b/libs/autocog/llama/model.cxx index 3ac7c58..ea8f7e7 100644 --- a/libs/autocog/llama/model.cxx +++ b/libs/autocog/llama/model.cxx @@ -55,7 +55,7 @@ Model::~Model() { if (this->id == 0) { // NOP } else { - for (auto* ctx : this->contexts) if (ctx) llama_free(ctx); + // for (auto* ctx : this->contexts) if (ctx) llama_free(ctx); contexts.clear(); if (this->model) llama_model_free(model); } From 9cc8282c127fe07d2b83de11cca486da054425c4 Mon Sep 17 00:00:00 2001 From: Tristan Vanderbruggen Date: Tue, 26 Aug 2025 07:50:50 -0700 Subject: [PATCH 17/51] Added a test harness for laguage parsing --- tests/parse_sta_files.py | 199 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 tests/parse_sta_files.py diff --git a/tests/parse_sta_files.py b/tests/parse_sta_files.py new file mode 100644 index 0000000..90e652b --- /dev/null +++ b/tests/parse_sta_files.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +""" +Test harness for parsing all .sta files in a directory tree. +Usage: python tests/parse_sta_files.py [directory] +If no directory is specified, uses current directory. +""" + +import os +import sys +import argparse +from pathlib import Path +from autocog.sta.frontend import frontend + + +def find_sta_files(directory): + """Recursively find all .sta files in the given directory.""" + sta_files = [] + + for root, dirs, files in os.walk(directory): + for file in files: + if file.endswith('.sta'): + sta_files.append(Path(root) / file) + + return sorted(sta_files) + + +def parse_sta_file(filepath): + """ + Parse a single .sta file and return success status with error details. + + Returns: + tuple: (success: bool, error_msg: str or None) + """ + try: + with open(filepath, 'r') as f: + content = f.read() + + # Parse the STA file + ast = frontend(content) + return (True, None) + + except FileNotFoundError: + return (False, f"File not found: {filepath}") + except Exception as e: + # Get the error type and message + error_type = type(e).__name__ + error_msg = str(e) + + # Try to extract line number if available in the error + if hasattr(e, 'line'): + return (False, f"{error_type} at line {e.line}: {error_msg}") + else: + return (False, f"{error_type}: {error_msg}") + + +def get_relative_path(filepath, base_dir): + """Get a nice relative path for display.""" + try: + return filepath.relative_to(base_dir) + except ValueError: + # If filepath is not relative to base_dir, return absolute path + return filepath + + +def print_file_snippet(filepath, max_lines=5): + """Print the first few lines of a file for context.""" + try: + with open(filepath, 'r') as f: + lines = f.readlines()[:max_lines] + + print(" File content (first {} lines):".format(min(len(lines), max_lines))) + for i, line in enumerate(lines, 1): + print(f" {i:3}: {line.rstrip()}") + + if len(lines) == max_lines: + print(f" ... ({filepath.stat().st_size} bytes total)") + except Exception as e: + print(f" Could not read file content: {e}") + + +def main(): + # Set up argument parser + parser = argparse.ArgumentParser( + description='Parse all .sta files in a directory tree to validate syntax.' + ) + parser.add_argument( + 'directory', + nargs='?', + default='.', + help='Directory to search for .sta files (default: current directory)' + ) + parser.add_argument( + '--verbose', '-v', + action='store_true', + help='Show file snippets for failed parses' + ) + parser.add_argument( + '--quiet', '-q', + action='store_true', + help='Only show summary, not individual file results' + ) + parser.add_argument( + '--fail-fast', '-f', + action='store_true', + help='Stop on first parse failure' + ) + + args = parser.parse_args() + + # Convert directory to Path object + search_dir = Path(args.directory).resolve() + + if not search_dir.exists(): + print(f"Error: Directory '{search_dir}' does not exist") + sys.exit(1) + + if not search_dir.is_dir(): + print(f"Error: '{search_dir}' is not a directory") + sys.exit(1) + + # Header + print("=" * 70) + print("AutoCog STA File Parser Test Harness") + print("=" * 70) + print(f"Searching for .sta files in: {search_dir}") + print() + + # Find all .sta files + sta_files = find_sta_files(search_dir) + + if not sta_files: + print("No .sta files found in the specified directory") + sys.exit(0) + + print(f"Found {len(sta_files)} .sta file(s)") + print("-" * 70) + print() + + # Process each file + passed = 0 + failed = 0 + failed_files = [] + + for filepath in sta_files: + relative_path = get_relative_path(filepath, search_dir) + + # Parse the file + success, error_msg = parse_sta_file(filepath) + + if success: + passed += 1 + if not args.quiet: + print(f"✓ {relative_path}") + else: + failed += 1 + failed_files.append((relative_path, error_msg)) + + if not args.quiet: + print(f"✗ {relative_path}") + print(f" Error: {error_msg}") + + if args.verbose: + print_file_snippet(filepath) + + print() + + if args.fail_fast: + print("\nStopping due to --fail-fast flag") + break + + # Summary + print() + print("=" * 70) + print("Summary") + print("=" * 70) + print(f"Total files: {len(sta_files)}") + print(f"Passed: {passed} ({passed/len(sta_files)*100:.1f}%)") + print(f"Failed: {failed} ({failed/len(sta_files)*100:.1f}%)") + + if failed > 0 and not args.quiet: + print() + print("Failed files:") + for filepath, error_msg in failed_files: + print(f" - {filepath}") + if args.verbose: + print(f" {error_msg}") + + print("=" * 70) + + # Exit with appropriate code + if failed > 0: + sys.exit(1) + else: + print("\nAll .sta files parsed successfully!") + sys.exit(0) + + +if __name__ == "__main__": + main() From 40d97e6e73057911d24435f9506a1d5a43f8cfa3 Mon Sep 17 00:00:00 2001 From: Tristan Date: Tue, 26 Aug 2025 19:31:45 -0700 Subject: [PATCH 18/51] Started working on a C++ implementation of the parser. --- Dockerfile.reflex | 21 + libs/autocog/parser/CMakeLists.txt | 71 + libs/autocog/parser/stl/binding.cxx | 333 + libs/autocog/parser/stl/diagnostic.cxx | 57 + libs/autocog/parser/stl/diagnostic.hxx | 29 + libs/autocog/parser/stl/ir.hxx | 151 + libs/autocog/parser/stl/ir/annot.hxx | 18 + libs/autocog/parser/stl/ir/channel.hxx | 31 + libs/autocog/parser/stl/ir/define.hxx | 14 + libs/autocog/parser/stl/ir/expr.hxx | 58 + libs/autocog/parser/stl/ir/flow.hxx | 18 + libs/autocog/parser/stl/ir/path.hxx | 20 + libs/autocog/parser/stl/ir/program.cxx | 22 + libs/autocog/parser/stl/ir/program.hxx | 47 + libs/autocog/parser/stl/ir/prompt.hxx | 20 + libs/autocog/parser/stl/ir/record.hxx | 17 + libs/autocog/parser/stl/ir/return.hxx | 14 + libs/autocog/parser/stl/ir/search.hxx | 17 + libs/autocog/parser/stl/ir/struct.hxx | 34 + libs/autocog/parser/stl/lexer.cxx | 292 + libs/autocog/parser/stl/lexer.hxx | 125 + libs/autocog/parser/stl/lexer.l | 108 + libs/autocog/parser/stl/location.hxx | 15 + libs/autocog/parser/stl/main.cxx | 115 + libs/autocog/parser/stl/parser.cxx | 90 + libs/autocog/parser/stl/parser.hxx | 44 + libs/autocog/parser/stl/token.cxx | 57 + libs/autocog/parser/stl/token.hxx | 72 + modules/autocog/sta/ir.py | 1 + tests/samples/tiny.sta | 15 + vendors/headers/nlohmann/json.hpp | 24765 +++++++++++++++++++++++ 31 files changed, 26691 insertions(+) create mode 100644 Dockerfile.reflex create mode 100644 libs/autocog/parser/CMakeLists.txt create mode 100644 libs/autocog/parser/stl/binding.cxx create mode 100644 libs/autocog/parser/stl/diagnostic.cxx create mode 100644 libs/autocog/parser/stl/diagnostic.hxx create mode 100644 libs/autocog/parser/stl/ir.hxx create mode 100644 libs/autocog/parser/stl/ir/annot.hxx create mode 100644 libs/autocog/parser/stl/ir/channel.hxx create mode 100644 libs/autocog/parser/stl/ir/define.hxx create mode 100644 libs/autocog/parser/stl/ir/expr.hxx create mode 100644 libs/autocog/parser/stl/ir/flow.hxx create mode 100644 libs/autocog/parser/stl/ir/path.hxx create mode 100644 libs/autocog/parser/stl/ir/program.cxx create mode 100644 libs/autocog/parser/stl/ir/program.hxx create mode 100644 libs/autocog/parser/stl/ir/prompt.hxx create mode 100644 libs/autocog/parser/stl/ir/record.hxx create mode 100644 libs/autocog/parser/stl/ir/return.hxx create mode 100644 libs/autocog/parser/stl/ir/search.hxx create mode 100644 libs/autocog/parser/stl/ir/struct.hxx create mode 100644 libs/autocog/parser/stl/lexer.cxx create mode 100644 libs/autocog/parser/stl/lexer.hxx create mode 100644 libs/autocog/parser/stl/lexer.l create mode 100644 libs/autocog/parser/stl/location.hxx create mode 100644 libs/autocog/parser/stl/main.cxx create mode 100644 libs/autocog/parser/stl/parser.cxx create mode 100644 libs/autocog/parser/stl/parser.hxx create mode 100644 libs/autocog/parser/stl/token.cxx create mode 100644 libs/autocog/parser/stl/token.hxx create mode 100644 tests/samples/tiny.sta create mode 100644 vendors/headers/nlohmann/json.hpp diff --git a/Dockerfile.reflex b/Dockerfile.reflex new file mode 100644 index 0000000..48a36e3 --- /dev/null +++ b/Dockerfile.reflex @@ -0,0 +1,21 @@ +FROM registry.access.redhat.com/ubi9/ubi:latest + +ENV PYTHONUNBUFFERED=1 + +RUN dnf --disablerepo=\* --enablerepo=ubi-\* install -y \ + gcc gcc-c++ \ + cmake \ + git && \ + dnf clean all + +# RE/Flex + +RUN git clone https://github.com/Genivia/RE-flex.git /tmp/reflex && \ + cd /tmp/reflex && \ + ./build && \ + ./allinstall.sh && \ + cp lib/*.so /usr/local/lib + +# cd /workspace/libs/autocog/parser/stl && reflex --header-file=lexer.hxx --outfile=lexer.cxx lexer.l + +WORKDIR /workspace diff --git a/libs/autocog/parser/CMakeLists.txt b/libs/autocog/parser/CMakeLists.txt new file mode 100644 index 0000000..745fc1e --- /dev/null +++ b/libs/autocog/parser/CMakeLists.txt @@ -0,0 +1,71 @@ +cmake_minimum_required(VERSION 3.12) +project(autocog_parser) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(Python COMPONENTS Interpreter Development REQUIRED) +find_package(pybind11 QUIET) +if(NOT pybind11_FOUND) + # Get pybind11 from Python + execute_process( + COMMAND ${Python_EXECUTABLE} -c "import pybind11; print(pybind11.get_cmake_dir())" + OUTPUT_VARIABLE pybind11_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) + if(pybind11_DIR) + find_package(pybind11 REQUIRED PATHS ${pybind11_DIR}) + else() + message(FATAL_ERROR "pybind11 not found. Install with: pip install pybind11") + endif() +endif() + +# RE/flex lexer generation +find_program(REFLEX reflex REQUIRED) +find_library(REFLEX_LIB reflex REQUIRED) + +# Generate lexer from .l file +add_custom_command( + OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/stl/lexer.cxx ${CMAKE_CURRENT_SOURCE_DIR}/stl/lexer.hxx + COMMAND ${REFLEX} --header-file=${CMAKE_CURRENT_SOURCE_DIR}/stl/lexer.hxx + --outfile=${CMAKE_CURRENT_SOURCE_DIR}/stl/lexer.cxx + ${CMAKE_CURRENT_SOURCE_DIR}/stl/lexer.l + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/stl/lexer.l + COMMENT "Generating lexer with RE/flex" +) + +add_library(autocog_parser_stl_lib STATIC stl/lexer.cxx stl/parser.cxx stl/token.cxx stl/diagnostic.cxx stl/ir/program.cxx) + +target_include_directories(autocog_parser_stl_lib PUBLIC ../.. ../../../vendors/headers) +if(REFLEX_LIB) + target_link_libraries(autocog_parser_stl_lib PUBLIC ${REFLEX_LIB}) +else() + message(FATAL_ERROR "RE/flex library not found, trying to link manually") +# target_link_libraries(autocog_parser_stl_lib PUBLIC reflex) +endif() +set_property(TARGET autocog_parser_stl_lib PROPERTY POSITION_INDEPENDENT_CODE ON) + +# Executable `autocog-parser-stl` + +add_executable(autocog_parser_stl_cli stl/main.cxx) +target_link_libraries(autocog_parser_stl_cli PUBLIC autocog_parser_stl_lib) +set_target_properties(autocog_parser_stl_cli PROPERTIES OUTPUT_NAME "autocog-parser-stl") +install(TARGETS autocog_parser_stl_cli DESTINATION bin) + +# Python module +#pybind11_add_module(stl_parser_cpp +# stl/binding.cxx +#) + +#target_link_libraries(stl_parser_cpp PRIVATE +# autocog_parser_stl_lib +#) + + + +# set_property(TARGET stl_parser_cpp PROPERTY POSITION_INDEPENDENT_CODE ON) + + +# install(TARGETS stl_parser_cpp DESTINATION ${Python_SITEARCH}) + diff --git a/libs/autocog/parser/stl/binding.cxx b/libs/autocog/parser/stl/binding.cxx new file mode 100644 index 0000000..c2925b3 --- /dev/null +++ b/libs/autocog/parser/stl/binding.cxx @@ -0,0 +1,333 @@ + +#include "autocog/parser/stl/parser.hxx" +#include "autocog/parser/stl/ir.hxx" + +#include +#include + +namespace py = pybind11; + +// Helper to convert C++ IR to Python dict +py::dict ir_to_dict(const autocog::parser::IrProgram& program) { + py::dict result; + + // Convert variables + py::list vars; + for (const auto& var : program.variables) { + py::dict v; + v["name"] = var.name; + + // Convert value variant + if (std::holds_alternative(var.value)) { + v["value"] = std::get(var.value); + } else if (std::holds_alternative(var.value)) { + v["value"] = std::get(var.value); + } else if (std::holds_alternative(var.value)) { + v["value"] = std::get(var.value); + } else if (std::holds_alternative(var.value)) { + v["value"] = std::get(var.value); + } else { + v["value"] = py::none(); + } + + v["is_argument"] = var.is_argument; + vars.append(v); + } + if (!program.variables.empty()) { + result["variables"] = vars; + } + + // Convert formats + py::list formats; + for (const auto& format : program.formats) { + py::dict f; + f["name"] = format.name; + + py::list fields; + for (const auto& field : format.fields) { + py::dict fld; + fld["name"] = field.name; + fld["type"] = static_cast(field.type); + if (!field.type_name.empty()) { + fld["type_name"] = field.type_name; + } + if (field.default_value.has_value()) { + const auto& val = field.default_value.value(); + if (std::holds_alternative(val)) { + fld["default_value"] = std::get(val); + } else if (std::holds_alternative(val)) { + fld["default_value"] = std::get(val); + } else if (std::holds_alternative(val)) { + fld["default_value"] = std::get(val); + } else if (std::holds_alternative(val)) { + fld["default_value"] = std::get(val); + } + } + if (field.array_size.has_value()) { + fld["array_size"] = field.array_size.value(); + } + fld["is_argument"] = field.is_argument; + if (!field.annotations.empty()) { + fld["annotations"] = field.annotations; + } + fields.append(fld); + } + f["fields"] = fields; + + if (!format.annotations.empty()) { + f["annotations"] = format.annotations; + } + + formats.append(f); + } + if (!program.formats.empty()) { + result["formats"] = formats; + } + + // Convert structs + py::list structs; + for (const auto& strct : program.structs) { + py::dict s; + s["name"] = strct.name; + + py::list fields; + for (const auto& field : strct.fields) { + py::dict fld; + fld["name"] = field.name; + fld["type"] = static_cast(field.type); + if (!field.type_name.empty()) { + fld["type_name"] = field.type_name; + } + if (field.array_size.has_value()) { + fld["array_size"] = field.array_size.value(); + } + fld["is_argument"] = field.is_argument; + if (!field.annotations.empty()) { + fld["annotations"] = field.annotations; + } + fields.append(fld); + } + s["fields"] = fields; + + if (!strct.annotations.empty()) { + s["annotations"] = strct.annotations; + } + + structs.append(s); + } + if (!program.structs.empty()) { + result["structs"] = structs; + } + + // Convert prompts + py::list prompts; + for (const auto& prompt : program.prompts) { + py::dict p; + p["name"] = prompt.name; + + // Fields + py::list fields; + for (const auto& field : prompt.fields) { + py::dict fld; + fld["name"] = field.name; + fld["type"] = static_cast(field.type); + if (!field.type_name.empty()) { + fld["type_name"] = field.type_name; + } + if (field.array_size.has_value()) { + fld["array_size"] = field.array_size.value(); + } + fld["is_argument"] = field.is_argument; + if (!field.annotations.empty()) { + fld["desc"] = field.annotations; // Python IR uses 'desc' + } + fields.append(fld); + } + p["fields"] = fields; + + // Channels + py::list channels; + for (const auto& channel : prompt.channels) { + py::dict ch; + ch["target_path"] = channel.target_path; + + if (channel.source_path.has_value()) { + ch["source_path"] = channel.source_path.value(); + } + + if (channel.call.has_value()) { + py::dict call; + const auto& c = channel.call.value(); + + if (c.extern_cog.has_value()) { + call["extern"] = c.extern_cog.value(); + } + if (c.entry_point.has_value()) { + call["entry"] = c.entry_point.value(); + } + if (!c.kwargs.empty()) { + call["kwargs"] = c.kwargs; + } + if (!c.maps.empty()) { + call["maps"] = c.maps; + } + if (!c.binds.empty()) { + call["binds"] = c.binds; + } + + ch["call"] = call; + } + + channels.append(ch); + } + if (!prompt.channels.empty()) { + p["channels"] = channels; + } + + // Flows + if (!prompt.flows.empty()) { + py::dict flows; + for (const auto& [tag, flow] : prompt.flows) { + py::dict f; + f["prompt"] = flow.target_prompt; // Python IR uses 'prompt' not 'target_prompt' + f["limit"] = flow.limit; + if (flow.alias.has_value()) { + f["alias"] = flow.alias.value(); + } + flows[tag.c_str()] = f; + } + p["flows"] = flows; + } + + // Return + if (prompt.return_stmt.has_value()) { + py::dict ret; + const auto& r = prompt.return_stmt.value(); + + if (r.alias.has_value()) { + ret["alias"] = r.alias.value(); + } + + py::list fields; + for (const auto& [path, rename] : r.fields) { + py::dict field; + field["path"] = path; + if (rename.has_value()) { + field["rename"] = rename.value(); + } + fields.append(field); + } + ret["fields"] = fields; + + p["return"] = ret; + } + + // Annotations (desc in Python) + if (!prompt.annotations.empty()) { + p["desc"] = prompt.annotations; + } + + prompts.append(p); + } + if (!program.prompts.empty()) { + result["prompts"] = prompts; + } + + // Global flows + if (!program.global_flows.empty()) { + py::dict flows; + for (const auto& [tag, flow] : program.global_flows) { + py::dict f; + f["prompt"] = flow.target_prompt; + f["limit"] = flow.limit; + if (flow.alias.has_value()) { + f["alias"] = flow.alias.value(); + } + flows[tag.c_str()] = f; + } + result["global_flows"] = flows; + } + + // Global annotations + if (!program.global_annotations.empty()) { + result["global_annotations"] = program.global_annotations; + } + + return result; +} + +PYBIND11_MODULE(stl_parser_cpp, m) { + m.doc() = "C++ STL parser for AutoCog"; + + // FieldType enum + py::enum_(m, "FieldType") + .value("TEXT", autocog::parser::FieldType::TEXT) + .value("SELECT", autocog::parser::FieldType::SELECT) + .value("GENERATE", autocog::parser::FieldType::GENERATE) + .value("INTEGER", autocog::parser::FieldType::INTEGER) + .value("BOOLEAN", autocog::parser::FieldType::BOOLEAN) + .value("FLOAT", autocog::parser::FieldType::FLOAT) + .value("CUSTOM", autocog::parser::FieldType::CUSTOM); + + // Diagnostic class + py::class_(m, "Diagnostic") + .def_readonly("level", &autocog::parser::Diagnostic::level) + .def_property_readonly("line", [](const autocog::parser::Diagnostic& d) { return d.location.line; }) + .def_property_readonly("column", [](const autocog::parser::Diagnostic& d) { return d.location.column; }) + .def_readonly("message", &autocog::parser::Diagnostic::message) + .def_readonly("source_line", &autocog::parser::Diagnostic::source_line) + .def_readonly("notes", &autocog::parser::Diagnostic::notes) + .def("format", &autocog::parser::Diagnostic::format) + .def("__str__", &autocog::parser::Diagnostic::format); + + // Diagnostic Level enum + py::enum_(m, "DiagnosticLevel") + .value("Error", autocog::parser::Diagnostic::Level::Error) + .value("Warning", autocog::parser::Diagnostic::Level::Warning) + .value("Note", autocog::parser::Diagnostic::Level::Note); + + // Main parsing functions + m.def("parse_file", [](const std::string& filename) { + std::vector diagnostics; + auto ir = autocog::parser::parse_file(filename, diagnostics); + + py::dict result; + result["success"] = (ir != nullptr); + result["diagnostics"] = diagnostics; + + if (ir) { + result["ir"] = ir_to_dict(*ir); + result["json"] = autocog::parser::ir_to_json(*ir); + } + + return result; + }, py::arg("filename"), + "Parse an STL file and return IR as Python dict"); + + m.def("parse_string", [](const std::string& source) { + std::vector diagnostics; + auto ir = autocog::parser::parse_string(source, diagnostics); + + py::dict result; + result["success"] = (ir != nullptr); + result["diagnostics"] = diagnostics; + + if (ir) { + result["ir"] = ir_to_dict(*ir); + result["json"] = autocog::parser::ir_to_json(*ir); + } + + return result; + }, py::arg("source"), + "Parse an STL string and return IR as Python dict"); + + m.def("validate_ir", [](py::dict ir_dict) { + // For now, just return true + // Full implementation would convert dict back to C++ IR and validate + return py::make_tuple(true, py::list()); + }, py::arg("ir"), + "Validate IR consistency"); + + // Version info + m.attr("__version__") = "1.0.0"; +} diff --git a/libs/autocog/parser/stl/diagnostic.cxx b/libs/autocog/parser/stl/diagnostic.cxx new file mode 100644 index 0000000..28ee044 --- /dev/null +++ b/libs/autocog/parser/stl/diagnostic.cxx @@ -0,0 +1,57 @@ + +#include "autocog/parser/stl/diagnostic.hxx" + +#include + +#include + +namespace autocog::parser { + +Diagnostic::Diagnostic(DiagnosticLevel const level_, std::string message_) : + level(level_), + message(message_), + source_line(std::nullopt), + location(std::nullopt), + notes() +{} + +Diagnostic::Diagnostic(DiagnosticLevel const level_, std::string message_, std::string source_line_, SourceLocation location_) : + level(level_), + message(message_), + source_line(source_line_), + location(location_), + notes() +{} + +std::string Diagnostic::format() const { + std::stringstream ss; + + // Main error message + if (location) { + ss << location.value().line << ":" << location.value().column << ": "; + } + switch (level) { + case DiagnosticLevel::Error: ss << "error: "; break; + case DiagnosticLevel::Warning: ss << "warning: "; break; + case DiagnosticLevel::Note: ss << "note: "; break; + } + ss << message << "\n"; + + // Source line with caret + if (source_line) { + ss << " " << source_line.value() << "\n"; + if (location) { + ss << " " << std::string(location.value().column - 1, ' ') << "^\n"; + } + } + + // Additional notes + for (const auto& note : notes) { + ss << " note: " << note << "\n"; + } + + return ss.str(); +} + +} + diff --git a/libs/autocog/parser/stl/diagnostic.hxx b/libs/autocog/parser/stl/diagnostic.hxx new file mode 100644 index 0000000..1fbe796 --- /dev/null +++ b/libs/autocog/parser/stl/diagnostic.hxx @@ -0,0 +1,29 @@ +#ifndef AUTOCOG_PARSER_STL_DIAGNOSTIC_HXX +#define AUTOCOG_PARSER_STL_DIAGNOSTIC_HXX + +#include "autocog/parser/stl/location.hxx" + +#include +#include +#include + +namespace autocog::parser { + +enum class DiagnosticLevel { Error, Warning, Note }; + +struct Diagnostic { + DiagnosticLevel const level; + std::string message; + std::optional source_line; + std::optional location; + std::vector notes; + + Diagnostic(DiagnosticLevel const level_, std::string message_); + Diagnostic(DiagnosticLevel const level_, std::string message_, std::string source_line_, SourceLocation location_); + + std::string format() const; +}; + +} + +#endif /* AUTOCOG_PARSER_STL_DIAGNOSTIC_HXX */ diff --git a/libs/autocog/parser/stl/ir.hxx b/libs/autocog/parser/stl/ir.hxx new file mode 100644 index 0000000..7de6826 --- /dev/null +++ b/libs/autocog/parser/stl/ir.hxx @@ -0,0 +1,151 @@ +#ifndef AUTOCOG_PARSER_STL_IR_HXX +#define AUTOCOG_PARSER_STL_IR_HXX + +#include "autocog/parser/stl/token.hxx" +#include "autocog/parser/stl/location.hxx" + +#include +#include +#include +#include +#include +#include +#include + +namespace autocog { namespace parser { + +class Lexer; +class Parser; +struct Diagnostic; + +struct SourceRange { + SourceLocation start; + SourceLocation stop; +}; + +enum class IrTag { + Program, + Import, + Entry, + Enum, + Choice, + Annotate, + Annotation, + Define, + Flow, + Edge, + Struct, + Field, + Search, + Param, + Return, + Record, + Format, + Prompt, + Channel, + Link, + Call, + Kwarg, + Path, + Step, + Expression, + Unary, + Binary, + Conditional, + Identifier, + Scalar, + Integer, + Float, + String, + Boolean +}; + +template +struct IrNode; + +template +struct IrData; + +template +struct BaseExec { + IrNode & node; + BaseExec(IrNode & node_) : node(node_) {} + + void semcheck(Parser const & parser, std::string filename, std::list & diagnostics) const; +}; + +template +struct IrExec : BaseExec { + IrExec(IrNode & node) : BaseExec(node) {} +}; + +template +struct IrNode { + static constexpr IrTag tag = tagT; + + IrNode() : data(), exec(*this) {} + + template + IrNode(Args&&... args) : data{std::forward(args)...}, exec(*this) {} + + template + static std::unique_ptr make(Args&&... args) { + return std::make_unique(std::forward(args)...); + } + + IrNode(const IrNode&) = delete; + IrNode& operator=(const IrNode&) = delete; + IrNode(IrNode&&) = delete; + IrNode& operator=(IrNode&&) = delete; + + std::optional location; + IrData data; + IrExec exec; +}; + +} } + +// Recursive expansion helpers +#define EXPAND(...) __VA_ARGS__ +#define FOR_EACH_1(what, x) what(x) +#define FOR_EACH_2(what, x, ...) what(x), EXPAND(FOR_EACH_1(what, __VA_ARGS__)) +#define FOR_EACH_3(what, x, ...) what(x), EXPAND(FOR_EACH_2(what, __VA_ARGS__)) +#define FOR_EACH_4(what, x, ...) what(x), EXPAND(FOR_EACH_3(what, __VA_ARGS__)) +#define FOR_EACH_5(what, x, ...) what(x), EXPAND(FOR_EACH_4(what, __VA_ARGS__)) +#define FOR_EACH_6(what, x, ...) what(x), EXPAND(FOR_EACH_5(what, __VA_ARGS__)) +#define FOR_EACH_7(what, x, ...) what(x), EXPAND(FOR_EACH_6(what, __VA_ARGS__)) +#define FOR_EACH_8(what, x, ...) what(x), EXPAND(FOR_EACH_7(what, __VA_ARGS__)) + +// Argument counting +#define GET_MACRO(_1,_2,_3,_4,_5,_6,_7,_8,NAME,...) NAME +#define FOR_EACH(action, ...) \ + GET_MACRO(__VA_ARGS__, FOR_EACH_8, FOR_EACH_7, FOR_EACH_6, FOR_EACH_5, \ + FOR_EACH_4, FOR_EACH_3, FOR_EACH_2, FOR_EACH_1)(action, __VA_ARGS__) + +#define NODE(tag) IrNode< IrTag::tag > +#define PNODE(tag) std::unique_ptr< IrNode< IrTag::tag > > +#define ONODE(tag) std::optional< IrNode< IrTag::tag > > +#define NODES(tag) std::list< NODE(tag) > +#define PNODES(tag) std::list< PNODE(tag) > +#define MAPPED(tag) std::unordered_map +#define VARIANT(...) std::variant + +#define DATA(tag) template <> struct IrData< IrTag::tag > +#define DATA_CTOR_EMPTY(tag) IrData< IrTag::tag >() +#define EXEC(tag) template <> struct IrExec< IrTag::tag > : BaseExec +#define EXEC_CTOR(tag) IrExec(IrNode & node) : BaseExec(node) + +#include "autocog/parser/stl/ir/expr.hxx" +#include "autocog/parser/stl/ir/path.hxx" +#include "autocog/parser/stl/ir/channel.hxx" +#include "autocog/parser/stl/ir/return.hxx" +#include "autocog/parser/stl/ir/search.hxx" +#include "autocog/parser/stl/ir/struct.hxx" +#include "autocog/parser/stl/ir/annot.hxx" +#include "autocog/parser/stl/ir/flow.hxx" +#include "autocog/parser/stl/ir/define.hxx" +#include "autocog/parser/stl/ir/prompt.hxx" +#include "autocog/parser/stl/ir/record.hxx" +#include "autocog/parser/stl/ir/program.hxx" + +#endif // AUTOCOG_PARSER_STL_IR_HXX diff --git a/libs/autocog/parser/stl/ir/annot.hxx b/libs/autocog/parser/stl/ir/annot.hxx new file mode 100644 index 0000000..9580c0c --- /dev/null +++ b/libs/autocog/parser/stl/ir/annot.hxx @@ -0,0 +1,18 @@ +#ifndef AUTOCOG_PARSER_STL_IR_ANNOT_HXX +#define AUTOCOG_PARSER_STL_IR_ANNOT_HXX + +namespace autocog { namespace parser { + +DATA(Annotation) { + ONODE(Path) path; + NODE(String) description; +}; + +DATA(Annotate) { + bool single_statement; //< Keyword followed by single element instead of block + NODES(Annotation) annotations; +}; + +} } + +#endif // AUTOCOG_PARSER_STL_IR_ANNOT_HXX diff --git a/libs/autocog/parser/stl/ir/channel.hxx b/libs/autocog/parser/stl/ir/channel.hxx new file mode 100644 index 0000000..377ca41 --- /dev/null +++ b/libs/autocog/parser/stl/ir/channel.hxx @@ -0,0 +1,31 @@ +#ifndef AUTOCOG_PARSER_STL_IR_CHANNEL_HXX +#define AUTOCOG_PARSER_STL_IR_CHANNEL_HXX + +namespace autocog { namespace parser { + +DATA(Kwarg) { + NODE(Identifier) name; + NODE(Path) source; + bool mapped; +}; + +DATA(Call) { + NODE(Identifier) extcog; + NODE(Identifier) entry; + NODES(Kwarg) kwargs; + MAPPED(Path) binds; +}; + +DATA(Link) { + NODE(Path) target; + VARIANT(Path, Call) source; +}; + +DATA(Channel) { + NODES(Link) links; +}; + +} // namespace parser +} // namespace autocog + +#endif // AUTOCOG_PARSER_STL_IR_CHANNEL_HXX diff --git a/libs/autocog/parser/stl/ir/define.hxx b/libs/autocog/parser/stl/ir/define.hxx new file mode 100644 index 0000000..e43c679 --- /dev/null +++ b/libs/autocog/parser/stl/ir/define.hxx @@ -0,0 +1,14 @@ +#ifndef AUTOCOG_PARSER_STL_IR_DEFINE_HXX +#define AUTOCOG_PARSER_STL_IR_DEFINE_HXX + +namespace autocog { namespace parser { + +DATA(Define) { + bool argument; + std::string name; + ONODE(Expression) init; +}; + +} } + +#endif // AUTOCOG_PARSER_STL_IR_DEFINE_HXX diff --git a/libs/autocog/parser/stl/ir/expr.hxx b/libs/autocog/parser/stl/ir/expr.hxx new file mode 100644 index 0000000..ddb951b --- /dev/null +++ b/libs/autocog/parser/stl/ir/expr.hxx @@ -0,0 +1,58 @@ +#ifndef AUTOCOG_PARSER_STL_IR_EXPR_HXX +#define AUTOCOG_PARSER_STL_IR_EXPR_HXX + +namespace autocog { +namespace parser { + +enum class UnaryKind { Not, Neg }; +enum class BinaryKind { Add, Sub, Mul, Div, Mod, And, Or, Xor }; + +DATA(Identifier) { + std::string name; +}; + +DATA(Integer) { + int value; +}; + +DATA(Float) { + float value; +}; + +DATA(Boolean) { + bool value; +}; + +DATA(String) { + bool is_format; + std::string value; +}; + +DATA(Scalar) { + VARIANT(Identifier, Integer, Float, Boolean, String) value; +}; + +DATA(Unary) { + UnaryKind kind; + PNODE(Expression) operand; +}; + +DATA(Binary) { + BinaryKind kind; + PNODE(Expression) lhs; + PNODE(Expression) rhs; +}; + +DATA(Conditional) { + PNODE(Expression) cond; + PNODE(Expression) e_true; + PNODE(Expression) e_false; +}; + +DATA(Expression) { + VARIANT(Scalar, Unary, Binary, Conditional, Identifier) expr; +}; + +} } + +#endif // AUTOCOG_PARSER_STL_IR_EXPR_HXX diff --git a/libs/autocog/parser/stl/ir/flow.hxx b/libs/autocog/parser/stl/ir/flow.hxx new file mode 100644 index 0000000..2b495b9 --- /dev/null +++ b/libs/autocog/parser/stl/ir/flow.hxx @@ -0,0 +1,18 @@ +#ifndef AUTOCOG_PARSER_STL_IR_FLOW_HXX +#define AUTOCOG_PARSER_STL_IR_FLOW_HXX + +namespace autocog { namespace parser { + +DATA(Edge) { + NODE(Identifier) prompt; + ONODE(String) label; +}; + +DATA(Flow) { + bool single_statement; //< Keyword followed by single element instead of block + NODES(Edge) edges; +}; + +} } + +#endif // AUTOCOG_PARSER_STL_IR_FLOW_HXX diff --git a/libs/autocog/parser/stl/ir/path.hxx b/libs/autocog/parser/stl/ir/path.hxx new file mode 100644 index 0000000..c19bd71 --- /dev/null +++ b/libs/autocog/parser/stl/ir/path.hxx @@ -0,0 +1,20 @@ +#ifndef AUTOCOG_PARSER_STL_IR_PATH_HXX +#define AUTOCOG_PARSER_STL_IR_PATH_HXX + +namespace autocog { namespace parser { + +DATA(Step) { + NODE(Identifier) field; + ONODE(Expression) lower; + ONODE(Expression) upper; +}; + +DATA(Path) { + bool input; + ONODE(Identifier) prompt; + NODES(Step) steps; +}; + +} } + +#endif // AUTOCOG_PARSER_STL_IR_PATH_HXX diff --git a/libs/autocog/parser/stl/ir/program.cxx b/libs/autocog/parser/stl/ir/program.cxx new file mode 100644 index 0000000..56bc72a --- /dev/null +++ b/libs/autocog/parser/stl/ir/program.cxx @@ -0,0 +1,22 @@ + + +#include "autocog/parser/stl/ir.hxx" +#include + +namespace autocog::parser { + +using IrExecProgram = IrExec; +using IrBaseProgram = BaseExec; + +void IrExecProgram::queue_imports(std::queue & queue) const { + for (auto & import: node.data.imports) { + queue.push(import.data.file.data.value); + } +} + +template <> +void IrBaseProgram::semcheck(Parser const & parser, std::string filename, std::list & diagnostics) const { + throw std::runtime_error("Not implemented yet : IrBaseProgram::semcheck()"); +} + +} diff --git a/libs/autocog/parser/stl/ir/program.hxx b/libs/autocog/parser/stl/ir/program.hxx new file mode 100644 index 0000000..b4f698c --- /dev/null +++ b/libs/autocog/parser/stl/ir/program.hxx @@ -0,0 +1,47 @@ +#ifndef AUTOCOG_PARSER_STL_IR_PROGRAM_HXX +#define AUTOCOG_PARSER_STL_IR_PROGRAM_HXX + +namespace autocog { namespace parser { + +DATA(Entry) { + NODE(Identifier) alias; + NODE(String) target; +}; + +DATA(Import) { + NODE(String) file; + NODES(Identifier) prompts; + NODES(Identifier) records; +}; + +DATA(Program) { +//DATA_CTOR_EMPTY(Program) : +// imports(), +// defines(), +// annotate(std::nullopt), +// search(std::nullopt), +// entries(), +// records(), +// prompts() +//{} + + NODES(Import) imports; + + MAPPED(Define) defines; + ONODE(Annotate) annotate; + ONODE(Search) search; + + MAPPED(Entry) entries; + MAPPED(Record) records; + MAPPED(Prompt) prompts; +}; + +EXEC(Program) { + EXEC_CTOR(Program) {} + + void queue_imports(std::queue & queue) const; +}; + +} } + +#endif // AUTOCOG_PARSER_STL_IR_PROGRAM_HXX diff --git a/libs/autocog/parser/stl/ir/prompt.hxx b/libs/autocog/parser/stl/ir/prompt.hxx new file mode 100644 index 0000000..0294c62 --- /dev/null +++ b/libs/autocog/parser/stl/ir/prompt.hxx @@ -0,0 +1,20 @@ +#ifndef AUTOCOG_PARSER_STL_IR_PROMPT_HXX +#define AUTOCOG_PARSER_STL_IR_PROMPT_HXX + +namespace autocog { namespace parser { + +DATA(Prompt) { + std::string name; + MAPPED(Define) defines; + ONODE(Annotate) annotate; + ONODE(Search) search; + + NODE(Struct) fields; + ONODE(Channel) channel; + ONODE(Flow) flow; + ONODE(Return) retstmt; +}; + +} } + +#endif // AUTOCOG_PARSER_STL_IR_PROMPT_HXX diff --git a/libs/autocog/parser/stl/ir/record.hxx b/libs/autocog/parser/stl/ir/record.hxx new file mode 100644 index 0000000..0ea093d --- /dev/null +++ b/libs/autocog/parser/stl/ir/record.hxx @@ -0,0 +1,17 @@ +#ifndef AUTOCOG_PARSER_STL_IR_RECORD_HXX +#define AUTOCOG_PARSER_STL_IR_RECORD_HXX + +namespace autocog { namespace parser { + +DATA(Record) { + std::string name; + MAPPED(Define) defines; + ONODE(Annotate) annotate; + ONODE(Search) search; + + VARIANT(Struct, Format) record; +}; + +} } + +#endif // AUTOCOG_PARSER_STL_IR_RECORD_HXX diff --git a/libs/autocog/parser/stl/ir/return.hxx b/libs/autocog/parser/stl/ir/return.hxx new file mode 100644 index 0000000..70e105a --- /dev/null +++ b/libs/autocog/parser/stl/ir/return.hxx @@ -0,0 +1,14 @@ +#ifndef AUTOCOG_PARSER_STL_IR_RETURN_HXX +#define AUTOCOG_PARSER_STL_IR_RETURN_HXX + +namespace autocog { namespace parser { + +DATA(Return) { + ONODE(String) label; + MAPPED(Path) fields; +}; + +} // namespace parser +} // namespace autocog + +#endif // AUTOCOG_PARSER_STL_IR_RETURN_HXX diff --git a/libs/autocog/parser/stl/ir/search.hxx b/libs/autocog/parser/stl/ir/search.hxx new file mode 100644 index 0000000..7a5ed5f --- /dev/null +++ b/libs/autocog/parser/stl/ir/search.hxx @@ -0,0 +1,17 @@ +#ifndef AUTOCOG_PARSER_STL_IR_SEARCH_HXX +#define AUTOCOG_PARSER_STL_IR_SEARCH_HXX + +namespace autocog { namespace parser { + +DATA(Param) { + NODES(Identifier) locator; + NODE(Expression) value; +}; + +DATA(Search) { + NODES(Param) params; +}; + +} } + +#endif // AUTOCOG_PARSER_STL_IR_SEARCH_HXX diff --git a/libs/autocog/parser/stl/ir/struct.hxx b/libs/autocog/parser/stl/ir/struct.hxx new file mode 100644 index 0000000..ce2174f --- /dev/null +++ b/libs/autocog/parser/stl/ir/struct.hxx @@ -0,0 +1,34 @@ +#ifndef AUTOCOG_PARSER_STL_IR_STRUCT_HXX +#define AUTOCOG_PARSER_STL_IR_STRUCT_HXX + +namespace autocog { namespace parser { + +DATA(Enum) { + NODES(String) enumerators; +}; + +enum class ChoiceKind { Repeat, Select }; + +DATA(Choice) { + ChoiceKind mode; + NODE(Path) source; +}; + +DATA(Format) { + VARIANT(Identifier, Enum, Choice) type; + NODES(Expression) args; + MAPPED(Expression) kwargs; +}; + +DATA(Struct) { + PNODES(Field) fields; +}; + +DATA(Field) { + std::string name; + VARIANT(Format, Struct) type; +}; + +} } + +#endif // AUTOCOG_PARSER_STL_IR_STRUCT_HXX diff --git a/libs/autocog/parser/stl/lexer.cxx b/libs/autocog/parser/stl/lexer.cxx new file mode 100644 index 0000000..dba0d8a --- /dev/null +++ b/libs/autocog/parser/stl/lexer.cxx @@ -0,0 +1,292 @@ +// /workspace/libs/autocog/parser/stl/lexer.cxx generated by reflex 6.0.0 from /workspace/libs/autocog/parser/stl/lexer.l + +#define REFLEX_VERSION "6.0.0" + +//////////////////////////////////////////////////////////////////////////////// +// // +// OPTIONS USED // +// // +//////////////////////////////////////////////////////////////////////////////// + +#undef REFLEX_OPTION_header_file +#undef REFLEX_OPTION_lex +#undef REFLEX_OPTION_lexer +#undef REFLEX_OPTION_namespace +#undef REFLEX_OPTION_noyywrap +#undef REFLEX_OPTION_outfile + +#define REFLEX_OPTION_header_file "/workspace/libs/autocog/parser/stl/lexer.hxx" +#define REFLEX_OPTION_lex lex +#define REFLEX_OPTION_lexer Lexer +#define REFLEX_OPTION_namespace autocog::parser +#define REFLEX_OPTION_noyywrap true +#define REFLEX_OPTION_outfile "/workspace/libs/autocog/parser/stl/lexer.cxx" + +//////////////////////////////////////////////////////////////////////////////// +// // +// SECTION 1: %top user code // +// // +//////////////////////////////////////////////////////////////////////////////// + +#line 1 "/workspace/libs/autocog/parser/stl/lexer.l" + +#include "autocog/parser/stl/token.hxx" +#include "autocog/parser/stl/location.hxx" +#include + + +//////////////////////////////////////////////////////////////////////////////// +// // +// LEXER CLASS INCLUDE // +// // +//////////////////////////////////////////////////////////////////////////////// + +#include "/workspace/libs/autocog/parser/stl/lexer.hxx" + +//////////////////////////////////////////////////////////////////////////////// +// // +// SECTION 2: rules // +// // +//////////////////////////////////////////////////////////////////////////////// + +int autocog::parser::Lexer::lex(void) +{ + static const char *REGEX_INITIAL = "(?m)(\\n)|([\\x09\\x0d\\x20]+)|((?:\\Q//\\E)[^\\x0a]*)|((?:\\Q/*\\E)(?:[^\\x2a]|\\*[^/])*(?:\\Q*/\\E))|((?:\\Qdefine\\E))|((?:\\Qargument\\E))|((?:\\Qformat\\E))|((?:\\Qstruct\\E))|((?:\\Qprompt\\E))|((?:\\Qchannel\\E))|((?:\\Qflow\\E))|((?:\\Qreturn\\E))|((?:\\Qannotate\\E))|((?:\\Qto\\E))|((?:\\Qfrom\\E))|((?:\\Qcall\\E))|((?:\\Qextern\\E))|((?:\\Qentry\\E))|((?:\\Qkwarg\\E))|((?:\\Qmap\\E))|((?:\\Qbind\\E))|((?:\\Qas\\E))|((?:\\Qis\\E))|((?:\\Qsearch\\E))|((?:\\Qtext\\E))|((?:\\Qselect\\E))|((?:\\Qrepeat\\E))|((?:\\Qtrue\\E))|((?:\\Qfalse\\E))|([A-Z_a-z][0-9A-Z_a-z]*)|([0-9]+)|([0-9]+\\.[0-9]+)|(\"(?:[^\"\\x5c]|\\\\.)*\")|('(?:[^'\\x5c]|\\\\.)*')|((?:\\Q{\\E))|((?:\\Q}\\E))|((?:\\Q[\\E))|((?:\\Q]\\E))|((?:\\Q(\\E))|((?:\\Q)\\E))|((?:\\Q;\\E))|((?:\\Q:\\E))|((?:\\Q,\\E))|((?:\\Q.\\E))|((?:\\Q=\\E))|((?:\\Q+\\E))|((?:\\Q-\\E))|((?:\\Q*\\E))|((?:\\Q/\\E))|(.)"; + static const reflex::Pattern PATTERN_INITIAL(REGEX_INITIAL); + if (!has_matcher()) + { + matcher(new Matcher(PATTERN_INITIAL, stdinit(), this)); + } + while (true) + { + switch (matcher().scan()) + { + case 0: + if (matcher().at_end()) + { +#line 106 "/workspace/libs/autocog/parser/stl/lexer.l" +{ return (int)TokenType::END_OF_FILE; } + + } + else + { + out().put(matcher().input()); + } + break; + case 1: // rule /workspace/libs/autocog/parser/stl/lexer.l:39: \n : +#line 39 "/workspace/libs/autocog/parser/stl/lexer.l" +{ newline(); } + break; + case 2: // rule /workspace/libs/autocog/parser/stl/lexer.l:40: [ \t\r]+ : +#line 40 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); } + + break; + case 3: // rule /workspace/libs/autocog/parser/stl/lexer.l:42: "//"[^\n]* : +#line 42 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); /* line comment */ } + break; + case 4: // rule /workspace/libs/autocog/parser/stl/lexer.l:43: "/*"([^*]|\*[^/])*"*/" : +#line 43 "/workspace/libs/autocog/parser/stl/lexer.l" +{ + for (size_t i = 0; i < size(); ++i) { + if (text()[i] == '\n') newline(); + else column_number++; + } +} + + break; + case 5: // rule /workspace/libs/autocog/parser/stl/lexer.l:51: "define" : +#line 51 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::DEFINE; } + break; + case 6: // rule /workspace/libs/autocog/parser/stl/lexer.l:52: "argument" : +#line 52 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::ARGUMENT; } + break; + case 7: // rule /workspace/libs/autocog/parser/stl/lexer.l:53: "format" : +#line 53 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::FORMAT; } + break; + case 8: // rule /workspace/libs/autocog/parser/stl/lexer.l:54: "struct" : +#line 54 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::STRUCT; } + break; + case 9: // rule /workspace/libs/autocog/parser/stl/lexer.l:55: "prompt" : +#line 55 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::PROMPT; } + break; + case 10: // rule /workspace/libs/autocog/parser/stl/lexer.l:56: "channel" : +#line 56 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::CHANNEL; } + break; + case 11: // rule /workspace/libs/autocog/parser/stl/lexer.l:57: "flow" : +#line 57 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::FLOW; } + break; + case 12: // rule /workspace/libs/autocog/parser/stl/lexer.l:58: "return" : +#line 58 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::RETURN; } + break; + case 13: // rule /workspace/libs/autocog/parser/stl/lexer.l:59: "annotate" : +#line 59 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::ANNOTATE; } + break; + case 14: // rule /workspace/libs/autocog/parser/stl/lexer.l:60: "to" : +#line 60 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::TO; } + break; + case 15: // rule /workspace/libs/autocog/parser/stl/lexer.l:61: "from" : +#line 61 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::FROM; } + break; + case 16: // rule /workspace/libs/autocog/parser/stl/lexer.l:62: "call" : +#line 62 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::CALL; } + break; + case 17: // rule /workspace/libs/autocog/parser/stl/lexer.l:63: "extern" : +#line 63 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::EXTERN; } + break; + case 18: // rule /workspace/libs/autocog/parser/stl/lexer.l:64: "entry" : +#line 64 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::ENTRY; } + break; + case 19: // rule /workspace/libs/autocog/parser/stl/lexer.l:65: "kwarg" : +#line 65 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::KWARG; } + break; + case 20: // rule /workspace/libs/autocog/parser/stl/lexer.l:66: "map" : +#line 66 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::MAP; } + break; + case 21: // rule /workspace/libs/autocog/parser/stl/lexer.l:67: "bind" : +#line 67 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::BIND; } + break; + case 22: // rule /workspace/libs/autocog/parser/stl/lexer.l:68: "as" : +#line 68 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::AS; } + break; + case 23: // rule /workspace/libs/autocog/parser/stl/lexer.l:69: "is" : +#line 69 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::IS; } + break; + case 24: // rule /workspace/libs/autocog/parser/stl/lexer.l:70: "search" : +#line 70 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::SEARCH; } + break; + case 25: // rule /workspace/libs/autocog/parser/stl/lexer.l:71: "text" : +#line 71 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::TEXT; } + break; + case 26: // rule /workspace/libs/autocog/parser/stl/lexer.l:72: "select" : +#line 72 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::SELECT; } + break; + case 27: // rule /workspace/libs/autocog/parser/stl/lexer.l:73: "repeat" : +#line 73 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::REPEAT; } + break; + case 28: // rule /workspace/libs/autocog/parser/stl/lexer.l:74: "true" : +#line 74 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::BOOLEAN_LITERAL; } + break; + case 29: // rule /workspace/libs/autocog/parser/stl/lexer.l:75: "false" : +#line 75 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::BOOLEAN_LITERAL; } + + break; + case 30: // rule /workspace/libs/autocog/parser/stl/lexer.l:78: [a-zA-Z_][a-zA-Z0-9_]* : +#line 78 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::IDENTIFIER; } + break; + case 31: // rule /workspace/libs/autocog/parser/stl/lexer.l:79: [0-9]+ : +#line 79 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::INTEGER_LITERAL; } + break; + case 32: // rule /workspace/libs/autocog/parser/stl/lexer.l:80: [0-9]+\.[0-9]+ : +#line 80 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::FLOAT_LITERAL; } + break; + case 33: // rule /workspace/libs/autocog/parser/stl/lexer.l:81: \"([^"\\]|\\.)*\" : +#line 81 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::STRING_LITERAL; } + break; + case 34: // rule /workspace/libs/autocog/parser/stl/lexer.l:82: '([^'\\]|\\.)*' : +#line 82 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::STRING_LITERAL; } + + break; + case 35: // rule /workspace/libs/autocog/parser/stl/lexer.l:85: "{" : +#line 85 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::LBRACE; } + break; + case 36: // rule /workspace/libs/autocog/parser/stl/lexer.l:86: "}" : +#line 86 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::RBRACE; } + break; + case 37: // rule /workspace/libs/autocog/parser/stl/lexer.l:87: "[" : +#line 87 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::LSQUARE; } + break; + case 38: // rule /workspace/libs/autocog/parser/stl/lexer.l:88: "]" : +#line 88 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::RSQUARE; } + break; + case 39: // rule /workspace/libs/autocog/parser/stl/lexer.l:89: "(" : +#line 89 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::LPAREN; } + break; + case 40: // rule /workspace/libs/autocog/parser/stl/lexer.l:90: ")" : +#line 90 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::RPAREN; } + break; + case 41: // rule /workspace/libs/autocog/parser/stl/lexer.l:91: ";" : +#line 91 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::SEMICOLON; } + break; + case 42: // rule /workspace/libs/autocog/parser/stl/lexer.l:92: ":" : +#line 92 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::COLON; } + break; + case 43: // rule /workspace/libs/autocog/parser/stl/lexer.l:93: "," : +#line 93 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::COMMA; } + break; + case 44: // rule /workspace/libs/autocog/parser/stl/lexer.l:94: "." : +#line 94 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::DOT; } + break; + case 45: // rule /workspace/libs/autocog/parser/stl/lexer.l:95: "=" : +#line 95 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::EQUAL; } + break; + case 46: // rule /workspace/libs/autocog/parser/stl/lexer.l:96: "+" : +#line 96 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::PLUS; } + break; + case 47: // rule /workspace/libs/autocog/parser/stl/lexer.l:97: "-" : +#line 97 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::MINUS; } + break; + case 48: // rule /workspace/libs/autocog/parser/stl/lexer.l:98: "*" : +#line 98 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::STAR; } + break; + case 49: // rule /workspace/libs/autocog/parser/stl/lexer.l:99: "/" : +#line 99 "/workspace/libs/autocog/parser/stl/lexer.l" +{ update_location(); return (int)TokenType::SLASH; } + + break; + case 50: // rule /workspace/libs/autocog/parser/stl/lexer.l:101: . : +#line 101 "/workspace/libs/autocog/parser/stl/lexer.l" +{ + update_location(); + return (int)TokenType::ERROR; +} + + break; + } + } +} diff --git a/libs/autocog/parser/stl/lexer.hxx b/libs/autocog/parser/stl/lexer.hxx new file mode 100644 index 0000000..c0cca57 --- /dev/null +++ b/libs/autocog/parser/stl/lexer.hxx @@ -0,0 +1,125 @@ +// /workspace/libs/autocog/parser/stl/lexer.hxx generated by reflex 6.0.0 from /workspace/libs/autocog/parser/stl/lexer.l + +#ifndef REFLEX__WORKSPACE_LIBS_AUTOCOG_PARSER_STL_LEXER_HXX +#define REFLEX__WORKSPACE_LIBS_AUTOCOG_PARSER_STL_LEXER_HXX +#define IN_HEADER 1 +#define REFLEX_VERSION "6.0.0" + +//////////////////////////////////////////////////////////////////////////////// +// // +// OPTIONS USED // +// // +//////////////////////////////////////////////////////////////////////////////// + +#undef REFLEX_OPTION_header_file +#undef REFLEX_OPTION_lex +#undef REFLEX_OPTION_lexer +#undef REFLEX_OPTION_namespace +#undef REFLEX_OPTION_noyywrap +#undef REFLEX_OPTION_outfile + +#define REFLEX_OPTION_header_file "/workspace/libs/autocog/parser/stl/lexer.hxx" +#define REFLEX_OPTION_lex lex +#define REFLEX_OPTION_lexer Lexer +#define REFLEX_OPTION_namespace autocog::parser +#define REFLEX_OPTION_noyywrap true +#define REFLEX_OPTION_outfile "/workspace/libs/autocog/parser/stl/lexer.cxx" + +//////////////////////////////////////////////////////////////////////////////// +// // +// SECTION 1: %top user code // +// // +//////////////////////////////////////////////////////////////////////////////// + +#line 1 "/workspace/libs/autocog/parser/stl/lexer.l" + +#include "autocog/parser/stl/token.hxx" +#include "autocog/parser/stl/location.hxx" +#include + + +//////////////////////////////////////////////////////////////////////////////// +// // +// REGEX MATCHER // +// // +//////////////////////////////////////////////////////////////////////////////// + +#include + +//////////////////////////////////////////////////////////////////////////////// +// // +// ABSTRACT LEXER CLASS // +// // +//////////////////////////////////////////////////////////////////////////////// + +#include + +//////////////////////////////////////////////////////////////////////////////// +// // +// LEXER CLASS // +// // +//////////////////////////////////////////////////////////////////////////////// + +namespace autocog { +namespace parser { + +class Lexer : public reflex::AbstractLexer { +#line 7 "/workspace/libs/autocog/parser/stl/lexer.l" + +private: + int line_number = 1; + int column_number = 1; + int last_column = 1; + unsigned current_offset = 0; + + void update_location() { + last_column = column_number; + column_number += size(); + current_offset += size(); + } + + void newline() { + line_number++; + last_column = column_number; + column_number = 1; + current_offset++; + } + +public: + autocog::parser::SourceLocation current_location() const { + return {line_number, last_column, current_offset}; + } + + public: + typedef reflex::AbstractLexer AbstractBaseLexer; + Lexer( + // a persistent source of input, empty by default + const reflex::Input& input = reflex::Input(), + // optional output stream, std::cout by default + std::ostream& os = std::cout) + : + AbstractBaseLexer(input, os) + { + } + static const int INITIAL = 0; + // the lexer function defined by SECTION 2 + virtual int lex(void); + // lexer functions accepting new input to scan + int lex(const reflex::Input& input) + { + in(input); + return lex(); + } + int lex(const reflex::Input& input, std::ostream *os) + { + in(input); + if (os) + out(*os); + return lex(); + } +}; + +} // namespace autocog +} // namespace parser + +#endif diff --git a/libs/autocog/parser/stl/lexer.l b/libs/autocog/parser/stl/lexer.l new file mode 100644 index 0000000..89a34ee --- /dev/null +++ b/libs/autocog/parser/stl/lexer.l @@ -0,0 +1,108 @@ +%top{ +#include "autocog/parser/stl/token.hxx" +#include "autocog/parser/stl/location.hxx" +#include +%} + +%class{ +private: + int line_number = 1; + int column_number = 1; + int last_column = 1; + unsigned current_offset = 0; + + void update_location() { + last_column = column_number; + column_number += size(); + current_offset += size(); + } + + void newline() { + line_number++; + last_column = column_number; + column_number = 1; + current_offset++; + } + +public: + autocog::parser::SourceLocation current_location() const { + return {line_number, last_column, current_offset}; + } +%} + +%option c++ noyywrap +%option lexer=Lexer +%option namespace=autocog::parser + +%% + +\n { newline(); } +[ \t\r]+ { update_location(); } + +"//"[^\n]* { update_location(); /* line comment */ } +"/*"([^*]|\*[^/])*"*/" { + for (size_t i = 0; i < size(); ++i) { + if (text()[i] == '\n') newline(); + else column_number++; + } +} + +// Keywords +"define" { update_location(); return (int)TokenType::DEFINE; } +"argument" { update_location(); return (int)TokenType::ARGUMENT; } +"format" { update_location(); return (int)TokenType::FORMAT; } +"struct" { update_location(); return (int)TokenType::STRUCT; } +"prompt" { update_location(); return (int)TokenType::PROMPT; } +"channel" { update_location(); return (int)TokenType::CHANNEL; } +"flow" { update_location(); return (int)TokenType::FLOW; } +"return" { update_location(); return (int)TokenType::RETURN; } +"annotate" { update_location(); return (int)TokenType::ANNOTATE; } +"to" { update_location(); return (int)TokenType::TO; } +"from" { update_location(); return (int)TokenType::FROM; } +"call" { update_location(); return (int)TokenType::CALL; } +"extern" { update_location(); return (int)TokenType::EXTERN; } +"entry" { update_location(); return (int)TokenType::ENTRY; } +"kwarg" { update_location(); return (int)TokenType::KWARG; } +"map" { update_location(); return (int)TokenType::MAP; } +"bind" { update_location(); return (int)TokenType::BIND; } +"as" { update_location(); return (int)TokenType::AS; } +"is" { update_location(); return (int)TokenType::IS; } +"search" { update_location(); return (int)TokenType::SEARCH; } +"text" { update_location(); return (int)TokenType::TEXT; } +"select" { update_location(); return (int)TokenType::SELECT; } +"repeat" { update_location(); return (int)TokenType::REPEAT; } +"true" { update_location(); return (int)TokenType::BOOLEAN_LITERAL; } +"false" { update_location(); return (int)TokenType::BOOLEAN_LITERAL; } + +// Identifiers and literals +[a-zA-Z_][a-zA-Z0-9_]* { update_location(); return (int)TokenType::IDENTIFIER; } +[0-9]+ { update_location(); return (int)TokenType::INTEGER_LITERAL; } +[0-9]+\.[0-9]+ { update_location(); return (int)TokenType::FLOAT_LITERAL; } +\"([^"\\]|\\.)*\" { update_location(); return (int)TokenType::STRING_LITERAL; } +'([^'\\]|\\.)*' { update_location(); return (int)TokenType::STRING_LITERAL; } + +// Operators and delimiters +"{" { update_location(); return (int)TokenType::LBRACE; } +"}" { update_location(); return (int)TokenType::RBRACE; } +"[" { update_location(); return (int)TokenType::LSQUARE; } +"]" { update_location(); return (int)TokenType::RSQUARE; } +"(" { update_location(); return (int)TokenType::LPAREN; } +")" { update_location(); return (int)TokenType::RPAREN; } +";" { update_location(); return (int)TokenType::SEMICOLON; } +":" { update_location(); return (int)TokenType::COLON; } +"," { update_location(); return (int)TokenType::COMMA; } +"." { update_location(); return (int)TokenType::DOT; } +"=" { update_location(); return (int)TokenType::EQUAL; } +"+" { update_location(); return (int)TokenType::PLUS; } +"-" { update_location(); return (int)TokenType::MINUS; } +"*" { update_location(); return (int)TokenType::STAR; } +"/" { update_location(); return (int)TokenType::SLASH; } + +. { + update_location(); + return (int)TokenType::ERROR; +} + +<> { return (int)TokenType::END_OF_FILE; } + +%% diff --git a/libs/autocog/parser/stl/location.hxx b/libs/autocog/parser/stl/location.hxx new file mode 100644 index 0000000..6f4e2e4 --- /dev/null +++ b/libs/autocog/parser/stl/location.hxx @@ -0,0 +1,15 @@ +#ifndef AUTOCOG_PARSER_STL_LOCATION_HXX +#define AUTOCOG_PARSER_STL_LOCATION_HXX + +namespace autocog::parser { + +// Source location tracking +struct SourceLocation { + int line; + int column; + unsigned offset; +}; + +} + +#endif /* AUTOCOG_PARSER_STL_LOCATION_HXX */ diff --git a/libs/autocog/parser/stl/main.cxx b/libs/autocog/parser/stl/main.cxx new file mode 100644 index 0000000..4e60c5f --- /dev/null +++ b/libs/autocog/parser/stl/main.cxx @@ -0,0 +1,115 @@ + +#include "autocog/parser/stl/diagnostic.hxx" +#include "autocog/parser/stl/parser.hxx" +#include "autocog/parser/stl/ir.hxx" + +#include +#include +#include +#include + +void print_usage(const char* program) { + std::cerr << "Usage: " << program << " [options] \n"; + std::cerr << "Options:\n"; + std::cerr << " -o Write JSON IR to file (default: stdout)\n"; + std::cerr << " -V Verbose\n"; + std::cerr << " -I Add to import search paths\n"; + std::cerr << " -h Show this help\n"; +} + +bool report_errors(std::list & diagnostics, unsigned & errors, unsigned & warnings, unsigned & notes) { + for (auto const & diag : diagnostics) { + std::cerr << diag.format() << std::endl; + switch (diag.level) { + case autocog::parser::DiagnosticLevel::Error: errors++; break; + case autocog::parser::DiagnosticLevel::Warning: warnings++; break; + case autocog::parser::DiagnosticLevel::Note: notes++; break; + } + } + + if (errors > 0) { + std::cerr << "Failed with " << errors << " error(s), " << warnings << " warning(s), and " << notes << " note(s).\n"; + } else if (warnings > 0) { + std::cerr << "Passed with " << warnings << " warning(s) and " << notes << " note(s).\n"; + } else if (notes > 0) { + std::cerr << "Passed with " << notes << " note(s).\n"; + } + diagnostics.clear(); + + return errors > 0; +} + +bool parse_args( + int argc, + char** argv, + std::queue & input_files, + std::string & output_file, + std::list & search_paths, + bool & verbose +) { + for (int i = 1; i < argc; ++i) { + if (std::strcmp(argv[i], "-h") == 0) { + print_usage(argv[0]); + return true; + } else if (std::strcmp(argv[i], "-o") == 0) { + if (i + 1 < argc) { + output_file = argv[++i]; + } else { + std::cerr << "Error: -o requires an argument" << std::endl; + return false; + } + } else if (std::strcmp(argv[i], "-I") == 0) { + if (i + 1 < argc) { + search_paths.push_back(argv[++i]); + } else { + std::cerr << "Error: -I requires an argument" << std::endl; + return false; + } + } else if (std::strcmp(argv[i], "-V") == 0) { + verbose = true; + } else if (argv[i][0] == '-') { + std::cerr << "Error: Unknown option " << argv[i] << std::endl; + print_usage(argv[0]); + return false; + } else { + input_files.push(argv[i]); + } + } + return true; +} + +int main(int argc, char** argv) { + + unsigned errors = 0; + unsigned warnings = 0; + unsigned notes = 0; + std::list diagnostics; + autocog::parser::Parser parser(diagnostics); + + // Parge CLI arguments + + std::string output_file; + std::list search_paths; + bool verbose = false; + if (!parse_args(argc, argv, parser.queue, output_file, search_paths, verbose)) return 1; + + if (parser.queue.empty()) { + std::cerr << "Error: No input file specified" << std::endl; + print_usage(argv[0]); + return 1; + } + + // Parse all files + + parser.parse(); + if (report_errors(diagnostics, errors, warnings, notes)) return 1; + + // Semantic Check + + for (auto const & [filepath,program]: parser.get()) { + program.exec.semcheck(parser, filepath, diagnostics); + } + if (report_errors(diagnostics, errors, warnings, notes)) return 1; + + return 0; +} diff --git a/libs/autocog/parser/stl/parser.cxx b/libs/autocog/parser/stl/parser.cxx new file mode 100644 index 0000000..133d4ca --- /dev/null +++ b/libs/autocog/parser/stl/parser.cxx @@ -0,0 +1,90 @@ + +#include "autocog/parser/stl/diagnostic.hxx" +#include "autocog/parser/stl/parser.hxx" +#include "autocog/parser/stl/lexer.hxx" + +#include + +namespace autocog::parser { + +static std::string get_line(std::string const & source, int line_pos) { + if (line_pos <= 0) throw std::runtime_error("In get_line(): line number must be greater than 0"); + std::stringstream ss(source); + int cnt = 0; + std::string line; + while (std::getline(ss, line)) { + cnt++; + if (cnt == line_pos) return line; + } + throw std::runtime_error("In get_line(): line number must be less than the number of lines in the file"); +} + +Parser::Parser(std::list & diagnostics_) : + queue(), + diagnostics(diagnostics_), + programs() +{} + +void Parser::parse() { + while (!queue.empty()) { + std::string filepath = queue.front(); + queue.pop(); + + std::ifstream file(filepath); // TODO file lookup in 'search_paths' + if (!file) { + std::ostringstream oss; + oss << "Cannot read file: " << filepath; + diagnostics.emplace_back(DiagnosticLevel::Error, oss.str()); + } else { + std::string source((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + parse(filepath, source); + file.close(); + programs[filepath].exec.queue_imports(queue); + } + } +} + +void Parser::parse(std::string const & name, std::string const & source) { + auto & program = programs[name]; + std::istringstream stream(source); + + Lexer lexer(stream); + TokenType current = (TokenType)(lexer.lex()); + switch (current) { +// case TokenType::PROMPT: +// throw std::runtime_error("Not implemented yet : Parser::parse()"); // TODO +// break; + default: + std::ostringstream oss; + oss << "Unexpected token " << token_type_name(current); + auto loc = lexer.current_location(); + auto line = get_line(source, loc.line); + diagnostics.emplace_back(DiagnosticLevel::Error, oss.str(), line, loc); + } +} + +MAPPED(Program) const & Parser::get() const { + return programs; +} + +bool Parser::has_prompt(std::string const & filename, std::string const & objname) const { + auto & program = programs.at(filename); + return program.data.prompts.find(objname) != program.data.prompts.end(); +} + +NODE(Prompt) const & Parser::get_prompt(std::string const & filename, std::string const & objname) const { + auto & program = programs.at(filename); + return program.data.prompts.at(objname); +} + +bool Parser::has_record(std::string const & filename, std::string const & objname) const { + auto & program = programs.at(filename); + return program.data.records.find(objname) != program.data.records.end(); +} + +NODE(Record) const & Parser::get_record(std::string const & filename, std::string const & objname) const { + auto & program = programs.at(filename); + return program.data.records.at(objname); +} + +} diff --git a/libs/autocog/parser/stl/parser.hxx b/libs/autocog/parser/stl/parser.hxx new file mode 100644 index 0000000..d95bb59 --- /dev/null +++ b/libs/autocog/parser/stl/parser.hxx @@ -0,0 +1,44 @@ +#ifndef AUTOCOG_PARSER_STL_PARSER_HXX +#define AUTOCOG_PARSER_STL_PARSER_HXX + +#include "autocog/parser/stl/token.hxx" +#include "autocog/parser/stl/ir.hxx" + +#include +#include +#include +#include + +namespace autocog::parser { + +class Lexer; +struct Diagnostic; + +class Parser { + public: + using Program = NODE(Program); + + std::queue queue; + + private: + std::list & diagnostics; + MAPPED(Program) programs; + + public: + Parser(std::list & diagnostics_); + + void parse(); + void parse(std::string const & filename, std::string const & source); + + MAPPED(Program) const & get() const; + + bool has_prompt(std::string const & filename, std::string const & objname) const; + NODE(Prompt) const & get_prompt(std::string const & filename, std::string const & objname) const; + + bool has_record(std::string const & filename, std::string const & objname) const; + NODE(Record) const & get_record(std::string const & filename, std::string const & objname) const; +}; + +} + +#endif // AUTOCOG_PARSER_STL_PARSER_HXX diff --git a/libs/autocog/parser/stl/token.cxx b/libs/autocog/parser/stl/token.cxx new file mode 100644 index 0000000..4dd7561 --- /dev/null +++ b/libs/autocog/parser/stl/token.cxx @@ -0,0 +1,57 @@ + +#include "autocog/parser/stl/token.hxx" + +namespace autocog::parser { + +// TODO move to token.cxx +const char * token_type_name(TokenType type) { + switch (type) { + case TokenType::DEFINE: return "define"; + case TokenType::ARGUMENT: return "argument"; + case TokenType::FORMAT: return "format"; + case TokenType::STRUCT: return "struct"; + case TokenType::PROMPT: return "prompt"; + case TokenType::CHANNEL: return "channel"; + case TokenType::FLOW: return "flow"; + case TokenType::RETURN: return "return"; + case TokenType::ANNOTATE: return "annotate"; + case TokenType::TO: return "to"; + case TokenType::FROM: return "from"; + case TokenType::CALL: return "call"; + case TokenType::EXTERN: return "extern"; + case TokenType::ENTRY: return "entry"; + case TokenType::KWARG: return "kwarg"; + case TokenType::MAP: return "map"; + case TokenType::BIND: return "bind"; + case TokenType::AS: return "as"; + case TokenType::IS: return "is"; + case TokenType::SEARCH: return "search"; + case TokenType::TEXT: return "text"; + case TokenType::SELECT: return "select"; + case TokenType::REPEAT: return "repeat"; + case TokenType::IDENTIFIER: return "identifier"; + case TokenType::STRING_LITERAL: return "string"; + case TokenType::INTEGER_LITERAL: return "integer literal"; + case TokenType::FLOAT_LITERAL: return "float literal"; + case TokenType::LBRACE: return "'{'"; + case TokenType::RBRACE: return "'}'"; + case TokenType::LSQUARE: return "'['"; + case TokenType::RSQUARE: return "']'"; + case TokenType::LPAREN: return "'('"; + case TokenType::RPAREN: return "')'"; + case TokenType::SEMICOLON: return "';'"; + case TokenType::COLON: return "':'"; + case TokenType::COMMA: return "','"; + case TokenType::DOT: return "'.'"; + case TokenType::EQUAL: return "'='"; + case TokenType::PLUS: return "'+'"; + case TokenType::MINUS: return "'-'"; + case TokenType::STAR: return "'*'"; + case TokenType::SLASH: return "'/'"; + case TokenType::ERROR: return "invalid token"; + case TokenType::END_OF_FILE: return "end of file"; + default: return "unknown"; + } +} + +} diff --git a/libs/autocog/parser/stl/token.hxx b/libs/autocog/parser/stl/token.hxx new file mode 100644 index 0000000..c81ecbb --- /dev/null +++ b/libs/autocog/parser/stl/token.hxx @@ -0,0 +1,72 @@ +#ifndef AUTOCOG_PARSER_STL_TOKEN_HXX +#define AUTOCOG_PARSER_STL_TOKEN_HXX + +#include +#include + +namespace autocog { +namespace parser { + +// Token types +enum class TokenType : int { + // Keywords + DEFINE, + ARGUMENT, + FORMAT, + STRUCT, + PROMPT, + CHANNEL, + FLOW, + RETURN, + ANNOTATE, + TO, + FROM, + CALL, + EXTERN, + ENTRY, + KWARG, + MAP, + BIND, + AS, + IS, + SEARCH, + TEXT, + SELECT, + REPEAT, + + // Identifiers and literals + IDENTIFIER, + STRING_LITERAL, + INTEGER_LITERAL, + FLOAT_LITERAL, + BOOLEAN_LITERAL, + + // Operators and delimiters + LBRACE, // { + RBRACE, // } + LSQUARE, // [ + RSQUARE, // ] + LPAREN, // ( + RPAREN, // ) + SEMICOLON, // ; + COLON, // : + COMMA, // , + DOT, // . + EQUAL, // = + PLUS, // + + MINUS, // - + STAR, // * + SLASH, // / + + // Special + ERROR, + END_OF_FILE +}; + +// Helper function to get token type name +const char* token_type_name(TokenType type); + +} // namespace parser +} // namespace autocog + +#endif // AUTOCOG_PARSER_STL_TOKEN_HXX diff --git a/modules/autocog/sta/ir.py b/modules/autocog/sta/ir.py index 3b3cc01..818051b 100644 --- a/modules/autocog/sta/ir.py +++ b/modules/autocog/sta/ir.py @@ -194,3 +194,4 @@ class Dataflow(Channel): class Input(Channel): src: SrcPath + diff --git a/tests/samples/tiny.sta b/tests/samples/tiny.sta new file mode 100644 index 0000000..5f9d367 --- /dev/null +++ b/tests/samples/tiny.sta @@ -0,0 +1,15 @@ +prompt main { + is { + statement is text<10, beams=3, width=3>; + correct is enum("True","False"); + } + channel { + to .statement from ?statement; + } + return { + from .correct; + } + annotate { + _ as "You are verifying whether statements are true or false."; + } +} diff --git a/vendors/headers/nlohmann/json.hpp b/vendors/headers/nlohmann/json.hpp new file mode 100644 index 0000000..8b72ea6 --- /dev/null +++ b/vendors/headers/nlohmann/json.hpp @@ -0,0 +1,24765 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + +/****************************************************************************\ + * Note on documentation: The source files contain links to the online * + * documentation of the public API at https://json.nlohmann.me. This URL * + * contains the most recent documentation and should also be applicable to * + * previous versions; documentation for deprecated functions is not * + * removed, but marked deprecated. See "Generate documentation" section in * + * file docs/README.md. * +\****************************************************************************/ + +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ + +#include // all_of, find, for_each +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#ifndef JSON_NO_IO + #include // istream, ostream +#endif // JSON_NO_IO +#include // random_access_iterator_tag +#include // unique_ptr +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap +#include // vector + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// This file contains all macro definitions affecting or depending on the ABI + +#ifndef JSON_SKIP_LIBRARY_VERSION_CHECK + #if defined(NLOHMANN_JSON_VERSION_MAJOR) && defined(NLOHMANN_JSON_VERSION_MINOR) && defined(NLOHMANN_JSON_VERSION_PATCH) + #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 11 || NLOHMANN_JSON_VERSION_PATCH != 3 + #warning "Already included a different version of the library!" + #endif + #endif +#endif + +#define NLOHMANN_JSON_VERSION_MAJOR 3 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_MINOR 11 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_PATCH 3 // NOLINT(modernize-macro-to-enum) + +#ifndef JSON_DIAGNOSTICS + #define JSON_DIAGNOSTICS 0 +#endif + +#ifndef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON + #define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0 +#endif + +#if JSON_DIAGNOSTICS + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag +#else + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS +#endif + +#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON + #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON _ldvcmp +#else + #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION + #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0 +#endif + +// Construct the namespace ABI tags component +#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b) json_abi ## a ## b +#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b) \ + NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b) + +#define NLOHMANN_JSON_ABI_TAGS \ + NLOHMANN_JSON_ABI_TAGS_CONCAT( \ + NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS, \ + NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON) + +// Construct the namespace version component +#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \ + _v ## major ## _ ## minor ## _ ## patch +#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(major, minor, patch) \ + NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) + +#if NLOHMANN_JSON_NAMESPACE_NO_VERSION +#define NLOHMANN_JSON_NAMESPACE_VERSION +#else +#define NLOHMANN_JSON_NAMESPACE_VERSION \ + NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(NLOHMANN_JSON_VERSION_MAJOR, \ + NLOHMANN_JSON_VERSION_MINOR, \ + NLOHMANN_JSON_VERSION_PATCH) +#endif + +// Combine namespace components +#define NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) a ## b +#define NLOHMANN_JSON_NAMESPACE_CONCAT(a, b) \ + NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) + +#ifndef NLOHMANN_JSON_NAMESPACE +#define NLOHMANN_JSON_NAMESPACE \ + nlohmann::NLOHMANN_JSON_NAMESPACE_CONCAT( \ + NLOHMANN_JSON_ABI_TAGS, \ + NLOHMANN_JSON_NAMESPACE_VERSION) +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_BEGIN +#define NLOHMANN_JSON_NAMESPACE_BEGIN \ + namespace nlohmann \ + { \ + inline namespace NLOHMANN_JSON_NAMESPACE_CONCAT( \ + NLOHMANN_JSON_ABI_TAGS, \ + NLOHMANN_JSON_NAMESPACE_VERSION) \ + { +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_END +#define NLOHMANN_JSON_NAMESPACE_END \ + } /* namespace (inline namespace) NOLINT(readability/namespace) */ \ + } // namespace nlohmann +#endif + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // transform +#include // array +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // nullptr_t +#include // exception +#if JSON_DIAGNOSTICS + #include // accumulate +#endif +#include // runtime_error +#include // to_string +#include // vector + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // array +#include // size_t +#include // uint8_t +#include // string + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // declval, pair +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +// https://en.cppreference.com/w/cpp/experimental/is_detected +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; + void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template class Op, class... Args> +using is_detected = typename detector::value_t; + +template class Op, class... Args> +struct is_detected_lazy : is_detected { }; + +template class Op, class... Args> +using detected_t = typename detector::type; + +template class Op, class... Args> +using detected_or = detector; + +template class Op, class... Args> +using detected_or_t = typename detected_or::type; + +template class Op, class... Args> +using is_detected_exact = std::is_same>; + +template class Op, class... Args> +using is_detected_convertible = + std::is_convertible, To>; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + + +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-FileCopyrightText: 2016-2021 Evan Nemerson +// SPDX-License-Identifier: MIT + +/* Hedley - https://nemequ.github.io/hedley + * Created by Evan Nemerson + */ + +#if !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < 15) +#if defined(JSON_HEDLEY_VERSION) + #undef JSON_HEDLEY_VERSION +#endif +#define JSON_HEDLEY_VERSION 15 + +#if defined(JSON_HEDLEY_STRINGIFY_EX) + #undef JSON_HEDLEY_STRINGIFY_EX +#endif +#define JSON_HEDLEY_STRINGIFY_EX(x) #x + +#if defined(JSON_HEDLEY_STRINGIFY) + #undef JSON_HEDLEY_STRINGIFY +#endif +#define JSON_HEDLEY_STRINGIFY(x) JSON_HEDLEY_STRINGIFY_EX(x) + +#if defined(JSON_HEDLEY_CONCAT_EX) + #undef JSON_HEDLEY_CONCAT_EX +#endif +#define JSON_HEDLEY_CONCAT_EX(a,b) a##b + +#if defined(JSON_HEDLEY_CONCAT) + #undef JSON_HEDLEY_CONCAT +#endif +#define JSON_HEDLEY_CONCAT(a,b) JSON_HEDLEY_CONCAT_EX(a,b) + +#if defined(JSON_HEDLEY_CONCAT3_EX) + #undef JSON_HEDLEY_CONCAT3_EX +#endif +#define JSON_HEDLEY_CONCAT3_EX(a,b,c) a##b##c + +#if defined(JSON_HEDLEY_CONCAT3) + #undef JSON_HEDLEY_CONCAT3 +#endif +#define JSON_HEDLEY_CONCAT3(a,b,c) JSON_HEDLEY_CONCAT3_EX(a,b,c) + +#if defined(JSON_HEDLEY_VERSION_ENCODE) + #undef JSON_HEDLEY_VERSION_ENCODE +#endif +#define JSON_HEDLEY_VERSION_ENCODE(major,minor,revision) (((major) * 1000000) + ((minor) * 1000) + (revision)) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MAJOR) + #undef JSON_HEDLEY_VERSION_DECODE_MAJOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MAJOR(version) ((version) / 1000000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MINOR) + #undef JSON_HEDLEY_VERSION_DECODE_MINOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MINOR(version) (((version) % 1000000) / 1000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_REVISION) + #undef JSON_HEDLEY_VERSION_DECODE_REVISION +#endif +#define JSON_HEDLEY_VERSION_DECODE_REVISION(version) ((version) % 1000) + +#if defined(JSON_HEDLEY_GNUC_VERSION) + #undef JSON_HEDLEY_GNUC_VERSION +#endif +#if defined(__GNUC__) && defined(__GNUC_PATCHLEVEL__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) +#elif defined(__GNUC__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0) +#endif + +#if defined(JSON_HEDLEY_GNUC_VERSION_CHECK) + #undef JSON_HEDLEY_GNUC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GNUC_VERSION) + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GNUC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION) + #undef JSON_HEDLEY_MSVC_VERSION +#endif +#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 140000000) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, (_MSC_FULL_VER % 100000) / 100) +#elif defined(_MSC_FULL_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10) +#elif defined(_MSC_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION_CHECK) + #undef JSON_HEDLEY_MSVC_VERSION_CHECK +#endif +#if !defined(JSON_HEDLEY_MSVC_VERSION) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (0) +#elif defined(_MSC_VER) && (_MSC_VER >= 1400) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch))) +#elif defined(_MSC_VER) && (_MSC_VER >= 1200) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch))) +#else + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_VER >= ((major * 100) + (minor))) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION) + #undef JSON_HEDLEY_INTEL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE) +#elif defined(__INTEL_COMPILER) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_VERSION) + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #undef JSON_HEDLEY_INTEL_CL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && defined(__ICL) + #define JSON_HEDLEY_INTEL_CL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER, __INTEL_COMPILER_UPDATE, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_CL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION) + #undef JSON_HEDLEY_PGI_VERSION +#endif +#if defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__) + #define JSON_HEDLEY_PGI_VERSION JSON_HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION_CHECK) + #undef JSON_HEDLEY_PGI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PGI_VERSION) + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PGI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #undef JSON_HEDLEY_SUNPRO_VERSION +#endif +#if defined(__SUNPRO_C) && (__SUNPRO_C > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10) +#elif defined(__SUNPRO_C) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C) & 0xf) +#elif defined(__SUNPRO_CC) && (__SUNPRO_CC > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), (__SUNPRO_CC & 0xf) * 10) +#elif defined(__SUNPRO_CC) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC) & 0xf) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION_CHECK) + #undef JSON_HEDLEY_SUNPRO_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_SUNPRO_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION +#endif +#if defined(__EMSCRIPTEN__) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION JSON_HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_EMSCRIPTEN_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION) + #undef JSON_HEDLEY_ARM_VERSION +#endif +#if defined(__CC_ARM) && defined(__ARMCOMPILER_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, (__ARMCOMPILER_VERSION % 10000) / 100) +#elif defined(__CC_ARM) && defined(__ARMCC_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, (__ARMCC_VERSION % 10000) / 100) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION_CHECK) + #undef JSON_HEDLEY_ARM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_ARM_VERSION) + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_ARM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION) + #undef JSON_HEDLEY_IBM_VERSION +#endif +#if defined(__ibmxl__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__) +#elif defined(__xlC__) && defined(__xlC_ver__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff) +#elif defined(__xlC__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION_CHECK) + #undef JSON_HEDLEY_IBM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IBM_VERSION) + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IBM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_VERSION) + #undef JSON_HEDLEY_TI_VERSION +#endif +#if \ + defined(__TI_COMPILER_VERSION__) && \ + ( \ + defined(__TMS470__) || defined(__TI_ARM__) || \ + defined(__MSP430__) || \ + defined(__TMS320C2000__) \ + ) +#if (__TI_COMPILER_VERSION__ >= 16000000) + #define JSON_HEDLEY_TI_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif +#endif + +#if defined(JSON_HEDLEY_TI_VERSION_CHECK) + #undef JSON_HEDLEY_TI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_VERSION) + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #undef JSON_HEDLEY_TI_CL2000_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C2000__) + #define JSON_HEDLEY_TI_CL2000_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL2000_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #undef JSON_HEDLEY_TI_CL430_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__MSP430__) + #define JSON_HEDLEY_TI_CL430_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL430_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL430_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #undef JSON_HEDLEY_TI_ARMCL_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && (defined(__TMS470__) || defined(__TI_ARM__)) + #define JSON_HEDLEY_TI_ARMCL_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION_CHECK) + #undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_ARMCL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #undef JSON_HEDLEY_TI_CL6X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C6X__) + #define JSON_HEDLEY_TI_CL6X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL6X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #undef JSON_HEDLEY_TI_CL7X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__C7000__) + #define JSON_HEDLEY_TI_CL7X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL7X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #undef JSON_HEDLEY_TI_CLPRU_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__PRU__) + #define JSON_HEDLEY_TI_CLPRU_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CLPRU_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION) + #undef JSON_HEDLEY_CRAY_VERSION +#endif +#if defined(_CRAYC) + #if defined(_RELEASE_PATCHLEVEL) + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL) + #else + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION_CHECK) + #undef JSON_HEDLEY_CRAY_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_CRAY_VERSION) + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_CRAY_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION) + #undef JSON_HEDLEY_IAR_VERSION +#endif +#if defined(__IAR_SYSTEMS_ICC__) + #if __VER__ > 1000 + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000)) + #else + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE(__VER__ / 100, __VER__ % 100, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION_CHECK) + #undef JSON_HEDLEY_IAR_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IAR_VERSION) + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IAR_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION) + #undef JSON_HEDLEY_TINYC_VERSION +#endif +#if defined(__TINYC__) + #define JSON_HEDLEY_TINYC_VERSION JSON_HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION_CHECK) + #undef JSON_HEDLEY_TINYC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TINYC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION) + #undef JSON_HEDLEY_DMC_VERSION +#endif +#if defined(__DMC__) + #define JSON_HEDLEY_DMC_VERSION JSON_HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION_CHECK) + #undef JSON_HEDLEY_DMC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_DMC_VERSION) + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_DMC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #undef JSON_HEDLEY_COMPCERT_VERSION +#endif +#if defined(__COMPCERT_VERSION__) + #define JSON_HEDLEY_COMPCERT_VERSION JSON_HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, __COMPCERT_VERSION__ % 100) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION_CHECK) + #undef JSON_HEDLEY_COMPCERT_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_COMPCERT_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION) + #undef JSON_HEDLEY_PELLES_VERSION +#endif +#if defined(__POCC__) + #define JSON_HEDLEY_PELLES_VERSION JSON_HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION_CHECK) + #undef JSON_HEDLEY_PELLES_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PELLES_VERSION) + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PELLES_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #undef JSON_HEDLEY_MCST_LCC_VERSION +#endif +#if defined(__LCC__) && defined(__LCC_MINOR__) + #define JSON_HEDLEY_MCST_LCC_VERSION JSON_HEDLEY_VERSION_ENCODE(__LCC__ / 100, __LCC__ % 100, __LCC_MINOR__) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION_CHECK) + #undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_MCST_LCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION) + #undef JSON_HEDLEY_GCC_VERSION +#endif +#if \ + defined(JSON_HEDLEY_GNUC_VERSION) && \ + !defined(__clang__) && \ + !defined(JSON_HEDLEY_INTEL_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_ARM_VERSION) && \ + !defined(JSON_HEDLEY_CRAY_VERSION) && \ + !defined(JSON_HEDLEY_TI_VERSION) && \ + !defined(JSON_HEDLEY_TI_ARMCL_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL430_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL2000_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL6X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL7X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CLPRU_VERSION) && \ + !defined(__COMPCERT__) && \ + !defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION JSON_HEDLEY_GNUC_VERSION +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_ATTRIBUTE +#endif +#if \ + defined(__has_attribute) && \ + ( \ + (!defined(JSON_HEDLEY_IAR_VERSION) || JSON_HEDLEY_IAR_VERSION_CHECK(8,5,9)) \ + ) +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute) +#else +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE +#endif +#if \ + defined(__has_cpp_attribute) && \ + defined(__cplusplus) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS +#endif +#if !defined(__cplusplus) || !defined(__has_cpp_attribute) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#elif \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) && \ + (!defined(JSON_HEDLEY_MSVC_VERSION) || JSON_HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(ns::attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_BUILTIN) + #undef JSON_HEDLEY_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin) +#else + #define JSON_HEDLEY_HAS_BUILTIN(builtin) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_BUILTIN) + #undef JSON_HEDLEY_GNUC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_BUILTIN) + #undef JSON_HEDLEY_GCC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_FEATURE) + #undef JSON_HEDLEY_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_HAS_FEATURE(feature) __has_feature(feature) +#else + #define JSON_HEDLEY_HAS_FEATURE(feature) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_FEATURE) + #undef JSON_HEDLEY_GNUC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_FEATURE) + #undef JSON_HEDLEY_GCC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_EXTENSION) + #undef JSON_HEDLEY_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_HAS_EXTENSION(extension) __has_extension(extension) +#else + #define JSON_HEDLEY_HAS_EXTENSION(extension) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_EXTENSION) + #undef JSON_HEDLEY_GNUC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_EXTENSION) + #undef JSON_HEDLEY_GCC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_WARNING) + #undef JSON_HEDLEY_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_HAS_WARNING(warning) __has_warning(warning) +#else + #define JSON_HEDLEY_HAS_WARNING(warning) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_WARNING) + #undef JSON_HEDLEY_GNUC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_WARNING) + #undef JSON_HEDLEY_GCC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + defined(__clang__) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,17) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(8,0,0) || \ + (JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) && defined(__C99_PRAGMA_OPERATOR)) + #define JSON_HEDLEY_PRAGMA(value) _Pragma(#value) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_PRAGMA(value) __pragma(value) +#else + #define JSON_HEDLEY_PRAGMA(value) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_PUSH) + #undef JSON_HEDLEY_DIAGNOSTIC_PUSH +#endif +#if defined(JSON_HEDLEY_DIAGNOSTIC_POP) + #undef JSON_HEDLEY_DIAGNOSTIC_POP +#endif +#if defined(__clang__) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push)) + #define JSON_HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop)) +#elif JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("pop") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("diag_push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("diag_pop") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_PUSH + #define JSON_HEDLEY_DIAGNOSTIC_POP +#endif + +/* JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat") +# if JSON_HEDLEY_HAS_WARNING("-Wc++17-extensions") +# if JSON_HEDLEY_HAS_WARNING("-Wc++1z-extensions") +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + _Pragma("clang diagnostic ignored \"-Wc++1z-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# endif +#endif +#if !defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(x) x +#endif + +#if defined(JSON_HEDLEY_CONST_CAST) + #undef JSON_HEDLEY_CONST_CAST +#endif +#if defined(__cplusplus) +# define JSON_HEDLEY_CONST_CAST(T, expr) (const_cast(expr)) +#elif \ + JSON_HEDLEY_HAS_WARNING("-Wcast-qual") || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_CONST_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_CONST_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_REINTERPRET_CAST) + #undef JSON_HEDLEY_REINTERPRET_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast(expr)) +#else + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_STATIC_CAST) + #undef JSON_HEDLEY_STATIC_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_STATIC_CAST(T, expr) (static_cast(expr)) +#else + #define JSON_HEDLEY_STATIC_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_CPP_CAST) + #undef JSON_HEDLEY_CPP_CAST +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wold-style-cast") +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wold-style-cast\"") \ + ((T) (expr)) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# elif JSON_HEDLEY_IAR_VERSION_CHECK(8,3,0) +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("diag_suppress=Pe137") \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_CPP_CAST(T, expr) ((T) (expr)) +# endif +#else +# define JSON_HEDLEY_CPP_CAST(T, expr) (expr) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wdeprecated-declarations") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warning(disable:1478 1786)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:1478 1786)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1216,1444,1445") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:4996)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1291,1718") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && !defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,symdeprecated,symdeprecated2)") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress=Pe1444,Pe1215") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warn(disable:2241)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("clang diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("warning(disable:161)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:161)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 1675") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("GCC diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:4068)) +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(16,9,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress=Pe161") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 161") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-attributes") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("clang diagnostic ignored \"-Wunknown-attributes\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("warning(disable:1292)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:1292)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:5030)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097,1098") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("error_messages(off,attrskipunsup)") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1173") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress=Pe1097") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wcast-qual") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("clang diagnostic ignored \"-Wcast-qual\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("warning(disable:2203 2331)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunused-function") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("clang diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("GCC diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(1,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION __pragma(warning(disable:4505)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("diag_suppress 3142") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif + +#if defined(JSON_HEDLEY_DEPRECATED) + #undef JSON_HEDLEY_DEPRECATED +#endif +#if defined(JSON_HEDLEY_DEPRECATED_FOR) + #undef JSON_HEDLEY_DEPRECATED_FOR +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated("Since " # since)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated("Since " #since "; use " #replacement)) +#elif \ + (JSON_HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__("Since " #since))) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__("Since " #since "; use " #replacement))) +#elif defined(__cplusplus) && (__cplusplus >= 201402L) + #define JSON_HEDLEY_DEPRECATED(since) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since)]]) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since "; use " #replacement)]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(deprecated) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_PELLES_VERSION_CHECK(6,50,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DEPRECATED(since) _Pragma("deprecated") + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma("deprecated") +#else + #define JSON_HEDLEY_DEPRECATED(since) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) +#endif + +#if defined(JSON_HEDLEY_UNAVAILABLE) + #undef JSON_HEDLEY_UNAVAILABLE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warning) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__("Not available until " #available_since))) +#else + #define JSON_HEDLEY_UNAVAILABLE(available_since) +#endif + +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT +#endif +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT_MSG) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) __attribute__((__warn_unused_result__)) +#elif (JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) >= 201907L) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard(msg)]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) +#elif defined(_Check_return_) /* SAL */ + #define JSON_HEDLEY_WARN_UNUSED_RESULT _Check_return_ + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) _Check_return_ +#else + #define JSON_HEDLEY_WARN_UNUSED_RESULT + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) +#endif + +#if defined(JSON_HEDLEY_SENTINEL) + #undef JSON_HEDLEY_SENTINEL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(sentinel) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position))) +#else + #define JSON_HEDLEY_SENTINEL(position) +#endif + +#if defined(JSON_HEDLEY_NO_RETURN) + #undef JSON_HEDLEY_NO_RETURN +#endif +#if JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NO_RETURN __noreturn +#elif \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L + #define JSON_HEDLEY_NO_RETURN _Noreturn +#elif defined(__cplusplus) && (__cplusplus >= 201103L) + #define JSON_HEDLEY_NO_RETURN JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[noreturn]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(noreturn) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,2,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_NO_RETURN _Pragma("does_not_return") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NO_RETURN _Pragma("FUNC_NEVER_RETURNS;") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NO_RETURN __attribute((noreturn)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#else + #define JSON_HEDLEY_NO_RETURN +#endif + +#if defined(JSON_HEDLEY_NO_ESCAPE) + #undef JSON_HEDLEY_NO_ESCAPE +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(noescape) + #define JSON_HEDLEY_NO_ESCAPE __attribute__((__noescape__)) +#else + #define JSON_HEDLEY_NO_ESCAPE +#endif + +#if defined(JSON_HEDLEY_UNREACHABLE) + #undef JSON_HEDLEY_UNREACHABLE +#endif +#if defined(JSON_HEDLEY_UNREACHABLE_RETURN) + #undef JSON_HEDLEY_UNREACHABLE_RETURN +#endif +#if defined(JSON_HEDLEY_ASSUME) + #undef JSON_HEDLEY_ASSUME +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_ASSUME(expr) __assume(expr) +#elif JSON_HEDLEY_HAS_BUILTIN(__builtin_assume) + #define JSON_HEDLEY_ASSUME(expr) __builtin_assume(expr) +#elif \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #if defined(__cplusplus) + #define JSON_HEDLEY_ASSUME(expr) std::_nassert(expr) + #else + #define JSON_HEDLEY_ASSUME(expr) _nassert(expr) + #endif +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(JSON_HEDLEY_ARM_VERSION))) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,5) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(10,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNREACHABLE() __builtin_unreachable() +#elif defined(JSON_HEDLEY_ASSUME) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif +#if !defined(JSON_HEDLEY_ASSUME) + #if defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, ((expr) ? 1 : (JSON_HEDLEY_UNREACHABLE(), 1))) + #else + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, expr) + #endif +#endif +#if defined(JSON_HEDLEY_UNREACHABLE) + #if \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (JSON_HEDLEY_STATIC_CAST(void, JSON_HEDLEY_ASSUME(0)), (value)) + #else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) JSON_HEDLEY_UNREACHABLE() + #endif +#else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (value) +#endif +#if !defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif + +JSON_HEDLEY_DIAGNOSTIC_PUSH +#if JSON_HEDLEY_HAS_WARNING("-Wpedantic") + #pragma clang diagnostic ignored "-Wpedantic" +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat-pedantic") && defined(__cplusplus) + #pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +#endif +#if JSON_HEDLEY_GCC_HAS_WARNING("-Wvariadic-macros",4,0,0) + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wvariadic-macros" + #elif defined(JSON_HEDLEY_GCC_VERSION) + #pragma GCC diagnostic ignored "-Wvariadic-macros" + #endif +#endif +#if defined(JSON_HEDLEY_NON_NULL) + #undef JSON_HEDLEY_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__))) +#else + #define JSON_HEDLEY_NON_NULL(...) +#endif +JSON_HEDLEY_DIAGNOSTIC_POP + +#if defined(JSON_HEDLEY_PRINTF_FORMAT) + #undef JSON_HEDLEY_PRINTF_FORMAT +#endif +#if defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && !defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(ms_printf, string_idx, first_to_check))) +#elif defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(gnu_printf, string_idx, first_to_check))) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(format) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(__printf__, string_idx, first_to_check))) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(6,0,0) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __declspec(vaformat(printf,string_idx,first_to_check)) +#else + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) +#endif + +#if defined(JSON_HEDLEY_CONSTEXPR) + #undef JSON_HEDLEY_CONSTEXPR +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_CONSTEXPR JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(constexpr) + #endif +#endif +#if !defined(JSON_HEDLEY_CONSTEXPR) + #define JSON_HEDLEY_CONSTEXPR +#endif + +#if defined(JSON_HEDLEY_PREDICT) + #undef JSON_HEDLEY_PREDICT +#endif +#if defined(JSON_HEDLEY_LIKELY) + #undef JSON_HEDLEY_LIKELY +#endif +#if defined(JSON_HEDLEY_UNLIKELY) + #undef JSON_HEDLEY_UNLIKELY +#endif +#if defined(JSON_HEDLEY_UNPREDICTABLE) + #undef JSON_HEDLEY_UNPREDICTABLE +#endif +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_unpredictable) + #define JSON_HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable((expr)) +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) && !defined(JSON_HEDLEY_PGI_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(9,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability( (expr), (value), (probability)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) __builtin_expect_with_probability(!!(expr), 1 , (probability)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) __builtin_expect_with_probability(!!(expr), 0 , (probability)) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect (!!(expr), 1 ) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect (!!(expr), 0 ) +#elif \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,27) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, expected, probability) \ + (((probability) >= 0.9) ? __builtin_expect((expr), (expected)) : (JSON_HEDLEY_STATIC_CAST(void, expected), (expr))) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \ + })) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \ + })) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#else +# define JSON_HEDLEY_PREDICT(expr, expected, probability) (JSON_HEDLEY_STATIC_CAST(void, expected), (expr)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_LIKELY(expr) (!!(expr)) +# define JSON_HEDLEY_UNLIKELY(expr) (!!(expr)) +#endif +#if !defined(JSON_HEDLEY_UNPREDICTABLE) + #define JSON_HEDLEY_UNPREDICTABLE(expr) JSON_HEDLEY_PREDICT(expr, 1, 0.5) +#endif + +#if defined(JSON_HEDLEY_MALLOC) + #undef JSON_HEDLEY_MALLOC +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(malloc) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_MALLOC __attribute__((__malloc__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_MALLOC _Pragma("returns_new_memory") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_MALLOC __declspec(restrict) +#else + #define JSON_HEDLEY_MALLOC +#endif + +#if defined(JSON_HEDLEY_PURE) + #undef JSON_HEDLEY_PURE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(pure) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,96,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PURE __attribute__((__pure__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) +# define JSON_HEDLEY_PURE _Pragma("does_not_write_global_data") +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) \ + ) +# define JSON_HEDLEY_PURE _Pragma("FUNC_IS_PURE;") +#else +# define JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_CONST) + #undef JSON_HEDLEY_CONST +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(const) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_CONST __attribute__((__const__)) +#elif \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_CONST _Pragma("no_side_effect") +#else + #define JSON_HEDLEY_CONST JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_RESTRICT) + #undef JSON_HEDLEY_RESTRICT +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT restrict +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,4) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + defined(__clang__) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RESTRICT __restrict +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,3,0) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT _Restrict +#else + #define JSON_HEDLEY_RESTRICT +#endif + +#if defined(JSON_HEDLEY_INLINE) + #undef JSON_HEDLEY_INLINE +#endif +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + (defined(__cplusplus) && (__cplusplus >= 199711L)) + #define JSON_HEDLEY_INLINE inline +#elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(6,2,0) + #define JSON_HEDLEY_INLINE __inline__ +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,1,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_INLINE __inline +#else + #define JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_ALWAYS_INLINE) + #undef JSON_HEDLEY_ALWAYS_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(always_inline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) +# define JSON_HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) JSON_HEDLEY_INLINE +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_ALWAYS_INLINE __forceinline +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) \ + ) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("FUNC_ALWAYS_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("inline=forced") +#else +# define JSON_HEDLEY_ALWAYS_INLINE JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_NEVER_INLINE) + #undef JSON_HEDLEY_NEVER_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(noinline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute__((__noinline__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(10,2,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("noinline") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("FUNC_CANNOT_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("inline=never") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute((noinline)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#else + #define JSON_HEDLEY_NEVER_INLINE +#endif + +#if defined(JSON_HEDLEY_PRIVATE) + #undef JSON_HEDLEY_PRIVATE +#endif +#if defined(JSON_HEDLEY_PUBLIC) + #undef JSON_HEDLEY_PUBLIC +#endif +#if defined(JSON_HEDLEY_IMPORT) + #undef JSON_HEDLEY_IMPORT +#endif +#if defined(_WIN32) || defined(__CYGWIN__) +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC __declspec(dllexport) +# define JSON_HEDLEY_IMPORT __declspec(dllimport) +#else +# if \ + JSON_HEDLEY_HAS_ATTRIBUTE(visibility) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + ( \ + defined(__TI_EABI__) && \ + ( \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) \ + ) \ + ) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PRIVATE __attribute__((__visibility__("hidden"))) +# define JSON_HEDLEY_PUBLIC __attribute__((__visibility__("default"))) +# else +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC +# endif +# define JSON_HEDLEY_IMPORT extern +#endif + +#if defined(JSON_HEDLEY_NO_THROW) + #undef JSON_HEDLEY_NO_THROW +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nothrow) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_THROW __attribute__((__nothrow__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NO_THROW __declspec(nothrow) +#else + #define JSON_HEDLEY_NO_THROW +#endif + +#if defined(JSON_HEDLEY_FALL_THROUGH) + #undef JSON_HEDLEY_FALL_THROUGH +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(fallthrough) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_FALL_THROUGH __attribute__((__fallthrough__)) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(clang,fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[clang::fallthrough]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[fallthrough]]) +#elif defined(__fallthrough) /* SAL */ + #define JSON_HEDLEY_FALL_THROUGH __fallthrough +#else + #define JSON_HEDLEY_FALL_THROUGH +#endif + +#if defined(JSON_HEDLEY_RETURNS_NON_NULL) + #undef JSON_HEDLEY_RETURNS_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__)) +#elif defined(_Ret_notnull_) /* SAL */ + #define JSON_HEDLEY_RETURNS_NON_NULL _Ret_notnull_ +#else + #define JSON_HEDLEY_RETURNS_NON_NULL +#endif + +#if defined(JSON_HEDLEY_ARRAY_PARAM) + #undef JSON_HEDLEY_ARRAY_PARAM +#endif +#if \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ + !defined(__STDC_NO_VLA__) && \ + !defined(__cplusplus) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_ARRAY_PARAM(name) (name) +#else + #define JSON_HEDLEY_ARRAY_PARAM(name) +#endif + +#if defined(JSON_HEDLEY_IS_CONSTANT) + #undef JSON_HEDLEY_IS_CONSTANT +#endif +#if defined(JSON_HEDLEY_REQUIRE_CONSTEXPR) + #undef JSON_HEDLEY_REQUIRE_CONSTEXPR +#endif +/* JSON_HEDLEY_IS_CONSTEXPR_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #undef JSON_HEDLEY_IS_CONSTEXPR_ +#endif +#if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_constant_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,19) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) && !defined(__cplusplus)) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr) +#endif +#if !defined(__cplusplus) +# if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,24) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0)), int*) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((intptr_t) ((expr) * 0)) : (int*) 0)), int*) +#endif +# elif \ + ( \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \ + !defined(JSON_HEDLEY_SUNPRO_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION)) || \ + (JSON_HEDLEY_HAS_EXTENSION(c_generic_selections) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,3,0) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0), int*: 1, void*: 0) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((intptr_t) * 0) : (int*) 0), int*: 1, void*: 0) +#endif +# elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + defined(JSON_HEDLEY_INTEL_VERSION) || \ + defined(JSON_HEDLEY_TINYC_VERSION) || \ + defined(JSON_HEDLEY_TI_ARMCL_VERSION) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(18,12,0) || \ + defined(JSON_HEDLEY_TI_CL2000_VERSION) || \ + defined(JSON_HEDLEY_TI_CL6X_VERSION) || \ + defined(JSON_HEDLEY_TI_CL7X_VERSION) || \ + defined(JSON_HEDLEY_TI_CLPRU_VERSION) || \ + defined(__clang__) +# define JSON_HEDLEY_IS_CONSTEXPR_(expr) ( \ + sizeof(void) != \ + sizeof(*( \ + 1 ? \ + ((void*) ((expr) * 0L) ) : \ +((struct { char v[sizeof(void) * 2]; } *) 1) \ + ) \ + ) \ + ) +# endif +#endif +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) JSON_HEDLEY_IS_CONSTEXPR_(expr) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (JSON_HEDLEY_IS_CONSTEXPR_(expr) ? (expr) : (-1)) +#else + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) (0) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (expr) +#endif + +#if defined(JSON_HEDLEY_BEGIN_C_DECLS) + #undef JSON_HEDLEY_BEGIN_C_DECLS +#endif +#if defined(JSON_HEDLEY_END_C_DECLS) + #undef JSON_HEDLEY_END_C_DECLS +#endif +#if defined(JSON_HEDLEY_C_DECL) + #undef JSON_HEDLEY_C_DECL +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_BEGIN_C_DECLS extern "C" { + #define JSON_HEDLEY_END_C_DECLS } + #define JSON_HEDLEY_C_DECL extern "C" +#else + #define JSON_HEDLEY_BEGIN_C_DECLS + #define JSON_HEDLEY_END_C_DECLS + #define JSON_HEDLEY_C_DECL +#endif + +#if defined(JSON_HEDLEY_STATIC_ASSERT) + #undef JSON_HEDLEY_STATIC_ASSERT +#endif +#if \ + !defined(__cplusplus) && ( \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || \ + (JSON_HEDLEY_HAS_FEATURE(c_static_assert) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(6,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + defined(_Static_assert) \ + ) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message) +#elif \ + (defined(__cplusplus) && (__cplusplus >= 201103L)) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(16,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(static_assert(expr, message)) +#else +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) +#endif + +#if defined(JSON_HEDLEY_NULL) + #undef JSON_HEDLEY_NULL +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_NULL JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(nullptr) + #elif defined(NULL) + #define JSON_HEDLEY_NULL NULL + #else + #define JSON_HEDLEY_NULL JSON_HEDLEY_STATIC_CAST(void*, 0) + #endif +#elif defined(NULL) + #define JSON_HEDLEY_NULL NULL +#else + #define JSON_HEDLEY_NULL ((void*) 0) +#endif + +#if defined(JSON_HEDLEY_MESSAGE) + #undef JSON_HEDLEY_MESSAGE +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_MESSAGE(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(message msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message msg) +#elif JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(_CRI message msg) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_WARNING) + #undef JSON_HEDLEY_WARNING +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_WARNING(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(clang warning msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,8,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(GCC warning msg) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_REQUIRE) + #undef JSON_HEDLEY_REQUIRE +#endif +#if defined(JSON_HEDLEY_REQUIRE_MSG) + #undef JSON_HEDLEY_REQUIRE_MSG +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(diagnose_if) +# if JSON_HEDLEY_HAS_WARNING("-Wgcc-compat") +# define JSON_HEDLEY_REQUIRE(expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), #expr, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), msg, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_REQUIRE(expr) __attribute__((diagnose_if(!(expr), #expr, "error"))) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) __attribute__((diagnose_if(!(expr), msg, "error"))) +# endif +#else +# define JSON_HEDLEY_REQUIRE(expr) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) +#endif + +#if defined(JSON_HEDLEY_FLAGS) + #undef JSON_HEDLEY_FLAGS +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(flag_enum) && (!defined(__cplusplus) || JSON_HEDLEY_HAS_WARNING("-Wbitfield-enum-conversion")) + #define JSON_HEDLEY_FLAGS __attribute__((__flag_enum__)) +#else + #define JSON_HEDLEY_FLAGS +#endif + +#if defined(JSON_HEDLEY_FLAGS_CAST) + #undef JSON_HEDLEY_FLAGS_CAST +#endif +#if JSON_HEDLEY_INTEL_VERSION_CHECK(19,0,0) +# define JSON_HEDLEY_FLAGS_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("warning(disable:188)") \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_FLAGS_CAST(T, expr) JSON_HEDLEY_STATIC_CAST(T, expr) +#endif + +#if defined(JSON_HEDLEY_EMPTY_BASES) + #undef JSON_HEDLEY_EMPTY_BASES +#endif +#if \ + (JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,23918) && !JSON_HEDLEY_MSVC_VERSION_CHECK(20,0,0)) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_EMPTY_BASES __declspec(empty_bases) +#else + #define JSON_HEDLEY_EMPTY_BASES +#endif + +/* Remaining macros are deprecated. */ + +#if defined(JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK +#endif +#if defined(__clang__) + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) (0) +#else + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_CLANG_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_BUILTIN) + #undef JSON_HEDLEY_CLANG_HAS_BUILTIN +#endif +#define JSON_HEDLEY_CLANG_HAS_BUILTIN(builtin) JSON_HEDLEY_HAS_BUILTIN(builtin) + +#if defined(JSON_HEDLEY_CLANG_HAS_FEATURE) + #undef JSON_HEDLEY_CLANG_HAS_FEATURE +#endif +#define JSON_HEDLEY_CLANG_HAS_FEATURE(feature) JSON_HEDLEY_HAS_FEATURE(feature) + +#if defined(JSON_HEDLEY_CLANG_HAS_EXTENSION) + #undef JSON_HEDLEY_CLANG_HAS_EXTENSION +#endif +#define JSON_HEDLEY_CLANG_HAS_EXTENSION(extension) JSON_HEDLEY_HAS_EXTENSION(extension) + +#if defined(JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_WARNING) + #undef JSON_HEDLEY_CLANG_HAS_WARNING +#endif +#define JSON_HEDLEY_CLANG_HAS_WARNING(warning) JSON_HEDLEY_HAS_WARNING(warning) + +#endif /* !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < X) */ + + +// This file contains all internal macro definitions (except those affecting ABI) +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// #include + + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// C++ language standard detection +// if the user manually specified the used c++ version this is skipped +#if !defined(JSON_HAS_CPP_20) && !defined(JSON_HAS_CPP_17) && !defined(JSON_HAS_CPP_14) && !defined(JSON_HAS_CPP_11) + #if (defined(__cplusplus) && __cplusplus >= 202002L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) + #define JSON_HAS_CPP_20 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 + #endif + // the cpp 11 flag is always specified because it is the minimal required version + #define JSON_HAS_CPP_11 +#endif + +#ifdef __has_include + #if __has_include() + #include + #endif +#endif + +#if !defined(JSON_HAS_FILESYSTEM) && !defined(JSON_HAS_EXPERIMENTAL_FILESYSTEM) + #ifdef JSON_HAS_CPP_17 + #if defined(__cpp_lib_filesystem) + #define JSON_HAS_FILESYSTEM 1 + #elif defined(__cpp_lib_experimental_filesystem) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif !defined(__has_include) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #endif + + // std::filesystem does not work on MinGW GCC 8: https://sourceforge.net/p/mingw-w64/bugs/737/ + #if defined(__MINGW32__) && defined(__GNUC__) && __GNUC__ == 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before GCC 8: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before Clang 7: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__clang_major__) && __clang_major__ < 7 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before MSVC 19.14: https://en.cppreference.com/w/cpp/compiler_support + #if defined(_MSC_VER) && _MSC_VER < 1914 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before iOS 13 + #if defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED < 130000 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before macOS Catalina + #if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + #endif +#endif + +#ifndef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 0 +#endif + +#ifndef JSON_HAS_FILESYSTEM + #define JSON_HAS_FILESYSTEM 0 +#endif + +#ifndef JSON_HAS_THREE_WAY_COMPARISON + #if defined(__cpp_impl_three_way_comparison) && __cpp_impl_three_way_comparison >= 201907L \ + && defined(__cpp_lib_three_way_comparison) && __cpp_lib_three_way_comparison >= 201907L + #define JSON_HAS_THREE_WAY_COMPARISON 1 + #else + #define JSON_HAS_THREE_WAY_COMPARISON 0 + #endif +#endif + +#ifndef JSON_HAS_RANGES + // ranges header shipping in GCC 11.1.0 (released 2021-04-27) has syntax error + #if defined(__GLIBCXX__) && __GLIBCXX__ == 20210427 + #define JSON_HAS_RANGES 0 + #elif defined(__cpp_lib_ranges) + #define JSON_HAS_RANGES 1 + #else + #define JSON_HAS_RANGES 0 + #endif +#endif + +#ifndef JSON_HAS_STATIC_RTTI + #if !defined(_HAS_STATIC_RTTI) || _HAS_STATIC_RTTI != 0 + #define JSON_HAS_STATIC_RTTI 1 + #else + #define JSON_HAS_STATIC_RTTI 0 + #endif +#endif + +#ifdef JSON_HAS_CPP_17 + #define JSON_INLINE_VARIABLE inline +#else + #define JSON_INLINE_VARIABLE +#endif + +#if JSON_HEDLEY_HAS_ATTRIBUTE(no_unique_address) + #define JSON_NO_UNIQUE_ADDRESS [[no_unique_address]] +#else + #define JSON_NO_UNIQUE_ADDRESS +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdocumentation" + #pragma clang diagnostic ignored "-Wdocumentation-unknown-command" +#endif + +// allow disabling exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #include + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// allow overriding assert +#if !defined(JSON_ASSERT) + #include // assert + #define JSON_ASSERT(x) assert(x) +#endif + +// allow to access some private functions (needed by the test suite) +#if defined(JSON_TESTS_PRIVATE) + #define JSON_PRIVATE_UNLESS_TESTED public +#else + #define JSON_PRIVATE_UNLESS_TESTED private +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [&j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template class ObjectType, \ + template class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template class AllocatorType, \ + template class JSONSerializer, \ + class BinaryType, \ + class CustomBaseClass> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json + +// Macros to simplify conversion from/to types + +#define NLOHMANN_JSON_EXPAND( x ) x +#define NLOHMANN_JSON_GET_MACRO(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63, _64, NAME,...) NAME +#define NLOHMANN_JSON_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \ + NLOHMANN_JSON_PASTE64, \ + NLOHMANN_JSON_PASTE63, \ + NLOHMANN_JSON_PASTE62, \ + NLOHMANN_JSON_PASTE61, \ + NLOHMANN_JSON_PASTE60, \ + NLOHMANN_JSON_PASTE59, \ + NLOHMANN_JSON_PASTE58, \ + NLOHMANN_JSON_PASTE57, \ + NLOHMANN_JSON_PASTE56, \ + NLOHMANN_JSON_PASTE55, \ + NLOHMANN_JSON_PASTE54, \ + NLOHMANN_JSON_PASTE53, \ + NLOHMANN_JSON_PASTE52, \ + NLOHMANN_JSON_PASTE51, \ + NLOHMANN_JSON_PASTE50, \ + NLOHMANN_JSON_PASTE49, \ + NLOHMANN_JSON_PASTE48, \ + NLOHMANN_JSON_PASTE47, \ + NLOHMANN_JSON_PASTE46, \ + NLOHMANN_JSON_PASTE45, \ + NLOHMANN_JSON_PASTE44, \ + NLOHMANN_JSON_PASTE43, \ + NLOHMANN_JSON_PASTE42, \ + NLOHMANN_JSON_PASTE41, \ + NLOHMANN_JSON_PASTE40, \ + NLOHMANN_JSON_PASTE39, \ + NLOHMANN_JSON_PASTE38, \ + NLOHMANN_JSON_PASTE37, \ + NLOHMANN_JSON_PASTE36, \ + NLOHMANN_JSON_PASTE35, \ + NLOHMANN_JSON_PASTE34, \ + NLOHMANN_JSON_PASTE33, \ + NLOHMANN_JSON_PASTE32, \ + NLOHMANN_JSON_PASTE31, \ + NLOHMANN_JSON_PASTE30, \ + NLOHMANN_JSON_PASTE29, \ + NLOHMANN_JSON_PASTE28, \ + NLOHMANN_JSON_PASTE27, \ + NLOHMANN_JSON_PASTE26, \ + NLOHMANN_JSON_PASTE25, \ + NLOHMANN_JSON_PASTE24, \ + NLOHMANN_JSON_PASTE23, \ + NLOHMANN_JSON_PASTE22, \ + NLOHMANN_JSON_PASTE21, \ + NLOHMANN_JSON_PASTE20, \ + NLOHMANN_JSON_PASTE19, \ + NLOHMANN_JSON_PASTE18, \ + NLOHMANN_JSON_PASTE17, \ + NLOHMANN_JSON_PASTE16, \ + NLOHMANN_JSON_PASTE15, \ + NLOHMANN_JSON_PASTE14, \ + NLOHMANN_JSON_PASTE13, \ + NLOHMANN_JSON_PASTE12, \ + NLOHMANN_JSON_PASTE11, \ + NLOHMANN_JSON_PASTE10, \ + NLOHMANN_JSON_PASTE9, \ + NLOHMANN_JSON_PASTE8, \ + NLOHMANN_JSON_PASTE7, \ + NLOHMANN_JSON_PASTE6, \ + NLOHMANN_JSON_PASTE5, \ + NLOHMANN_JSON_PASTE4, \ + NLOHMANN_JSON_PASTE3, \ + NLOHMANN_JSON_PASTE2, \ + NLOHMANN_JSON_PASTE1)(__VA_ARGS__)) +#define NLOHMANN_JSON_PASTE2(func, v1) func(v1) +#define NLOHMANN_JSON_PASTE3(func, v1, v2) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE2(func, v2) +#define NLOHMANN_JSON_PASTE4(func, v1, v2, v3) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE3(func, v2, v3) +#define NLOHMANN_JSON_PASTE5(func, v1, v2, v3, v4) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE4(func, v2, v3, v4) +#define NLOHMANN_JSON_PASTE6(func, v1, v2, v3, v4, v5) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE5(func, v2, v3, v4, v5) +#define NLOHMANN_JSON_PASTE7(func, v1, v2, v3, v4, v5, v6) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE6(func, v2, v3, v4, v5, v6) +#define NLOHMANN_JSON_PASTE8(func, v1, v2, v3, v4, v5, v6, v7) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE7(func, v2, v3, v4, v5, v6, v7) +#define NLOHMANN_JSON_PASTE9(func, v1, v2, v3, v4, v5, v6, v7, v8) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE8(func, v2, v3, v4, v5, v6, v7, v8) +#define NLOHMANN_JSON_PASTE10(func, v1, v2, v3, v4, v5, v6, v7, v8, v9) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE9(func, v2, v3, v4, v5, v6, v7, v8, v9) +#define NLOHMANN_JSON_PASTE11(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE10(func, v2, v3, v4, v5, v6, v7, v8, v9, v10) +#define NLOHMANN_JSON_PASTE12(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE11(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) +#define NLOHMANN_JSON_PASTE13(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE12(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) +#define NLOHMANN_JSON_PASTE14(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE13(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) +#define NLOHMANN_JSON_PASTE15(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE14(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) +#define NLOHMANN_JSON_PASTE16(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE15(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) +#define NLOHMANN_JSON_PASTE17(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE16(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) +#define NLOHMANN_JSON_PASTE18(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE17(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) +#define NLOHMANN_JSON_PASTE19(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE18(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) +#define NLOHMANN_JSON_PASTE20(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE19(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) +#define NLOHMANN_JSON_PASTE21(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE20(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) +#define NLOHMANN_JSON_PASTE22(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE21(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) +#define NLOHMANN_JSON_PASTE23(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE22(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) +#define NLOHMANN_JSON_PASTE24(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE23(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) +#define NLOHMANN_JSON_PASTE25(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE24(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) +#define NLOHMANN_JSON_PASTE26(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE25(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) +#define NLOHMANN_JSON_PASTE27(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE26(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) +#define NLOHMANN_JSON_PASTE28(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE27(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) +#define NLOHMANN_JSON_PASTE29(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE28(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) +#define NLOHMANN_JSON_PASTE30(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE29(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) +#define NLOHMANN_JSON_PASTE31(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE30(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) +#define NLOHMANN_JSON_PASTE32(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE31(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) +#define NLOHMANN_JSON_PASTE33(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE32(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) +#define NLOHMANN_JSON_PASTE34(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE33(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) +#define NLOHMANN_JSON_PASTE35(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE34(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) +#define NLOHMANN_JSON_PASTE36(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE35(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) +#define NLOHMANN_JSON_PASTE37(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE36(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) +#define NLOHMANN_JSON_PASTE38(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE37(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) +#define NLOHMANN_JSON_PASTE39(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE38(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) +#define NLOHMANN_JSON_PASTE40(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE39(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) +#define NLOHMANN_JSON_PASTE41(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE40(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) +#define NLOHMANN_JSON_PASTE42(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE41(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) +#define NLOHMANN_JSON_PASTE43(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE42(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) +#define NLOHMANN_JSON_PASTE44(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE43(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) +#define NLOHMANN_JSON_PASTE45(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE44(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) +#define NLOHMANN_JSON_PASTE46(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE45(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) +#define NLOHMANN_JSON_PASTE47(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE46(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) +#define NLOHMANN_JSON_PASTE48(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE47(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) +#define NLOHMANN_JSON_PASTE49(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE48(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) +#define NLOHMANN_JSON_PASTE50(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE49(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) +#define NLOHMANN_JSON_PASTE51(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE50(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) +#define NLOHMANN_JSON_PASTE52(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE51(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) +#define NLOHMANN_JSON_PASTE53(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE52(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) +#define NLOHMANN_JSON_PASTE54(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE53(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) +#define NLOHMANN_JSON_PASTE55(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE54(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) +#define NLOHMANN_JSON_PASTE56(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE55(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) +#define NLOHMANN_JSON_PASTE57(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE56(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) +#define NLOHMANN_JSON_PASTE58(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE57(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) +#define NLOHMANN_JSON_PASTE59(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE58(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) +#define NLOHMANN_JSON_PASTE60(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE59(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) +#define NLOHMANN_JSON_PASTE61(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE60(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) +#define NLOHMANN_JSON_PASTE62(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE61(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) +#define NLOHMANN_JSON_PASTE63(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE62(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) +#define NLOHMANN_JSON_PASTE64(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE63(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) + +#define NLOHMANN_JSON_TO(v1) nlohmann_json_j[#v1] = nlohmann_json_t.v1; +#define NLOHMANN_JSON_FROM(v1) nlohmann_json_j.at(#v1).get_to(nlohmann_json_t.v1); +#define NLOHMANN_JSON_FROM_WITH_DEFAULT(v1) nlohmann_json_t.v1 = nlohmann_json_j.value(#v1, nlohmann_json_default_obj.v1); + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_INTRUSIVE +@since version 3.9.0 +*/ +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE(Type, ...) \ + friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Type, ...) \ + friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE(Type, ...) \ + friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE +@since version 3.9.0 +*/ +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Type, ...) \ + inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE(Type, ...) \ + inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } + +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(Type, ...) \ + inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +// inspired from https://stackoverflow.com/a/26745591 +// allows to call any std function as if (e.g. with begin): +// using std::begin; begin(x); +// +// it allows using the detected idiom to retrieve the return type +// of such an expression +#define NLOHMANN_CAN_CALL_STD_FUNC_IMPL(std_name) \ + namespace detail { \ + using std::std_name; \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + } \ + \ + namespace detail2 { \ + struct std_name##_tag \ + { \ + }; \ + \ + template \ + std_name##_tag std_name(T&&...); \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + \ + template \ + struct would_call_std_##std_name \ + { \ + static constexpr auto const value = ::nlohmann::detail:: \ + is_detected_exact::value; \ + }; \ + } /* namespace detail2 */ \ + \ + template \ + struct would_call_std_##std_name : detail2::would_call_std_##std_name \ + { \ + } + +#ifndef JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_USE_IMPLICIT_CONVERSIONS 1 +#endif + +#if JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_EXPLICIT +#else + #define JSON_EXPLICIT explicit +#endif + +#ifndef JSON_DISABLE_ENUM_SERIALIZATION + #define JSON_DISABLE_ENUM_SERIALIZATION 0 +#endif + +#ifndef JSON_USE_GLOBAL_UDLS + #define JSON_USE_GLOBAL_UDLS 1 +#endif + +#if JSON_HAS_THREE_WAY_COMPARISON + #include // partial_ordering +#endif + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/////////////////////////// +// JSON type enumeration // +/////////////////////////// + +/*! +@brief the JSON type enumeration + +This enumeration collects the different JSON types. It is internally used to +distinguish the stored values, and the functions @ref basic_json::is_null(), +@ref basic_json::is_object(), @ref basic_json::is_array(), +@ref basic_json::is_string(), @ref basic_json::is_boolean(), +@ref basic_json::is_number() (with @ref basic_json::is_number_integer(), +@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()), +@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and +@ref basic_json::is_structured() rely on it. + +@note There are three enumeration entries (number_integer, number_unsigned, and +number_float), because the library distinguishes these three types for numbers: +@ref basic_json::number_unsigned_t is used for unsigned integers, +@ref basic_json::number_integer_t is used for signed integers, and +@ref basic_json::number_float_t is used for floating-point numbers or to +approximate integers which do not fit in the limits of their respective type. + +@sa see @ref basic_json::basic_json(const value_t value_type) -- create a JSON +value with the default value for a given type + +@since version 1.0.0 +*/ +enum class value_t : std::uint8_t +{ + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + binary, ///< binary array (ordered collection of bytes) + discarded ///< discarded by the parser callback function +}; + +/*! +@brief comparison operator for JSON types + +Returns an ordering that is similar to Python: +- order: null < boolean < number < object < array < string < binary +- furthermore, each type is not smaller than itself +- discarded values are not comparable +- binary is represented as a b"" string in python and directly comparable to a + string; however, making a binary array directly comparable with a string would + be surprising behavior in a JSON file. + +@since version 1.0.0 +*/ +#if JSON_HAS_THREE_WAY_COMPARISON + inline std::partial_ordering operator<=>(const value_t lhs, const value_t rhs) noexcept // *NOPAD* +#else + inline bool operator<(const value_t lhs, const value_t rhs) noexcept +#endif +{ + static constexpr std::array order = {{ + 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, + 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */, + 6 /* binary */ + } + }; + + const auto l_index = static_cast(lhs); + const auto r_index = static_cast(rhs); +#if JSON_HAS_THREE_WAY_COMPARISON + if (l_index < order.size() && r_index < order.size()) + { + return order[l_index] <=> order[r_index]; // *NOPAD* + } + return std::partial_ordering::unordered; +#else + return l_index < order.size() && r_index < order.size() && order[l_index] < order[r_index]; +#endif +} + +// GCC selects the built-in operator< over an operator rewritten from +// a user-defined spaceship operator +// Clang, MSVC, and ICC select the rewritten candidate +// (see GCC bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105200) +#if JSON_HAS_THREE_WAY_COMPARISON && defined(__GNUC__) +inline bool operator<(const value_t lhs, const value_t rhs) noexcept +{ + return std::is_lt(lhs <=> rhs); // *NOPAD* +} +#endif + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief replace all occurrences of a substring by another string + +@param[in,out] s the string to manipulate; changed so that all + occurrences of @a f are replaced with @a t +@param[in] f the substring to replace with @a t +@param[in] t the string to replace @a f + +@pre The search string @a f must not be empty. **This precondition is +enforced with an assertion.** + +@since version 2.0.0 +*/ +template +inline void replace_substring(StringType& s, const StringType& f, + const StringType& t) +{ + JSON_ASSERT(!f.empty()); + for (auto pos = s.find(f); // find first occurrence of f + pos != StringType::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t, and + pos = s.find(f, pos + t.size())) // find next occurrence of f + {} +} + +/*! + * @brief string escaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to escape + * @return escaped string + * + * Note the order of escaping "~" to "~0" and "/" to "~1" is important. + */ +template +inline StringType escape(StringType s) +{ + replace_substring(s, StringType{"~"}, StringType{"~0"}); + replace_substring(s, StringType{"/"}, StringType{"~1"}); + return s; +} + +/*! + * @brief string unescaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to unescape + * @return unescaped string + * + * Note the order of escaping "~1" to "/" and "~0" to "~" is important. + */ +template +static void unescape(StringType& s) +{ + replace_substring(s, StringType{"~1"}, StringType{"/"}); + replace_substring(s, StringType{"~0"}, StringType{"~"}); +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // size_t + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-FileCopyrightText: 2018 The Abseil Authors +// SPDX-License-Identifier: MIT + + + +#include // array +#include // size_t +#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type +#include // index_sequence, make_index_sequence, index_sequence_for + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template +using uncvref_t = typename std::remove_cv::type>::type; + +#ifdef JSON_HAS_CPP_14 + +// the following utilities are natively available in C++14 +using std::enable_if_t; +using std::index_sequence; +using std::make_index_sequence; +using std::index_sequence_for; + +#else + +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +// The following code is taken from https://github.com/abseil/abseil-cpp/blob/10cb35e459f5ecca5b2ff107635da0bfa41011b4/absl/utility/utility.h +// which is part of Google Abseil (https://github.com/abseil/abseil-cpp), licensed under the Apache License 2.0. + +//// START OF CODE FROM GOOGLE ABSEIL + +// integer_sequence +// +// Class template representing a compile-time integer sequence. An instantiation +// of `integer_sequence` has a sequence of integers encoded in its +// type through its template arguments (which is a common need when +// working with C++11 variadic templates). `absl::integer_sequence` is designed +// to be a drop-in replacement for C++14's `std::integer_sequence`. +// +// Example: +// +// template< class T, T... Ints > +// void user_function(integer_sequence); +// +// int main() +// { +// // user_function's `T` will be deduced to `int` and `Ints...` +// // will be deduced to `0, 1, 2, 3, 4`. +// user_function(make_integer_sequence()); +// } +template +struct integer_sequence +{ + using value_type = T; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +// index_sequence +// +// A helper template for an `integer_sequence` of `size_t`, +// `absl::index_sequence` is designed to be a drop-in replacement for C++14's +// `std::index_sequence`. +template +using index_sequence = integer_sequence; + +namespace utility_internal +{ + +template +struct Extend; + +// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency. +template +struct Extend, SeqSize, 0> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)... >; +}; + +template +struct Extend, SeqSize, 1> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)..., 2 * SeqSize >; +}; + +// Recursion helper for 'make_integer_sequence'. +// 'Gen::type' is an alias for 'integer_sequence'. +template +struct Gen +{ + using type = + typename Extend < typename Gen < T, N / 2 >::type, N / 2, N % 2 >::type; +}; + +template +struct Gen +{ + using type = integer_sequence; +}; + +} // namespace utility_internal + +// Compile-time sequences of integers + +// make_integer_sequence +// +// This template alias is equivalent to +// `integer_sequence`, and is designed to be a drop-in +// replacement for C++14's `std::make_integer_sequence`. +template +using make_integer_sequence = typename utility_internal::Gen::type; + +// make_index_sequence +// +// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`, +// and is designed to be a drop-in replacement for C++14's +// `std::make_index_sequence`. +template +using make_index_sequence = make_integer_sequence; + +// index_sequence_for +// +// Converts a typename pack into an index sequence of the same length, and +// is designed to be a drop-in replacement for C++14's +// `std::index_sequence_for()` +template +using index_sequence_for = make_index_sequence; + +//// END OF CODE FROM GOOGLE ABSEIL + +#endif + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template +struct static_const +{ + static JSON_INLINE_VARIABLE constexpr T value{}; +}; + +#ifndef JSON_HAS_CPP_17 + template + constexpr T static_const::value; +#endif + +template +inline constexpr std::array make_array(Args&& ... args) +{ + return std::array {{static_cast(std::forward(args))...}}; +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // numeric_limits +#include // false_type, is_constructible, is_integral, is_same, true_type +#include // declval +#include // tuple +#include // char_traits + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // random_access_iterator_tag + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template +struct iterator_types {}; + +template +struct iterator_types < + It, + void_t> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template +struct iterator_traits +{ +}; + +template +struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> + : iterator_types +{ +}; + +template +struct iterator_traits::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN + +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin); + +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN + +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end); + +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + +#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ + #define INCLUDE_NLOHMANN_JSON_FWD_HPP_ + + #include // int64_t, uint64_t + #include // map + #include // allocator + #include // string + #include // vector + + // #include + + + /*! + @brief namespace for Niels Lohmann + @see https://github.com/nlohmann + @since version 1.0.0 + */ + NLOHMANN_JSON_NAMESPACE_BEGIN + + /*! + @brief default JSONSerializer template argument + + This serializer ignores the template arguments and uses ADL + ([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) + for serialization. + */ + template + struct adl_serializer; + + /// a class to store JSON values + /// @sa https://json.nlohmann.me/api/basic_json/ + template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer, + class BinaryType = std::vector, // cppcheck-suppress syntaxError + class CustomBaseClass = void> + class basic_json; + + /// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document + /// @sa https://json.nlohmann.me/api/json_pointer/ + template + class json_pointer; + + /*! + @brief default specialization + @sa https://json.nlohmann.me/api/json/ + */ + using json = basic_json<>; + + /// @brief a minimal map-like container that preserves insertion order + /// @sa https://json.nlohmann.me/api/ordered_map/ + template + struct ordered_map; + + /// @brief specialization that maintains the insertion order of object keys + /// @sa https://json.nlohmann.me/api/ordered_json/ + using ordered_json = basic_json; + + NLOHMANN_JSON_NAMESPACE_END + +#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ + + +NLOHMANN_JSON_NAMESPACE_BEGIN +/*! +@brief detail namespace with internal helper functions + +This namespace collects functions that should not be exposed, +implementations of some @ref basic_json methods, and meta-programming helpers. + +@since version 2.1.0 +*/ +namespace detail +{ + +///////////// +// helpers // +///////////// + +// Note to maintainers: +// +// Every trait in this file expects a non CV-qualified type. +// The only exceptions are in the 'aliases for detected' section +// (i.e. those of the form: decltype(T::member_function(std::declval()))) +// +// In this case, T has to be properly CV-qualified to constraint the function arguments +// (e.g. to_json(BasicJsonType&, const T&)) + +template struct is_basic_json : std::false_type {}; + +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct is_basic_json : std::true_type {}; + +// used by exceptions create() member functions +// true_type for pointer to possibly cv-qualified basic_json or std::nullptr_t +// false_type otherwise +template +struct is_basic_json_context : + std::integral_constant < bool, + is_basic_json::type>::type>::value + || std::is_same::value > +{}; + +////////////////////// +// json_ref helpers // +////////////////////// + +template +class json_ref; + +template +struct is_json_ref : std::false_type {}; + +template +struct is_json_ref> : std::true_type {}; + +////////////////////////// +// aliases for detected // +////////////////////////// + +template +using mapped_type_t = typename T::mapped_type; + +template +using key_type_t = typename T::key_type; + +template +using value_type_t = typename T::value_type; + +template +using difference_type_t = typename T::difference_type; + +template +using pointer_t = typename T::pointer; + +template +using reference_t = typename T::reference; + +template +using iterator_category_t = typename T::iterator_category; + +template +using to_json_function = decltype(T::to_json(std::declval()...)); + +template +using from_json_function = decltype(T::from_json(std::declval()...)); + +template +using get_template_function = decltype(std::declval().template get()); + +// trait checking if JSONSerializer::from_json(json const&, udt&) exists +template +struct has_from_json : std::false_type {}; + +// trait checking if j.get is valid +// use this trait instead of std::is_constructible or std::is_convertible, +// both rely on, or make use of implicit conversions, and thus fail when T +// has several constructors/operator= (see https://github.com/nlohmann/json/issues/958) +template +struct is_getable +{ + static constexpr bool value = is_detected::value; +}; + +template +struct has_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if JSONSerializer::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template +struct has_non_default_from_json : std::false_type {}; + +template +struct has_non_default_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if BasicJsonType::json_serializer::to_json exists +// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion. +template +struct has_to_json : std::false_type {}; + +template +struct has_to_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +template +using detect_key_compare = typename T::key_compare; + +template +struct has_key_compare : std::integral_constant::value> {}; + +// obtains the actual object key comparator +template +struct actual_object_comparator +{ + using object_t = typename BasicJsonType::object_t; + using object_comparator_t = typename BasicJsonType::default_object_comparator_t; + using type = typename std::conditional < has_key_compare::value, + typename object_t::key_compare, object_comparator_t>::type; +}; + +template +using actual_object_comparator_t = typename actual_object_comparator::type; + +///////////////// +// char_traits // +///////////////// + +// Primary template of char_traits calls std char_traits +template +struct char_traits : std::char_traits +{}; + +// Explicitly define char traits for unsigned char since it is not standard +template<> +struct char_traits : std::char_traits +{ + using char_type = unsigned char; + using int_type = uint64_t; + + // Redefine to_int_type function + static int_type to_int_type(char_type c) noexcept + { + return static_cast(c); + } + + static char_type to_char_type(int_type i) noexcept + { + return static_cast(i); + } + + static constexpr int_type eof() noexcept + { + return static_cast(EOF); + } +}; + +// Explicitly define char traits for signed char since it is not standard +template<> +struct char_traits : std::char_traits +{ + using char_type = signed char; + using int_type = uint64_t; + + // Redefine to_int_type function + static int_type to_int_type(char_type c) noexcept + { + return static_cast(c); + } + + static char_type to_char_type(int_type i) noexcept + { + return static_cast(i); + } + + static constexpr int_type eof() noexcept + { + return static_cast(EOF); + } +}; + +/////////////////// +// is_ functions // +/////////////////// + +// https://en.cppreference.com/w/cpp/types/conjunction +template struct conjunction : std::true_type { }; +template struct conjunction : B { }; +template +struct conjunction +: std::conditional(B::value), conjunction, B>::type {}; + +// https://en.cppreference.com/w/cpp/types/negation +template struct negation : std::integral_constant < bool, !B::value > { }; + +// Reimplementation of is_constructible and is_default_constructible, due to them being broken for +// std::pair and std::tuple until LWG 2367 fix (see https://cplusplus.github.io/LWG/lwg-defects.html#2367). +// This causes compile errors in e.g. clang 3.5 or gcc 4.9. +template +struct is_default_constructible : std::is_default_constructible {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + +template +struct is_constructible : std::is_constructible {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_iterator_traits : std::false_type {}; + +template +struct is_iterator_traits> +{ + private: + using traits = iterator_traits; + + public: + static constexpr auto value = + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value; +}; + +template +struct is_range +{ + private: + using t_ref = typename std::add_lvalue_reference::type; + + using iterator = detected_t; + using sentinel = detected_t; + + // to be 100% correct, it should use https://en.cppreference.com/w/cpp/iterator/input_or_output_iterator + // and https://en.cppreference.com/w/cpp/iterator/sentinel_for + // but reimplementing these would be too much work, as a lot of other concepts are used underneath + static constexpr auto is_iterator_begin = + is_iterator_traits>::value; + + public: + static constexpr bool value = !std::is_same::value && !std::is_same::value && is_iterator_begin; +}; + +template +using iterator_t = enable_if_t::value, result_of_begin())>>; + +template +using range_value_t = value_type_t>>; + +// The following implementation of is_complete_type is taken from +// https://blogs.msdn.microsoft.com/vcblog/2015/12/02/partial-support-for-expression-sfinae-in-vs-2015-update-1/ +// and is written by Xiang Fan who agreed to using it in this library. + +template +struct is_complete_type : std::false_type {}; + +template +struct is_complete_type : std::true_type {}; + +template +struct is_compatible_object_type_impl : std::false_type {}; + +template +struct is_compatible_object_type_impl < + BasicJsonType, CompatibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + // macOS's is_constructible does not play well with nonesuch... + static constexpr bool value = + is_constructible::value && + is_constructible::value; +}; + +template +struct is_compatible_object_type + : is_compatible_object_type_impl {}; + +template +struct is_constructible_object_type_impl : std::false_type {}; + +template +struct is_constructible_object_type_impl < + BasicJsonType, ConstructibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + static constexpr bool value = + (is_default_constructible::value && + (std::is_move_assignable::value || + std::is_copy_assignable::value) && + (is_constructible::value && + std::is_same < + typename object_t::mapped_type, + typename ConstructibleObjectType::mapped_type >::value)) || + (has_from_json::value || + has_non_default_from_json < + BasicJsonType, + typename ConstructibleObjectType::mapped_type >::value); +}; + +template +struct is_constructible_object_type + : is_constructible_object_type_impl {}; + +template +struct is_compatible_string_type +{ + static constexpr auto value = + is_constructible::value; +}; + +template +struct is_constructible_string_type +{ + // launder type through decltype() to fix compilation failure on ICPC +#ifdef __INTEL_COMPILER + using laundered_type = decltype(std::declval()); +#else + using laundered_type = ConstructibleStringType; +#endif + + static constexpr auto value = + conjunction < + is_constructible, + is_detected_exact>::value; +}; + +template +struct is_compatible_array_type_impl : std::false_type {}; + +template +struct is_compatible_array_type_impl < + BasicJsonType, CompatibleArrayType, + enable_if_t < + is_detected::value&& + is_iterator_traits>>::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 + !std::is_same>::value >> +{ + static constexpr bool value = + is_constructible>::value; +}; + +template +struct is_compatible_array_type + : is_compatible_array_type_impl {}; + +template +struct is_constructible_array_type_impl : std::false_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t::value >> + : std::true_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t < !std::is_same::value&& + !is_compatible_string_type::value&& + is_default_constructible::value&& +(std::is_move_assignable::value || + std::is_copy_assignable::value)&& +is_detected::value&& +is_iterator_traits>>::value&& +is_detected::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 +!std::is_same>::value&& + is_complete_type < + detected_t>::value >> +{ + using value_type = range_value_t; + + static constexpr bool value = + std::is_same::value || + has_from_json::value || + has_non_default_from_json < + BasicJsonType, + value_type >::value; +}; + +template +struct is_constructible_array_type + : is_constructible_array_type_impl {}; + +template +struct is_compatible_integer_type_impl : std::false_type {}; + +template +struct is_compatible_integer_type_impl < + RealIntegerType, CompatibleNumberIntegerType, + enable_if_t < std::is_integral::value&& + std::is_integral::value&& + !std::is_same::value >> +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits; + using CompatibleLimits = std::numeric_limits; + + static constexpr auto value = + is_constructible::value && + CompatibleLimits::is_integer && + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template +struct is_compatible_integer_type + : is_compatible_integer_type_impl {}; + +template +struct is_compatible_type_impl: std::false_type {}; + +template +struct is_compatible_type_impl < + BasicJsonType, CompatibleType, + enable_if_t::value >> +{ + static constexpr bool value = + has_to_json::value; +}; + +template +struct is_compatible_type + : is_compatible_type_impl {}; + +template +struct is_constructible_tuple : std::false_type {}; + +template +struct is_constructible_tuple> : conjunction...> {}; + +template +struct is_json_iterator_of : std::false_type {}; + +template +struct is_json_iterator_of : std::true_type {}; + +template +struct is_json_iterator_of : std::true_type +{}; + +// checks if a given type T is a template specialization of Primary +template