Skip to content

Commit

Permalink
Html example, situation queue fix, minors. (#5)
Browse files Browse the repository at this point in the history
* Html parser example. Queue fix. Minor fixes.

* Version bump.
  • Loading branch information
peter-winter authored Nov 24, 2021
1 parent 5961076 commit 33f06f4
Show file tree
Hide file tree
Showing 2 changed files with 140 additions and 19 deletions.
42 changes: 23 additions & 19 deletions ctpg.hpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#ifndef CTPG_H
#define CTPG_H

constexpr const char* version_str = "1.1.0";
constexpr const char* version_str = "1.1.1";

#include <utility>
#include <type_traits>
Expand Down Expand Up @@ -729,11 +729,10 @@ class string_term : public term
static const size_t dfa_size = (DataSize - 1) * 2;
static const bool is_trivial = true;

template<size_t N>
constexpr string_term(const char (&str)[N], int precedence = 0, associativity a = associativity::no_assoc):
constexpr string_term(const char (&str)[DataSize], int precedence = 0, associativity a = associativity::no_assoc):
term(precedence, a)
{
utils::copy_array(data, str, std::make_index_sequence<N>{});
utils::copy_array(data, str, std::make_index_sequence<DataSize>{});
}

constexpr const char* get_id() const { return data; }
Expand Down Expand Up @@ -1836,9 +1835,9 @@ class parser<
}

private:
static const size_t max_states = MaxStatesUsage::template value<sizeof...(Terms), Rules::n...>;
static const size_t max_rule_element_count = meta::max_v<1, Rules::n...>;
static const size_t term_count = sizeof...(Terms) + 1;
static const size_t max_states = MaxStatesUsage::template value<term_count, Rules::n..., 1>;
static const size16_t eof_idx = sizeof...(Terms);
static const size_t nterm_count = sizeof...(NTermValueType) + 1;
static const size16_t fake_root_idx = sizeof...(NTermValueType);
Expand Down Expand Up @@ -1867,7 +1866,10 @@ class parser<
size32_t idx;
};

using situation_queue_t = stdex::cqueue<situation_queue_entry, situation_address_space_size>;
using situation_queue_t = stdex::cqueue<
situation_queue_entry,
max_states * max_states
>;

struct symbol
{
Expand Down Expand Up @@ -2158,23 +2160,26 @@ class parser<
situation_info root_situation_info{ root_rule_idx, 0, eof_idx };
size32_t idx = make_situation_idx(root_situation_info);
state_count = 1;
add_situation_to_state(0, idx);

situation_queue_t situation_queue = { };

add_situation_to_state(0, idx, situation_queue);

while (!situation_queue.empty())
{
const auto& entry = situation_queue.top();
analyze_situation(entry.state_idx, entry.idx);
situation_queue.pop();
analyze_situation(entry.state_idx, entry.idx, situation_queue);
}
}

constexpr void analyze_situation(size16_t state_idx, size32_t idx)
constexpr void analyze_situation(size16_t state_idx, size32_t idx, situation_queue_t& situation_queue)
{
situation_closure(state_idx, idx);
situation_transition(state_idx, idx);
situation_closure(state_idx, idx, situation_queue);
situation_transition(state_idx, idx, situation_queue);
}

constexpr void add_situation_to_state(size16_t state_idx, size32_t idx)
constexpr void add_situation_to_state(size16_t state_idx, size32_t idx, situation_queue_t& situation_queue)
{
state& s = states[state_idx];
if (!s.test(idx))
Expand All @@ -2184,7 +2189,7 @@ class parser<
}
}

constexpr void situation_closure(size16_t state_idx, size32_t idx)
constexpr void situation_closure(size16_t state_idx, size32_t idx, situation_queue_t& situation_queue)
{
situation_info addr = make_situation_info(idx);
const rule_info& ri = rule_infos[addr.rule_info_idx];
Expand All @@ -2204,7 +2209,7 @@ class parser<
if (first.test(t))
{
size32_t new_s_idx = make_situation_idx(situation_info{ size16_t(s.start + i), 0, t });
add_situation_to_state(state_idx, new_s_idx);
add_situation_to_state(state_idx, new_s_idx, situation_queue);
}
}
}
Expand All @@ -2227,7 +2232,7 @@ class parser<
return parse_table_entry_kind::shift;
}

constexpr void situation_transition(size16_t state_idx, size32_t idx)
constexpr void situation_transition(size16_t state_idx, size32_t idx, situation_queue_t& situation_queue)
{
situation_info addr = make_situation_info(idx);
const rule_info& ri = rule_infos[addr.rule_info_idx];
Expand Down Expand Up @@ -2263,7 +2268,7 @@ class parser<

if (entry.has_shift)
{
add_situation_to_state(entry.shift, new_idx);
add_situation_to_state(entry.shift, new_idx, situation_queue);
return;
}

Expand Down Expand Up @@ -2291,7 +2296,7 @@ class parser<
entry.kind = parse_table_entry_kind::shift;
};
entry.set_shift(new_state_idx);
add_situation_to_state(new_state_idx, new_idx);
add_situation_to_state(new_state_idx, new_idx, situation_queue);
}

constexpr const char* get_symbol_name(const symbol& s) const
Expand Down Expand Up @@ -2540,7 +2545,6 @@ class parser<
state states[max_states] = { };
parse_table_entry parse_table[max_states][term_count + nterm_count] = {};
size16_t state_count = 0;
situation_queue_t situation_queue = { };
int term_precedences[term_count] = { };
associativity term_associativities[term_count] = { };
int rule_precedences[rule_count] = { };
Expand Down Expand Up @@ -2570,7 +2574,7 @@ struct use_max_states
struct deduce_max_states
{
template<size_t TermCount, size_t... RuleSizes>
static const size_t value = ((0 + ... + (RuleSizes + 1)) + 2) * (TermCount + 1);
static const size_t value = (0 + ... + (RuleSizes + 1)) * TermCount;
};

template<typename Root, typename Terms, typename NTerms, typename Rules>
Expand Down
117 changes: 117 additions & 0 deletions examples/html.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/*
HTML parser
It only deals with matching tags parsing for simplification.
Texts and attributes are a trivial task to add
This example shows how to take advantage of the fact that the parser is defined in c++.
Here both terms and rules are generated from tag names
*/
#include "../ctpg.hpp"

#include <iostream>
#include <vector>

using namespace ctpg;
using namespace ctpg::ftors;
using namespace ctpg::buffers;

struct html_block;
using list_type = std::vector<html_block>;

class html_block
{
public:
html_block(std::string_view tag, list_type&& list):
name(tag), inner_blocks(std::move(list))
{}

private:
std::string name;
list_type inner_blocks;
};

template<std::size_t N>
struct tag
{
constexpr tag(const char (&name)[N])
{
open_tag_str[0] = '<';
for (auto i = 0u; i < N - 1; ++i) // skip terminating 0
open_tag_str[i + 1] = name[i];
open_tag_str[N] = '>';

close_tag_str[0] = '<';
close_tag_str[1] = '/';
for (auto i = 0u; i < N - 1; ++i) // skip terminating 0
close_tag_str[i + 2] = name[i];
close_tag_str[N + 1] = '>';
}

constexpr const auto& get_open_tag_str() const { return open_tag_str; }
constexpr const auto& get_close_tag_str() const { return close_tag_str; }

char open_tag_str[N + 2] = {0};
char close_tag_str[N + 3] = {0};
};

constexpr auto tag_tuple = std::make_tuple(
tag("div"),
tag("span"),
tag("a"),
tag("button")
);

constexpr nterm<html_block> block("block");
constexpr nterm<list_type> block_list("list");

template<typename Tags>
constexpr auto create_terms(Tags tags)
{
return std::apply(
[](auto... tag)
{
return ctpg::terms(tag.get_open_tag_str()..., tag.get_close_tag_str()...);
},
tags
);
}

template<typename Tags>
constexpr auto create_rules(Tags tags)
{
return std::apply(
[](auto... tag)
{
// create rule in the form:
// block("<tag>" block_list "</tag>") for each tag
// and add last two rules not depending on tags
return ctpg::rules(
block(tag.get_open_tag_str(), block_list, tag.get_close_tag_str())
>= [](auto tag, auto&& list, skip){ return html_block(tag, std::move(list)); }...,
block_list()
>= create<list_type>{},
block_list(block_list, block)
>= [](auto&& list, auto&& block){ list.emplace_back(std::move(block)); return std::move(list); }
);
},
tags
);
}

constexpr parser p(
block, // root
create_terms(tag_tuple), // terms
nterms(block, block_list), // nonterms
create_rules(tag_tuple) // rules
);

int main(int argc, char const *argv[])
{
if (argc != 2)
{
p.write_diag_str(std::cout);
return 0;
}

auto res = p.parse(parse_options{}.set_verbose(), string_buffer(argv[1]), std::cerr);
return 0;
}

0 comments on commit 33f06f4

Please sign in to comment.