diff --git a/src/checker.cpp b/src/checker.cpp index 7c9888c..872b32a 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -23,16 +23,14 @@ namespace divvun { const std::u16string from_bytes(const std::string& s) { - std::wstring_convert, char16_t> utf16conv; - return utf16conv.from_bytes(s); + return fromUtf8(s); } // CheckerSpec CheckerSpec::CheckerSpec(const std::string& file) : pImpl( new PipeSpec(file) ) { - // std::wstring_convert, char16_t> utf16conv; // for(const auto& k : pImpl->pnodes) { - // std::cerr << "init " << utf16conv.to_bytes(k.first.c_str()) < CheckerSpec::pipeNames() const { - std::wstring_convert, char16_t> utf16conv; std::set keys; for(const auto& it : pImpl->pnodes) { - keys.insert(utf16conv.to_bytes(it.first)); + keys.insert(toUtf8(it.first)); } return keys; } @@ -60,9 +57,8 @@ std::unique_ptr CheckerSpec::getChecker(const std::string& pipename, bo // ArCheckerSpec ArCheckerSpec::ArCheckerSpec(const std::string& file) : pImpl( readArPipeSpec(file) ) { - // std::wstring_convert, char16_t> utf16conv; // for(const auto& k : pImpl->spec->pnodes) { - // std::cerr << "init " << utf16conv.to_bytes(k.first.c_str()) < ArCheckerSpec::pipeNames() const { - std::wstring_convert, char16_t> utf16conv; std::set keys; for(const auto& it : pImpl->spec->pnodes) { - keys.insert(utf16conv.to_bytes(it.first)); + keys.insert(toUtf8(it.first)); } return keys; } diff --git a/src/divvun-blanktag.1 b/src/divvun-blanktag.1 index 0c6f908..db3ca00 100644 --- a/src/divvun-blanktag.1 +++ b/src/divvun-blanktag.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH DIVVUN-BLANKTAG "1" "January 2018" "divvun-gramcheck" "User Commands" +.TH DIVVUN-BLANKTAG "1" "February 2018" "divvun-gramcheck" "User Commands" .SH NAME divvun-blanktag \- manual page for divvun-blanktag - Divvun gramcheck version 0.2.0 .SH DESCRIPTION diff --git a/src/divvun-cgspell.1 b/src/divvun-cgspell.1 index 08c1926..59f179b 100644 --- a/src/divvun-cgspell.1 +++ b/src/divvun-cgspell.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH DIVVUN-CGSPELL "1" "January 2018" "divvun-gramcheck" "User Commands" +.TH DIVVUN-CGSPELL "1" "February 2018" "divvun-gramcheck" "User Commands" .SH NAME divvun-cgspell \- manual page for divvun-cgspell - Divvun gramcheck version 0.2.0 .SH DESCRIPTION diff --git a/src/divvun-checker.1 b/src/divvun-checker.1 index 92d25e7..18db7c7 100644 --- a/src/divvun-checker.1 +++ b/src/divvun-checker.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH DIVVUN-CHECKER "1" "January 2018" "divvun-gramcheck" "User Commands" +.TH DIVVUN-CHECKER "1" "February 2018" "divvun-gramcheck" "User Commands" .SH NAME divvun-checker \- manual page for divvun-checker - Divvun gramcheck version 0.2.0 .SH DESCRIPTION diff --git a/src/divvun-gen-sh.1 b/src/divvun-gen-sh.1 index a3ebe05..70d0843 100644 --- a/src/divvun-gen-sh.1 +++ b/src/divvun-gen-sh.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH DIVVUN-GEN-SH "1" "January 2018" "divvun-gramcheck" "User Commands" +.TH DIVVUN-GEN-SH "1" "February 2018" "divvun-gramcheck" "User Commands" .SH NAME divvun-gen-sh \- manual page for divvun-gen-sh - Divvun gramcheck version 0.2.0 .SH DESCRIPTION diff --git a/src/divvun-gen-xmlschemas.1 b/src/divvun-gen-xmlschemas.1 index decd7bd..972fc30 100644 --- a/src/divvun-gen-xmlschemas.1 +++ b/src/divvun-gen-xmlschemas.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH DIVVUN-GEN-XMLSCHEMAS "1" "January 2018" "divvun-gramcheck" "User Commands" +.TH DIVVUN-GEN-XMLSCHEMAS "1" "February 2018" "divvun-gramcheck" "User Commands" .SH NAME divvun-gen-xmlschemas \- manual page for divvun-gen-xmlschemas - Divvun gramcheck version 0.2.0 .SH DESCRIPTION diff --git a/src/divvun-suggest.1 b/src/divvun-suggest.1 index 7a2003f..8db75b5 100644 --- a/src/divvun-suggest.1 +++ b/src/divvun-suggest.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH DIVVUN-SUGGEST "1" "January 2018" "divvun-gramcheck" "User Commands" +.TH DIVVUN-SUGGEST "1" "February 2018" "divvun-gramcheck" "User Commands" .SH NAME divvun-suggest \- manual page for divvun-suggest - Divvun gramcheck version 0.2.0 .SH DESCRIPTION diff --git a/src/divvun-validate-pipespec.1 b/src/divvun-validate-pipespec.1 index a833021..48fef1f 100644 --- a/src/divvun-validate-pipespec.1 +++ b/src/divvun-validate-pipespec.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH DIVVUN-VALIDATE-PIPESPEC "1" "January 2018" "divvun-gramcheck" "User Commands" +.TH DIVVUN-VALIDATE-PIPESPEC "1" "February 2018" "divvun-gramcheck" "User Commands" .SH NAME divvun-validate-pipespec \- manual page for divvun-validate-pipespec - Divvun gramcheck version 0.2.0 .SH DESCRIPTION diff --git a/src/divvun-validate-suggest.1 b/src/divvun-validate-suggest.1 index c5d86ca..6b853d7 100644 --- a/src/divvun-validate-suggest.1 +++ b/src/divvun-validate-suggest.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH DIVVUN-VALIDATE-SUGGEST "1" "January 2018" "divvun-gramcheck" "User Commands" +.TH DIVVUN-VALIDATE-SUGGEST "1" "February 2018" "divvun-gramcheck" "User Commands" .SH NAME divvun-validate-suggest \- manual page for divvun-validate-suggest - Divvun gramcheck version 0.2.0 .SH DESCRIPTION diff --git a/src/json.hpp b/src/json.hpp index 3c55d5b..ad49138 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -25,9 +25,9 @@ #include #include #include - #include -#include + +#include "util.hpp" namespace json { @@ -42,7 +42,6 @@ inline const std::string uhex(const int i) inline const std::string esc(const std::u16string& str) { std::vector os; - std::wstring_convert, char16_t> utf16conv; for (const char16_t& c : str) { switch(c) { case '"': @@ -74,7 +73,7 @@ inline const std::string esc(const std::u16string& str) { } } } - return utf16conv.to_bytes(std::u16string(os.begin(), os.end())); + return divvun::toUtf8(std::u16string(os.begin(), os.end())); } inline const std::string str(const std::u16string& s) diff --git a/src/main_checker.cpp b/src/main_checker.cpp index 074c180..11b39ef 100644 --- a/src/main_checker.cpp +++ b/src/main_checker.cpp @@ -25,23 +25,23 @@ using divvun::variant; using divvun::Pipeline; +using divvun::toUtf8; +using divvun::fromUtf8; variant getPipelineXml(const std::string& path, const std::u16string& pipename, bool verbose) { - std::wstring_convert, char16_t> utf16conv; const std::unique_ptr spec(new divvun::PipeSpec(path)); if(spec->pnodes.find(pipename) == spec->pnodes.end()) { - std::cerr << "divvun-checker: ERROR: Couldn't find pipe " << utf16conv.to_bytes(pipename) << " in " << path << std::endl; + std::cerr << "divvun-checker: ERROR: Couldn't find pipe " << toUtf8(pipename) << " in " << path << std::endl; return EXIT_FAILURE; } return Pipeline(spec, pipename, verbose); } variant getPipelineAr(const std::string& path, const std::u16string& pipename, bool verbose) { - std::wstring_convert, char16_t> utf16conv; const auto& ar_spec = divvun::readArPipeSpec(path); if(ar_spec->spec->pnodes.find(pipename) == ar_spec->spec->pnodes.end()) { - std::cerr << "divvun-checker: ERROR: Couldn't find pipe " << utf16conv.to_bytes(pipename) << " in " << path << std::endl; + std::cerr << "divvun-checker: ERROR: Couldn't find pipe " << toUtf8(pipename) << " in " << path << std::endl; return EXIT_FAILURE; } return Pipeline(ar_spec, pipename, verbose); @@ -49,10 +49,9 @@ variant getPipelineAr(const std::string& path, const std::u16stri int printNamesXml(const std::string& path, bool verbose) { const std::unique_ptr spec(new divvun::PipeSpec(path)); - std::wstring_convert, char16_t> utf16conv; std::cout << "Please specify a pipeline variant with the -n/--variant option. Available variants in pipespec:" << std::endl; for(const auto& p : spec->pnodes) { - const auto& name = utf16conv.to_bytes(p.first.c_str()); + const auto& name = toUtf8(p.first.c_str()); std::cout << name << std::endl; } return EXIT_SUCCESS; @@ -60,10 +59,9 @@ int printNamesXml(const std::string& path, bool verbose) { int printNamesAr(const std::string& path, bool verbose) { const auto& ar_spec = divvun::readArPipeSpec(path); - std::wstring_convert, char16_t> utf16conv; std::cout << "Please specify a pipeline variant with the -n/--variant option. Available variants in archive:" << std::endl; for(const auto& p : ar_spec->spec->pnodes) { - const auto& name = utf16conv.to_bytes(p.first.c_str()); + const auto& name = toUtf8(p.first.c_str()); std::cout << name << std::endl; } return EXIT_SUCCESS; @@ -81,7 +79,6 @@ int run(Pipeline& pipeline) { void printPrefs(const Pipeline& pipeline) { using namespace divvun; - std::wstring_convert, char16_t> utf16conv; std::cout << "== Available preferences ==" << std::endl; for(const auto& lp : pipeline.prefs) { const Lang& lang = lp.first; @@ -89,16 +86,16 @@ void printPrefs(const Pipeline& pipeline) { const Prefs& prefs = lp.second; std::cout << "==== Toggles: ====" << std::endl; for(const auto& id : prefs.toggleIds) { - std::cout << "- [ ] " << utf16conv.to_bytes(id.first) << " \t" << utf16conv.to_bytes(id.second) << std::endl; + std::cout << "- [ ] " << toUtf8(id.first) << " \t" << toUtf8(id.second) << std::endl; } for(const auto& re : prefs.toggleRes) { - std::cout << "- [ ] [regex] \t" << utf16conv.to_bytes(re.second) << std::endl; + std::cout << "- [ ] [regex] \t" << toUtf8(re.second) << std::endl; } std::cout << "==== Options: ====" << std::endl; for(const Option& o : prefs.options) { std::cout << "- " << o.name << " (" << o.type << "):" << std::endl; for(const auto& c : o.choices) { - std::cout << "- ( ) " << utf16conv.to_bytes(c.first) << " \t" << utf16conv.to_bytes(c.second) << std::endl; + std::cout << "- ( ) " << toUtf8(c.first) << " \t" << toUtf8(c.second) << std::endl; } } } @@ -108,7 +105,6 @@ int main(int argc, char ** argv) { try { - std::wstring_convert, char16_t> utf16conv; cxxopts::Options options(argv[0], " - generate grammar checker suggestions from a CG stream"); options.add_options() @@ -157,9 +153,8 @@ int main(int argc, char ** argv) auto ignores = std::set(); if(options.count("ignore")) { - std::wstring_convert, char16_t> utf16conv; for(const auto& ignore : divvun::split(options["ignore"].as(), ',')) { - ignores.insert(utf16conv.from_bytes(ignore)); + ignores.insert(fromUtf8(ignore)); } } @@ -172,7 +167,7 @@ int main(int argc, char ** argv) return printNamesXml(specfile, verbose); } else { - const auto& pipename = utf16conv.from_bytes(options["variant"].as()); + const auto& pipename = fromUtf8(options["variant"].as()); return getPipelineXml(specfile, pipename, verbose).match( [] (int r) { return r; }, [&](Pipeline& p) { @@ -196,7 +191,7 @@ int main(int argc, char ** argv) return printNamesAr(archive, verbose); } else { - const auto& pipename = utf16conv.from_bytes(options["variant"].as()); + const auto& pipename = fromUtf8(options["variant"].as()); return getPipelineAr(archive, pipename, verbose).match( [] (int r) { return r; }, [&](Pipeline& p) { diff --git a/src/main_gen_sh.cpp b/src/main_gen_sh.cpp index 705967c..44f9686 100644 --- a/src/main_gen_sh.cpp +++ b/src/main_gen_sh.cpp @@ -27,7 +27,6 @@ int main(int argc, char ** argv) { try { - std::wstring_convert, char16_t> utf16conv; cxxopts::Options options(argv[0], "BIN - generate shell script to run checker pipeline from XML pipespec"); options.add_options() @@ -79,7 +78,7 @@ int main(int argc, char ** argv) std::cerr << argv[0] << " ERROR: Specify either --variant or --dir, not both!" << std::endl; return EXIT_FAILURE; } - const auto& pipename = utf16conv.from_bytes(options["variant"].as()); + const auto& pipename = divvun::fromUtf8(options["variant"].as()); divvun::writePipeSpecSh(specfile, pipename, json, std::cout); return EXIT_SUCCESS; } diff --git a/src/pipeline.cpp b/src/pipeline.cpp index afd6b24..7f0d6c7 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -178,10 +178,9 @@ Pipeline Pipeline::mkPipeline(const unique_ptr& ar_spec, const u16st // TODO: Move into a lib-general init function? Or can I call this safely once per CGCmd? throw std::runtime_error("libdivvun: ERROR: Couldn't initialise ICU for vislcg3!"); } - std::wstring_convert, char16_t> utf16conv; string locale = spec->language; for (const pugi::xml_node& cmd: spec->pnodes.at(pipename).children()) { - const auto& name = utf16conv.from_bytes(cmd.name()); + const auto& name = fromUtf8(cmd.name()); std::unordered_map args; for (const pugi::xml_node& arg: cmd.children()) { args[arg.name()] = arg.attribute("n").value(); @@ -243,7 +242,7 @@ Pipeline Pipeline::mkPipeline(const unique_ptr& ar_spec, const u16st suggestcmd = s; } else if(name == u"sh") { - // const auto& prog = utf16conv.from_bytes(cmd.attribute("prog").value()); + // const auto& prog = fromUtf8(cmd.attribute("prog").value()); // cmds.emplace_back(new ShCmd(prog, args, verbose)); } else if(name == u"prefs") { @@ -262,10 +261,9 @@ Pipeline Pipeline::mkPipeline(const unique_ptr& spec, const u16string& // TODO: Move into a lib-general init function? Or can I call this safely once per CGCmd? throw std::runtime_error("libdivvun: ERROR: Couldn't initialise ICU for vislcg3!"); } - std::wstring_convert, char16_t> utf16conv; string locale = spec->language; for (const pugi::xml_node& cmd: spec->pnodes.at(pipename).children()) { - const auto& name = utf16conv.from_bytes(cmd.name()); + const auto& name = fromUtf8(cmd.name()); std::unordered_map args; for (const pugi::xml_node& arg: cmd.children()) { args[arg.name()] = arg.attribute("n").value(); @@ -298,7 +296,7 @@ Pipeline Pipeline::mkPipeline(const unique_ptr& spec, const u16string& suggestcmd = s; } else if(name == u"sh") { - // const auto& prog = utf16conv.from_bytes(cmd.attribute("prog").value()); + // const auto& prog = fromUtf8(cmd.attribute("prog").value()); // cmds.emplace_back(new ShCmd(prog, args, verbose)); } else if(name == u"prefs") { diff --git a/src/pipeline.hpp b/src/pipeline.hpp index 452aeed..a6f0930 100644 --- a/src/pipeline.hpp +++ b/src/pipeline.hpp @@ -213,10 +213,10 @@ inline void parsePrefs(LocalisedPrefs& prefs, const pugi::xml_node& cmd) { const auto name = pref.attribute("name").value(); std::unordered_map> lems; for (const pugi::xml_node& option: pref.children()) { - const auto errId = utf16conv.from_bytes(option.attribute("err-id").value()); + const auto errId = fromUtf8(option.attribute("err-id").value()); for (const pugi::xml_node& label: option.children()) { const auto lang = label.attribute("xml:lang").value(); - const auto msg = utf16conv.from_bytes(label.text().get()); // or xml_raw_cdata(label); + const auto msg = fromUtf8(label.text().get()); // or xml_raw_cdata(label); lems[lang][errId] = msg; } } diff --git a/src/pipespec.cpp b/src/pipespec.cpp index da356bb..ae8d828 100644 --- a/src/pipespec.cpp +++ b/src/pipespec.cpp @@ -30,7 +30,7 @@ PipeSpec::PipeSpec(const string& file) { language = "se"; // reasonable default } for (pugi::xml_node pipeline: doc.child("pipespec").children("pipeline")) { - const u16string& pipename = utf16conv.from_bytes(pipeline.attribute("name").value()); + const u16string& pipename = fromUtf8(pipeline.attribute("name").value()); auto pr = std::make_pair(pipename, pipeline); pnodes[pipename] = pipeline; } @@ -51,7 +51,7 @@ PipeSpec::PipeSpec(pugi::char_t* buff, size_t size) { language = "se"; // reasonable default } for (pugi::xml_node pipeline: doc.child("pipespec").children("pipeline")) { - const u16string& pipename = utf16conv.from_bytes(pipeline.attribute("name").value()); + const u16string& pipename = fromUtf8(pipeline.attribute("name").value()); auto pr = std::make_pair(pipename, pipeline); pnodes[pipename] = pipeline; } @@ -97,7 +97,7 @@ void validatePipespecCmd(const pugi::xml_node& cmd, const std::unordered_map command not implemented yet!"); - // const auto& prog = utf16conv.from_bytes(cmd.attribute("prog").value()); + // const auto& prog = fromUtf8(cmd.attribute("prog").value()); } else if(name == "prefs") { // pass @@ -299,7 +299,7 @@ void writePipeSpecShDir(const string& specfile, bool json, const string& modesdi const std::unique_ptr spec(new PipeSpec(specfile)); const auto dir = dirname(abspath(specfile)); for(const auto& p : spec->pnodes) { - const auto& pipename = utf16conv.to_bytes(p.first); + const auto& pipename = toUtf8(p.first); writePipeSpecShDirOne(toPipeSpecShVector(dir, *spec, p.first, false, json), pipename, modesdir, diff --git a/src/suggest.cpp b/src/suggest.cpp index 7af6225..bec28ec 100644 --- a/src/suggest.cpp +++ b/src/suggest.cpp @@ -87,18 +87,17 @@ enum LineType { const MsgMap readMessagesXml(pugi::xml_document& doc, pugi::xml_parse_result& result) { MsgMap msgs; - std::wstring_convert, char16_t> utf16conv; if (result) { for (pugi::xml_node def: doc.child("errors").child("defaults").children("default")) { // std::cerr << "defaults" << std::endl; for (pugi::xml_node child: def.child("header").children("title")) { - const auto& msg = utf16conv.from_bytes(xml_raw_cdata(child)); + const auto& msg = fromUtf8(xml_raw_cdata(child)); const auto& lang = child.attribute("xml:lang").value(); for (pugi::xml_node e: def.child("ids").children("e")) { // e_value assumes we only ever have one PCDATA element here: - const auto& errtype = utf16conv.from_bytes(e.attribute("id").value()); - // std::cerr << utf16conv.to_bytes(errtype) << std::endl; + const auto& errtype = fromUtf8(e.attribute("id").value()); + // std::cerr << toUtf8(errtype) << std::endl; if(msgs[lang].first.count(errtype) != 0) { std::cerr << "divvun-suggest: WARNING: Duplicate titles for " << e.attribute("id").value() << std::endl; } @@ -113,8 +112,8 @@ const MsgMap readMessagesXml(pugi::xml_document& doc, pugi::xml_parse_result& re for (pugi::xml_node error: doc.child("errors").children("error")) { for (pugi::xml_node child: error.child("header").children("title")) { // child_value assumes we only ever have one PCDATA element here: - const auto& errtype = utf16conv.from_bytes(error.attribute("id").value()); - const auto& msg = utf16conv.from_bytes(xml_raw_cdata(child)); + const auto& errtype = fromUtf8(error.attribute("id").value()); + const auto& msg = fromUtf8(xml_raw_cdata(child)); const auto& lang = child.attribute("xml:lang").value(); if(msgs[lang].first.count(errtype) != 0) { std::cerr << "divvun-suggest: WARNING: Duplicate titles for " << error.attribute("id").value() << std::endl; @@ -154,7 +153,6 @@ const MsgMap Suggest::readMessages(const string& file) { const Reading proc_subreading(const string& line) { - std::wstring_convert, char16_t> utf16conv; Reading r; const auto& lemma_beg = line.find("\""); const auto& lemma_end = line.find("\" ", lemma_beg); @@ -190,7 +188,7 @@ const Reading proc_subreading(const string& line) { r.link = true; } else { - r.errtype = utf16conv.from_bytes(result[2]); + r.errtype = fromUtf8(result[2]); } } else if(result[3].length() != 0 && result[4].length() != 0) { @@ -219,13 +217,12 @@ const Reading proc_subreading(const string& line) { const auto& tagsplus = join(gentags, "+"); r.ana = lemma+"+"+tagsplus; if(r.suggestwf) { - r.sforms.emplace_back(utf16conv.from_bytes(r.wf)); + r.sforms.emplace_back(fromUtf8(r.wf)); } return r; }; const Reading proc_reading(const hfst::HfstTransducer& t, const string& line) { - std::wstring_convert, char16_t> utf16conv; stringstream ss(line); string subline; std::deque subs; @@ -257,7 +254,7 @@ const Reading proc_reading(const hfst::HfstTransducer& t, const string& line) { form << symbol; } } - r.sforms.emplace_back(utf16conv.from_bytes(form.str())); + r.sforms.emplace_back(fromUtf8(form.str())); } } return r; @@ -378,7 +375,6 @@ variant Suggest::cohort_errs(const ErrId& err_id, if(cohort_empty(c) || c.added || ignores.find(err_id) != ignores.end()) { return Nothing(); } - std::wstring_convert, char16_t> utf16conv; u16string msg; for(const auto& mlang : sortedmsglangs) { if(msg.empty() && mlang != locale) { @@ -391,7 +387,7 @@ variant Suggest::cohort_errs(const ErrId& err_id, else { for(const auto& p : lmsgs.second) { std::match_results result; - const auto& et = utf16conv.to_bytes(err_id.c_str()); + const auto& et = toUtf8(err_id.c_str()); std::regex_match(et.c_str(), result, p.first); if(!result.empty() && // Only consider full matches: @@ -419,7 +415,7 @@ variant Suggest::cohort_errs(const ErrId& err_id, } rel_on_match(r.rels, MSG_TEMPLATE_REL, sentence, [&] (const string& relname, size_t i_t, const Cohort& trg) { - replaceAll(msg, utf16conv.from_bytes(relname.c_str()), trg.form); + replaceAll(msg, fromUtf8(relname.c_str()), trg.form); }); } auto beg = c.pos; @@ -542,10 +538,6 @@ Sentence run_sentence(std::istream& is, const hfst::HfstTransducer& t, const Msg Sentence sentence; sentence.runstate = eof; - // TODO: could use http://utfcpp.sourceforge.net, but it's not in macports; - // and ICU seems overkill just for iterating codepoints - std::wstring_convert, char16_t> utf16conv; - string line; string readinglines; std::getline(is, line); // TODO: Why do I need at least one getline before os<< after flushing? @@ -577,20 +569,20 @@ Sentence run_sentence(std::istream& is, const hfst::HfstTransducer& t, const Msg } if(!c.added) { pos += c.form.size(); - sentence.text << utf16conv.to_bytes(c.form); + sentence.text << toUtf8(c.form); } c = DEFAULT_COHORT; } if (!result.empty() && result[2].length() != 0) { // wordform - c.form = utf16conv.from_bytes(result[2]); + c.form = fromUtf8(result[2]); } else if(!result.empty() && result[3].length() != 0) { // reading readinglines += line + "\n"; } else if(!result.empty() && result[6].length() != 0) { // blank const auto blank = clean_blank(result[6]); - pos += utf16conv.from_bytes(blank).size(); + pos += fromUtf8(blank).size(); sentence.text << blank; } else if(!result.empty() && result[7].length() != 0) { // flush @@ -624,7 +616,7 @@ Sentence run_sentence(std::istream& is, const hfst::HfstTransducer& t, const Msg } if(!c.added) { pos += c.form.size(); - sentence.text << utf16conv.to_bytes(c.form); + sentence.text << toUtf8(c.form); } return sentence; } @@ -707,8 +699,7 @@ void expand_errs(vector& errs, const u16string& text) { } vector Suggest::mk_errs(const Sentence &sentence) { - std::wstring_convert, char16_t> utf16conv; - const auto& text = utf16conv.from_bytes(sentence.text.str()); + const auto& text = fromUtf8(sentence.text.str()); vector errs; for(const auto& c : sentence.cohorts) { std::map> c_errs; @@ -740,7 +731,6 @@ vector Suggest::run_errs(std::istream& is) RunState Suggest::run_json(std::istream& is, std::ostream& os) { - std::wstring_convert, char16_t> utf16conv; json::sanity_test(); Sentence sentence = run_sentence(is, *generator, msgs); @@ -764,7 +754,7 @@ RunState Suggest::run_json(std::istream& is, std::ostream& os) wantsep = true; } os << "]" - << "," << json::key(u"text") << json::str(utf16conv.from_bytes(sentence.text.str())) + << "," << json::key(u"text") << json::str(fromUtf8(sentence.text.str())) << "}"; if(sentence.runstate == flushing) { os << '\0'; @@ -775,7 +765,7 @@ RunState Suggest::run_json(std::istream& is, std::ostream& os) } -void print_cg_reading(const string& readinglines, std::ostream& os, const hfst::HfstTransducer& t, std::wstring_convert, char16_t>& utf16conv) { +void print_cg_reading(const string& readinglines, std::ostream& os, const hfst::HfstTransducer& t) { os << readinglines; const auto& reading = proc_reading(t, readinglines); if(reading.suggest) { @@ -793,7 +783,7 @@ void print_cg_reading(const string& readinglines, std::ostream& os, const hfst:: else { const auto& errtype = reading.errtype; if(!errtype.empty()) { - os << utf16conv.to_bytes(errtype) << std::endl; + os << toUtf8(errtype) << std::endl; } } @@ -801,7 +791,6 @@ void print_cg_reading(const string& readinglines, std::ostream& os, const hfst:: void run_cg(std::istream& is, std::ostream& os, const hfst::HfstTransducer& t) { - std::wstring_convert, char16_t> utf16conv; // Simple debug function; only subreading state kept between lines string readinglines; for (string line;std::getline(is, line);) { @@ -809,7 +798,7 @@ void run_cg(std::istream& is, std::ostream& os, const hfst::HfstTransducer& t) std::regex_match(line.c_str(), result, CG_LINE); if(!readinglines.empty() && (result.empty() || result[3].length() <= 1)) { - print_cg_reading(readinglines, os, t, utf16conv); + print_cg_reading(readinglines, os, t); readinglines = ""; } @@ -826,7 +815,7 @@ void run_cg(std::istream& is, std::ostream& os, const hfst::HfstTransducer& t) } } if(!readinglines.empty()) { - print_cg_reading(readinglines, os, t, utf16conv); + print_cg_reading(readinglines, os, t); } } diff --git a/src/suggest.hpp b/src/suggest.hpp index bfd226f..b9b7acd 100644 --- a/src/suggest.hpp +++ b/src/suggest.hpp @@ -20,7 +20,6 @@ #define fe64e9a18486d375_SUGGEST_H #include -#include #include #include #include diff --git a/src/utf8.h b/src/utf8.h new file mode 100644 index 0000000..82b13f5 --- /dev/null +++ b/src/utf8.h @@ -0,0 +1,34 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "utf8/checked.h" +#include "utf8/unchecked.h" + +#endif // header guard diff --git a/src/utf8/checked.h b/src/utf8/checked.h new file mode 100644 index 0000000..2aef583 --- /dev/null +++ b/src/utf8/checked.h @@ -0,0 +1,327 @@ +// Copyright 2006-2016 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" +#include + +namespace utf8 +{ + // Base for the exceptions that may be thrown from the library + class exception : public ::std::exception { + }; + + // Exceptions that may be thrown from the library functions. + class invalid_code_point : public exception { + uint32_t cp; + public: + invalid_code_point(uint32_t codepoint) : cp(codepoint) {} + virtual const char* what() const throw() { return "Invalid code point"; } + uint32_t code_point() const {return cp;} + }; + + class invalid_utf8 : public exception { + uint8_t u8; + public: + invalid_utf8 (uint8_t u) : u8(u) {} + virtual const char* what() const throw() { return "Invalid UTF-8"; } + uint8_t utf8_octet() const {return u8;} + }; + + class invalid_utf16 : public exception { + uint16_t u16; + public: + invalid_utf16 (uint16_t u) : u16(u) {} + virtual const char* what() const throw() { return "Invalid UTF-16"; } + uint16_t utf16_word() const {return u16;} + }; + + class not_enough_room : public exception { + public: + virtual const char* what() const throw() { return "Not enough space"; } + }; + + /// The library API - functions intended to be called by the users + + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (!utf8::internal::is_code_point_valid(cp)) + throw invalid_code_point(cp); + + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement) + { + while (start != end) { + octet_iterator sequence_start = start; + internal::utf_error err_code = utf8::internal::validate_next(start, end); + switch (err_code) { + case internal::UTF8_OK : + for (octet_iterator it = sequence_start; it != start; ++it) + *out++ = *it; + break; + case internal::NOT_ENOUGH_ROOM: + throw not_enough_room(); + case internal::INVALID_LEAD: + out = utf8::append (replacement, out); + ++start; + break; + case internal::INCOMPLETE_SEQUENCE: + case internal::OVERLONG_SEQUENCE: + case internal::INVALID_CODE_POINT: + out = utf8::append (replacement, out); + ++start; + // just one replacement mark for the sequence + while (start != end && utf8::internal::is_trail(*start)) + ++start; + break; + } + } + return out; + } + + template + inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) + { + static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd); + return utf8::replace_invalid(start, end, out, replacement_marker); + } + + template + uint32_t next(octet_iterator& it, octet_iterator end) + { + uint32_t cp = 0; + internal::utf_error err_code = utf8::internal::validate_next(it, end, cp); + switch (err_code) { + case internal::UTF8_OK : + break; + case internal::NOT_ENOUGH_ROOM : + throw not_enough_room(); + case internal::INVALID_LEAD : + case internal::INCOMPLETE_SEQUENCE : + case internal::OVERLONG_SEQUENCE : + throw invalid_utf8(*it); + case internal::INVALID_CODE_POINT : + throw invalid_code_point(cp); + } + return cp; + } + + template + uint32_t peek_next(octet_iterator it, octet_iterator end) + { + return utf8::next(it, end); + } + + template + uint32_t prior(octet_iterator& it, octet_iterator start) + { + // can't do much if it == start + if (it == start) + throw not_enough_room(); + + octet_iterator end = it; + // Go back until we hit either a lead octet or start + while (utf8::internal::is_trail(*(--it))) + if (it == start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + return utf8::peek_next(it, end); + } + + /// Deprecated in versions that include "prior" + template + uint32_t previous(octet_iterator& it, octet_iterator pass_start) + { + octet_iterator end = it; + while (utf8::internal::is_trail(*(--it))) + if (it == pass_start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + octet_iterator temp = it; + return utf8::next(temp, end); + } + + template + void advance (octet_iterator& it, distance_type n, octet_iterator end) + { + for (distance_type i = 0; i < n; ++i) + utf8::next(it, end); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::next(first, last); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + if (start != end) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + if (utf8::internal::is_trail_surrogate(trail_surrogate)) + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + else + throw invalid_utf16(static_cast(trail_surrogate)); + } + else + throw invalid_utf16(static_cast(cp)); + + } + // Lone trail surrogate + else if (utf8::internal::is_trail_surrogate(cp)) + throw invalid_utf16(static_cast(cp)); + + result = utf8::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start < end) { + uint32_t cp = utf8::next(start, end); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = utf8::next(start, end); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + octet_iterator range_start; + octet_iterator range_end; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it, + const octet_iterator& rangestart, + const octet_iterator& rangeend) : + it(octet_it), range_start(rangestart), range_end(rangeend) + { + if (it < range_start || it > range_end) + throw std::out_of_range("Invalid utf-8 iterator position"); + } + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::next(temp, range_end); + } + bool operator == (const iterator& rhs) const + { + if (range_start != rhs.range_start || range_end != rhs.range_end) + throw std::logic_error("Comparing utf-8 iterators defined with different ranges"); + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + utf8::next(it, range_end); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + utf8::next(it, range_end); + return temp; + } + iterator& operator -- () + { + utf8::prior(it, range_start); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::prior(it, range_start); + return temp; + } + }; // class iterator + +} // namespace utf8 + +#endif //header guard + + diff --git a/src/utf8/core.h b/src/utf8/core.h new file mode 100644 index 0000000..ae0f367 --- /dev/null +++ b/src/utf8/core.h @@ -0,0 +1,332 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include + +namespace utf8 +{ + // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers + // You may need to change them to match your system. + // These typedefs have the same names as ones from cstdint, or boost/cstdint + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + +// Helper code - not intended to be directly called by the library users. May be changed at any time +namespace internal +{ + // Unicode constants + // Leading (high) surrogates: 0xd800 - 0xdbff + // Trailing (low) surrogates: 0xdc00 - 0xdfff + const uint16_t LEAD_SURROGATE_MIN = 0xd800u; + const uint16_t LEAD_SURROGATE_MAX = 0xdbffu; + const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u; + const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu; + const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); + const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; + + // Maximum valid value for a Unicode code point + const uint32_t CODE_POINT_MAX = 0x0010ffffu; + + template + inline uint8_t mask8(octet_type oc) + { + return static_cast(0xff & oc); + } + template + inline uint16_t mask16(u16_type oc) + { + return static_cast(0xffff & oc); + } + template + inline bool is_trail(octet_type oc) + { + return ((utf8::internal::mask8(oc) >> 6) == 0x2); + } + + template + inline bool is_lead_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX); + } + + template + inline bool is_trail_surrogate(u16 cp) + { + return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_code_point_valid(u32 cp) + { + return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp)); + } + + template + inline typename std::iterator_traits::difference_type + sequence_length(octet_iterator lead_it) + { + uint8_t lead = utf8::internal::mask8(*lead_it); + if (lead < 0x80) + return 1; + else if ((lead >> 5) == 0x6) + return 2; + else if ((lead >> 4) == 0xe) + return 3; + else if ((lead >> 3) == 0x1e) + return 4; + else + return 0; + } + + template + inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length) + { + if (cp < 0x80) { + if (length != 1) + return true; + } + else if (cp < 0x800) { + if (length != 2) + return true; + } + else if (cp < 0x10000) { + if (length != 3) + return true; + } + + return false; + } + + enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT}; + + /// Helper for get_sequence_x + template + utf_error increase_safely(octet_iterator& it, octet_iterator end) + { + if (++it == end) + return NOT_ENOUGH_ROOM; + + if (!utf8::internal::is_trail(*it)) + return INCOMPLETE_SEQUENCE; + + return UTF8_OK; + } + + #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;} + + /// get_sequence_x functions decode utf-8 sequences of the length x + template + utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + return UTF8_OK; + } + + template + utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f); + + return UTF8_OK; + } + + template + utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + template + utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (utf8::internal::mask8(*it) << 6) & 0xfff; + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR + + template + utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + // Save the original value of it so we can go back in case of failure + // Of course, it does not make much sense with i.e. stream iterators + octet_iterator original_it = it; + + uint32_t cp = 0; + // Determine the sequence length based on the lead octet + typedef typename std::iterator_traits::difference_type octet_difference_type; + const octet_difference_type length = utf8::internal::sequence_length(it); + + // Get trail octets and calculate the code point + utf_error err = UTF8_OK; + switch (length) { + case 0: + return INVALID_LEAD; + case 1: + err = utf8::internal::get_sequence_1(it, end, cp); + break; + case 2: + err = utf8::internal::get_sequence_2(it, end, cp); + break; + case 3: + err = utf8::internal::get_sequence_3(it, end, cp); + break; + case 4: + err = utf8::internal::get_sequence_4(it, end, cp); + break; + } + + if (err == UTF8_OK) { + // Decoding succeeded. Now, security checks... + if (utf8::internal::is_code_point_valid(cp)) { + if (!utf8::internal::is_overlong_sequence(cp, length)){ + // Passed! Return here. + code_point = cp; + ++it; + return UTF8_OK; + } + else + err = OVERLONG_SEQUENCE; + } + else + err = INVALID_CODE_POINT; + } + + // Failure branch - restore the original value of the iterator + it = original_it; + return err; + } + + template + inline utf_error validate_next(octet_iterator& it, octet_iterator end) { + uint32_t ignored; + return utf8::internal::validate_next(it, end, ignored); + } + +} // namespace internal + + /// The library API - functions intended to be called by the users + + // Byte order mark + const uint8_t bom[] = {0xef, 0xbb, 0xbf}; + + template + octet_iterator find_invalid(octet_iterator start, octet_iterator end) + { + octet_iterator result = start; + while (result != end) { + utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end); + if (err_code != internal::UTF8_OK) + return result; + } + return result; + } + + template + inline bool is_valid(octet_iterator start, octet_iterator end) + { + return (utf8::find_invalid(start, end) == end); + } + + template + inline bool starts_with_bom (octet_iterator it, octet_iterator end) + { + return ( + ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) && + ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) && + ((it != end) && (utf8::internal::mask8(*it)) == bom[2]) + ); + } + + //Deprecated in release 2.3 + template + inline bool is_bom (octet_iterator it) + { + return ( + (utf8::internal::mask8(*it++)) == bom[0] && + (utf8::internal::mask8(*it++)) == bom[1] && + (utf8::internal::mask8(*it)) == bom[2] + ); + } +} // namespace utf8 + +#endif // header guard + + diff --git a/src/utf8/unchecked.h b/src/utf8/unchecked.h new file mode 100644 index 0000000..cb24271 --- /dev/null +++ b/src/utf8/unchecked.h @@ -0,0 +1,228 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" + +namespace utf8 +{ + namespace unchecked + { + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f)| 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + uint32_t next(octet_iterator& it) + { + uint32_t cp = utf8::internal::mask8(*it); + typename std::iterator_traits::difference_type length = utf8::internal::sequence_length(it); + switch (length) { + case 1: + break; + case 2: + it++; + cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); + break; + case 3: + ++it; + cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + ++it; + cp += (*it) & 0x3f; + break; + case 4: + ++it; + cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + ++it; + cp += (utf8::internal::mask8(*it) << 6) & 0xfff; + ++it; + cp += (*it) & 0x3f; + break; + } + ++it; + return cp; + } + + template + uint32_t peek_next(octet_iterator it) + { + return utf8::unchecked::next(it); + } + + template + uint32_t prior(octet_iterator& it) + { + while (utf8::internal::is_trail(*(--it))) ; + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + + // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous) + template + inline uint32_t previous(octet_iterator& it) + { + return utf8::unchecked::prior(it); + } + + template + void advance (octet_iterator& it, distance_type n) + { + for (distance_type i = 0; i < n; ++i) + utf8::unchecked::next(it); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::unchecked::next(first); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + } + result = utf8::unchecked::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start < end) { + uint32_t cp = utf8::unchecked::next(start); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::unchecked::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = utf8::unchecked::next(start); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it): it(octet_it) {} + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + bool operator == (const iterator& rhs) const + { + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + ::std::advance(it, utf8::internal::sequence_length(it)); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + ::std::advance(it, utf8::internal::sequence_length(it)); + return temp; + } + iterator& operator -- () + { + utf8::unchecked::prior(it); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::unchecked::prior(it); + return temp; + } + }; // class iterator + + } // namespace utf8::unchecked +} // namespace utf8 + + +#endif // header guard + diff --git a/src/util.hpp b/src/util.hpp index ce640d1..c0d2a4f 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -34,7 +34,7 @@ #include #include -#include +#include "utf8.h" namespace divvun { @@ -48,6 +48,18 @@ const std::basic_regex CG_LINE ("^" using StringVec = std::vector; +inline const std::string toUtf8(const std::u16string& from) { + std::string to; + utf8::utf16to8(from.begin(), from.end(), std::back_inserter(to)); + return to; +} + +inline const std::u16string fromUtf8(const std::string& from) { + std::u16string to; + utf8::utf8to16(from.begin(), from.end(), std::back_inserter(to)); + return to; +} + template inline const std::string join_quoted(const Container& ss, const std::string& delim=" ") { std::ostringstream os; @@ -57,6 +69,7 @@ inline const std::string join_quoted(const Container& ss, const std::string& del str.size() - delim.size()); } +/* Join a container of bytestrings by delim */ template inline const std::string join(const Container& ss, const std::string& delim=" ") { std::ostringstream os; @@ -66,12 +79,12 @@ inline const std::string join(const Container& ss, const std::string& delim=" ") str.size() - delim.size()); } +/* Join a container of u16strings by delim */ template inline const std::string u16join(const Container& ss, const std::string& delim=" ") { std::ostringstream os; - std::wstring_convert, char16_t> utf16conv; for(const auto& s : ss) { - os << utf16conv.to_bytes(s) << ","; + os << toUtf8(s) << ","; } const auto& str = os.str(); return str.substr(0,