➕ dep on utfcpp; <codecvt> gone from c++17

not in macports nor RPM's, so copying in headers instead (2-clause license)
divvun · Feb 2, 2018 · 5fc3012 · 5fc3012
1 parent 3367ff4
commit 5fc3012
Show file tree

Hide file tree

Showing 22 changed files with 996 additions and 88 deletions.
diff --git a/src/checker.cpp b/src/checker.cpp
@@ -23,27 +23,24 @@ namespace divvun {
 
 
 const std::u16string from_bytes(const std::string& s) {
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
-	return utf16conv.from_bytes(s);
+	return fromUtf8(s);
 }
 
 // CheckerSpec
 CheckerSpec::CheckerSpec(const std::string& file) : pImpl( new PipeSpec(file) )
 {
-	// std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 	// for(const auto& k : pImpl->pnodes) {
-		// std::cerr << "init " << utf16conv.to_bytes(k.first.c_str()) <<std::endl;
+		// std::cerr << "init " << toUtf8(k.first.c_str()) <<std::endl;
 	// }
 }
 CheckerSpec::~CheckerSpec()
 {
 }
 const std::set<std::string> CheckerSpec::pipeNames() const
 {
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 	std::set<std::string> keys;
 	for(const auto& it : pImpl->pnodes) {
-		keys.insert(utf16conv.to_bytes(it.first));
+		keys.insert(toUtf8(it.first));
 	}
 	return keys;
 }
@@ -60,20 +57,18 @@ std::unique_ptr<Checker> CheckerSpec::getChecker(const std::string& pipename, bo
 // ArCheckerSpec
 ArCheckerSpec::ArCheckerSpec(const std::string& file) : pImpl( readArPipeSpec(file) )
 {
-	// std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 	// for(const auto& k : pImpl->spec->pnodes) {
-	// 	std::cerr << "init " << utf16conv.to_bytes(k.first.c_str()) <<std::endl;
+	// 	std::cerr << "init " << toUtf8(k.first.c_str()) <<std::endl;
 	// }
 }
 ArCheckerSpec::~ArCheckerSpec()
 {
 }
 const std::set<std::string> ArCheckerSpec::pipeNames() const
 {
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 	std::set<std::string> keys;
 	for(const auto& it : pImpl->spec->pnodes) {
-		keys.insert(utf16conv.to_bytes(it.first));
+		keys.insert(toUtf8(it.first));
 	}
 	return keys;
 }

diff --git a/src/divvun-blanktag.1 b/src/divvun-blanktag.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH DIVVUN-BLANKTAG "1" "January 2018" "divvun-gramcheck" "User Commands"
+.TH DIVVUN-BLANKTAG "1" "February 2018" "divvun-gramcheck" "User Commands"
 .SH NAME
 divvun-blanktag \- manual page for divvun-blanktag - Divvun gramcheck version 0.2.0
 .SH DESCRIPTION

diff --git a/src/divvun-cgspell.1 b/src/divvun-cgspell.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH DIVVUN-CGSPELL "1" "January 2018" "divvun-gramcheck" "User Commands"
+.TH DIVVUN-CGSPELL "1" "February 2018" "divvun-gramcheck" "User Commands"
 .SH NAME
 divvun-cgspell \- manual page for divvun-cgspell - Divvun gramcheck version 0.2.0
 .SH DESCRIPTION

diff --git a/src/divvun-checker.1 b/src/divvun-checker.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH DIVVUN-CHECKER "1" "January 2018" "divvun-gramcheck" "User Commands"
+.TH DIVVUN-CHECKER "1" "February 2018" "divvun-gramcheck" "User Commands"
 .SH NAME
 divvun-checker \- manual page for divvun-checker - Divvun gramcheck version 0.2.0
 .SH DESCRIPTION

diff --git a/src/divvun-gen-sh.1 b/src/divvun-gen-sh.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH DIVVUN-GEN-SH "1" "January 2018" "divvun-gramcheck" "User Commands"
+.TH DIVVUN-GEN-SH "1" "February 2018" "divvun-gramcheck" "User Commands"
 .SH NAME
 divvun-gen-sh \- manual page for divvun-gen-sh - Divvun gramcheck version 0.2.0
 .SH DESCRIPTION

diff --git a/src/divvun-gen-xmlschemas.1 b/src/divvun-gen-xmlschemas.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH DIVVUN-GEN-XMLSCHEMAS "1" "January 2018" "divvun-gramcheck" "User Commands"
+.TH DIVVUN-GEN-XMLSCHEMAS "1" "February 2018" "divvun-gramcheck" "User Commands"
 .SH NAME
 divvun-gen-xmlschemas \- manual page for divvun-gen-xmlschemas - Divvun gramcheck version 0.2.0
 .SH DESCRIPTION

diff --git a/src/divvun-suggest.1 b/src/divvun-suggest.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH DIVVUN-SUGGEST "1" "January 2018" "divvun-gramcheck" "User Commands"
+.TH DIVVUN-SUGGEST "1" "February 2018" "divvun-gramcheck" "User Commands"
 .SH NAME
 divvun-suggest \- manual page for divvun-suggest - Divvun gramcheck version 0.2.0
 .SH DESCRIPTION

diff --git a/src/divvun-validate-pipespec.1 b/src/divvun-validate-pipespec.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH DIVVUN-VALIDATE-PIPESPEC "1" "January 2018" "divvun-gramcheck" "User Commands"
+.TH DIVVUN-VALIDATE-PIPESPEC "1" "February 2018" "divvun-gramcheck" "User Commands"
 .SH NAME
 divvun-validate-pipespec \- manual page for divvun-validate-pipespec - Divvun gramcheck version 0.2.0
 .SH DESCRIPTION

diff --git a/src/divvun-validate-suggest.1 b/src/divvun-validate-suggest.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH DIVVUN-VALIDATE-SUGGEST "1" "January 2018" "divvun-gramcheck" "User Commands"
+.TH DIVVUN-VALIDATE-SUGGEST "1" "February 2018" "divvun-gramcheck" "User Commands"
 .SH NAME
 divvun-validate-suggest \- manual page for divvun-validate-suggest - Divvun gramcheck version 0.2.0
 .SH DESCRIPTION

diff --git a/src/json.hpp b/src/json.hpp
@@ -25,9 +25,9 @@
 #include <sstream>
 #include <iostream>
 #include <iomanip>
-
 #include <locale>
-#include <codecvt>
+
+#include "util.hpp"
 
 namespace json {
 
@@ -42,7 +42,6 @@ inline const std::string uhex(const int i)
 
 inline const std::string esc(const std::u16string& str) {
 	std::vector<char16_t> os;
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 	for (const char16_t& c : str) {
 		switch(c) {
 			case '"':
@@ -74,7 +73,7 @@ inline const std::string esc(const std::u16string& str) {
 				}
 		}
 	}
-	return utf16conv.to_bytes(std::u16string(os.begin(), os.end()));
+	return divvun::toUtf8(std::u16string(os.begin(), os.end()));
 }
 
 inline const std::string str(const std::u16string& s)

diff --git a/src/main_checker.cpp b/src/main_checker.cpp
@@ -25,45 +25,43 @@
 
 using divvun::variant;
 using divvun::Pipeline;
+using divvun::toUtf8;
+using divvun::fromUtf8;
 
 
 variant<int, Pipeline> getPipelineXml(const std::string& path, const std::u16string& pipename, bool verbose) {
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 	const std::unique_ptr<divvun::PipeSpec> spec(new divvun::PipeSpec(path));
 	if(spec->pnodes.find(pipename) == spec->pnodes.end()) {
-		std::cerr << "divvun-checker: ERROR: Couldn't find pipe " << utf16conv.to_bytes(pipename) << " in " << path << std::endl;
+		std::cerr << "divvun-checker: ERROR: Couldn't find pipe " << toUtf8(pipename) << " in " << path << std::endl;
 		return EXIT_FAILURE;
 	}
 	return Pipeline(spec, pipename, verbose);
 }
 
 variant<int, Pipeline> getPipelineAr(const std::string& path, const std::u16string& pipename, bool verbose) {
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 	const auto& ar_spec = divvun::readArPipeSpec(path);
 	if(ar_spec->spec->pnodes.find(pipename) == ar_spec->spec->pnodes.end()) {
-		std::cerr << "divvun-checker: ERROR: Couldn't find pipe " << utf16conv.to_bytes(pipename) << " in " << path << std::endl;
+		std::cerr << "divvun-checker: ERROR: Couldn't find pipe " << toUtf8(pipename) << " in " << path << std::endl;
 		return EXIT_FAILURE;
 	}
 	return Pipeline(ar_spec, pipename, verbose);
 }
 
 int printNamesXml(const std::string& path, bool verbose) {
 	const std::unique_ptr<divvun::PipeSpec> spec(new divvun::PipeSpec(path));
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 	std::cout << "Please specify a pipeline variant with the -n/--variant option. Available variants in pipespec:" << std::endl;
 	for(const auto& p : spec->pnodes) {
-		const auto& name = utf16conv.to_bytes(p.first.c_str());
+		const auto& name = toUtf8(p.first.c_str());
 		std::cout << name << std::endl;
 	}
 	return EXIT_SUCCESS;
 }
 
 int printNamesAr(const std::string& path, bool verbose) {
 	const auto& ar_spec = divvun::readArPipeSpec(path);
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 	std::cout << "Please specify a pipeline variant with the -n/--variant option. Available variants in archive:" << std::endl;
 	for(const auto& p : ar_spec->spec->pnodes) {
-		const auto& name = utf16conv.to_bytes(p.first.c_str());
+		const auto& name = toUtf8(p.first.c_str());
 		std::cout << name << std::endl;
 	}
 	return EXIT_SUCCESS;
@@ -81,24 +79,23 @@ int run(Pipeline& pipeline) {
 
 void printPrefs(const Pipeline& pipeline) {
 	using namespace divvun;
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 	std::cout << "== Available preferences ==" << std::endl;
 	for(const auto& lp : pipeline.prefs) {
 		const Lang& lang = lp.first;
 		std::cout << std::endl << "=== with messages localised for '" << lang << "' ===" << std::endl;
 		const Prefs& prefs = lp.second;
 		std::cout << "==== Toggles: ====" << std::endl;
 		for(const auto& id : prefs.toggleIds) {
-			std::cout << "- [ ] " << utf16conv.to_bytes(id.first) << " \t" << utf16conv.to_bytes(id.second) << std::endl;
+			std::cout << "- [ ] " << toUtf8(id.first) << " \t" << toUtf8(id.second) << std::endl;
 		}
 		for(const auto& re : prefs.toggleRes) {
-			std::cout << "- [ ] [regex] \t" << utf16conv.to_bytes(re.second) << std::endl;
+			std::cout << "- [ ] [regex] \t" << toUtf8(re.second) << std::endl;
 		}
 		std::cout << "==== Options: ====" << std::endl;
 		for(const Option& o : prefs.options) {
 			std::cout << "- " << o.name << " (" << o.type << "):" << std::endl;
 			for(const auto& c : o.choices) {
-				std::cout << "- ( ) " << utf16conv.to_bytes(c.first) << " \t" << utf16conv.to_bytes(c.second) << std::endl;
+				std::cout << "- ( ) " << toUtf8(c.first) << " \t" << toUtf8(c.second) << std::endl;
 			}
 		}
 	}
@@ -108,7 +105,6 @@ int main(int argc, char ** argv)
 {
 	try
 	{
-		std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 		cxxopts::Options options(argv[0], " - generate grammar checker suggestions from a CG stream");
 
 		options.add_options()
@@ -157,9 +153,8 @@ int main(int argc, char ** argv)
 
 		auto ignores = std::set<divvun::ErrId>();
 		if(options.count("ignore")) {
-			std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 			for(const auto& ignore : divvun::split(options["ignore"].as<std::string>(), ',')) {
-				ignores.insert(utf16conv.from_bytes(ignore));
+				ignores.insert(fromUtf8(ignore));
 			}
 		}
 
@@ -172,7 +167,7 @@ int main(int argc, char ** argv)
 				return printNamesXml(specfile, verbose);
 			}
 			else {
-				const auto& pipename = utf16conv.from_bytes(options["variant"].as<std::string>());
+				const auto& pipename = fromUtf8(options["variant"].as<std::string>());
 				return getPipelineXml(specfile, pipename, verbose).match(
 					[]       (int r) { return r; },
 					[&](Pipeline& p) {
@@ -196,7 +191,7 @@ int main(int argc, char ** argv)
 				return printNamesAr(archive, verbose);
 			}
 			else {
-				const auto& pipename = utf16conv.from_bytes(options["variant"].as<std::string>());
+				const auto& pipename = fromUtf8(options["variant"].as<std::string>());
 				return getPipelineAr(archive, pipename, verbose).match(
 					[]       (int r) { return r; },
 					[&](Pipeline& p) {

diff --git a/src/main_gen_sh.cpp b/src/main_gen_sh.cpp
@@ -27,7 +27,6 @@ int main(int argc, char ** argv)
 {
 	try
 	{
-		std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 		cxxopts::Options options(argv[0], "BIN - generate shell script to run checker pipeline from XML pipespec");
 
 		options.add_options()
@@ -79,7 +78,7 @@ int main(int argc, char ** argv)
 					std::cerr << argv[0] << " ERROR: Specify either --variant or --dir, not both!" << std::endl;
 					return EXIT_FAILURE;
 				}
-				const auto& pipename = utf16conv.from_bytes(options["variant"].as<std::string>());
+				const auto& pipename = divvun::fromUtf8(options["variant"].as<std::string>());
 				divvun::writePipeSpecSh(specfile, pipename, json, std::cout);
 				return EXIT_SUCCESS;
 			}

diff --git a/src/pipeline.cpp b/src/pipeline.cpp
@@ -178,10 +178,9 @@ Pipeline Pipeline::mkPipeline(const unique_ptr<ArPipeSpec>& ar_spec, const u16st
 		// TODO: Move into a lib-general init function? Or can I call this safely once per CGCmd?
 		throw std::runtime_error("libdivvun: ERROR: Couldn't initialise ICU for vislcg3!");
 	}
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 	string locale = spec->language;
 	for (const pugi::xml_node& cmd: spec->pnodes.at(pipename).children()) {
-		const auto& name = utf16conv.from_bytes(cmd.name());
+		const auto& name = fromUtf8(cmd.name());
 		std::unordered_map<string, string> args;
 		for (const pugi::xml_node& arg: cmd.children()) {
 			args[arg.name()] = arg.attribute("n").value();
@@ -243,7 +242,7 @@ Pipeline Pipeline::mkPipeline(const unique_ptr<ArPipeSpec>& ar_spec, const u16st
 			suggestcmd = s;
 		}
 		else if(name == u"sh") {
-			// const auto& prog = utf16conv.from_bytes(cmd.attribute("prog").value());
+			// const auto& prog = fromUtf8(cmd.attribute("prog").value());
 			// cmds.emplace_back(new ShCmd(prog, args, verbose));
 		}
 		else if(name == u"prefs") {
@@ -262,10 +261,9 @@ Pipeline Pipeline::mkPipeline(const unique_ptr<PipeSpec>& spec, const u16string&
 		// TODO: Move into a lib-general init function? Or can I call this safely once per CGCmd?
 		throw std::runtime_error("libdivvun: ERROR: Couldn't initialise ICU for vislcg3!");
 	}
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf16conv;
 	string locale = spec->language;
 	for (const pugi::xml_node& cmd: spec->pnodes.at(pipename).children()) {
-		const auto& name = utf16conv.from_bytes(cmd.name());
+		const auto& name = fromUtf8(cmd.name());
 		std::unordered_map<string, string> args;
 		for (const pugi::xml_node& arg: cmd.children()) {
 			args[arg.name()] = arg.attribute("n").value();
@@ -298,7 +296,7 @@ Pipeline Pipeline::mkPipeline(const unique_ptr<PipeSpec>& spec, const u16string&
 			suggestcmd = s;
 		}
 		else if(name == u"sh") {
-			// const auto& prog = utf16conv.from_bytes(cmd.attribute("prog").value());
+			// const auto& prog = fromUtf8(cmd.attribute("prog").value());
 			// cmds.emplace_back(new ShCmd(prog, args, verbose));
 		}
 		else if(name == u"prefs") {

diff --git a/src/pipeline.hpp b/src/pipeline.hpp
@@ -213,10 +213,10 @@ inline void parsePrefs(LocalisedPrefs& prefs, const pugi::xml_node& cmd) {
 		const auto name = pref.attribute("name").value();
 		std::unordered_map<Lang, std::unordered_map<ErrId, Msg>> lems;
 		for (const pugi::xml_node& option: pref.children()) {
-			const auto errId = utf16conv.from_bytes(option.attribute("err-id").value());
+			const auto errId = fromUtf8(option.attribute("err-id").value());
 			for (const pugi::xml_node& label: option.children()) {
 				const auto lang = label.attribute("xml:lang").value();
-				const auto msg = utf16conv.from_bytes(label.text().get()); // or xml_raw_cdata(label);
+				const auto msg = fromUtf8(label.text().get()); // or xml_raw_cdata(label);
 				lems[lang][errId] = msg;
 			}
 		}

diff --git a/src/pipespec.cpp b/src/pipespec.cpp
@@ -30,7 +30,7 @@ PipeSpec::PipeSpec(const string& file) {
 			language = "se"; // reasonable default
 		}
 		for (pugi::xml_node pipeline: doc.child("pipespec").children("pipeline")) {
-			const u16string& pipename = utf16conv.from_bytes(pipeline.attribute("name").value());
+			const u16string& pipename = fromUtf8(pipeline.attribute("name").value());
 			auto pr = std::make_pair(pipename, pipeline);
 			pnodes[pipename] = pipeline;
 		}
@@ -51,7 +51,7 @@ PipeSpec::PipeSpec(pugi::char_t* buff, size_t size) {
 			language = "se"; // reasonable default
 		}
 		for (pugi::xml_node pipeline: doc.child("pipespec").children("pipeline")) {
-			const u16string& pipename = utf16conv.from_bytes(pipeline.attribute("name").value());
+			const u16string& pipename = fromUtf8(pipeline.attribute("name").value());
 			auto pr = std::make_pair(pipename, pipeline);
 			pnodes[pipename] = pipeline;
 		}
@@ -97,7 +97,7 @@ void validatePipespecCmd(const pugi::xml_node& cmd, const std::unordered_map<str
 	}
 	else if(name == "sh") {
 		throw std::runtime_error("<sh> command not implemented yet!");
-		// const auto& prog = utf16conv.from_bytes(cmd.attribute("prog").value());
+		// const auto& prog = fromUtf8(cmd.attribute("prog").value());
 	}
 	else if(name == "prefs") {
 		// pass
@@ -299,7 +299,7 @@ void writePipeSpecShDir(const string& specfile, bool json, const string& modesdi
 	const std::unique_ptr<PipeSpec> spec(new PipeSpec(specfile));
 	const auto dir = dirname(abspath(specfile));
 	for(const auto& p : spec->pnodes) {
-		const auto& pipename = utf16conv.to_bytes(p.first);
+		const auto& pipename = toUtf8(p.first);
 		writePipeSpecShDirOne(toPipeSpecShVector(dir, *spec, p.first, false, json),
 				      pipename,
 				      modesdir,