Check UTF-8 encoding, and convert strings to UTF-8

Handle conversion errors properly by inserting � instead.  Add a binary
header to provide portable way to write binary literals (I like them) Also
add a test file.  ref #2404

1 files changed, 21 insertions, 0 deletions

diff --git a/src/utils/encoding.hpp b/src/utils/encoding.hpp
new file mode 100644
index 0000000..362f1df
--- /dev/null
+++ b/src/utils/encoding.hpp
@@ -0,0 +1,21 @@
+#ifndef ENCODING_INCLUDED
+# define ENCODING_INCLUDED
+
+#include <string>
+
+namespace utils
+{
+  /**
+   * Returns true if the given null-terminated string is valid utf-8.
+   *
+   * Based on http://en.wikipedia.org/wiki/UTF-8#Description
+   */
+  bool is_valid_utf8(const char* s);
+  /**
+   * Convert the given string (encoded is "encoding") into valid utf-8.
+   * If some decoding fails, insert an utf-8 placeholder character instead.
+   */
+  std::string convert_to_utf8(const std::string& str, const char* encoding);
+}
+
+#endif // ENCODING_INCLUDED