added tokenizer samples

test/test_sw_tokenizer.cpp

@@ -1,4 +1,4 @@
-#include "gtest/gtest.h"
+#include "gtest/gtest.h"
 #include <fstream>
 #include <kiwi/Kiwi.h>
 #include <kiwi/SwTokenizer.h>
@@ -165,6 +165,24 @@ TEST(KiwiSwTokenizer, Builder)
 	}
 }
 
+TEST(KiwiSwTokenizer, EncodeError)
+{
+	SwTokenizer tokenizer;
+	{
+		std::ifstream ifs{ "tokenizers/kor.16k.json" };
+		tokenizer = SwTokenizer::load(reuseKiwiInstance(), ifs);
+	}
+
+	for (auto c : {
+		u8"또는 “𡆮”으로 새겨야 하는 것",
+		})
+	{
+		auto encoded = tokenizer.encode(c);
+		auto decoded = tokenizer.decode(encoded);
+		//EXPECT_EQ(decoded, c);
+	}
+}
+
 TEST(KiwiSwTokenizer, BasicEncodeAndDecode)
 {
 	SwTokenizer tokenizer;