mirror of
https://github.com/bab2min/Kiwi.git
synced 2026-06-17 01:54:27 +00:00
Update tools
This commit is contained in:
bab2min
parent
f351ebb995
commit
35464359d6
8 changed files with 828 additions and 28 deletions
14
Kiwi.sln
14
Kiwi.sln
|
|
@ -29,6 +29,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "build_cong", "vsproj\build_
|
|||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "diff_tokens", "vsproj\diff_tokens.vcxproj", "{EBA7E78E-01A2-42A4-B335-97D3DEFF9882}"
|
||||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "self_label", "vsproj\self_label.vcxproj", "{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|ARM64 = Debug|ARM64
|
||||
|
|
@ -159,6 +161,18 @@ Global
|
|||
{EBA7E78E-01A2-42A4-B335-97D3DEFF9882}.Release|x64.Build.0 = Release|x64
|
||||
{EBA7E78E-01A2-42A4-B335-97D3DEFF9882}.Release|x86.ActiveCfg = Release|Win32
|
||||
{EBA7E78E-01A2-42A4-B335-97D3DEFF9882}.Release|x86.Build.0 = Release|Win32
|
||||
{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}.Debug|ARM64.ActiveCfg = Debug|ARM64
|
||||
{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}.Debug|ARM64.Build.0 = Debug|ARM64
|
||||
{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}.Debug|x64.Build.0 = Debug|x64
|
||||
{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}.Debug|x86.ActiveCfg = Debug|Win32
|
||||
{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}.Debug|x86.Build.0 = Debug|Win32
|
||||
{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}.Release|ARM64.ActiveCfg = Release|ARM64
|
||||
{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}.Release|ARM64.Build.0 = Release|ARM64
|
||||
{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}.Release|x64.ActiveCfg = Release|x64
|
||||
{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}.Release|x64.Build.0 = Release|x64
|
||||
{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}.Release|x86.ActiveCfg = Release|Win32
|
||||
{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}.Release|x86.Build.0 = Release|Win32
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ int main(int argc, const char* argv[])
|
|||
{
|
||||
tutils::setUTF8Output();
|
||||
|
||||
CmdLine cmd{ "Kiwi PCLanguageModel Builder", ' ', "0.21.0" };
|
||||
CmdLine cmd{ "Kiwi CoNgram Builder", ' ', "0.21.0" };
|
||||
|
||||
ValueArg<string> mdef{ "m", "morpheme-def", "morpheme definition", true, "", "string" };
|
||||
ValueArg<string> cdef{ "c", "context-def", "context definition", true, "", "string" };
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#include <fstream>
|
||||
|
||||
#include <kiwi/Kiwi.h>
|
||||
#include <kiwi/TypoTransformer.h>
|
||||
#include <tclap/CmdLine.h>
|
||||
|
||||
#include "toolUtils.h"
|
||||
|
|
@ -10,13 +11,54 @@ using namespace std;
|
|||
using namespace kiwi;
|
||||
using namespace TCLAP;
|
||||
|
||||
Kiwi loadKiwiFromArg(const string& model, const string& modelType, size_t numThreads = 2)
|
||||
Kiwi loadKiwiFromArg(const string& model, const string& modelType,
|
||||
size_t numThreads = 2,
|
||||
bool normCoda = true,
|
||||
bool zCoda = true,
|
||||
bool loadMultiDict = true,
|
||||
float typoCostWeight = 0.f,
|
||||
bool bTypo = false,
|
||||
bool cTypo = false,
|
||||
bool lTypo = false,
|
||||
Dialect allowedDialects = Dialect::standard)
|
||||
{
|
||||
ModelType kiwiModelType = tutils::parseModelType(modelType);
|
||||
BuildOption opt = BuildOption::default_;
|
||||
opt &= ~BuildOption::loadMultiDict;
|
||||
KiwiBuilder builder{ model, numThreads < 2 ? 2 : numThreads, opt, kiwiModelType };
|
||||
return builder.build();
|
||||
|
||||
if (!loadMultiDict) opt &= ~BuildOption::loadMultiDict;
|
||||
KiwiBuilder builder{ model, numThreads < 2 ? 2 : numThreads, opt, kiwiModelType, allowedDialects };
|
||||
|
||||
if (typoCostWeight > 0 && !bTypo && !cTypo && !lTypo)
|
||||
{
|
||||
bTypo = true;
|
||||
}
|
||||
else if (typoCostWeight == 0)
|
||||
{
|
||||
bTypo = false;
|
||||
cTypo = false;
|
||||
lTypo = false;
|
||||
}
|
||||
|
||||
auto typo = getDefaultTypoSet(DefaultTypoSet::withoutTypo);
|
||||
|
||||
if (bTypo)
|
||||
{
|
||||
typo |= getDefaultTypoSet(DefaultTypoSet::basicTypoSet);
|
||||
}
|
||||
|
||||
if (cTypo)
|
||||
{
|
||||
typo |= getDefaultTypoSet(DefaultTypoSet::continualTypoSet);
|
||||
}
|
||||
|
||||
if (lTypo)
|
||||
{
|
||||
typo |= getDefaultTypoSet(DefaultTypoSet::lengtheningTypoSet);
|
||||
}
|
||||
|
||||
auto kiwi = builder.build(typo);
|
||||
kiwi.setTypoCostWeight(typoCostWeight);
|
||||
return kiwi;
|
||||
}
|
||||
|
||||
inline bool isEqual(const TokenInfo* a, size_t aSize, const TokenInfo* b, size_t bSize, bool ignoreTag = false)
|
||||
|
|
@ -32,7 +74,13 @@ inline bool isEqual(const TokenInfo* a, size_t aSize, const TokenInfo* b, size_t
|
|||
|
||||
inline ostream& operator<<(ostream& ostr, const TokenInfo& token)
|
||||
{
|
||||
return ostr << utf16To8(token.str) << '/' << tagToString(token.tag);
|
||||
ostr << utf16To8(token.str);
|
||||
if (token.senseId && !isSpecialClass(token.tag))
|
||||
{
|
||||
ostr << "__" << (int)token.senseId;
|
||||
}
|
||||
ostr << '/' << tagToString(token.tag);
|
||||
return ostr;
|
||||
}
|
||||
|
||||
bool printDiffTokens(ostream& ostr, const string& raw, const TokenInfo* a, size_t aSize, const TokenInfo* b, size_t bSize, bool ignoreTag = false, bool showSame = false)
|
||||
|
|
@ -131,7 +179,11 @@ pair<size_t, size_t> diffTokens(ostream& ostr, const string& raw, const TokenRes
|
|||
return { diff, total };
|
||||
}
|
||||
|
||||
pair<size_t, size_t> diffInputs(Kiwi& kiwiA, Kiwi& kiwiB, const string& inputs, ostream& ostr, bool sentenceLevel, bool ignoreTag = false, bool showSame = false)
|
||||
pair<size_t, size_t> diffInputs(Kiwi& kiwiA, Kiwi& kiwiB, const string& inputs, ostream& ostr, bool sentenceLevel,
|
||||
bool ignoreTag = false, bool showSame = false,
|
||||
Dialect allowedDialects = Dialect::standard,
|
||||
float dialectCost = 6.f
|
||||
)
|
||||
{
|
||||
ifstream ifs{ inputs };
|
||||
if (!ifs)
|
||||
|
|
@ -145,6 +197,8 @@ pair<size_t, size_t> diffInputs(Kiwi& kiwiA, Kiwi& kiwiB, const string& inputs,
|
|||
auto* poolB = kiwiB.getThreadPool();
|
||||
size_t diff = 0, total = 0;
|
||||
|
||||
AnalyzeOption option{ Match::allWithNormalizing, nullptr, false, allowedDialects, dialectCost };
|
||||
|
||||
while (getline(ifs, line))
|
||||
{
|
||||
while (futures.size() > kiwiA.getNumThreads() * 2)
|
||||
|
|
@ -161,8 +215,8 @@ pair<size_t, size_t> diffInputs(Kiwi& kiwiA, Kiwi& kiwiB, const string& inputs,
|
|||
|
||||
futures.emplace_back(
|
||||
line,
|
||||
poolA->enqueue([&, line](size_t tid) { return kiwiA.analyze(line, Match::allWithNormalizing);}),
|
||||
poolB->enqueue([&, line](size_t tid) { return kiwiB.analyze(line, Match::allWithNormalizing);})
|
||||
poolA->enqueue([&, line](size_t tid) { return kiwiA.analyze(line, option);}),
|
||||
poolB->enqueue([&, line](size_t tid) { return kiwiB.analyze(line, option);})
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -202,6 +256,8 @@ int main(int argc, const char* argv[])
|
|||
SwitchArg bTypo{ "", "btypo", "make basic-typo-tolerant model", false };
|
||||
SwitchArg cTypo{ "", "ctypo", "make continual-typo-tolerant model", false };
|
||||
SwitchArg lTypo{ "", "ltypo", "make lengthening-typo-tolerant model", false };
|
||||
ValueArg<string> dialect{ "d", "dialect", "allowed dialect", false, "standard", "string" };
|
||||
ValueArg<float> dialectCost{ "", "dialect-cost", "dialect cost", false, 6.f, "float" };
|
||||
UnlabeledMultiArg<string> inputs{ "inputs", "targets", false, "string" };
|
||||
|
||||
cmd.add(modelA);
|
||||
|
|
@ -214,6 +270,15 @@ int main(int argc, const char* argv[])
|
|||
cmd.add(sentence);
|
||||
cmd.add(ignoreTag);
|
||||
cmd.add(showSame);
|
||||
cmd.add(noNormCoda);
|
||||
cmd.add(noZCoda);
|
||||
cmd.add(noMulti);
|
||||
cmd.add(typoWeight);
|
||||
cmd.add(bTypo);
|
||||
cmd.add(cTypo);
|
||||
cmd.add(lTypo);
|
||||
cmd.add(dialect);
|
||||
cmd.add(dialectCost);
|
||||
|
||||
try
|
||||
{
|
||||
|
|
@ -225,8 +290,27 @@ int main(int argc, const char* argv[])
|
|||
return -1;
|
||||
}
|
||||
|
||||
Kiwi kiwiA = loadKiwiFromArg(modelA, modelAType, numThreads);
|
||||
Kiwi kiwiB = loadKiwiFromArg(modelB, modelBType, numThreads);
|
||||
Dialect parsedDialect = parseDialects(dialect.getValue());
|
||||
|
||||
Kiwi kiwiA = loadKiwiFromArg(modelA, modelAType, numThreads,
|
||||
!noNormCoda,
|
||||
!noZCoda,
|
||||
!noMulti,
|
||||
typoWeight,
|
||||
bTypo,
|
||||
cTypo,
|
||||
lTypo,
|
||||
parsedDialect);
|
||||
|
||||
Kiwi kiwiB = loadKiwiFromArg(modelB, modelBType, numThreads,
|
||||
!noNormCoda,
|
||||
!noZCoda,
|
||||
!noMulti,
|
||||
typoWeight,
|
||||
bTypo,
|
||||
cTypo,
|
||||
lTypo,
|
||||
parsedDialect);
|
||||
|
||||
unique_ptr<ofstream> ofstr;
|
||||
ostream* ostr = &cout;
|
||||
|
|
@ -242,7 +326,7 @@ int main(int argc, const char* argv[])
|
|||
{
|
||||
cout << "input: " << input << " ";
|
||||
cout.flush();
|
||||
auto p = diffInputs(kiwiA, kiwiB, input, *ostr, sentence, ignoreTag, showSame);
|
||||
auto p = diffInputs(kiwiA, kiwiB, input, *ostr, sentence, ignoreTag, showSame, parsedDialect, dialectCost);
|
||||
cout << "(diff: " << p.first << " / " << p.second << ")" << endl;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,9 +8,14 @@
|
|||
using namespace std;
|
||||
using namespace kiwi;
|
||||
|
||||
void printResult(Kiwi& kw, const string& line, int topn, bool score, ostream& out)
|
||||
void printResult(Kiwi& kw,
|
||||
const string& line,
|
||||
int topn,
|
||||
bool score,
|
||||
AnalyzeOption option,
|
||||
ostream& out)
|
||||
{
|
||||
for (auto& result : kw.analyze(line, topn, Match::allWithNormalizing))
|
||||
for (auto& result : kw.analyze(line, topn, option))
|
||||
{
|
||||
for (auto& t : result.first)
|
||||
{
|
||||
|
|
@ -22,13 +27,54 @@ void printResult(Kiwi& kw, const string& line, int topn, bool score, ostream& ou
|
|||
if (topn > 1) out << endl;
|
||||
}
|
||||
|
||||
int run(const string& modelPath, bool benchmark, const string& output, const string& user, int topn, int tolerance, float typos, bool score, bool largest, const vector<string>& input)
|
||||
int run(const string& modelPath, bool benchmark, const string& output, const string& user, int topn, int tolerance,
|
||||
float typoCostWeight,
|
||||
bool bTypo,
|
||||
bool cTypo,
|
||||
bool lTypo,
|
||||
Dialect allowedDialects,
|
||||
float dialectCost,
|
||||
bool score,
|
||||
ModelType modelType,
|
||||
const vector<string>& input)
|
||||
{
|
||||
try
|
||||
{
|
||||
tutils::Timer timer;
|
||||
size_t lines = 0, bytes = 0;
|
||||
Kiwi kw = KiwiBuilder{ modelPath, 1, BuildOption::default_, largest ? ModelType::largest : ModelType::none }.build(typos > 0 ? DefaultTypoSet::basicTypoSet : DefaultTypoSet::withoutTypo);
|
||||
|
||||
if (typoCostWeight > 0 && !bTypo && !cTypo && !lTypo)
|
||||
{
|
||||
bTypo = true;
|
||||
}
|
||||
else if (typoCostWeight == 0)
|
||||
{
|
||||
bTypo = false;
|
||||
cTypo = false;
|
||||
lTypo = false;
|
||||
}
|
||||
|
||||
auto typo = getDefaultTypoSet(DefaultTypoSet::withoutTypo);
|
||||
|
||||
if (bTypo)
|
||||
{
|
||||
typo |= getDefaultTypoSet(DefaultTypoSet::basicTypoSet);
|
||||
}
|
||||
|
||||
if (cTypo)
|
||||
{
|
||||
typo |= getDefaultTypoSet(DefaultTypoSet::continualTypoSet);
|
||||
}
|
||||
|
||||
if (lTypo)
|
||||
{
|
||||
typo |= getDefaultTypoSet(DefaultTypoSet::lengtheningTypoSet);
|
||||
}
|
||||
|
||||
|
||||
Kiwi kw = KiwiBuilder{ modelPath, 1, BuildOption::default_, modelType }.build(typo);
|
||||
|
||||
AnalyzeOption option{ Match::allWithNormalizing, nullptr, false, allowedDialects, dialectCost };
|
||||
|
||||
cout << "Kiwi v" << KIWI_VERSION_STRING << endl;
|
||||
if (tolerance)
|
||||
|
|
@ -36,10 +82,10 @@ int run(const string& modelPath, bool benchmark, const string& output, const str
|
|||
kw.setSpaceTolerance(tolerance);
|
||||
cout << "SpaceTolerance: " << tolerance << endl;
|
||||
}
|
||||
if (typos > 0)
|
||||
if (typoCostWeight > 0)
|
||||
{
|
||||
kw.setTypoCostWeight(typos);
|
||||
cout << "Typo Correction Cost Weight: " << typos << endl;
|
||||
kw.setTypoCostWeight(typoCostWeight);
|
||||
cout << "Typo Correction Cost Weight: " << typoCostWeight << endl;
|
||||
}
|
||||
|
||||
if (benchmark)
|
||||
|
|
@ -65,14 +111,14 @@ int run(const string& modelPath, bool benchmark, const string& output, const str
|
|||
#ifdef _WIN32
|
||||
for (wstring line; (cout << ">> ").flush(), getline(wcin, line);)
|
||||
{
|
||||
printResult(kw, utf16To8((const char16_t*)line.c_str()), topn, score, *out);
|
||||
printResult(kw, utf16To8((const char16_t*)line.c_str()), topn, score, option, *out);
|
||||
++lines;
|
||||
bytes += line.size();
|
||||
}
|
||||
#else
|
||||
for (string line; (cout << ">> ").flush(), getline(cin, line);)
|
||||
{
|
||||
printResult(kw, line, topn, score, *out);
|
||||
printResult(kw, line, topn, score, option, *out);
|
||||
++lines;
|
||||
bytes += line.size();
|
||||
}
|
||||
|
|
@ -91,7 +137,7 @@ int run(const string& modelPath, bool benchmark, const string& output, const str
|
|||
|
||||
for (string line; getline(in, line);)
|
||||
{
|
||||
printResult(kw, line, topn, score, *out);
|
||||
printResult(kw, line, topn, score, option, *out);
|
||||
++lines;
|
||||
bytes += line.size();
|
||||
}
|
||||
|
|
@ -125,26 +171,36 @@ int main(int argc, const char* argv[])
|
|||
CmdLine cmd{ "Kiwi CLI", ' ', KIWI_VERSION_STRING};
|
||||
|
||||
ValueArg<string> model{ "m", "model", "Kiwi model path", true, "", "string" };
|
||||
ValueArg<string> modelType{ "", "model-type", "Model Type", false, "none", "string" };
|
||||
SwitchArg benchmark{ "e", "benchmark", "benchmark performance" };
|
||||
ValueArg<string> output{ "o", "output", "output file path", false, "", "string" };
|
||||
ValueArg<string> user{ "u", "user", "user dictionary path", false, "", "string" };
|
||||
ValueArg<int> topn{ "n", "topn", "top-n of result", false, 1, "int > 0" };
|
||||
ValueArg<int> tolerance{ "t", "tolerance", "space tolerance of Kiwi", false, 0, "int >= 0" };
|
||||
ValueArg<float> typos{ "", "typos", "typo cost weight", false, 0.f, "float >= 0" };
|
||||
SwitchArg largest{ "", "largest", "use largest model" };
|
||||
ValueArg<float> typoWeight{ "", "typo", "typo weight", false, 0.f, "float" };
|
||||
SwitchArg bTypo{ "", "btypo", "make basic-typo-tolerant model", false };
|
||||
SwitchArg cTypo{ "", "ctypo", "make continual-typo-tolerant model", false };
|
||||
SwitchArg lTypo{ "", "ltypo", "make lengthening-typo-tolerant model", false };
|
||||
ValueArg<string> dialect{ "d", "dialect", "allowed dialect", false, "standard", "string" };
|
||||
ValueArg<float> dialectCost{ "", "dialect-cost", "dialect cost", false, 6.f, "float" };
|
||||
SwitchArg score{ "s", "score", "print score together" };
|
||||
UnlabeledMultiArg<string> files{ "inputs", "input files", false, "string" };
|
||||
|
||||
cmd.add(model);
|
||||
cmd.add(modelType);
|
||||
cmd.add(benchmark);
|
||||
cmd.add(output);
|
||||
cmd.add(user);
|
||||
cmd.add(topn);
|
||||
cmd.add(tolerance);
|
||||
cmd.add(typoWeight);
|
||||
cmd.add(bTypo);
|
||||
cmd.add(cTypo);
|
||||
cmd.add(lTypo);
|
||||
cmd.add(dialect);
|
||||
cmd.add(dialectCost);
|
||||
cmd.add(score);
|
||||
cmd.add(files);
|
||||
cmd.add(typos);
|
||||
cmd.add(largest);
|
||||
|
||||
try
|
||||
{
|
||||
|
|
@ -155,6 +211,19 @@ int main(int argc, const char* argv[])
|
|||
cerr << "error: " << e.error() << " for arg " << e.argId() << endl;
|
||||
return -1;
|
||||
}
|
||||
return run(model, benchmark, output, user, topn, tolerance, typos, score, largest, files.getValue());
|
||||
|
||||
Dialect parsedDialect = parseDialects(dialect.getValue());
|
||||
ModelType kiwiModelType = tutils::parseModelType(modelType);
|
||||
|
||||
return run(model, benchmark, output, user, topn, tolerance,
|
||||
typoWeight,
|
||||
bTypo,
|
||||
cTypo,
|
||||
lTypo,
|
||||
parsedDialect,
|
||||
dialectCost,
|
||||
score,
|
||||
kiwiModelType,
|
||||
files.getValue());
|
||||
}
|
||||
|
||||
|
|
|
|||
341
tools/self_label.cpp
Normal file
341
tools/self_label.cpp
Normal file
|
|
@ -0,0 +1,341 @@
|
|||
#include <iostream>
|
||||
#include <fstream>
|
||||
|
||||
#include <kiwi/Kiwi.h>
|
||||
#include <kiwi/TypoTransformer.h>
|
||||
#include <tclap/CmdLine.h>
|
||||
|
||||
#include "toolUtils.h"
|
||||
|
||||
using namespace std;
|
||||
using namespace kiwi;
|
||||
using namespace TCLAP;
|
||||
|
||||
Kiwi loadKiwiFromArg(const string& model,
|
||||
const string& modelType,
|
||||
const string& customDict,
|
||||
size_t numThreads = 2,
|
||||
bool normCoda = true,
|
||||
bool zCoda = true,
|
||||
bool loadMultiDict = true,
|
||||
float typoCostWeight = 0.f,
|
||||
bool bTypo = false,
|
||||
bool cTypo = false,
|
||||
bool lTypo = false,
|
||||
Dialect allowedDialects = Dialect::standard
|
||||
)
|
||||
{
|
||||
ModelType kiwiModelType = tutils::parseModelType(modelType);
|
||||
BuildOption opt = BuildOption::default_;
|
||||
|
||||
if (!loadMultiDict) opt &= ~BuildOption::loadMultiDict;
|
||||
KiwiBuilder builder{ model, numThreads < 2 ? 2 : numThreads, opt, kiwiModelType, allowedDialects };
|
||||
|
||||
if (!customDict.empty())
|
||||
{
|
||||
const size_t addedCnt = builder.loadDictionary(customDict);
|
||||
cerr << "Loaded custom dictionary: " << customDict << " (+" << addedCnt << " words)" << endl;
|
||||
}
|
||||
|
||||
if (typoCostWeight > 0 && !bTypo && !cTypo && !lTypo)
|
||||
{
|
||||
bTypo = true;
|
||||
}
|
||||
else if (typoCostWeight == 0)
|
||||
{
|
||||
bTypo = false;
|
||||
cTypo = false;
|
||||
lTypo = false;
|
||||
}
|
||||
|
||||
auto typo = getDefaultTypoSet(DefaultTypoSet::withoutTypo);
|
||||
|
||||
if (bTypo)
|
||||
{
|
||||
typo |= getDefaultTypoSet(DefaultTypoSet::basicTypoSet);
|
||||
}
|
||||
|
||||
if (cTypo)
|
||||
{
|
||||
typo |= getDefaultTypoSet(DefaultTypoSet::continualTypoSet);
|
||||
}
|
||||
|
||||
if (lTypo)
|
||||
{
|
||||
typo |= getDefaultTypoSet(DefaultTypoSet::lengtheningTypoSet);
|
||||
}
|
||||
|
||||
auto kiwi = builder.build(typo);
|
||||
if (typoCostWeight > 0)
|
||||
{
|
||||
kiwi.setTypoCostWeight(typoCostWeight);
|
||||
}
|
||||
return kiwi;
|
||||
}
|
||||
|
||||
|
||||
inline ostream& operator<<(ostream& ostr, const TokenInfo& token)
|
||||
{
|
||||
ostr << utf16To8(token.str);
|
||||
if (token.senseId && !isSpecialClass(token.tag))
|
||||
{
|
||||
ostr << "__" << (int)token.senseId;
|
||||
}
|
||||
ostr << '\t' << tagToString(token.tag);
|
||||
return ostr;
|
||||
}
|
||||
|
||||
UnorderedMap<pair<u16string, POSTag>, float> loadMorphemeRef(const string& path)
|
||||
{
|
||||
UnorderedMap<pair<u16string, POSTag>, float> refMap;
|
||||
ifstream ifs{ path };
|
||||
if (!ifs)
|
||||
{
|
||||
cerr << "Cannot open morpheme ref file: " << path << endl;
|
||||
return refMap;
|
||||
}
|
||||
string line;
|
||||
|
||||
vector<tuple<u16string, POSTag, uint64_t>> cnts;
|
||||
uint64_t totalCount = 0;
|
||||
while (getline(ifs, line))
|
||||
{
|
||||
auto parts = tutils::split(line, '\t');
|
||||
if (parts.size() < 3) continue;
|
||||
u16string form = utf8To16(string{ parts[0] });
|
||||
POSTag tag = toPOSTag(utf8To16(string{ parts[1] }));
|
||||
size_t count = stoull(string{ parts[2] });
|
||||
cnts.emplace_back(form, tag, count);
|
||||
totalCount += count;
|
||||
}
|
||||
|
||||
for (auto& t : cnts)
|
||||
{
|
||||
const float prob = (float)(log((double)(get<2>(t) + 1)) - log((double)totalCount));
|
||||
refMap.emplace(make_pair(move(get<0>(t)), get<1>(t)), prob);
|
||||
}
|
||||
return refMap;
|
||||
}
|
||||
|
||||
bool writeResult(ostream& ostr, const string& raw, const TokenResult& result,
|
||||
float filterThreshold = 5,
|
||||
const UnorderedMap<pair<u16string, POSTag>, float>& morphemeRef = {}
|
||||
)
|
||||
{
|
||||
auto& tokens = result.first;
|
||||
|
||||
if (filterThreshold > 0)
|
||||
{
|
||||
for (auto& token : tokens)
|
||||
{
|
||||
if ((token.tag == POSTag::nng || token.tag == POSTag::nnp) && token.morph->getForm().empty())
|
||||
{
|
||||
//cerr << "Low quality analysis detected. Skipped: " << utf16To8(token.str) << endl;
|
||||
return false;
|
||||
}
|
||||
if (isJClass(token.tag) || isEClass(token.tag))
|
||||
{
|
||||
auto it = morphemeRef.find(make_pair(token.str, token.tag));
|
||||
if (it != morphemeRef.end() && token.score < it->second - filterThreshold)
|
||||
{
|
||||
//cerr << "Low quality analysis detected. Skipped: " << utf16To8(token.str) << endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto utf16raw = utf8To16(raw);
|
||||
|
||||
if (tokens.empty()) return false;
|
||||
|
||||
size_t prevIdx = 0;
|
||||
uint32_t prevWordPosition = tokens[0].wordPosition;
|
||||
uint32_t prevSentPosition = tokens[0].sentPosition;
|
||||
|
||||
for (size_t i = 1; i < tokens.size(); ++i)
|
||||
{
|
||||
if ((prevWordPosition == tokens[i].wordPosition) &&
|
||||
(prevSentPosition == tokens[i].sentPosition))
|
||||
{
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
const size_t start = tokens[prevIdx].position, end = tokens[i - 1].position + tokens[i - 1].length;
|
||||
|
||||
ostr << utf16To8(utf16raw.substr(start, end - start));
|
||||
for (size_t j = prevIdx; j < i; ++j)
|
||||
{
|
||||
ostr << '\t' << tokens[j];
|
||||
}
|
||||
ostr << '\n';
|
||||
|
||||
prevIdx = i;
|
||||
prevWordPosition = tokens[i].wordPosition;
|
||||
prevSentPosition = tokens[i].sentPosition;
|
||||
}
|
||||
}
|
||||
|
||||
if (prevIdx < tokens.size())
|
||||
{
|
||||
const size_t start = tokens[prevIdx].position, end = tokens.back().position + tokens.back().length;
|
||||
ostr << utf16To8(utf16raw.substr(start, end - start));
|
||||
for (size_t j = prevIdx; j < tokens.size(); ++j)
|
||||
{
|
||||
ostr << '\t' << tokens[j];
|
||||
}
|
||||
ostr << '\n';
|
||||
}
|
||||
ostr << endl;
|
||||
return true;
|
||||
}
|
||||
|
||||
pair<size_t, size_t> selfLabel(Kiwi& kiwi, const string& inputs, ostream& ostr,
|
||||
Dialect allowedDialects = Dialect::standard,
|
||||
float dialectCost = 6.f,
|
||||
const UnorderedMap<pair<u16string, POSTag>, float>& morphemeRef = {},
|
||||
size_t skipOffset = 0
|
||||
)
|
||||
{
|
||||
ifstream ifs{ inputs };
|
||||
if (!ifs)
|
||||
{
|
||||
cerr << "Cannot open " << inputs << endl;
|
||||
return {0, 0};
|
||||
}
|
||||
string line;
|
||||
deque<tuple<string, future<TokenResult>>> futures;
|
||||
auto* pool = kiwi.getThreadPool();
|
||||
size_t total = 0, printed = 0;
|
||||
|
||||
AnalyzeOption option{ Match::allWithNormalizing | Match::splitSaisiot, nullptr, false, allowedDialects, dialectCost };
|
||||
|
||||
for (size_t n = 0; getline(ifs, line); ++n)
|
||||
{
|
||||
if (n < skipOffset) continue;
|
||||
while (futures.size() > kiwi.getNumThreads() * 8)
|
||||
{
|
||||
auto rawInput = move(get<0>(futures.front()));
|
||||
auto result = get<1>(futures.front()).get();
|
||||
futures.pop_front();
|
||||
|
||||
printed += writeResult(ostr, rawInput, result, true, morphemeRef) ? 1 : 0;
|
||||
++total;
|
||||
if (total % 100 == 0)
|
||||
{
|
||||
cerr << printed << " / " << (total + skipOffset) << " lines processed." << endl;
|
||||
}
|
||||
}
|
||||
|
||||
futures.emplace_back(
|
||||
line,
|
||||
pool->enqueue([&, line](size_t tid) { return kiwi.analyze(line, option);})
|
||||
);
|
||||
}
|
||||
|
||||
while (!futures.empty())
|
||||
{
|
||||
auto rawInput = move(get<0>(futures.front()));
|
||||
auto result = get<1>(futures.front()).get();
|
||||
futures.pop_front();
|
||||
|
||||
printed += writeResult(ostr, rawInput, result, true, morphemeRef) ? 1 : 0;
|
||||
++total;
|
||||
if (total % 100 == 0)
|
||||
{
|
||||
cerr << printed << " / " << (total + skipOffset) << " lines processed." << endl;
|
||||
}
|
||||
}
|
||||
return { printed, total };
|
||||
}
|
||||
|
||||
int main(int argc, const char* argv[])
|
||||
{
|
||||
tutils::setUTF8Output();
|
||||
|
||||
CmdLine cmd{ "Kiwi Diff Tokenizations" };
|
||||
|
||||
ValueArg<string> model{ "", "model", "Model A path", true, "", "string" };
|
||||
ValueArg<string> modelType{ "", "model-type", "Model A Type", false, "none", "string" };
|
||||
ValueArg<string> customDict{ "c", "custom-dict", "Custom dictionary path", false, "", "string" };
|
||||
ValueArg<string> output{ "o", "output", "output path", false, "", "string" };
|
||||
ValueArg<size_t> numThreads{ "t", "threads", "number of threads", false, 2, "int" };
|
||||
SwitchArg noNormCoda{ "", "no-normcoda", "without normalizing coda", false };
|
||||
SwitchArg noZCoda{ "", "no-zcoda", "without z-coda", false };
|
||||
SwitchArg noMulti{ "", "no-multi", "turn off multi dict", false };
|
||||
ValueArg<float> typoWeight{ "", "typo", "typo weight", false, 0.f, "float" };
|
||||
SwitchArg bTypo{ "", "btypo", "make basic-typo-tolerant model", false };
|
||||
SwitchArg cTypo{ "", "ctypo", "make continual-typo-tolerant model", false };
|
||||
SwitchArg lTypo{ "", "ltypo", "make lengthening-typo-tolerant model", false };
|
||||
ValueArg<string> dialect{ "d", "dialect", "allowed dialect", false, "standard", "string" };
|
||||
ValueArg<float> dialectCost{ "", "dialect-cost", "dialect cost", false, 6.f, "float" };
|
||||
ValueArg<string> morphemeRef{ "r", "morpheme-ref", "reference morpheme def", false, "", "string" };
|
||||
ValueArg<size_t> skipOffset{ "", "skip-offset", "skip offset for input files", false, 0, "int" };
|
||||
UnlabeledMultiArg<string> inputs{ "inputs", "targets", false, "string" };
|
||||
|
||||
cmd.add(model);
|
||||
cmd.add(modelType);
|
||||
cmd.add(customDict);
|
||||
cmd.add(output);
|
||||
cmd.add(numThreads);
|
||||
cmd.add(noNormCoda);
|
||||
cmd.add(noZCoda);
|
||||
cmd.add(noMulti);
|
||||
cmd.add(typoWeight);
|
||||
cmd.add(bTypo);
|
||||
cmd.add(cTypo);
|
||||
cmd.add(lTypo);
|
||||
cmd.add(dialect);
|
||||
cmd.add(dialectCost);
|
||||
cmd.add(morphemeRef);
|
||||
cmd.add(skipOffset);
|
||||
cmd.add(inputs);
|
||||
|
||||
try
|
||||
{
|
||||
cmd.parse(argc, argv);
|
||||
}
|
||||
catch (const ArgException& e)
|
||||
{
|
||||
cerr << "error: " << e.error() << " for arg " << e.argId() << endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
Dialect parsedDialect = parseDialects(dialect.getValue());
|
||||
|
||||
Kiwi kiwi = loadKiwiFromArg(
|
||||
model,
|
||||
modelType,
|
||||
customDict,
|
||||
numThreads,
|
||||
!noNormCoda,
|
||||
!noZCoda,
|
||||
!noMulti,
|
||||
typoWeight,
|
||||
bTypo,
|
||||
cTypo,
|
||||
lTypo,
|
||||
parsedDialect);
|
||||
|
||||
UnorderedMap<pair<u16string, POSTag>, float> morphemeRefMap;
|
||||
if (!morphemeRef.getValue().empty())
|
||||
{
|
||||
morphemeRefMap = loadMorphemeRef(morphemeRef);
|
||||
}
|
||||
|
||||
unique_ptr<ofstream> ofstr;
|
||||
ostream* ostr = &cout;
|
||||
if (!output.getValue().empty())
|
||||
{
|
||||
ofstr = std::make_unique<ofstream>(output);
|
||||
ostr = ofstr.get();
|
||||
}
|
||||
|
||||
for (auto& input : inputs)
|
||||
{
|
||||
cout << "input: " << input << endl;
|
||||
auto p = selfLabel(kiwi, input, *ostr, parsedDialect, dialectCost, morphemeRefMap, skipOffset);
|
||||
cout << "Completed. " << p.first << " / " << p.second << " lines written." << endl;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,9 @@
|
|||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <string_view>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
|
|
@ -146,4 +149,56 @@ namespace tutils
|
|||
throw std::invalid_argument{ "Invalid model type" };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template<class BaseStr, class BaseChr, class OutIterator>
|
||||
OutIterator split(BaseStr&& s, BaseChr delim, OutIterator result, size_t maxSplit = -1, BaseChr delimEscape = 0)
|
||||
{
|
||||
size_t p = 0, e = 0;
|
||||
for (size_t i = 0; i < maxSplit; ++i)
|
||||
{
|
||||
size_t t = s.find(delim, p);
|
||||
if (t == s.npos)
|
||||
{
|
||||
*(result++) = std::basic_string_view<BaseChr>{ &s[e] , s.size() - e };
|
||||
return result;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (delimEscape && delimEscape != delim && t > 0 && s[t - 1] == delimEscape)
|
||||
{
|
||||
p = t + 1;
|
||||
}
|
||||
else if (delimEscape && delimEscape == delim && t < s.size() - 1 && s[t + 1] == delimEscape)
|
||||
{
|
||||
p = t + 2;
|
||||
}
|
||||
else
|
||||
{
|
||||
*(result++) = std::basic_string_view<BaseChr>{ &s[e] , t - e };
|
||||
p = t + 1;
|
||||
e = t + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
*(result++) = std::basic_string_view<BaseChr>{ &s[e] , s.size() - e };
|
||||
return result;
|
||||
}
|
||||
|
||||
template<class BaseChr, class Trait>
|
||||
inline std::vector<std::basic_string_view<BaseChr, Trait>> split(std::basic_string_view<BaseChr, Trait> s, BaseChr delim, BaseChr delimEscape = 0)
|
||||
{
|
||||
std::vector<std::basic_string_view<BaseChr, Trait>> ret;
|
||||
split(s, delim, std::back_inserter(ret), -1, delimEscape);
|
||||
return ret;
|
||||
}
|
||||
|
||||
template<class BaseChr, class Trait, class Alloc>
|
||||
inline std::vector<std::basic_string_view<BaseChr, Trait>> split(const std::basic_string<BaseChr, Trait, Alloc>& s, BaseChr delim, BaseChr delimEscape = 0)
|
||||
{
|
||||
std::vector<std::basic_string_view<BaseChr, Trait>> ret;
|
||||
split(s, delim, std::back_inserter(ret), -1, delimEscape);
|
||||
return ret;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -96,7 +96,6 @@
|
|||
<ClInclude Include="..\src\SkipBigramTrainer.hpp" />
|
||||
<ClInclude Include="..\src\SkipBigramModel.hpp" />
|
||||
<ClInclude Include="..\src\SortUtils.hpp" />
|
||||
<ClInclude Include="..\src\string_view.hpp" />
|
||||
<ClInclude Include="..\src\StrUtils.h" />
|
||||
<ClInclude Include="..\src\UnicodeCase.h" />
|
||||
</ItemGroup>
|
||||
|
|
|
|||
238
vsproj/self_label.vcxproj
Normal file
238
vsproj/self_label.vcxproj
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|ARM64">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>ARM64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Debug|Win32">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|ARM64">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>ARM64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|Win32">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|x64">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
</ItemGroup>
|
||||
<PropertyGroup Label="Globals">
|
||||
<VCProjectVersion>15.0</VCProjectVersion>
|
||||
<ProjectGuid>{0CB53E9D-99A3-4BC9-9C32-E7BEE58910EC}</ProjectGuid>
|
||||
<Keyword>Win32Proj</Keyword>
|
||||
<RootNamespace>self_label</RootNamespace>
|
||||
<WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v143</PlatformToolset>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v143</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v143</PlatformToolset>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v143</PlatformToolset>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v143</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v143</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="Shared">
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<PropertyGroup Label="UserMacros" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<IncludePath>$(SolutionDir)third_party/mimalloc/include;$(SolutionDir)third_party/tclap/include;$(SolutionDir)include\;$(VC_IncludePath);$(WindowsSDK_IncludePath)</IncludePath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<IncludePath>$(SolutionDir)third_party/mimalloc/include;$(SolutionDir)third_party/tclap/include;$(SolutionDir)include\;$(VC_IncludePath);$(WindowsSDK_IncludePath)</IncludePath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<IncludePath>$(SolutionDir)third_party/mimalloc/include;$(SolutionDir)third_party/tclap/include;$(SolutionDir)include\;$(VC_IncludePath);$(WindowsSDK_IncludePath)</IncludePath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<IncludePath>$(SolutionDir)third_party/mimalloc/include;$(SolutionDir)third_party/tclap/include;$(SolutionDir)include\;$(VC_IncludePath);$(WindowsSDK_IncludePath)</IncludePath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<IncludePath>$(SolutionDir)third_party/mimalloc/include;$(SolutionDir)third_party/tclap/include;$(SolutionDir)include\;$(VC_IncludePath);$(WindowsSDK_IncludePath)</IncludePath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<IncludePath>$(SolutionDir)third_party/mimalloc/include;$(SolutionDir)third_party/tclap/include;$(SolutionDir)include\;$(VC_IncludePath);$(WindowsSDK_IncludePath)</IncludePath>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<PrecompiledHeader>NotUsing</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>KIWI_USE_MIMALLOC;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalOptions>/Qvec-report:1 /utf-8 /bigobj %(AdditionalOptions)</AdditionalOptions>
|
||||
<EnableEnhancedInstructionSet>AdvancedVectorExtensions512</EnableEnhancedInstructionSet>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<LanguageStandard>stdcpp17</LanguageStandard>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<Profile>true</Profile>
|
||||
</Link>
|
||||
<ProjectReference />
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<PrecompiledHeader>NotUsing</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>KIWI_USE_MIMALLOC;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalOptions>/Qvec-report:1 /utf-8 /bigobj %(AdditionalOptions)</AdditionalOptions>
|
||||
<EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<Profile>true</Profile>
|
||||
</Link>
|
||||
<ProjectReference />
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>NotUsing</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>KIWI_USE_MIMALLOC;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
|
||||
<AdditionalOptions>/Qvec-report:1 /utf-8 /bigobj %(AdditionalOptions)</AdditionalOptions>
|
||||
<LanguageStandard>stdcpp17</LanguageStandard>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>NotUsing</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>KIWI_USE_MIMALLOC;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
|
||||
<AdditionalOptions>/Qvec-report:1 /utf-8 /bigobj %(AdditionalOptions)</AdditionalOptions>
|
||||
<LanguageStandard>stdcpp17</LanguageStandard>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
</Link>
|
||||
<ProjectReference />
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>NotUsing</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>KIWI_USE_MIMALLOC;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
|
||||
<AdditionalOptions>/Qvec-report:1 /utf-8 /bigobj %(AdditionalOptions)</AdditionalOptions>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
</Link>
|
||||
<ProjectReference />
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<PrecompiledHeader>NotUsing</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>KIWI_USE_MIMALLOC;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<AdditionalOptions>/Qvec-report:1 /utf-8 /bigobj %(AdditionalOptions)</AdditionalOptions>
|
||||
<LanguageStandard>stdcpp17</LanguageStandard>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<Profile>true</Profile>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="kiwi_shared_library.vcxproj">
|
||||
<Project>{f790bc37-2732-4ed1-9ca5-7248bed3588e}</Project>
|
||||
</ProjectReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\tools\self_label.cpp" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
</ImportGroup>
|
||||
</Project>
|
||||
Loading…
Add table
Add a link
Reference in a new issue