mirrors/doublecmd
2
0
Fork 0
mirror of https://github.com/doublecmd/doublecmd.git synced 2026-06-21 09:58:13 +00:00
Code Issues Projects Releases Packages Wiki Activity

FIX: Disable UCS2BELangModel and UCS2LELangModel, it is often mistaken

Browse source
This commit is contained in:
Alexander Koblov 2019-06-09 12:58:02 +00:00
commit 80d795cfaf
3 changed files with 26 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

12
components/chsdet/ReadMe.txt
View file

@ -6,8 +6,13 @@ Based on Mozilla's i18n component - https://dxr.mozilla.org/mozilla/source/exten
-----------State
Version 0.2.9 stable.
The original version can be found at http://chsdet.sourceforge.net.
https://sourceforge.net/p/doublecmd/code/HEAD/tree/trunk/components/chsdet/.
Copyright (C) 2011-2019 Alexander Koblov
The latest version can be found at https://sourceforge.net/p/doublecmd/code/HEAD/tree/trunk/components/chsdet/.
-----------Original
Based on
Charset Detector - http://chsdet.sourceforge.net
Copyright (C) 2006-2013 Nikolaj Yakowlew
-----------Requirements
Charset Detector doesn't need any external components.
@ -88,6 +93,3 @@ begin
FreeAndNil(Detector);
end;
end;
Copyright (C) 2006-2013 Nikolaj Yakowlew
Copyright (C) 2011-2019 Alexander Koblov

6
components/chsdet/src/MBUnicodeMultiProber.pas
View file

@ -61,8 +61,8 @@ begin
inherited Create;
AddCharsetModel(UTF8LangModel);
AddCharsetModel(UCS2BELangModel);
AddCharsetModel(UCS2LELangModel);
// AddCharsetModel(UCS2BELangModel);
// AddCharsetModel(UCS2LELangModel);
end;
destructor TMBUnicodeMultiProber.Destroy;
@ -143,4 +143,4 @@ end;
// mDetectedCharset := UNKNOWN_CHARSET;
//end;
end.
end.

19
components/chsdet/src/nsUniversalDetector.pas
View file

@ -178,12 +178,25 @@ begin
if aLen > 0 then
mGotData := TRUE;
(*If the data starts with BOM, it should be Unicode, but we continue check*)
(*If the data starts with BOM, we know it is UTF*)
if mStart then
begin
mStart := FALSE;
CheckBOM(aBuf, aLen);
if CheckBOM(aBuf, aLen) > 0 then
begin
case mDetectedBOM of
BOM_UTF8: mDetectedCharset := UTF8_CHARSET;
BOM_UTF16_LE: mDetectedCharset := UTF16_LE_CHARSET;
BOM_UTF16_BE: mDetectedCharset := UTF16_BE_CHARSET;
BOM_UCS4_LE: mDetectedCharset := UTF32_LE_CHARSET;
BOM_UCS4_BE: mDetectedCharset := UTF32_BE_CHARSET;
BOM_UCS4_2143: mDetectedCharset := UCS4_LE_CHARSET;
BOM_UCS4_3412: mDetectedCharset := UCS4_BE_CHARSET
end;
mDone := TRUE;
Result := NS_OK;
Exit;
end;
end; {if mStart}
for i := 0 to Pred(aLen) do