Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

uchardet.dll が存在したらそれを使って文字エンコーディングの検出が行われるように処理追加 #1726

Merged
merged 4 commits into from
Sep 28, 2021
Merged

uchardet.dll が存在したらそれを使って文字エンコーディングの検出が行われるように処理追加 #1726

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fb12156
uchardet.dll が存在したらそれを使って文字エンコーディングの検出が行われるように処理追加
beru Sep 20, 2021
33b87cd
delete default constructor of CUchardet class as it can be implicitly…
beru Sep 20, 2021
bec3780
SonarCloud の Code Smells 数を減らす為の変更
beru Sep 20, 2021
34d4bc8
CharsetDetector を親フォルダに移動
beru Sep 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions sakura/sakura.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,7 @@
<ClInclude Include="..\sakura_core\charset\CEuc.h" />
<ClInclude Include="..\sakura_core\charset\charcode.h" />
<ClInclude Include="..\sakura_core\charset\charset.h" />
<ClInclude Include="..\sakura_core\charset\CharsetDetector.h" />
<ClInclude Include="..\sakura_core\charset\CJis.h" />
<ClInclude Include="..\sakura_core\charset\CLatin1.h" />
<ClInclude Include="..\sakura_core\charset\codechecker.h" />
Expand All @@ -299,7 +300,6 @@
<ClInclude Include="..\sakura_core\charset\CUnicodeBe.h" />
<ClInclude Include="..\sakura_core\charset\CUtf7.h" />
<ClInclude Include="..\sakura_core\charset\CUtf8.h" />
<ClInclude Include="..\sakura_core\charset\icu4c\CharsetDetector.h" />
<ClInclude Include="..\sakura_core\CHokanMgr.h" />
<ClInclude Include="..\sakura_core\CKeyWordSetMgr.h" />
<ClInclude Include="..\sakura_core\CLoadAgent.h" />
Expand Down Expand Up @@ -415,6 +415,7 @@
<ClInclude Include="..\sakura_core\extmodule\CHtmlHelp.h" />
<ClInclude Include="..\sakura_core\extmodule\CIcu4cI18n.h" />
<ClInclude Include="..\sakura_core\extmodule\CMigemo.h" />
<ClInclude Include="..\sakura_core\extmodule\CUchardet.h" />
<ClInclude Include="..\sakura_core\extmodule\CUxTheme.h" />
<ClInclude Include="..\sakura_core\Funccode_define.h" />
<ClInclude Include="..\sakura_core\Funccode_enum.h" />
Expand Down Expand Up @@ -638,6 +639,7 @@
<ClCompile Include="..\sakura_core\charset\CEuc.cpp" />
<ClCompile Include="..\sakura_core\charset\charcode.cpp" />
<ClCompile Include="..\sakura_core\charset\charset.cpp" />
<ClCompile Include="..\sakura_core\charset\CharsetDetector.cpp" />
<ClCompile Include="..\sakura_core\charset\CJis.cpp" />
<ClCompile Include="..\sakura_core\charset\CLatin1.cpp" />
<ClCompile Include="..\sakura_core\charset\codechecker.cpp" />
Expand All @@ -647,7 +649,6 @@
<ClCompile Include="..\sakura_core\charset\CUnicodeBe.cpp" />
<ClCompile Include="..\sakura_core\charset\CUtf7.cpp" />
<ClCompile Include="..\sakura_core\charset\CUtf8.cpp" />
<ClCompile Include="..\sakura_core\charset\icu4c\CharsetDetector.cpp" />
<ClCompile Include="..\sakura_core\CHokanMgr.cpp" />
<ClCompile Include="..\sakura_core\CKeyWordSetMgr.cpp" />
<ClCompile Include="..\sakura_core\CLoadAgent.cpp" />
Expand Down Expand Up @@ -783,6 +784,7 @@
<ClCompile Include="..\sakura_core\extmodule\CHtmlHelp.cpp" />
<ClCompile Include="..\sakura_core\extmodule\CIcu4cI18n.cpp" />
<ClCompile Include="..\sakura_core\extmodule\CMigemo.cpp" />
<ClCompile Include="..\sakura_core\extmodule\CUchardet.cpp" />
<ClCompile Include="..\sakura_core\extmodule\CUxTheme.cpp" />
<ClCompile Include="..\sakura_core\func\CFuncKeyWnd.cpp" />
<ClCompile Include="..\sakura_core\func\CFuncLookup.cpp" />
Expand Down
21 changes: 12 additions & 9 deletions sakura/sakura.vcxproj.filters
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,6 @@
<Filter Include="Cpp Source Files\uiparts">
<UniqueIdentifier>{930f3f82-ab3f-49e3-af4a-d4f9c2d51f46}</UniqueIdentifier>
</Filter>
<Filter Include="Cpp Source Files\charset\icu4c">
<UniqueIdentifier>{e4629f85-3be8-4dda-80db-1be310929433}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\sakura_core\Funccode_define.h">
Expand Down Expand Up @@ -1085,9 +1082,6 @@
<ClInclude Include="..\sakura_core\extmodule\CIcu4cI18n.h">
<Filter>Cpp Source Files\extmodule</Filter>
</ClInclude>
<ClInclude Include="..\sakura_core\charset\icu4c\CharsetDetector.h">
<Filter>Cpp Source Files\charset\icu4c</Filter>
</ClInclude>
<ClInclude Include="..\sakura_core\GrepInfo.h">
<Filter>Cpp Source Files</Filter>
</ClInclude>
Expand All @@ -1112,6 +1106,12 @@
<ClInclude Include="..\sakura_core\recent\SShare_History.h">
<Filter>Cpp Source Files\recent</Filter>
</ClInclude>
<ClInclude Include="..\sakura_core\extmodule\CUchardet.h">
<Filter>Cpp Source Files\extmodule</Filter>
</ClInclude>
<ClInclude Include="..\sakura_core\charset\CharsetDetector.h">
<Filter>Cpp Source Files\charset</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="..\resource\auto_scroll_center.cur">
Expand Down Expand Up @@ -2282,9 +2282,6 @@
<ClCompile Include="..\sakura_core\extmodule\CIcu4cI18n.cpp">
<Filter>Cpp Source Files\extmodule</Filter>
</ClCompile>
<ClCompile Include="..\sakura_core\charset\icu4c\CharsetDetector.cpp">
<Filter>Cpp Source Files\charset\icu4c</Filter>
</ClCompile>
<ClCompile Include="..\sakura_core\EditInfo.cpp">
<Filter>Cpp Source Files</Filter>
</ClCompile>
Expand All @@ -2306,6 +2303,12 @@
<ClCompile Include="..\sakura_core\convert\CConvert_CodeToSjis.cpp">
<Filter>Cpp Source Files\convert</Filter>
</ClCompile>
<ClCompile Include="..\sakura_core\extmodule\CUchardet.cpp">
<Filter>Cpp Source Files\extmodule</Filter>
</ClCompile>
<ClCompile Include="..\sakura_core\charset\CharsetDetector.cpp">
<Filter>Cpp Source Files\charset</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<Image Include="..\resource\auto_scroll_center.bmp">
Expand Down
1 change: 1 addition & 0 deletions sakura_core/StdAfx.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@
#include <initializer_list>
#include <list>
#include <map>
#include <unordered_map>
#include <memory>
#include <optional>
#include <string>
Expand Down
2 changes: 1 addition & 1 deletion sakura_core/charset/CCodeMediator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
*/
#include "StdAfx.h"
#include "charset/CCodeMediator.h"
#include "charset/icu4c/CharsetDetector.h"
#include "charset/CharsetDetector.h"
#include "charset/CESI.h"
#include "io/CBinaryStream.h"
#include "mem/CMemory.h"
Expand Down
258 changes: 258 additions & 0 deletions sakura_core/charset/CharsetDetector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,258 @@
/*! @file */
/*
Copyright (C) 2018-2021, Sakura Editor Organization

This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.

Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:

1. The origin of this software must not be misrepresented;
you must not claim that you wrote the original software.
If you use this software in a product, an acknowledgment
in the product documentation would be appreciated but is
not required.

2. Altered source versions must be plainly marked as such,
and must not be misrepresented as being the original software.

3. This notice may not be removed or altered from any source
distribution.
*/
#include "StdAfx.h"
#include "CharsetDetector.h"

// https://www.freedesktop.org/wiki/Software/uchardet/
// https://docs.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
static const std::unordered_map<std::string, uint16_t> map_charsetToCodePage = {
// International (Unicode)
{ "UTF-32BE", 12001, },
{ "UTF-32LE", 12000, },
// Arabic
{ "ISO-8859-6", 28596, },
{ "WINDOWS-1256", 1256, },
// Bulgarian
{ "ISO-8859-5", 28595, },
{ "WINDOWS-1251", 1251, },
// Chinese
{ "ISO-2022-CN", 50227, },
{ "BIG5", 950, },
{ "EUC-TW", 51950, },
{ "GB18030", 54936, },
{ "HZ-GB-2312", 52936, },
// Croatian
{ "ISO-8859-2", 28592, },
{ "ISO-8859-13", 28603, },
{ "ISO-8859-16", 28606, },
{ "WINDOWS-1250", 1250, },
{ "IBM852", 852, },
{ "MAC-CENTRALEUROPE", 10029, },
// Czech
{ "WINDOWS-1250", 1250, },
{ "ISO-8859-2", 28592, },
{ "IBM852", 852, },
{ "MAC-CENTRALEUROPE", 10029, },
// Danish
{ "ISO-8859-1", 28591, },
{ "ISO-8859-15", 28605, },
{ "WINDOWS-1252", 1252, },
// English
{ "ASCII", 20127, },
// Esperanto
{ "ISO-8859-3", 28593, },
// Estonian
{ "ISO-8859-4", 28594, },
{ "ISO-8859-13", 28603, },
{ "ISO-8859-13", 28603, },
{ "WINDOWS-1252", 1252, },
{ "WINDOWS-1257", 1257, },
// Finnish
{ "ISO-8859-1", 28591, },
{ "ISO-8859-4", 28594, },
{ "ISO-8859-9", 28599, },
{ "ISO-8859-13", 28603, },
{ "ISO-8859-15", 28605, },
{ "WINDOWS-1252", 1252, },
// French
{ "ISO-8859-1", 28591, },
{ "ISO-8859-15", 28605, },
{ "WINDOWS-1252", 1252, },
// German
{ "ISO-8859-1", 28591, },
{ "WINDOWS-1252", 1252, },
// Greek
{ "ISO-8859-7", 28597, },
{ "WINDOWS-1253", 1253, },
// Hebrew
{ "ISO-8859-8", 28598, },
{ "WINDOWS-1255", 1255, },
// Hungarian
{ "ISO-8859-2", 28592, },
{ "WINDOWS-1250", 1250, },
// Irish Gaelic
{ "ISO-8859-1", 28591, },
{ "ISO-8859-9", 28599, },
{ "ISO-8859-15", 28605, },
{ "WINDOWS-1252", 1252, },
// Italian
{ "ISO-8859-1", 28591, },
{ "ISO-8859-3", 28593, },
{ "ISO-8859-9", 28599, },
{ "ISO-8859-15", 28605, },
{ "WINDOWS-1252", 1252, },
// Japanese
{ "ISO-2022-JP", 50220, },
{ "SHIFT_JIS", 932, },
{ "EUC-JP", 20932, },
// Korean
{ "ISO-2022-KR", 50225, },
{ "EUC-KR", 51949, },
// Lithuanian
{ "ISO-8859-4", 28594, },
{ "ISO-8859-10", 28600, },
{ "ISO-8859-13", 28603, },
// Latvian
{ "ISO-8859-4", 28594, },
{ "ISO-8859-10", 28600, },
{ "ISO-8859-13", 28603, },
// Maltese
{ "ISO-8859-3", 28593, },
// Polish
{ "ISO-8859-2", 28592, },
{ "ISO-8859-13", 28603, },
{ "ISO-8859-16", 28606, },
{ "WINDOWS-1250", 1250, },
{ "IBM852", 852, },
{ "MAC-CENTRALEUROPE", 10029, },
// Portuguese
{ "ISO-8859-1", 28591, },
{ "ISO-8859-9", 28599, },
{ "ISO-8859-15", 28605, },
{ "WINDOWS-1252", 1252, },
// Romanian
{ "ISO-8859-2", 28592, },
{ "ISO-8859-16", 28606, },
{ "WINDOWS-1250", 1250, },
{ "IBM852", 852, },
// Russian
{ "ISO-8859-5", 28595, },
{ "KOI8-R", 20866, },
{ "WINDOWS-1251", 1251, },
{ "MAC-CYRILLIC", 10007, },
{ "IBM866", 866, },
{ "IBM855", 855, },
// Slovak
{ "WINDOWS-1250", 1250, },
{ "ISO-8859-2", 28592, },
{ "IBM852", 852, },
{ "MAC-CENTRALEUROPE", 10029, },
// Slovene
{ "ISO-8859-2", 28592, },
{ "ISO-8859-16", 28606, },
{ "WINDOWS-1250", 1250, },
{ "IBM852", 852, },
{ "MAC-CENTRALEUROPE", 10029, },
// Spanish
{ "ISO-8859-1", 28591, },
{ "ISO-8859-15", 28605, },
{ "WINDOWS-1252", 1252, },
// Swedish
{ "ISO-8859-1", 28591, },
{ "ISO-8859-4", 28594, },
{ "ISO-8859-9", 28599, },
{ "ISO-8859-15", 28605, },
{ "WINDOWS-1252", 1252, },
// Thai
{ "TIS-620", 874, },
{ "ISO-8859-11", 28601, },
// Turkish
{ "ISO-8859-3", 28593, },
{ "ISO-8859-9", 28599, },
// Vietnamese
//{ "VISCII", , },
{ "Windows-1258", 1258, },
// Others
{ "WINDOWS-1252", 1252, },
};

CharsetDetector::CharsetDetector() noexcept
: _icuin()
, _csd(nullptr)
{
_icuin.InitDll();
_uchardet.InitDll();
}

CharsetDetector::~CharsetDetector() noexcept
{
if (_icuin.IsAvailable()) {
_icuin.ucsdet_close(_csd);
}
if (_uchardet.IsAvailable()) {
_uchardet.uchardet_delete(_ud);
}
}

ECodeType CharsetDetector::Detect(const std::string_view& bytes)
{
if (_icuin.IsAvailable()) {
UErrorCode status = U_ZERO_ERROR;
_csd = _icuin.ucsdet_open(&status);
if (status != U_ZERO_ERROR) {
return CODE_ERROR;
}

_icuin.ucsdet_setText(_csd, bytes.data(), bytes.length(), &status);
if (status != U_ZERO_ERROR) {
return CODE_ERROR;
}

const auto csm = _icuin.ucsdet_detect(_csd, &status);
if (status != U_ZERO_ERROR) {
return CODE_ERROR;
}

std::string_view name = _icuin.ucsdet_getName(csm, &status);
if (status != U_ZERO_ERROR) {
return CODE_ERROR;
}

// 文字セット名⇒サクラエディタ内部コードの変換
if (name == "UTF-8") return CODE_UTF8;
if (name == "SHIFT_JIS") return CODE_SJIS;
if (name == "UTF-16BE") return CODE_UNICODEBE;
if (name == "UTF-16LE") return CODE_UNICODE;
if (name == "EUC-JP") return CODE_EUC;
if (name == "ISO-2022-JP") return CODE_JIS;
if (name == "UTF-7") return CODE_UTF7;
if (name == "ISO-8859-1") return CODE_LATIN1;

return CODE_ERROR;
}
if (_uchardet.IsAvailable()) {
if (!_ud) {
_ud = _uchardet.uchardet_new();
}
if (!_ud) {
return CODE_ERROR;
}
_uchardet.uchardet_reset(_ud);
if (_uchardet.uchardet_handle_data(_ud, bytes.data(), bytes.length()) != 0) {
return CODE_ERROR;
}
_uchardet.uchardet_data_end(_ud);
std::string_view name = _uchardet.uchardet_get_charset(_ud);
std::string str(name);
std::transform(str.begin(), str.end(), str.begin(), ::toupper);
auto it = map_charsetToCodePage.find(str);
if (it == map_charsetToCodePage.end()) {
return CODE_ERROR;
}else {
return (ECodeType)it->second;
}
}
return CODE_ERROR;
}
Loading