Merge 4e6bb2d into d578ff4

sakura-editor · Sep 20, 2021 · 5a17b76 · 5a17b76
2 parents d578ff4 + 4e6bb2d
commit 5a17b76
Show file tree

Hide file tree

Showing 6 changed files with 184 additions and 24 deletions.
diff --git a/sakura/sakura.vcxproj b/sakura/sakura.vcxproj
@@ -415,6 +415,7 @@
     <ClInclude Include="..\sakura_core\extmodule\CHtmlHelp.h" />
     <ClInclude Include="..\sakura_core\extmodule\CIcu4cI18n.h" />
     <ClInclude Include="..\sakura_core\extmodule\CMigemo.h" />
+    <ClInclude Include="..\sakura_core\extmodule\CUchardet.h" />
     <ClInclude Include="..\sakura_core\extmodule\CUxTheme.h" />
     <ClInclude Include="..\sakura_core\Funccode_define.h" />
     <ClInclude Include="..\sakura_core\Funccode_enum.h" />
@@ -783,6 +784,7 @@
     <ClCompile Include="..\sakura_core\extmodule\CHtmlHelp.cpp" />
     <ClCompile Include="..\sakura_core\extmodule\CIcu4cI18n.cpp" />
     <ClCompile Include="..\sakura_core\extmodule\CMigemo.cpp" />
+    <ClCompile Include="..\sakura_core\extmodule\CUchardet.cpp" />
     <ClCompile Include="..\sakura_core\extmodule\CUxTheme.cpp" />
     <ClCompile Include="..\sakura_core\func\CFuncKeyWnd.cpp" />
     <ClCompile Include="..\sakura_core\func\CFuncLookup.cpp" />

diff --git a/sakura/sakura.vcxproj.filters b/sakura/sakura.vcxproj.filters
@@ -1112,6 +1112,9 @@
     <ClInclude Include="..\sakura_core\recent\SShare_History.h">
       <Filter>Cpp Source Files\recent</Filter>
     </ClInclude>
+    <ClInclude Include="..\sakura_core\extmodule\CUchardet.h">
+      <Filter>Cpp Source Files\extmodule</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\resource\auto_scroll_center.cur">
@@ -2306,6 +2309,9 @@
     <ClCompile Include="..\sakura_core\convert\CConvert_CodeToSjis.cpp">
       <Filter>Cpp Source Files\convert</Filter>
     </ClCompile>
+    <ClCompile Include="..\sakura_core\extmodule\CUchardet.cpp">
+      <Filter>Cpp Source Files\extmodule</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <Image Include="..\resource\auto_scroll_center.bmp">

diff --git a/sakura_core/charset/icu4c/CharsetDetector.cpp b/sakura_core/charset/icu4c/CharsetDetector.cpp
@@ -30,39 +30,21 @@ CharsetDetector::CharsetDetector() noexcept
 	, _csd(nullptr)
 {
 	_icuin.InitDll();
+	_uchardet.InitDll();
 }
 
 CharsetDetector::~CharsetDetector() noexcept
 {
 	if (_icuin.IsAvailable()) {
 		_icuin.ucsdet_close(_csd);
 	}
+	if (_uchardet.IsAvailable()) {
+		_uchardet.uchardet_delete(_ud);
+	}
 }
 
-ECodeType CharsetDetector::Detect(const std::string_view& bytes)
+static ECodeType name2code(std::string_view name)
 {
-	UErrorCode status = U_ZERO_ERROR;
-
-	_csd = _icuin.ucsdet_open(&status);
-	if (status != U_ZERO_ERROR) {
-		return CODE_ERROR;
-	}
-
-	_icuin.ucsdet_setText(_csd, bytes.data(), bytes.length(), &status);
-	if (status != U_ZERO_ERROR) {
-		return CODE_ERROR;
-	}
-
-	const auto csm = _icuin.ucsdet_detect(_csd, &status);
-	if (status != U_ZERO_ERROR) {
-		return CODE_ERROR;
-	}
-
-	std::string_view name = _icuin.ucsdet_getName(csm, &status);
-	if (status != U_ZERO_ERROR) {
-		return CODE_ERROR;
-	}
-
 	// 文字セット名⇒サクラエディタ内部コードの変換
 	if (name == "UTF-8") return CODE_UTF8;
 	if (name == "SHIFT_JIS") return CODE_SJIS;
@@ -72,6 +54,56 @@ ECodeType CharsetDetector::Detect(const std::string_view& bytes)
 	if (name == "ISO-2022-JP") return CODE_JIS;
 	if (name == "UTF-7") return CODE_UTF7;
 	if (name == "ISO-8859-1") return CODE_LATIN1;
+	// ここから下は数が多いのでどうしたものか…
+	// https://www.freedesktop.org/wiki/Software/uchardet/
+	// https://docs.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
+	if (name == "GB2312") return (ECodeType)936;
+	if (name == "BIG5") return (ECodeType)950;
+	if (name == "ISO-2022-KR") return (ECodeType)50225;
+	if (name == "GB18030") return (ECodeType)54936;
+	return CODE_ERROR;
+}
+
+ECodeType CharsetDetector::Detect(const std::string_view& bytes)
+{
+	if (_icuin.IsAvailable()) {
+		UErrorCode status = U_ZERO_ERROR;
+		_csd = _icuin.ucsdet_open(&status);
+		if (status != U_ZERO_ERROR) {
+			return CODE_ERROR;
+		}
+
+		_icuin.ucsdet_setText(_csd, bytes.data(), bytes.length(), &status);
+		if (status != U_ZERO_ERROR) {
+			return CODE_ERROR;
+		}
+
+		const auto csm = _icuin.ucsdet_detect(_csd, &status);
+		if (status != U_ZERO_ERROR) {
+			return CODE_ERROR;
+		}
 
+		std::string_view name = _icuin.ucsdet_getName(csm, &status);
+		if (status != U_ZERO_ERROR) {
+			return CODE_ERROR;
+		}
+		return name2code(name);
+	}
+	if (_uchardet.IsAvailable()) {
+		if (!_ud) {
+			_ud = _uchardet.uchardet_new();
+		}
+		if (!_ud) {
+			return CODE_ERROR;
+		}
+		_uchardet.uchardet_reset(_ud);
+		if (_uchardet.uchardet_handle_data(_ud, bytes.data(), bytes.length()) != 0) {
+			return CODE_ERROR;
+		}
+		_uchardet.uchardet_data_end(_ud);
+		std::string_view name = _uchardet.uchardet_get_charset(_ud);
+		auto code = name2code(name);
+		return code;
+	}
 	return CODE_ERROR;
 }
diff --git a/sakura_core/charset/icu4c/CharsetDetector.h b/sakura_core/charset/icu4c/CharsetDetector.h
@@ -29,6 +29,7 @@
 #include <string_view>
 
 #include "extmodule/CIcu4cI18n.h"
+#include "extmodule/CUchardet.h"
 
 /*!
  * @brief 文字コード検出クラス
@@ -38,12 +39,15 @@ class CharsetDetector final
 	CIcu4cI18n _icuin;
 	UCharsetDetector* _csd;
 
+	CUchardet _uchardet;
+	uchardet_t _ud = nullptr;
+
 public:
 	CharsetDetector() noexcept;
 	~CharsetDetector() noexcept;
 
 	bool IsAvailable() const noexcept {
-		return _icuin.IsAvailable();
+		return _icuin.IsAvailable() || _uchardet.IsAvailable();
 	}
 
 	ECodeType Detect(const std::string_view& bytes);

diff --git a/sakura_core/extmodule/CUchardet.cpp b/sakura_core/extmodule/CUchardet.cpp
@@ -0,0 +1,58 @@
+/*! @file */
+/*
+	Copyright (C) 2018-2021, Sakura Editor Organization
+
+	This software is provided 'as-is', without any express or implied
+	warranty. In no event will the authors be held liable for any damages
+	arising from the use of this software.
+
+	Permission is granted to anyone to use this software for any purpose,
+	including commercial applications, and to alter it and redistribute it
+	freely, subject to the following restrictions:
+
+		1. The origin of this software must not be misrepresented;
+		   you must not claim that you wrote the original software.
+		   If you use this software in a product, an acknowledgment
+		   in the product documentation would be appreciated but is
+		   not required.
+
+		2. Altered source versions must be plainly marked as such,
+		   and must not be misrepresented as being the original software.
+
+		3. This notice may not be removed or altered from any source
+		   distribution.
+*/
+#include "StdAfx.h"
+#include "CUchardet.h"
+
+/*!
+ * @brief DLLの名前を返す
+ */
+LPCWSTR CUchardet::GetDllNameImp( [[maybe_unused]] int index )
+{
+	return L"uchardet.dll";
+}
+
+/*!
+	DLLの初期化
+
+	関数のアドレスを取得してメンバに保管する．
+
+	@retval true 成功
+	@retval false アドレス取得に失敗
+*/
+bool CUchardet::InitDllImp()
+{
+	// DLL内関数名リスト
+	const ImportTable table[] = {
+		{ &_uchardet_new,			"uchardet_new" },
+		{ &_uchardet_delete,		"uchardet_delete" },
+		{ &_uchardet_handle_data,	"uchardet_handle_data" },
+		{ &_uchardet_data_end,		"uchardet_data_end" },
+		{ &_uchardet_reset,			"uchardet_reset" },
+		{ &_uchardet_get_charset,	"uchardet_get_charset" },
+		{ nullptr, 0 }
+	};
+	return RegisterEntries(table);
+}
+
diff --git a/sakura_core/extmodule/CUchardet.h b/sakura_core/extmodule/CUchardet.h
@@ -0,0 +1,58 @@
+/*! @file */
+/*
+	Copyright (C) 2018-2021, Sakura Editor Organization
+
+	This software is provided 'as-is', without any express or implied
+	warranty. In no event will the authors be held liable for any damages
+	arising from the use of this software.
+
+	Permission is granted to anyone to use this software for any purpose,
+	including commercial applications, and to alter it and redistribute it
+	freely, subject to the following restrictions:
+
+		1. The origin of this software must not be misrepresented;
+		   you must not claim that you wrote the original software.
+		   If you use this software in a product, an acknowledgment
+		   in the product documentation would be appreciated but is
+		   not required.
+
+		2. Altered source versions must be plainly marked as such,
+		   and must not be misrepresented as being the original software.
+
+		3. This notice may not be removed or altered from any source
+		   distribution.
+*/
+#pragma once
+
+#include "CDllHandler.h"
+
+typedef struct uchardet * uchardet_t;
+
+/*!
+ * uchardet ライブラリ(uchardet.dll) をラップするクラス
+ */
+class CUchardet final : public CDllImp
+{
+public:
+	// DLL関数ポインタ
+	uchardet_t (*_uchardet_new)(void) = nullptr;
+	void (*_uchardet_delete)(uchardet_t ud) = nullptr;
+	int (*_uchardet_handle_data)(uchardet_t ud, const char * data, size_t len) = nullptr;
+	void (*_uchardet_data_end)(uchardet_t ud) = nullptr;
+	void (*_uchardet_reset)(uchardet_t ud) = nullptr;
+	const char * (*_uchardet_get_charset)(uchardet_t ud) = nullptr;
+
+protected:
+	// CDllImpインタフェース
+	LPCWSTR GetDllNameImp(int nIndex) override;
+	bool InitDllImp() override;
+
+public:
+	uchardet_t uchardet_new(void) const { return _uchardet_new(); }
+	void uchardet_delete(uchardet_t ud) const { _uchardet_delete(ud); }
+	int uchardet_handle_data(uchardet_t ud, const char * data, size_t len) const { return _uchardet_handle_data(ud, data, len); }
+	void uchardet_data_end(uchardet_t ud) const { _uchardet_data_end(ud); }
+	void uchardet_reset(uchardet_t ud) const { _uchardet_reset(ud); }
+	const char * uchardet_get_charset(uchardet_t ud) const { return _uchardet_get_charset(ud); }
+};
+