Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

UTF32BEの文字コード変換クラスが7bit ASCIIを取り込めない問題に対処する #1627

Merged
merged 2 commits into from
Apr 16, 2021
Merged

UTF32BEの文字コード変換クラスが7bit ASCIIを取り込めない問題に対処する #1627

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
532755e
UTF32BEのテストを追加
berryzplus Apr 14, 2021
ae87b19
CCodePage::S_UTF32BEToUnicodeが7bitASCIIを変換できない不具合を修正
berryzplus Apr 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions sakura_core/charset/CCodePage.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -569,7 +569,7 @@ int CCodePage::S_UTF32BEToUnicode( const char* pSrc, int nSrcLen, wchar_t* pDst,
for(; i < nSrcLen; ){
if( i + 3 < nSrcLen ){
if( pSrcByte[i+1] == 0x00 && pSrcByte[i] == 0x00 ){
wchar_t c = static_cast<wchar_t>(pSrcByte[i+1] << 8 | pSrcByte[i]);
wchar_t c = static_cast<wchar_t>(pSrcByte[i+2] << 8 | pSrcByte[i+3]);
if( IsUtf16SurrogHi(c) || IsUtf16SurrogLow(c) ){
// サロゲート断片。バイトごとに出力する)
nDstUseLen += 4;
Expand Down Expand Up @@ -597,7 +597,7 @@ int CCodePage::S_UTF32BEToUnicode( const char* pSrc, int nSrcLen, wchar_t* pDst,
for(; i < nSrcLen; ){
if( i + 3 < nSrcLen ){
if( pSrcByte[i] == 0x00 && pSrcByte[i+1] == 0x00 ){
wchar_t c = static_cast<wchar_t>(pSrcByte[i+1] << 8 | pSrcByte[i]);
wchar_t c = static_cast<wchar_t>(pSrcByte[i+2] << 8 | pSrcByte[i+3]);
if( IsUtf16SurrogHi(c) || IsUtf16SurrogLow(c) ){
nDstUseCharLen = 4;
if( nDstUseLen + nDstUseCharLen <= nDstLen ){
Expand Down
48 changes: 48 additions & 0 deletions tests/unittests/test-ccodebase.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,54 @@ TEST(CCodeBase, codeUtf32Le)

//! googletestの出力に文字セットIDを出力させる
std::ostream& operator << (std::ostream& os, const ECodeType& eCodeType);
/*!
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

空行の入れ忘れでしょうか？

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

・・・どちらかというと、テストを挿入する場所を間違ってますね。

普通に考えて、452行目の手前に入れるのが適切な気がします。

このPRはこのまま進めて、なんかのついでに直してしまおうと思います。

* @brief 文字コード変換のテスト
*/
TEST(CCodeBase, codeUtf32Be)
{
const auto eCodeType = (ECodeType)12001;
auto pCodeBase = CCodeFactory::CreateCodeBase( eCodeType );

// 7bit ASCII範囲（等価変換）
constexpr auto& mbsAscii = "\x01\x02\x03\x04\x05\x06\a\b\t\n\v\f\r\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7F";
constexpr auto& wcsAscii = L"\x01\x02\x03\x04\x05\x06\a\b\t\n\v\f\r\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7F";

// ビッグエンディアンのバイナリを作成
std::basic_string<uint32_t> bin;
for( const auto ch : mbsAscii ){
bin.append( 1, ::_byteswap_ulong( ch ) );
}

bool bComplete1_1 = false;
auto encoded = pCodeBase->CodeToUnicode( BinarySequenceView( reinterpret_cast<const std::byte*>(bin.data()), bin.size() * sizeof(decltype(bin)::value_type)), &bComplete1_1 );
ASSERT_STREQ( wcsAscii, encoded.GetStringPtr() );
ASSERT_TRUE( bComplete1_1 );

bool bComplete1_2 = false;
auto decoded = pCodeBase->UnicodeToCode( encoded, &bComplete1_2 );
ASSERT_EQ( 0, memcmp( bin.data(), decoded.data(), decoded.size() ) );
ASSERT_TRUE( bComplete1_2 );

// かな漢字の変換（UTF-32BE仕様）
constexpr const auto& wcsKanaKanji = L"ｶﾅかなカナ漢字";

// ビッグエンディアンのバイナリを作成
bin.clear();
for( const auto ch : wcsKanaKanji ){
bin.append( 1, ::_byteswap_ulong( ch ) );
}

bool bComplete2_1 = false;
auto encoded2 = pCodeBase->CodeToUnicode( BinarySequenceView( reinterpret_cast<const std::byte*>(bin.data()), bin.size() * sizeof(decltype(bin)::value_type) ), &bComplete2_1 );
ASSERT_STREQ( wcsKanaKanji, encoded2.GetStringPtr() );
ASSERT_TRUE( bComplete2_1 );

bool bComplete2_2 = false;
auto decoded2 = pCodeBase->UnicodeToCode( encoded2, &bComplete2_2 );
ASSERT_EQ( 0, memcmp( bin.data(), decoded2.data(), decoded2.size() ) );
ASSERT_TRUE( bComplete2_2 );
}


//! EOLテストのためのフィクスチャクラス
class EolTest : public ::testing::TestWithParam<ECodeType> {};
Expand Down