diff --git a/.circleci/scripts/run-tests-cpp.sh b/.circleci/scripts/run-tests-cpp.sh index a6cd3d7aca..4c8c6cea31 100755 --- a/.circleci/scripts/run-tests-cpp.sh +++ b/.circleci/scripts/run-tests-cpp.sh @@ -2,6 +2,10 @@ set -euo pipefail +pushd runtime/Cpp +ctest +popd + pushd runtime-testsuite echo "running maven tests..." if [ $GROUP == "LEXER" ]; then diff --git a/.github/scripts/run-tests-cpp.sh b/.github/scripts/run-tests-cpp.sh index 8b95026e2e..c246ef9aca 100755 --- a/.github/scripts/run-tests-cpp.sh +++ b/.github/scripts/run-tests-cpp.sh @@ -2,8 +2,11 @@ set -euo pipefail -cd runtime-testsuite/ +pushd runtime/Cpp +ctest +popd +pushd runtime-testsuite if [ $GROUP == "LEXER" ]; then mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=cpp.** test elif [ $GROUP == "PARSER" ]; then @@ -11,5 +14,6 @@ elif [ $GROUP == "PARSER" ]; then elif [ $GROUP == "RECURSION" ]; then mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=cpp.** test else - mvn -q -Dtest=cpp.* test + mvn -q -Dtest=cpp.* test fi +popd diff --git a/.travis/run-tests-cpp.sh b/.travis/run-tests-cpp.sh index e95dc8922d..6fb7367079 100755 --- a/.travis/run-tests-cpp.sh +++ b/.travis/run-tests-cpp.sh @@ -2,13 +2,18 @@ set -euo pipefail +pushd runtime/Cpp +ctest +popd + +pushd runtime-testsuite if [ $GROUP == "LEXER" ]; then - mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=cpp.* test + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=cpp.* test elif [ $GROUP == "PARSER" ]; then - mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=cpp.* test + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=cpp.* test elif [ $GROUP == "RECURSION" ]; then - mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=cpp.* test + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=cpp.* test else - mvn -q -Dtest=cpp.* test + mvn -q -Dtest=cpp.* test fi - +popd diff --git a/runtime/Cpp/CMakeLists.txt b/runtime/Cpp/CMakeLists.txt index 67879ecc33..302cd4a783 100644 --- a/runtime/Cpp/CMakeLists.txt +++ b/runtime/Cpp/CMakeLists.txt @@ -1,6 +1,8 @@ # -*- mode:cmake -*- -cmake_minimum_required (VERSION 2.8) -# 2.8 needed because of ExternalProject +cmake_minimum_required (VERSION 3.14) +# 3.14 needed because of FetchContent + +enable_testing() # Detect build type, fallback to release and throw a warning if use didn't specify any if(NOT CMAKE_BUILD_TYPE) diff --git a/runtime/Cpp/deploy-macos.sh b/runtime/Cpp/deploy-macos.sh index cf977652d9..770d06fa1b 100755 --- a/runtime/Cpp/deploy-macos.sh +++ b/runtime/Cpp/deploy-macos.sh @@ -4,18 +4,6 @@ rm -f -R antlr4-runtime build lib 2> /dev/null rm antlr4-cpp-runtime-macos.zip 2> /dev/null -# Get utf8 dependency. -mkdir -p runtime/thirdparty 2> /dev/null -pushd runtime/thirdparty -if [ ! -d utfcpp ] -then - git clone https://github.com/nemtrif/utfcpp.git utfcpp - pushd utfcpp - git checkout tags/v3.1.1 - popd -fi -popd - # Binaries xcodebuild -project runtime/antlrcpp.xcodeproj \ -target antlr4 \ @@ -35,9 +23,6 @@ rm -f -R antlr4-runtime pushd runtime/src find . -name '*.h' | cpio -pdm ../../antlr4-runtime popd -pushd runtime/thirdparty/utfcpp/source -find . -name '*.h' | cpio -pdm ../../../../antlr4-runtime -popd # Zip up and clean up zip -r antlr4-cpp-runtime-macos.zip antlr4-runtime lib diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt index c8b16c6cf9..77994cdb44 100644 --- a/runtime/Cpp/runtime/CMakeLists.txt +++ b/runtime/Cpp/runtime/CMakeLists.txt @@ -35,45 +35,35 @@ add_custom_target(make_lib_output_dir ALL add_dependencies(antlr4_shared make_lib_output_dir) add_dependencies(antlr4_static make_lib_output_dir) -find_package(utf8cpp QUIET) +include(FetchContent) -set(INSTALL_utf8cpp FALSE) +FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/e2239ee6043f73722e7aa812a459f54a28552929.zip +) -if (utf8cpp_FOUND) - target_link_libraries(antlr4_shared utf8cpp) - target_link_libraries(antlr4_static utf8cpp) -else() +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) - # older utf8cpp doesn't define the package above - find_path(utf8cpp_HEADER utf8.h - PATH_SUFFIXES utf8cpp - ) +FetchContent_MakeAvailable(googletest) - if (utf8cpp_HEADER) - include_directories(${utf8cpp_HEADER}) - else() - include(${CMAKE_ROOT}/Modules/ExternalProject.cmake) - set(THIRDPARTY_DIR ${CMAKE_BINARY_DIR}/runtime/thirdparty) - set(UTFCPP_DIR ${THIRDPARTY_DIR}/utfcpp) - ExternalProject_Add( - utf8cpp - GIT_REPOSITORY "https://github.com/nemtrif/utfcpp" - GIT_TAG "v3.1.1" - SOURCE_DIR ${UTFCPP_DIR} - UPDATE_DISCONNECTED 1 - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${UTFCPP_DIR}/install -DUTF8_TESTS=off -DUTF8_SAMPLES=off - STEP_TARGETS build) - - include_directories( - ${UTFCPP_DIR}/install/include/utf8cpp - ${UTFCPP_DIR}/install/include/utf8cpp/utf8 - ) - - add_dependencies(antlr4_shared utf8cpp) - add_dependencies(antlr4_static utf8cpp) - set(INSTALL_utf8cpp TRUE) - endif() -endif() +file(GLOB libantlrcpp_TESTS + "${PROJECT_SOURCE_DIR}/runtime/tests/*.cpp" +) + +add_executable( + antlr4_tests + ${libantlrcpp_TESTS} +) + +target_link_libraries( + antlr4_tests + antlr4_static + gtest_main +) + +include(GoogleTest) + +gtest_discover_tests(antlr4_tests) if(CMAKE_SYSTEM_NAME MATCHES "Linux") target_link_libraries(antlr4_shared ${UUID_LIBRARIES}) @@ -152,16 +142,3 @@ install(DIRECTORY "${PROJECT_SOURCE_DIR}/runtime/src/" COMPONENT dev FILES_MATCHING PATTERN "*.h" ) - -if (INSTALL_utf8cpp) - install(FILES "${UTFCPP_DIR}/source/utf8.h" - DESTINATION "include/antlr4-runtime") - install(DIRECTORY "${UTFCPP_DIR}/source/utf8" - DESTINATION "include/antlr4-runtime" - COMPONENT dev - FILES_MATCHING PATTERN "*.h" - ) -endif() - - - diff --git a/runtime/Cpp/runtime/src/ANTLRInputStream.cpp b/runtime/Cpp/runtime/src/ANTLRInputStream.cpp index 85fea92192..b6470af9b7 100755 --- a/runtime/Cpp/runtime/src/ANTLRInputStream.cpp +++ b/runtime/Cpp/runtime/src/ANTLRInputStream.cpp @@ -9,7 +9,7 @@ #include "misc/Interval.h" #include "IntStream.h" -#include "support/StringUtils.h" +#include "support/Utf8.h" #include "support/CPPUtils.h" #include "ANTLRInputStream.h" @@ -35,28 +35,37 @@ ANTLRInputStream::ANTLRInputStream(std::istream &stream): ANTLRInputStream() { load(stream); } -void ANTLRInputStream::load(const std::string &input) { - load(input.data(), input.size()); +void ANTLRInputStream::load(const std::string &input, bool lenient) { + load(input.data(), input.size(), lenient); } -void ANTLRInputStream::load(const char *data, size_t length) { +void ANTLRInputStream::load(const char *data, size_t length, bool lenient) { // Remove the UTF-8 BOM if present. const char *bom = "\xef\xbb\xbf"; - if (length >= 3 && strncmp(data, bom, 3) == 0) - _data = antlrcpp::utf8_to_utf32(data + 3, data + length); - else - _data = antlrcpp::utf8_to_utf32(data, data + length); + if (length >= 3 && strncmp(data, bom, 3) == 0) { + data += 3; + length -= 3; + } + if (lenient) { + _data = Utf8::lenientDecode(std::string_view(data, length)); + } else { + auto maybe_utf32 = Utf8::strictDecode(std::string_view(data, length)); + if (!maybe_utf32.has_value()) { + throw IllegalArgumentException("UTF-8 string contains an illegal byte sequence"); + } + _data = std::move(maybe_utf32).value(); + } p = 0; } -void ANTLRInputStream::load(std::istream &stream) { +void ANTLRInputStream::load(std::istream &stream, bool lenient) { if (!stream.good() || stream.eof()) // No fail, bad or EOF. return; _data.clear(); std::string s((std::istreambuf_iterator(stream)), std::istreambuf_iterator()); - load(s.data(), s.length()); + load(s.data(), s.length(), lenient); } void ANTLRInputStream::reset() { @@ -144,7 +153,11 @@ std::string ANTLRInputStream::getText(const Interval &interval) { return ""; } - return antlrcpp::utf32_to_utf8(_data.substr(start, count)); + auto maybeUtf8 = Utf8::strictEncode(std::u32string_view(_data).substr(start, count)); + if (!maybeUtf8.has_value()) { + throw IllegalArgumentException("Input stream contains invalid Unicode code points"); + } + return std::move(maybeUtf8).value(); } std::string ANTLRInputStream::getSourceName() const { @@ -155,7 +168,11 @@ std::string ANTLRInputStream::getSourceName() const { } std::string ANTLRInputStream::toString() const { - return antlrcpp::utf32_to_utf8(_data); + auto maybeUtf8 = Utf8::strictEncode(_data); + if (!maybeUtf8.has_value()) { + throw IllegalArgumentException("Input stream contains invalid Unicode code points"); + } + return std::move(maybeUtf8).value(); } void ANTLRInputStream::InitializeInstanceFields() { diff --git a/runtime/Cpp/runtime/src/ANTLRInputStream.h b/runtime/Cpp/runtime/src/ANTLRInputStream.h index cfe5ad9bcc..413eadefa4 100755 --- a/runtime/Cpp/runtime/src/ANTLRInputStream.h +++ b/runtime/Cpp/runtime/src/ANTLRInputStream.h @@ -18,7 +18,7 @@ namespace antlr4 { protected: /// The data being scanned. // UTF-32 - UTF32String _data; + std::u32string _data; /// 0..n-1 index into string of next char size_t p; @@ -34,9 +34,13 @@ namespace antlr4 { ANTLRInputStream(const char *data, size_t length); ANTLRInputStream(std::istream &stream); - virtual void load(const std::string &input); - virtual void load(const char *data, size_t length); - virtual void load(std::istream &stream); + virtual void load(const std::string &input, bool lenient); + virtual void load(const char *data, size_t length, bool lenient); + virtual void load(std::istream &stream, bool lenient); + + virtual void load(const std::string &input) { load(input, false); } + virtual void load(const char *data, size_t length) { load(data, length, false); } + virtual void load(std::istream &stream) { load(stream, false); } /// Reset the stream so that it's in the same state it was /// when the object was created *except* the data array is not diff --git a/runtime/Cpp/runtime/src/UnbufferedCharStream.cpp b/runtime/Cpp/runtime/src/UnbufferedCharStream.cpp index 1f18d38431..3993938032 100755 --- a/runtime/Cpp/runtime/src/UnbufferedCharStream.cpp +++ b/runtime/Cpp/runtime/src/UnbufferedCharStream.cpp @@ -5,7 +5,7 @@ #include "misc/Interval.h" #include "Exceptions.h" -#include "support/StringUtils.h" +#include "support/Utf8.h" #include "UnbufferedCharStream.h" @@ -195,7 +195,11 @@ std::string UnbufferedCharStream::getText(const misc::Interval &interval) { } // convert from absolute to local index size_t i = interval.a - bufferStartIndex; - return utf32_to_utf8(_data.substr(i, interval.length())); + auto maybeUtf8 = Utf8::strictEncode(std::u32string_view(_data).substr(i, interval.length())); + if (!maybeUtf8.has_value()) { + throw IllegalArgumentException("Unbuffered stream contains invalid Unicode code points"); + } + return std::move(maybeUtf8).value(); } size_t UnbufferedCharStream::getBufferStartIndex() const { diff --git a/runtime/Cpp/runtime/src/antlr4-common.h b/runtime/Cpp/runtime/src/antlr4-common.h index fd3c7cf03d..6a3639329a 100644 --- a/runtime/Cpp/runtime/src/antlr4-common.h +++ b/runtime/Cpp/runtime/src/antlr4-common.h @@ -7,39 +7,34 @@ #include #include -#include #include +#include +#include #include +#include +#include +#include +#include #include #include #include #include -#include #include #include #include +#include #include -#include -#include -#include #include #include #include #include #include #include +#include #include #include #include #include -#include -#include -#include -#include - -#ifndef USE_UTF8_INSTEAD_OF_CODECVT - #include -#endif // Defines for the Guid class and other platform dependent stuff. #ifdef _WIN32 @@ -61,17 +56,6 @@ typedef __int32 ssize_t; #endif - #if _MSC_VER >= 1900 && _MSC_VER < 2000 - // VS 2015 has a known bug when using std::codecvt_utf8 - // so we have to temporarily use __int32 instead. - // https://connect.microsoft.com/VisualStudio/feedback/details/1403302/unresolved-external-when-using-codecvt-utf8 - typedef std::basic_string<__int32> i32string; - - typedef i32string UTF32String; - #else - typedef std::u32string UTF32String; - #endif - #ifdef ANTLR4CPP_EXPORTS #define ANTLR4CPP_PUBLIC __declspec(dllexport) #else @@ -82,16 +66,7 @@ #endif #endif - #if defined(_MSC_VER) && !defined(__clang__) - // clang-cl should escape this to prevent [ignored-attributes]. - namespace std { - class ANTLR4CPP_PUBLIC exception; // Prevents warning C4275 from MSVC. - } // namespace std - #endif - #elif defined(__APPLE__) - typedef std::u32string UTF32String; - #define GUID_CFUUID #if __GNUC__ >= 4 #define ANTLR4CPP_PUBLIC __attribute__ ((visibility ("default"))) @@ -99,8 +74,6 @@ #define ANTLR4CPP_PUBLIC #endif #else - typedef std::u32string UTF32String; - #define GUID_LIBUUID #if __GNUC__ >= 6 #define ANTLR4CPP_PUBLIC __attribute__ ((visibility ("default"))) diff --git a/runtime/Cpp/runtime/src/support/StringUtils.cpp b/runtime/Cpp/runtime/src/support/StringUtils.cpp index 15e7b83683..1f07ba05c2 100644 --- a/runtime/Cpp/runtime/src/support/StringUtils.cpp +++ b/runtime/Cpp/runtime/src/support/StringUtils.cpp @@ -7,40 +7,15 @@ namespace antlrcpp { -void replaceAll(std::string& str, std::string const& from, std::string const& to) -{ - if (from.empty()) - return; - - size_t start_pos = 0; - while ((start_pos = str.find(from, start_pos)) != std::string::npos) { - str.replace(start_pos, from.length(), to); - start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx'. + void replaceAll(std::string& str, std::string_view from, std::string_view to) { + if (from.empty()) + return; + + size_t start_pos = 0; + while ((start_pos = str.find(from, start_pos)) != std::string::npos) { + str.replace(start_pos, from.length(), to); + start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx'. + } } -} - -std::string ws2s(std::wstring const& wstr) { -#ifndef USE_UTF8_INSTEAD_OF_CODECVT - std::wstring_convert> converter; - std::string narrow = converter.to_bytes(wstr); -#else - std::string narrow; - utf8::utf32to8(wstr.begin(), wstr.end(), std::back_inserter(narrow)); -#endif - - return narrow; -} - -std::wstring s2ws(const std::string &str) { -#ifndef USE_UTF8_INSTEAD_OF_CODECVT - std::wstring_convert> converter; - std::wstring wide = converter.from_bytes(str); -#else - std::wstring wide; - utf8::utf8to32(str.begin(), str.end(), std::back_inserter(wide)); -#endif - - return wide; -} } // namespace antrlcpp diff --git a/runtime/Cpp/runtime/src/support/StringUtils.h b/runtime/Cpp/runtime/src/support/StringUtils.h index d00cc52d9c..c34de0b4c8 100644 --- a/runtime/Cpp/runtime/src/support/StringUtils.h +++ b/runtime/Cpp/runtime/src/support/StringUtils.h @@ -7,70 +7,8 @@ #include "antlr4-common.h" -#ifdef USE_UTF8_INSTEAD_OF_CODECVT -#include "utf8.h" -#endif - namespace antlrcpp { - // For all conversions utf8 <-> utf32. - // I wouldn't prefer wstring_convert because: according to - // https://en.cppreference.com/w/cpp/locale/wstring_convert, - // wstring_convert is deprecated in C++17. - // utfcpp (https://github.com/nemtrif/utfcpp) is a substitution. -#ifndef USE_UTF8_INSTEAD_OF_CODECVT - // VS 2015 and VS 2017 have different bugs in std::codecvt_utf8 (VS 2013 works fine). - #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000 - typedef std::wstring_convert, __int32> UTF32Converter; - #else - typedef std::wstring_convert, char32_t> UTF32Converter; - #endif -#endif - - // The conversion functions fails in VS2017, so we explicitly use a workaround. - template - inline std::string utf32_to_utf8(T const& data) - { - #ifndef USE_UTF8_INSTEAD_OF_CODECVT - // Don't make the converter static or we have to serialize access to it. - thread_local UTF32Converter converter; - - #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000 - const auto p = reinterpret_cast(data.data()); - return converter.to_bytes(p, p + data.size()); - #else - return converter.to_bytes(data); - #endif - #else - std::string narrow; - utf8::utf32to8(data.begin(), data.end(), std::back_inserter(narrow)); - return narrow; - #endif - } - - inline UTF32String utf8_to_utf32(const char* first, const char* last) - { - #ifndef USE_UTF8_INSTEAD_OF_CODECVT - thread_local UTF32Converter converter; - - #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000 - auto r = converter.from_bytes(first, last); - i32string s = reinterpret_cast(r.data()); - return s; - #else - std::u32string s = converter.from_bytes(first, last); - return s; - #endif - #else - UTF32String wide; - utf8::utf8to32(first, last, std::back_inserter(wide)); - return wide; - #endif - } - - void replaceAll(std::string &str, std::string const& from, std::string const& to); + void replaceAll(std::string &str, std::string_view from, std::string_view to); - // string <-> wstring conversion (UTF-16), e.g. for use with Window's wide APIs. - ANTLR4CPP_PUBLIC std::string ws2s(std::wstring const& wstr); - ANTLR4CPP_PUBLIC std::wstring s2ws(std::string const& str); } diff --git a/runtime/Cpp/runtime/src/support/Unicode.h b/runtime/Cpp/runtime/src/support/Unicode.h new file mode 100644 index 0000000000..f847ce2462 --- /dev/null +++ b/runtime/Cpp/runtime/src/support/Unicode.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2021 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +#pragma once + +namespace antlrcpp { + + class Unicode final { + public: + static constexpr char32_t REPLACEMENT_CHARACTER = 0xfffd; + + static constexpr bool isValid(char32_t codePoint) { + return codePoint < 0xd800 || (codePoint > 0xdfff && codePoint <= 0x10ffff); + } + + private: + Unicode() = delete; + + Unicode(const Unicode&) = delete; + + Unicode& operator=(const Unicode&) = delete; + }; + +} diff --git a/runtime/Cpp/runtime/src/support/Utf8.cpp b/runtime/Cpp/runtime/src/support/Utf8.cpp new file mode 100644 index 0000000000..294e9f1b21 --- /dev/null +++ b/runtime/Cpp/runtime/src/support/Utf8.cpp @@ -0,0 +1,242 @@ +/* Copyright (c) 2021 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +#include +#include + +#include "support/Utf8.h" +#include "support/Unicode.h" + +// The below implementation is based off of https://github.com/google/cel-cpp/internal/utf8.cc, +// which is itself based off of https://go.googlesource.com/go/+/refs/heads/master/src/unicode/utf8/utf8.go. +// If for some reason you feel the need to copy this implementation, please retain a comment +// referencing the two source files and giving credit, as well as maintaining any and all +// obligations required by the BSD 3-clause license that governs this file. + +namespace antlrcpp { + +namespace { + +#undef SELF + constexpr uint8_t SELF = 0x80; + +#undef LOW + constexpr uint8_t LOW = 0x80; +#undef HIGH + constexpr uint8_t HIGH = 0xbf; + +#undef MASKX + constexpr uint8_t MASKX = 0x3f; +#undef MASK2 + constexpr uint8_t MASK2 = 0x1f; +#undef MASK3 + constexpr uint8_t MASK3 = 0xf; +#undef MASK4 + constexpr uint8_t MASK4 = 0x7; + +#undef TX + constexpr uint8_t TX = 0x80; +#undef T2 + constexpr uint8_t T2 = 0xc0; +#undef T3 + constexpr uint8_t T3 = 0xe0; +#undef T4 + constexpr uint8_t T4 = 0xf0; + +#undef XX + constexpr uint8_t XX = 0xf1; +#undef AS + constexpr uint8_t AS = 0xf0; +#undef S1 + constexpr uint8_t S1 = 0x02; +#undef S2 + constexpr uint8_t S2 = 0x13; +#undef S3 + constexpr uint8_t S3 = 0x03; +#undef S4 + constexpr uint8_t S4 = 0x23; +#undef S5 + constexpr uint8_t S5 = 0x34; +#undef S6 + constexpr uint8_t S6 = 0x04; +#undef S7 + constexpr uint8_t S7 = 0x44; + + // NOLINTBEGIN + // clang-format off +#undef LEADING + constexpr uint8_t LEADING[256] = { + // 1 2 3 4 5 6 7 8 9 A B C D E F + AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x00-0x0F + AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x10-0x1F + AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x20-0x2F + AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x30-0x3F + AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x40-0x4F + AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x50-0x5F + AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x60-0x6F + AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x70-0x7F + // 1 2 3 4 5 6 7 8 9 A B C D E F + XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0x80-0x8F + XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0x90-0x9F + XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0xA0-0xAF + XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0xB0-0xBF + XX, XX, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, // 0xC0-0xCF + S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, // 0xD0-0xDF + S2, S3, S3, S3, S3, S3, S3, S3, S3, S3, S3, S3, S3, S4, S3, S3, // 0xE0-0xEF + S5, S6, S6, S6, S7, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0xF0-0xFF + }; + // clang-format on + // NOLINTEND + +#undef ACCEPT + constexpr std::pair ACCEPT[16] = { + {LOW, HIGH}, {0xa0, HIGH}, {LOW, 0x9f}, {0x90, HIGH}, + {LOW, 0x8f}, {0x0, 0x0}, {0x0, 0x0}, {0x0, 0x0}, + {0x0, 0x0}, {0x0, 0x0}, {0x0, 0x0}, {0x0, 0x0}, + {0x0, 0x0}, {0x0, 0x0}, {0x0, 0x0}, {0x0, 0x0}, + }; + +} // namespace + + std::pair Utf8::decode(std::string_view input) { + assert(!input.empty()); + const auto b = static_cast(input.front()); + input.remove_prefix(1); + if (b < SELF) { + return {static_cast(b), 1}; + } + const auto leading = LEADING[b]; + if (leading == XX) { + return {Unicode::REPLACEMENT_CHARACTER, 1}; + } + auto size = static_cast(leading & 7) - 1; + if (size > input.size()) { + return {Unicode::REPLACEMENT_CHARACTER, 1}; + } + const auto& accept = ACCEPT[leading >> 4]; + const auto b1 = static_cast(input.front()); + input.remove_prefix(1); + if (b1 < accept.first || b1 > accept.second) { + return {Unicode::REPLACEMENT_CHARACTER, 1}; + } + if (size <= 1) { + return {(static_cast(b & MASK2) << 6) | + static_cast(b1 & MASKX), + 2}; + } + const auto b2 = static_cast(input.front()); + input.remove_prefix(1); + if (b2 < LOW || b2 > HIGH) { + return {Unicode::REPLACEMENT_CHARACTER, 1}; + } + if (size <= 2) { + return {(static_cast(b & MASK3) << 12) | + (static_cast(b1 & MASKX) << 6) | + static_cast(b2 & MASKX), + 3}; + } + const auto b3 = static_cast(input.front()); + input.remove_prefix(1); + if (b3 < LOW || b3 > HIGH) { + return {Unicode::REPLACEMENT_CHARACTER, 1}; + } + return {(static_cast(b & MASK4) << 18) | + (static_cast(b1 & MASKX) << 12) | + (static_cast(b2 & MASKX) << 6) | + static_cast(b3 & MASKX), + 4}; + } + + std::optional Utf8::strictDecode(std::string_view input) { + std::u32string output; + char32_t codePoint; + size_t codeUnits; + output.reserve(input.size()); // Worst case is each byte is a single Unicode code point. + for (size_t index = 0; index < input.size(); index += codeUnits) { + std::tie(codePoint, codeUnits) = Utf8::decode(input.substr(index)); + if (codePoint == Unicode::REPLACEMENT_CHARACTER && codeUnits == 1) { + // Condition is only met when an illegal byte sequence is encountered. See Utf8::decode. + return std::nullopt; + } + output.push_back(codePoint); + } + output.shrink_to_fit(); + return output; + } + + std::u32string Utf8::lenientDecode(std::string_view input) { + std::u32string output; + char32_t codePoint; + size_t codeUnits; + output.reserve(input.size()); // Worst case is each byte is a single Unicode code point. + for (size_t index = 0; index < input.size(); index += codeUnits) { + std::tie(codePoint, codeUnits) = Utf8::decode(input.substr(index)); + output.push_back(codePoint); + } + output.shrink_to_fit(); + return output; + } + + std::string& Utf8::encode(std::string* buffer, char32_t codePoint) { + assert(buffer != nullptr); + if (!Unicode::isValid(codePoint)) { + codePoint = Unicode::REPLACEMENT_CHARACTER; + } + if (codePoint <= 0x7f) { + buffer->push_back(static_cast(static_cast(codePoint))); + } else if (codePoint <= 0x7ff) { + buffer->push_back( + static_cast(T2 | static_cast(codePoint >> 6))); + buffer->push_back( + static_cast(TX | (static_cast(codePoint) & MASKX))); + } else if (codePoint <= 0xffff) { + buffer->push_back( + static_cast(T3 | static_cast(codePoint >> 12))); + buffer->push_back(static_cast( + TX | (static_cast(codePoint >> 6) & MASKX))); + buffer->push_back( + static_cast(TX | (static_cast(codePoint) & MASKX))); + } else { + buffer->push_back( + static_cast(T4 | static_cast(codePoint >> 18))); + buffer->push_back(static_cast( + TX | (static_cast(codePoint >> 12) & MASKX))); + buffer->push_back(static_cast( + TX | (static_cast(codePoint >> 6) & MASKX))); + buffer->push_back( + static_cast(TX | (static_cast(codePoint) & MASKX))); + } + return *buffer; + } + + std::optional Utf8::strictEncode(std::u32string_view input) { + std::string output; + output.reserve(input.size() * 4); // Worst case is each Unicode code point encodes to 4 bytes. + for (size_t index = 0; index < input.size(); index++) { + char32_t codePoint = input[index]; + if (!Unicode::isValid(codePoint)) { + return std::nullopt; + } + Utf8::encode(&output, codePoint); + } + output.shrink_to_fit(); + return output; + } + + std::string Utf8::lenientEncode(std::u32string_view input) { + std::string output; + output.reserve(input.size() * 4); // Worst case is each Unicode code point encodes to 4 bytes. + for (size_t index = 0; index < input.size(); index++) { + char32_t codePoint = input[index]; + if (!Unicode::isValid(codePoint)) { + codePoint = Unicode::REPLACEMENT_CHARACTER; + } + Utf8::encode(&output, codePoint); + } + output.shrink_to_fit(); + return output; + } + +} diff --git a/runtime/Cpp/runtime/src/support/Utf8.h b/runtime/Cpp/runtime/src/support/Utf8.h new file mode 100644 index 0000000000..67de236612 --- /dev/null +++ b/runtime/Cpp/runtime/src/support/Utf8.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2021 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +#pragma once + +#include +#include +#include +#include + +namespace antlrcpp { + + class Utf8 final { + public: + // Decodes the next code point, returning the decoded code point and the number + // of code units (a.k.a. bytes) consumed. In the event that an invalid code unit + // sequence is returned the replacement character, U+FFFD, is returned with a + // code unit count of 1. As U+FFFD requires 3 code units when encoded, this can + // be used to differentiate valid input from malformed input. + static std::pair decode(std::string_view input); + + // Decodes the given UTF-8 encoded input into a string of code points. + static std::optional strictDecode(std::string_view input); + + // Decodes the given UTF-8 encoded input into a string of code points. Unlike strictDecode(), + // each byte in an illegal byte sequence is replaced with the Unicode replacement character, + // U+FFFD. + static std::u32string lenientDecode(std::string_view input); + + // Encodes the given code point and appends it to the buffer. If the code point + // is an unpaired surrogate or outside of the valid Unicode range it is replaced + // with the replacement character, U+FFFD. + static std::string& encode(std::string *buffer, char32_t codePoint); + + // Encodes the given Unicode code point string as UTF-8. + static std::optional strictEncode(std::u32string_view input); + + // Encodes the given Unicode code point string as UTF-8. Unlike strictEncode(), + // each invalid Unicode code point is replaced with the Unicode replacement character, U+FFFD. + static std::string lenientEncode(std::u32string_view input); + + private: + Utf8() = delete; + + Utf8(const Utf8&) = delete; + + Utf8& operator=(const Utf8&) = delete; + }; + +} diff --git a/runtime/Cpp/runtime/tests/Utf8Test.cpp b/runtime/Cpp/runtime/tests/Utf8Test.cpp new file mode 100644 index 0000000000..e7e415a53f --- /dev/null +++ b/runtime/Cpp/runtime/tests/Utf8Test.cpp @@ -0,0 +1,110 @@ +#include +#include + +#include "gtest/gtest.h" +#include "support/Utf8.h" + +namespace antlrcpp { +namespace { + + struct Utf8EncodeTestCase final { + char32_t code_point; + std::string_view code_units; + }; + + using Utf8EncodeTest = testing::TestWithParam; + + TEST_P(Utf8EncodeTest, Compliance) { + const Utf8EncodeTestCase& test_case = GetParam(); + std::string result; + EXPECT_EQ(Utf8::encode(&result, test_case.code_point), test_case.code_units); + } + + INSTANTIATE_TEST_SUITE_P(Utf8EncodeTest, Utf8EncodeTest, + testing::ValuesIn({ + {0x0000, std::string_view("\x00", 1)}, + {0x0001, "\x01"}, + {0x007e, "\x7e"}, + {0x007f, "\x7f"}, + {0x0080, "\xc2\x80"}, + {0x0081, "\xc2\x81"}, + {0x00bf, "\xc2\xbf"}, + {0x00c0, "\xc3\x80"}, + {0x00c1, "\xc3\x81"}, + {0x00c8, "\xc3\x88"}, + {0x00d0, "\xc3\x90"}, + {0x00e0, "\xc3\xa0"}, + {0x00f0, "\xc3\xb0"}, + {0x00f8, "\xc3\xb8"}, + {0x00ff, "\xc3\xbf"}, + {0x0100, "\xc4\x80"}, + {0x07ff, "\xdf\xbf"}, + {0x0400, "\xd0\x80"}, + {0x0800, "\xe0\xa0\x80"}, + {0x0801, "\xe0\xa0\x81"}, + {0x1000, "\xe1\x80\x80"}, + {0xd000, "\xed\x80\x80"}, + {0xd7ff, "\xed\x9f\xbf"}, + {0xe000, "\xee\x80\x80"}, + {0xfffe, "\xef\xbf\xbe"}, + {0xffff, "\xef\xbf\xbf"}, + {0x10000, "\xf0\x90\x80\x80"}, + {0x10001, "\xf0\x90\x80\x81"}, + {0x40000, "\xf1\x80\x80\x80"}, + {0x10fffe, "\xf4\x8f\xbf\xbe"}, + {0x10ffff, "\xf4\x8f\xbf\xbf"}, + {0xFFFD, "\xef\xbf\xbd"}, + })); + + struct Utf8DecodeTestCase final { + char32_t code_point; + std::string_view code_units; + }; + + using Utf8DecodeTest = testing::TestWithParam; + + TEST_P(Utf8DecodeTest, Compliance) { + const Utf8DecodeTestCase& test_case = GetParam(); + auto [code_point, code_units] = Utf8::decode(test_case.code_units); + EXPECT_EQ(code_units, test_case.code_units.size()); + EXPECT_EQ(code_point, test_case.code_point); + } + + INSTANTIATE_TEST_SUITE_P(Utf8DecodeTest, Utf8DecodeTest, + testing::ValuesIn({ + {0x0000, std::string_view("\x00", 1)}, + {0x0001, "\x01"}, + {0x007e, "\x7e"}, + {0x007f, "\x7f"}, + {0x0080, "\xc2\x80"}, + {0x0081, "\xc2\x81"}, + {0x00bf, "\xc2\xbf"}, + {0x00c0, "\xc3\x80"}, + {0x00c1, "\xc3\x81"}, + {0x00c8, "\xc3\x88"}, + {0x00d0, "\xc3\x90"}, + {0x00e0, "\xc3\xa0"}, + {0x00f0, "\xc3\xb0"}, + {0x00f8, "\xc3\xb8"}, + {0x00ff, "\xc3\xbf"}, + {0x0100, "\xc4\x80"}, + {0x07ff, "\xdf\xbf"}, + {0x0400, "\xd0\x80"}, + {0x0800, "\xe0\xa0\x80"}, + {0x0801, "\xe0\xa0\x81"}, + {0x1000, "\xe1\x80\x80"}, + {0xd000, "\xed\x80\x80"}, + {0xd7ff, "\xed\x9f\xbf"}, + {0xe000, "\xee\x80\x80"}, + {0xfffe, "\xef\xbf\xbe"}, + {0xffff, "\xef\xbf\xbf"}, + {0x10000, "\xf0\x90\x80\x80"}, + {0x10001, "\xf0\x90\x80\x81"}, + {0x40000, "\xf1\x80\x80\x80"}, + {0x10fffe, "\xf4\x8f\xbf\xbe"}, + {0x10ffff, "\xf4\x8f\xbf\xbf"}, + {0xFFFD, "\xef\xbf\xbd"}, + })); + +} +}