From bbe2a8b289d1c8a1dce2a3702a9c4b81670fb96d Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Thu, 1 Mar 2018 18:58:23 +0100 Subject: [PATCH 01/23] JSON parsing fix from steem PR 2178 --- src/io/json.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/io/json.cpp b/src/io/json.cpp index a53b3a572..288f40d76 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -390,8 +390,9 @@ namespace fc { skip_white_space(in); variant var; - while( signed char c = in.peek() ) + while( true ) { + signed char c = in.peek(); switch( c ) { case ' ': From b12205caf6d01a5062e6d73e4e8eac966c8ce454 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Fri, 2 Mar 2018 21:13:03 +0100 Subject: [PATCH 02/23] Minor optimization --- include/fc/smart_ref_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/fc/smart_ref_impl.hpp b/include/fc/smart_ref_impl.hpp index 7c31ceaa3..fd696941b 100644 --- a/include/fc/smart_ref_impl.hpp +++ b/include/fc/smart_ref_impl.hpp @@ -89,7 +89,7 @@ namespace fc { template T& smart_ref::operator = ( smart_ref&& u ) { if( &u == this ) return *impl; - if( impl ) delete impl; + delete impl; impl = u.impl; u.impl = nullptr; return *impl; From 0c757094884c0835e09c1e89cae950c644222925 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sat, 3 Mar 2018 11:00:21 +0100 Subject: [PATCH 03/23] Added unit tests for stringstream, [io]fstream and buffered_[io]stream --- tests/CMakeLists.txt | 1 + tests/io/stream_tests.cpp | 178 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 tests/io/stream_tests.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bf7a01d6a..48f7a05b8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -49,6 +49,7 @@ add_executable( all_tests all_tests.cpp crypto/dh_test.cpp crypto/rand_test.cpp crypto/sha_tests.cpp + io/stream_tests.cpp network/http/websocket_test.cpp thread/task_cancel.cpp thread/thread_tests.cpp diff --git a/tests/io/stream_tests.cpp b/tests/io/stream_tests.cpp new file mode 100644 index 000000000..038c4198d --- /dev/null +++ b/tests/io/stream_tests.cpp @@ -0,0 +1,178 @@ +#include + +#include +#include +#include +#include +#include + +#include + +BOOST_AUTO_TEST_SUITE(stream_tests) + +BOOST_AUTO_TEST_CASE(stringstream_test) +{ + const fc::string constant( "Hello", 6 ); // includes trailing \0 + fc::string writable( "World" ); + fc::stringstream in1( constant ); + fc::stringstream in2( writable ); + fc::stringstream out; + + std::shared_ptr buf( new char[15], [](char* p){ delete[] p; } ); + *buf = 'w'; + in2.writesome( buf, 1, 0 ); + + BOOST_CHECK_EQUAL( 3, in1.readsome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 3, out.writesome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 'l', in1.peek() ); + BOOST_CHECK_EQUAL( 3, in1.readsome( buf, 4, 0 ) ); + BOOST_CHECK_EQUAL( '\0', (&(*buf))[2] ); + BOOST_CHECK_EQUAL( 2, out.writesome( buf, 2, 0 ) ); + *buf = ' '; + out.writesome( buf, 1, 0 ); + BOOST_CHECK_THROW( in1.readsome( buf, 3, 0 ), fc::eof_exception ); + BOOST_CHECK_EQUAL( 5, in2.readsome( buf, 6, 0 ) ); + BOOST_CHECK_EQUAL( 5, out.writesome( buf, 5, 0 ) ); + BOOST_CHECK_THROW( in2.readsome( buf, 3, 0 ), fc::eof_exception ); + + BOOST_CHECK_EQUAL( "Hello world", out.str() ); + BOOST_CHECK_THROW( in1.peek(), fc::eof_exception ); + BOOST_CHECK( in1.eof() ); + BOOST_CHECK_THROW( in2.readsome( buf, 3, 0 ), fc::eof_exception ); + // BOOST_CHECK( in2.eof() ); // fails, apparently readsome doesn't set eof +} + +BOOST_AUTO_TEST_CASE(buffered_stringstream_test) +{ + const fc::string constant( "Hello", 6 ); // includes trailing \0 + fc::string writable( "World" ); + fc::istream_ptr in1( new fc::stringstream( constant ) ); + std::shared_ptr in2( new fc::stringstream( writable ) ); + std::shared_ptr out1( new fc::stringstream() ); + fc::buffered_istream bin1( in1 ); + fc::buffered_istream bin2( in2 ); + fc::buffered_ostream bout( out1 ); + + std::shared_ptr buf( new char[15], [](char* p){ delete[] p; } ); + *buf = 'w'; + in2->writesome( buf, 1, 0 ); + + BOOST_CHECK_EQUAL( 3, bin1.readsome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 3, bout.writesome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 'l', bin1.peek() ); + BOOST_CHECK_EQUAL( 3, bin1.readsome( buf, 4, 0 ) ); + BOOST_CHECK_EQUAL( '\0', (&(*buf))[2] ); + BOOST_CHECK_EQUAL( 2, bout.writesome( buf, 2, 0 ) ); + *buf = ' '; + bout.writesome( buf, 1, 0 ); + BOOST_CHECK_THROW( bin1.readsome( buf, 3, 0 ), fc::eof_exception ); + BOOST_CHECK_EQUAL( 5, bin2.readsome( buf, 6, 0 ) ); + BOOST_CHECK_EQUAL( 5, bout.writesome( buf, 5, 0 ) ); + BOOST_CHECK_THROW( bin2.readsome( buf, 3, 0 ), fc::eof_exception ); + + bout.flush(); + + BOOST_CHECK_EQUAL( "Hello world", out1->str() ); +} + +BOOST_AUTO_TEST_CASE(fstream_test) +{ + fc::temp_file inf1( fc::temp_directory_path(), true ); + fc::temp_file inf2( fc::temp_directory_path(), true ); + fc::temp_file outf( fc::temp_directory_path(), true ); + + { + std::fstream init( inf1.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.write( "Hello", 6 ); // includes trailing \0 + init.close(); + + init.open( inf2.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.write( "world", 5 ); + init.close(); + + init.open( outf.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.close(); + } + + fc::ifstream in1( inf1.path() ); + fc::ifstream in2( inf2.path() ); + fc::ofstream out( outf.path() ); + + std::shared_ptr buf( new char[15], [](char* p){ delete[] p; } ); + BOOST_CHECK_EQUAL( 3, in1.readsome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 3, out.writesome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 3, in1.readsome( buf, 4, 0 ) ); + BOOST_CHECK_EQUAL( '\0', (&(*buf))[2] ); + BOOST_CHECK_EQUAL( 2, out.writesome( buf, 2, 0 ) ); + *buf = ' '; + out.writesome( buf, 1, 0 ); + BOOST_CHECK_THROW( in1.readsome( buf, 3, 0 ), fc::eof_exception ); + BOOST_CHECK_EQUAL( 5, in2.readsome( buf, 6, 0 ) ); + BOOST_CHECK_EQUAL( 5, out.writesome( buf, 5, 0 ) ); + BOOST_CHECK_THROW( in2.readsome( buf, 3, 0 ), fc::eof_exception ); + + { + out.flush(); + std::fstream test( outf.path().to_native_ansi_path(), std::fstream::in ); + BOOST_CHECK_EQUAL( 11, test.readsome( (&(*buf)), 11 ) ); + BOOST_CHECK_EQUAL( "Hello world", std::string( (&(*buf)), 11 ) ); + BOOST_CHECK_EQUAL( 0, test.readsome( (&(*buf)), 11 ) ); + test.close(); + } + + BOOST_CHECK( in1.eof() ); + BOOST_CHECK( in2.eof() ); +} + +BOOST_AUTO_TEST_CASE(buffered_fstream_test) +{ + fc::temp_file inf1( fc::temp_directory_path(), true ); + fc::temp_file inf2( fc::temp_directory_path(), true ); + fc::temp_file outf( fc::temp_directory_path(), true ); + + { + std::fstream init( inf1.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.write( "Hello", 6 ); // includes trailing \0 + init.close(); + + init.open( inf2.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.write( "world", 5 ); + init.close(); + + init.open( outf.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.close(); + } + + fc::istream_ptr in1( new fc::ifstream( inf1.path() ) ); + fc::istream_ptr in2( new fc::ifstream( inf2.path() ) ); + fc::ostream_ptr out( new fc::ofstream( outf.path() ) ); + fc::buffered_istream bin1( in1 ); + fc::buffered_istream bin2( in2 ); + fc::buffered_ostream bout( out ); + + std::shared_ptr buf( new char[15], [](char* p){ delete[] p; } ); + + BOOST_CHECK_EQUAL( 3, bin1.readsome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 3, bout.writesome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 'l', bin1.peek() ); + BOOST_CHECK_EQUAL( 3, bin1.readsome( buf, 4, 0 ) ); + BOOST_CHECK_EQUAL( '\0', (&(*buf))[2] ); + BOOST_CHECK_EQUAL( 2, bout.writesome( buf, 2, 0 ) ); + *buf = ' '; + bout.writesome( buf, 1, 0 ); + BOOST_CHECK_THROW( bin1.readsome( buf, 3, 0 ), fc::eof_exception ); + BOOST_CHECK_EQUAL( 5, bin2.readsome( buf, 6, 0 ) ); + BOOST_CHECK_EQUAL( 5, bout.writesome( buf, 5, 0 ) ); + BOOST_CHECK_THROW( bin2.readsome( buf, 3, 0 ), fc::eof_exception ); + + { + bout.flush(); + std::fstream test( outf.path().to_native_ansi_path(), std::fstream::in ); + BOOST_CHECK_EQUAL( 11, test.readsome( (&(*buf)), 11 ) ); + BOOST_CHECK_EQUAL( "Hello world", std::string( (&(*buf)), 11 ) ); + BOOST_CHECK_EQUAL( 0, test.readsome( (&(*buf)), 11 ) ); + test.close(); + } +} + +BOOST_AUTO_TEST_SUITE_END() From 72e96e3c1d53b4e8d54b1aa0d7b2146d18fa622d Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sat, 3 Mar 2018 18:40:02 +0100 Subject: [PATCH 04/23] Added first json test case --- tests/CMakeLists.txt | 1 + tests/io/json_tests.cpp | 117 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 tests/io/json_tests.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 48f7a05b8..fe1b412ee 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -49,6 +49,7 @@ add_executable( all_tests all_tests.cpp crypto/dh_test.cpp crypto/rand_test.cpp crypto/sha_tests.cpp + io/json_tests.cpp io/stream_tests.cpp network/http/websocket_test.cpp thread/task_cancel.cpp diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp new file mode 100644 index 000000000..8db3e08fc --- /dev/null +++ b/tests/io/json_tests.cpp @@ -0,0 +1,117 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include + +BOOST_AUTO_TEST_SUITE(json_tests) + +static void replace_some( std::string& str ) +{ + for( size_t i = 0; i < str.length(); i++ ) + if( str[i] == '\1' ) str[i] = '\0'; + else if( str[i] == '\'' ) str[i] = '"'; +} + +static void test_fail_string( const std::string& str ) +{ + try { + fc::json::from_string( str ); + BOOST_FAIL( "json::from_string('" + str + "') failed" ); + } catch( const fc::parse_error_exception& ) { // ignore, ok + } catch( const fc::eof_exception& ) { // ignore, ok + } FC_CAPTURE_LOG_AND_RETHROW( ("json::from_string failed")(str) ) +} + +static void test_fail_stream( const std::string& str ) +{ + fc::temp_file file( fc::temp_directory_path(), true ); + { + std::fstream init( file.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.write( str.c_str(), str.length() ); + init.close(); + } + try { + fc::istream_ptr in( new fc::ifstream( file.path() ) ); + fc::buffered_istream bin( in ); + fc::json::from_stream( bin ); + BOOST_FAIL( "json::from_stream('" + str + "') failed using ifstream" ); + } catch( const fc::parse_error_exception& ) { // ignore, ok + } catch( const fc::eof_exception& ) { // ignore, ok + } FC_CAPTURE_LOG_AND_RETHROW( ("json::from_stream failed using ifstream")(str) ) + try { + fc::istream_ptr in( new fc::stringstream( str ) ); + fc::buffered_istream bin( in ); + fc::json::from_stream( bin ); + BOOST_FAIL( "json::from_stream('" + str + "') failed using stringstream" ); + } catch( const fc::parse_error_exception& ) { // ignore, ok + } catch( const fc::eof_exception& ) { // ignore, ok + } FC_CAPTURE_LOG_AND_RETHROW( ("json::from_stream failed using stringstream")(str) ) +} + +static void test_fail_file( const std::string& str ) +{ + fc::temp_file file( fc::temp_directory_path(), true ); + { + std::fstream init( file.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.write( str.c_str(), str.length() ); + init.close(); + } + try { + fc::json::from_file( file.path() ); + BOOST_FAIL( "json::from_file('" + str + "') failed using" ); + } catch( const fc::parse_error_exception& ) { // ignore, ok + } catch( const fc::eof_exception& ) { // ignore, ok + } FC_CAPTURE_LOG_AND_RETHROW( ("json::from_file failed")(str) ) +} + +BOOST_AUTO_TEST_CASE(imbalanced_test) +{ + std::vector tests + { // for easier handling and better readability, in the following test + // strings ' is used instead of " and \1 instead of \0 + "{", + "{'", + "{'}", + "{'a'", + "{'a':", + "{'a':5", + "[", + "['", + "[']", + "[ 13", + "' end", + "{ 13: }", + "{\1", + "{\1}", + "{'\1", + "{'\1}", + "{'a'\1", + "{'a'\1}", + "{'a': \1", + "{'a': \1}", + "[\1", + "[\1]", + "['\1", + "['\1]", + "[ 13\1", + "[ 13\1]", + "' end\1" + }; + + for( std::string test : tests ) + { + replace_some( test ); + test_fail_string( test ); + test_fail_stream( test ); + test_fail_file( test ); + } +} + + +BOOST_AUTO_TEST_SUITE_END() From 52f68107350e047ee040d6abf77d319e25d7743f Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sat, 3 Mar 2018 21:59:28 +0100 Subject: [PATCH 05/23] Fixed from_file --- src/io/json.cpp | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/src/io/json.cpp b/src/io/json.cpp index 288f40d76..2c453cfad 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -798,23 +798,9 @@ namespace fc } variant json::from_file( const fc::path& p, parse_type ptype ) { - //auto tmp = std::make_shared( p, ifstream::binary ); - //auto tmp = std::make_shared( p.generic_string().c_str(), std::ios::binary ); - //buffered_istream bi( tmp ); - boost::filesystem::ifstream bi( p, std::ios::binary ); - switch( ptype ) - { - case legacy_parser: - return variant_from_stream( bi ); - case legacy_parser_with_string_doubles: - return variant_from_stream( bi ); - case strict_parser: - return json_relaxed::variant_from_stream( bi ); - case relaxed_parser: - return json_relaxed::variant_from_stream( bi ); - default: - FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); - } + fc::istream_ptr in( new fc::ifstream( p ) ); + fc::buffered_istream bin( in ); + return from_stream( bin, ptype ); } variant json::from_stream( buffered_istream& in, parse_type ptype ) { From 4d782e894f9866e87ca82448526881666ebd3082 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sat, 3 Mar 2018 22:03:23 +0100 Subject: [PATCH 06/23] Deduplicate some code --- src/io/json.cpp | 41 +++++++---------------------------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/src/io/json.cpp b/src/io/json.cpp index 2c453cfad..ca1c7fb95 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -461,21 +461,9 @@ namespace fc { try { check_string_depth( utf8_str ); - fc::stringstream in( utf8_str ); - //in.exceptions( std::ifstream::eofbit ); - switch( ptype ) - { - case legacy_parser: - return variant_from_stream( in ); - case legacy_parser_with_string_doubles: - return variant_from_stream( in ); - case strict_parser: - return json_relaxed::variant_from_stream( in ); - case relaxed_parser: - return json_relaxed::variant_from_stream( in ); - default: - FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); - } + fc::istream_ptr in( new fc::stringstream( utf8_str ) ); + fc::buffered_istream bin( in ); + return from_stream( bin, ptype ); } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) } variants json::variants_from_string( const std::string& utf8_str, parse_type ptype ) @@ -838,25 +826,10 @@ namespace fc bool json::is_valid( const std::string& utf8_str, parse_type ptype ) { if( utf8_str.size() == 0 ) return false; - fc::stringstream in( utf8_str ); - switch( ptype ) - { - case legacy_parser: - variant_from_stream( in ); - break; - case legacy_parser_with_string_doubles: - variant_from_stream( in ); - break; - case strict_parser: - json_relaxed::variant_from_stream( in ); - break; - case relaxed_parser: - json_relaxed::variant_from_stream( in ); - break; - default: - FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); - } - try { in.peek(); } catch ( const eof_exception& e ) { return true; } + fc::istream_ptr in( new fc::stringstream( utf8_str ) ); + fc::buffered_istream bin( in ); + from_stream( bin, ptype ); + try { bin.peek(); } catch ( const eof_exception& e ) { return true; } return false; } From 8c09ff09fc4a4ee47ee9f4e530064d1d460f2094 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Mon, 5 Mar 2018 21:09:39 +0100 Subject: [PATCH 07/23] Added json write/validate/read test --- tests/io/json_tests.cpp | 173 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp index 8db3e08fc..35ff11097 100644 --- a/tests/io/json_tests.cpp +++ b/tests/io/json_tests.cpp @@ -1,5 +1,8 @@ #include +#include +#include + #include #include #include @@ -75,6 +78,7 @@ BOOST_AUTO_TEST_CASE(imbalanced_test) std::vector tests { // for easier handling and better readability, in the following test // strings ' is used instead of " and \1 instead of \0 + "", "{", "{'", "{'}", @@ -87,6 +91,7 @@ BOOST_AUTO_TEST_CASE(imbalanced_test) "[ 13", "' end", "{ 13: }", + "\1", "{\1", "{\1}", "{'\1", @@ -113,5 +118,173 @@ BOOST_AUTO_TEST_CASE(imbalanced_test) } } +static bool equal( const fc::variant& a, const fc::variant& b ) +{ + auto a_type = a.get_type(); + auto b_type = b.get_type(); + if( a_type == fc::variant::type_id::int64_type && a.as() > 0 ) + a_type = fc::variant::type_id::uint64_type; + if( b_type == fc::variant::type_id::int64_type && b.as() > 0 ) + b_type = fc::variant::type_id::uint64_type; + if( a_type != b_type ) return false; + switch( a_type ) + { + case fc::variant::type_id::null_type: return true; + case fc::variant::type_id::int64_type: return a.as() == b.as(); + case fc::variant::type_id::uint64_type: return a.as() == b.as(); + case fc::variant::type_id::double_type: return a.as() == b.as(); + case fc::variant::type_id::bool_type: return a.as() == b.as(); + case fc::variant::type_id::string_type: return a.as() == b.as(); + case fc::variant::type_id::array_type: + if( a.get_array().size() != b.get_array().size() ) return false; + else + { + std::vector::const_iterator b_it = b.get_array().begin(); + for( const auto& a_it : a.get_array() ) + { + if( !equal( a_it, *b_it ) ) return false; + b_it++; + } + } + return true; + case fc::variant::type_id::object_type: + if( a.get_object().size() != b.get_object().size() ) return false; + for( const auto& a_it : a.get_object() ) + { + const auto& b_obj = b.get_object().find( a_it.key() ); + if( b_obj == b.get_object().end() || !equal( a_it.value(), b_obj->value() ) ) return false; + } + return true; + case fc::variant::type_id::blob_type: + default: + FC_THROW_EXCEPTION( fc::invalid_arg_exception, "Unsupported variant type: " + a.get_type() ); + } +} + +static void test_recursive( const fc::variant& v ) +{ try { + const std::string json = fc::json::to_string( v, fc::json::output_formatting::legacy_generator ); + BOOST_CHECK( fc::json::is_valid( json ) ); + BOOST_CHECK( !fc::json::is_valid( json + " " ) ); + + const std::string pretty = fc::json::to_pretty_string( v, fc::json::output_formatting::legacy_generator ); + BOOST_CHECK( fc::json::is_valid( pretty ) ); + BOOST_CHECK( !fc::json::is_valid( pretty + " " ) ); + + fc::temp_file file( fc::temp_directory_path(), true ); + fc::json::save_to_file( v, file.path(), false, fc::json::output_formatting::legacy_generator ); + + BOOST_CHECK( equal( v, fc::json::from_string( json + " " ) ) ); + BOOST_CHECK( equal( v, fc::json::from_string( pretty + " " ) ) ); + BOOST_CHECK( equal( v, fc::json::from_file( file.path() ) ) ); + + if( v.get_type() == fc::variant::type_id::array_type ) + for( const auto& item : v.get_array() ) + test_recursive( item ); + else if( v.get_type() == fc::variant::type_id::object_type ) + for( const auto& item : v.get_object() ) + test_recursive( item.value() ); +} FC_CAPTURE_LOG_AND_RETHROW( (v) ) } + +BOOST_AUTO_TEST_CASE(structured_test) +{ + fc::variant_object v_empty_obj; + fc::variants v_empty_array; + fc::variant v_null; + fc::variant v_true( true ); + fc::variant v_false( false ); + fc::variant v_empty_str( "" ); + fc::variant v_str( "false" ); + fc::variant v_int8_1( (int8_t) 1 ); + fc::variant v_int8_2( (int8_t) -2 ); + fc::variant v_uint8_1( (int8_t) 1 ); + fc::variant v_int16_1( (int16_t) 1 ); + fc::variant v_int16_2( (int16_t) -2 ); + fc::variant v_uint16_1( (int16_t) 1 ); + fc::variant v_int32_1( (int32_t) 1 ); + fc::variant v_int32_2( (int32_t) -2 ); + fc::variant v_uint32_1( (int32_t) 1 ); + fc::variant v_int64_1( (int8_t) 1 ); + fc::variant v_int64_2( (int8_t) -2 ); + fc::variant v_uint64_1( (int8_t) 1 ); + fc::variant v_float_1( 0.0f ); + fc::variant v_float_2( -2.0f ); + fc::variant v_double_1( 0.0d ); + fc::variant v_double_2( -2.0d ); + fc::variants v_small_array + { + v_empty_obj, + v_empty_array, + v_null, + v_true, + v_false, + v_empty_str + }; + fc::mutable_variant_object v_small_obj; + v_small_obj( "", v_empty_str ) + ( "1", v_empty_array ) + ( "2", v_null ) + ( "a", v_true ) + ( "b", v_false ) + ( "x", v_small_array ) + ( "y", v_empty_obj ); + fc::variants v_big_array + { + v_empty_obj, + v_empty_array, + v_null, + v_true, + v_false, + v_empty_str, + v_str, + v_int8_1, + v_int8_2, + v_uint8_1, + v_int16_1, + v_int16_2, + v_uint16_1, + v_int32_1, + v_int32_2, + v_uint32_1, + v_int64_1, + v_int64_2, + v_uint64_1, + v_float_1, + v_float_2, + v_double_1, + v_double_2, + v_small_array, + v_small_obj + }; + fc::mutable_variant_object v_big_obj; + v_big_obj( "v_empty_obj", v_empty_obj ) + ( "v_empty_array", v_empty_array ) + ( "v_null", v_null ) + ( "v_true", v_true ) + ( "v_false", v_false ) + ( "v_empty_str", v_empty_str ) + ( "v_str", v_str ) + ( "v_int8_1", v_int8_1 ) + ( "v_int8_2", v_int8_2 ) + ( "v_uint8_1", v_uint8_1 ) + ( "v_int16_1", v_int16_1 ) + ( "v_int16_2", v_int16_2 ) + ( "v_uint16_1", v_uint16_1 ) + ( "v_int32_1", v_int32_1 ) + ( "v_int32_2", v_int32_2 ) + ( "v_uint32_1", v_uint32_1 ) + ( "v_int64_1", v_int64_1 ) + ( "v_int64_2", v_int64_2 ) + ( "v_uint64_1", v_uint64_1 ) + ( "v_float_1", v_float_1 ) + ( "v_float_2", v_float_2 ) + ( "v_double_1", v_double_1 ) + ( "v_double_2", v_double_2 ) + ( "v_small_array", v_small_array ) + ( "v_small_obj", v_small_obj ); + v_big_array.push_back( v_big_obj ); + + test_recursive( v_big_array ); +} BOOST_AUTO_TEST_SUITE_END() From e37d9a50514122853791c43befd50286fbfd1a4a Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Mon, 5 Mar 2018 21:57:11 +0100 Subject: [PATCH 08/23] Code cleanups and simplifications --- src/io/json.cpp | 151 +++++++++++++++++------------------------------- 1 file changed, 54 insertions(+), 97 deletions(-) diff --git a/src/io/json.cpp b/src/io/json.cpp index ca1c7fb95..99feab97f 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -5,7 +5,6 @@ #include #include #include -//#include #include #include #include @@ -179,7 +178,6 @@ namespace fc "Expected '{', but read '${char}'", ("char",string(&c, &c + 1)) ); in.get(); - skip_white_space(in); while( in.peek() != '}' ) { if( in.peek() == ',' ) @@ -199,7 +197,6 @@ namespace fc auto val = variant_from_stream( in ); obj(std::move(key),std::move(val)); - skip_white_space(in); } if( in.peek() == '}' ) { @@ -227,7 +224,6 @@ namespace fc if( in.peek() != '[' ) FC_THROW_EXCEPTION( parse_error_exception, "Expected '['" ); in.get(); - skip_white_space(in); while( in.peek() != ']' ) { @@ -238,7 +234,6 @@ namespace fc } if( skip_white_space(in) ) continue; ar.push_back( variant_from_stream(in) ); - skip_white_space(in); } if( in.peek() != ']' ) FC_THROW_EXCEPTION( parse_error_exception, "Expected ']' after parsing ${variant}", @@ -276,6 +271,7 @@ namespace fc if (dot) FC_THROW_EXCEPTION(parse_error_exception, "Can't parse a number with two decimal places"); dot = true; + [[fallthrough]]; case '0': case '1': case '2': @@ -299,10 +295,10 @@ namespace fc } } catch (fc::eof_exception&) - { + { // EOF ends the loop } catch (const std::ios_base::failure&) - { + { // read error ends the loop } fc::string str = ss.str(); if (str == "-." || str == ".") // check the obviously wrong things we could have encountered @@ -379,7 +375,7 @@ namespace fc // make out ("falfe") // A strict JSON parser would signal this as an error, but we // will just treat the malformed token as an un-quoted string. - return str + stringFromToken(in);; + return str + stringFromToken(in); } } } @@ -389,53 +385,42 @@ namespace fc variant variant_from_stream( T& in ) { skip_white_space(in); - variant var; - while( true ) + signed char c = in.peek(); + switch( c ) { - signed char c = in.peek(); - switch( c ) - { - case ' ': - case '\t': - case '\n': - case '\r': - in.get(); - continue; - case '"': - return stringFromStream( in ); - case '{': - return objectFromStream( in ); - case '[': - return arrayFromStream( in ); - case '-': - case '.': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - return number_from_stream( in ); - // null, true, false, or 'warning' / string - case 'n': - case 't': - case 'f': - return token_from_stream( in ); - case 0x04: // ^D end of transmission - case EOF: - case 0: - FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" ); - default: - FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", - ("c", c)("s", stringFromToken(in)) ); - } + case '"': + return stringFromStream( in ); + case '{': + return objectFromStream( in ); + case '[': + return arrayFromStream( in ); + case '-': + case '.': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return number_from_stream( in ); + // null, true, false, or 'warning' / string + case 'n': + case 't': + case 'f': + return token_from_stream( in ); + case 0x04: // ^D end of transmission + case EOF: + FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" ); + case 0: + default: + FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", + ("c", c)("s", stringFromToken(in)) ); } - return variant(); - } + } /** the purpose of this check is to verify that we will not get a stack overflow in the recursive descent parser */ @@ -467,33 +452,18 @@ namespace fc } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) } variants json::variants_from_string( const std::string& utf8_str, parse_type ptype ) - { try { - check_string_depth( utf8_str ); + { variants result; - fc::stringstream in( utf8_str ); - //in.exceptions( std::ifstream::eofbit ); try { + check_string_depth( utf8_str ); + fc::stringstream in( utf8_str ); while( true ) - { - // result.push_back( variant_from_stream( in )); - result.push_back(json_relaxed::variant_from_stream( in )); - } - } catch ( const fc::eof_exception& ){} - return result; - } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) } - /* - void toUTF8( const char str, ostream& os ) - { - // validate str == valid utf8 - utf8::replace_invalid( &str, &str + 1, ostream_iterator(os) ); + result.push_back(json_relaxed::variant_from_stream( in )); + } catch ( const fc::eof_exception& ) { + return result; + } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) } - void toUTF8( const wchar_t c, ostream& os ) - { - utf8::utf16to8( &c, (&c)+1, ostream_iterator(os) ); - } - */ - /** * Convert '\t', '\a', '\n', '\\' and '"' to "\t\a\n\\\"" * @@ -563,7 +533,6 @@ namespace fc default: os << *itr; - //toUTF8( *itr, os ); } } os << '"'; @@ -616,27 +585,19 @@ namespace fc os << "null"; return; case variant::int64_type: - { - int64_t i = v.as_int64(); if( format == json::stringify_large_ints_and_doubles && - i > 0xffffffff ) + v.as_int64() > 0xffffffff ) os << '"'< 0xffffffff ) + v.as_uint64() > 0xffffffff ) os << '"'< Date: Mon, 5 Mar 2018 22:36:01 +0100 Subject: [PATCH 09/23] Code deduplication --- include/fc/io/json_relaxed.hpp | 76 ++-------------------------------- src/io/json.cpp | 27 +++++++++--- 2 files changed, 24 insertions(+), 79 deletions(-) diff --git a/include/fc/io/json_relaxed.hpp b/include/fc/io/json_relaxed.hpp index e4876f259..26a89a529 100644 --- a/include/fc/io/json_relaxed.hpp +++ b/include/fc/io/json_relaxed.hpp @@ -564,84 +564,14 @@ namespace fc { namespace json_relaxed template variant_object objectFromStream( T& in ) { - mutable_variant_object obj; - try - { - char c = in.peek(); - if( c != '{' ) - FC_THROW_EXCEPTION( parse_error_exception, - "Expected '{', but read '${char}'", - ("char",string(&c, &c + 1)) ); - in.get(); - skip_white_space(in); - while( in.peek() != '}' ) - { - if( in.peek() == ',' ) - { - in.get(); - continue; - } - if( skip_white_space(in) ) continue; - string key = json_relaxed::stringFromStream( in ); - skip_white_space(in); - if( in.peek() != ':' ) - { - FC_THROW_EXCEPTION( parse_error_exception, "Expected ':' after key \"${key}\"", - ("key", key) ); - } - in.get(); - auto val = json_relaxed::variant_from_stream( in ); - - obj(std::move(key),std::move(val)); - skip_white_space(in); - } - if( in.peek() == '}' ) - { - in.get(); - return obj; - } - FC_THROW_EXCEPTION( parse_error_exception, "Expected '}' after ${variant}", ("variant", obj ) ); - } - catch( const fc::eof_exception& e ) - { - FC_THROW_EXCEPTION( parse_error_exception, "Unexpected EOF: ${e}", ("e", e.to_detail_string() ) ); - } - catch( const std::ios_base::failure& e ) - { - FC_THROW_EXCEPTION( parse_error_exception, "Unexpected EOF: ${e}", ("e", e.what() ) ); - } FC_RETHROW_EXCEPTIONS( warn, "Error parsing object" ); + return objectFromStreamBase( in, []( T& in ){ return json_relaxed::stringFromStream( in ); }, + []( T& in ){ return json_relaxed::variant_from_stream( in ); } ); } template variants arrayFromStream( T& in ) { - variants ar; - try - { - if( in.peek() != '[' ) - FC_THROW_EXCEPTION( parse_error_exception, "Expected '['" ); - in.get(); - skip_white_space(in); - - while( in.peek() != ']' ) - { - if( in.peek() == ',' ) - { - in.get(); - continue; - } - if( skip_white_space(in) ) continue; - ar.push_back( json_relaxed::variant_from_stream(in) ); - skip_white_space(in); - } - if( in.peek() != ']' ) - FC_THROW_EXCEPTION( parse_error_exception, "Expected ']' after parsing ${variant}", - ("variant", ar) ); - - in.get(); - } FC_RETHROW_EXCEPTIONS( warn, "Attempting to parse array ${array}", - ("array", ar ) ); - return ar; + return arrayFromStreamBase( in, []( T& in ){ return json_relaxed::variant_from_stream( in ); } ); } template diff --git a/src/io/json.cpp b/src/io/json.cpp index 99feab97f..8f4f2c1f9 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -19,7 +19,9 @@ namespace fc template fc::string stringFromStream( T& in ); template bool skip_white_space( T& in ); template fc::string stringFromToken( T& in ); + template variant_object objectFromStreamBase( T& in, std::function& get_key, std::function& get_value ); template variant_object objectFromStream( T& in ); + template variants arrayFromStreamBase( T& in, std::function& get_value ); template variants arrayFromStream( T& in ); template variant number_from_stream( T& in ); template variant token_from_stream( T& in ); @@ -166,8 +168,8 @@ namespace fc ("token", token.str() ) ); } - template - variant_object objectFromStream( T& in ) + template + variant_object objectFromStreamBase( T& in, std::string (*get_key)(T&), variant (*get_value)(T&) ) { mutable_variant_object obj; try @@ -186,7 +188,7 @@ namespace fc continue; } if( skip_white_space(in) ) continue; - string key = stringFromStream( in ); + string key = get_key( in ); skip_white_space(in); if( in.peek() != ':' ) { @@ -194,7 +196,7 @@ namespace fc ("key", key) ); } in.get(); - auto val = variant_from_stream( in ); + auto val = get_value( in ); obj(std::move(key),std::move(val)); } @@ -216,7 +218,14 @@ namespace fc } template - variants arrayFromStream( T& in ) + variant_object objectFromStream( T& in ) + { + return objectFromStreamBase( in, []( T& in ){ return stringFromStream( in ); }, + []( T& in ){ return variant_from_stream( in ); } ); + } + + template + variants arrayFromStreamBase( T& in, variant (*get_value)(T&) ) { variants ar; try @@ -233,7 +242,7 @@ namespace fc continue; } if( skip_white_space(in) ) continue; - ar.push_back( variant_from_stream(in) ); + ar.push_back( get_value(in) ); } if( in.peek() != ']' ) FC_THROW_EXCEPTION( parse_error_exception, "Expected ']' after parsing ${variant}", @@ -245,6 +254,12 @@ namespace fc return ar; } + template + variants arrayFromStream( T& in ) + { + return arrayFromStreamBase( in, []( T& in ){ return variant_from_stream( in ); } ); + } + template variant number_from_stream( T& in ) { From cb9c61fa2d20e85c1002a2cfd755262c288ba6fa Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Tue, 6 Mar 2018 22:13:17 +0100 Subject: [PATCH 10/23] Avoid legacy_generator --- tests/io/json_tests.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp index 35ff11097..25bd975ea 100644 --- a/tests/io/json_tests.cpp +++ b/tests/io/json_tests.cpp @@ -126,7 +126,15 @@ static bool equal( const fc::variant& a, const fc::variant& b ) a_type = fc::variant::type_id::uint64_type; if( b_type == fc::variant::type_id::int64_type && b.as() > 0 ) b_type = fc::variant::type_id::uint64_type; - if( a_type != b_type ) return false; + if( a_type != b_type ) + { + if( ( a_type == fc::variant::type_id::double_type + && b_type == fc::variant::type_id::string_type ) + || ( a_type == fc::variant::type_id::string_type + && b_type == fc::variant::type_id::double_type ) ) + return a.as() == b.as(); + return false; + } switch( a_type ) { case fc::variant::type_id::null_type: return true; @@ -163,16 +171,16 @@ static bool equal( const fc::variant& a, const fc::variant& b ) static void test_recursive( const fc::variant& v ) { try { - const std::string json = fc::json::to_string( v, fc::json::output_formatting::legacy_generator ); + const std::string json = fc::json::to_string( v ); BOOST_CHECK( fc::json::is_valid( json ) ); BOOST_CHECK( !fc::json::is_valid( json + " " ) ); - const std::string pretty = fc::json::to_pretty_string( v, fc::json::output_formatting::legacy_generator ); + const std::string pretty = fc::json::to_pretty_string( v ); BOOST_CHECK( fc::json::is_valid( pretty ) ); BOOST_CHECK( !fc::json::is_valid( pretty + " " ) ); fc::temp_file file( fc::temp_directory_path(), true ); - fc::json::save_to_file( v, file.path(), false, fc::json::output_formatting::legacy_generator ); + fc::json::save_to_file( v, file.path(), false ); BOOST_CHECK( equal( v, fc::json::from_string( json + " " ) ) ); BOOST_CHECK( equal( v, fc::json::from_string( pretty + " " ) ) ); From 1ae3cc2fad06aa17bdaf1f5fc7713df0a1f24c92 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Tue, 6 Mar 2018 22:14:30 +0100 Subject: [PATCH 11/23] Make unused parsers compile-time optional --- include/fc/io/json.hpp | 4 ++++ include/fc/io/json_relaxed.hpp | 1 - src/io/json.cpp | 8 +++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/include/fc/io/json.hpp b/include/fc/io/json.hpp index 8a46d134f..10b5314b2 100644 --- a/include/fc/io/json.hpp +++ b/include/fc/io/json.hpp @@ -18,14 +18,18 @@ namespace fc enum parse_type { legacy_parser = 0, +#ifdef WITH_EXOTIC_JSON_PARSERS strict_parser = 1, relaxed_parser = 2, legacy_parser_with_string_doubles = 3 +#endif }; enum output_formatting { stringify_large_ints_and_doubles = 0, +#ifdef WITH_EXOTIC_JSON_PARSERS legacy_generator = 1 +#endif }; static ostream& to_stream( ostream& out, const fc::string&); diff --git a/include/fc/io/json_relaxed.hpp b/include/fc/io/json_relaxed.hpp index 26a89a529..dd6db7326 100644 --- a/include/fc/io/json_relaxed.hpp +++ b/include/fc/io/json_relaxed.hpp @@ -671,5 +671,4 @@ namespace fc { namespace json_relaxed } return variant(); } - } } // fc::json_relaxed diff --git a/src/io/json.cpp b/src/io/json.cpp index 8f4f2c1f9..d3296c9b4 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -319,7 +319,11 @@ namespace fc if (str == "-." || str == ".") // check the obviously wrong things we could have encountered FC_THROW_EXCEPTION(parse_error_exception, "Can't parse token \"${token}\" as a JSON numeric constant", ("token", str)); if( dot ) - return parser_type == json::legacy_parser_with_string_doubles ? variant(str) : variant(to_double(str)); + return +#ifdef WITH_EXOTIC_JSON_PARSERS + parser_type == json::legacy_parser_with_string_doubles ? variant(str) : +#endif + variant(to_double(str)); if( neg ) return to_int64(str); return to_uint64(str); @@ -768,12 +772,14 @@ namespace fc { case legacy_parser: return variant_from_stream( in ); +#ifdef WITH_EXOTIC_JSON_PARSERS case legacy_parser_with_string_doubles: return variant_from_stream( in ); case strict_parser: return json_relaxed::variant_from_stream( in ); case relaxed_parser: return json_relaxed::variant_from_stream( in ); +#endif default: FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); } From b4f3947cb1bd9b5194c9988ecf6ab2977e24d547 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Tue, 6 Mar 2018 22:47:02 +0100 Subject: [PATCH 12/23] Include variants_from_string in parser test --- tests/io/json_tests.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp index 25bd975ea..d86b743a0 100644 --- a/tests/io/json_tests.cpp +++ b/tests/io/json_tests.cpp @@ -182,6 +182,14 @@ static void test_recursive( const fc::variant& v ) fc::temp_file file( fc::temp_directory_path(), true ); fc::json::save_to_file( v, file.path(), false ); + fc::variants list = fc::json::variants_from_string( json ); + BOOST_CHECK_EQUAL( 1, list.size() ); + BOOST_CHECK( equal( v, list[0] ) ); + + list = fc::json::variants_from_string( pretty ); + BOOST_CHECK_EQUAL( 1, list.size() ); + BOOST_CHECK( equal( v, list[0] ) ); + BOOST_CHECK( equal( v, fc::json::from_string( json + " " ) ) ); BOOST_CHECK( equal( v, fc::json::from_string( pretty + " " ) ) ); BOOST_CHECK( equal( v, fc::json::from_file( file.path() ) ) ); From 4bb8bf7832b29a3be6f3c1ef1002e11ffbd11270 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Tue, 6 Mar 2018 22:47:59 +0100 Subject: [PATCH 13/23] Fixed relaxed parser wrt "" input --- include/fc/io/json_relaxed.hpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/include/fc/io/json_relaxed.hpp b/include/fc/io/json_relaxed.hpp index dd6db7326..36f11c802 100644 --- a/include/fc/io/json_relaxed.hpp +++ b/include/fc/io/json_relaxed.hpp @@ -104,8 +104,15 @@ namespace fc { namespace json_relaxed if( in.peek() == q ) { in.get(); - if( in.peek() != q ) - return fc::string(); + try + { + if( in.peek() != q ) + return fc::string(); + } + catch( const fc::eof_exception& e ) + { + return fc::string(); + } // triple quote processing if( strict ) From d336af82a68ed03d9b1ebd22b3df9be846f9fd26 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Tue, 6 Mar 2018 23:01:40 +0100 Subject: [PATCH 14/23] Applied variant_from_stream fix from regular to relaxed --- include/fc/io/json_relaxed.hpp | 90 ++++++++++++++++------------------ 1 file changed, 41 insertions(+), 49 deletions(-) diff --git a/include/fc/io/json_relaxed.hpp b/include/fc/io/json_relaxed.hpp index 36f11c802..1ed424b8f 100644 --- a/include/fc/io/json_relaxed.hpp +++ b/include/fc/io/json_relaxed.hpp @@ -626,56 +626,48 @@ namespace fc { namespace json_relaxed variant variant_from_stream( T& in ) { skip_white_space(in); - variant var; - while( signed char c = in.peek() ) + signed char c = in.peek(); + switch( c ) { - switch( c ) - { - case ' ': - case '\t': - case '\n': - case '\r': - in.get(); - continue; - case '"': - return json_relaxed::stringFromStream( in ); - case '{': - return json_relaxed::objectFromStream( in ); - case '[': - return json_relaxed::arrayFromStream( in ); - case '-': - case '+': - case '.': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - return json_relaxed::numberFromStream( in ); - // null, true, false, or 'warning' / string - case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': - case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': - case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': - case 'y': case 'z': - case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': - case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': - case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': - case 'Y': case 'Z': - case '_': case '/': - return json_relaxed::wordFromStream( in ); - case 0x04: // ^D end of transmission - case EOF: - FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" ); - default: - FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", - ("c", c)("s", stringFromToken(in)) ); - } + case '"': + return json_relaxed::stringFromStream( in ); + case '{': + return json_relaxed::objectFromStream( in ); + case '[': + return json_relaxed::arrayFromStream( in ); + case '-': + case '+': + case '.': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return json_relaxed::numberFromStream( in ); + // null, true, false, or 'warning' / string + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': + case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': + case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': + case 'y': case 'z': + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': + case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': + case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': + case 'Y': case 'Z': + case '_': case '/': + return json_relaxed::wordFromStream( in ); + case 0x04: // ^D end of transmission + case EOF: + FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" ); + case 0: + default: + FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", + ("c", c)("s", stringFromToken(in)) ); } - return variant(); } + } } // fc::json_relaxed From a7e0c887dbc7a33ba8eec470d05711022cbc4530 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 7 Mar 2018 15:19:42 +0100 Subject: [PATCH 15/23] Added test case for recursion depth 240 --- tests/io/json_tests.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp index d86b743a0..4e4cb5cd3 100644 --- a/tests/io/json_tests.cpp +++ b/tests/io/json_tests.cpp @@ -75,6 +75,10 @@ static void test_fail_file( const std::string& str ) BOOST_AUTO_TEST_CASE(imbalanced_test) { + std::string open40("[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[["); + std::string close40("]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]"); + std::string open80 = open40 + open40; + std::string close80 = close40 + close40; std::vector tests { // for easier handling and better readability, in the following test // strings ' is used instead of " and \1 instead of \0 @@ -106,7 +110,9 @@ BOOST_AUTO_TEST_CASE(imbalanced_test) "['\1]", "[ 13\1", "[ 13\1]", - "' end\1" + "' end\1", + open80 + "'" + close80 + close80 + "'," + open80 + open80 + + close80 + close80 + close80 }; for( std::string test : tests ) From 90137d400df93c9bd41f1ebb187c01102e4f9c78 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 7 Mar 2018 15:20:31 +0100 Subject: [PATCH 16/23] Fix for recursion depth limitation --- include/fc/io/json_relaxed.hpp | 21 +++++++------ src/io/json.cpp | 57 ++++++++++++---------------------- 2 files changed, 32 insertions(+), 46 deletions(-) diff --git a/include/fc/io/json_relaxed.hpp b/include/fc/io/json_relaxed.hpp index 1ed424b8f..a9196c232 100644 --- a/include/fc/io/json_relaxed.hpp +++ b/include/fc/io/json_relaxed.hpp @@ -21,7 +21,7 @@ namespace fc { namespace json_relaxed { template - variant variant_from_stream( T& in ); + variant variant_from_stream( T& in, uint32_t depth ); template fc::string tokenFromStream( T& in ) @@ -569,16 +569,18 @@ namespace fc { namespace json_relaxed } FC_CAPTURE_AND_RETHROW( (token) ) } template - variant_object objectFromStream( T& in ) + variant_object objectFromStream( T& in, uint32_t depth ) { - return objectFromStreamBase( in, []( T& in ){ return json_relaxed::stringFromStream( in ); }, - []( T& in ){ return json_relaxed::variant_from_stream( in ); } ); + std::function get_key = []( T& in ){ return json_relaxed::stringFromStream( in ); }; + std::function get_value = [depth]( T& in ){ return json_relaxed::variant_from_stream( in, depth ); }; + return objectFromStreamBase( in, get_key, get_value ); } template - variants arrayFromStream( T& in ) + variants arrayFromStream( T& in, uint32_t depth ) { - return arrayFromStreamBase( in, []( T& in ){ return json_relaxed::variant_from_stream( in ); } ); + std::function get_value = [depth]( T& in ){ return json_relaxed::variant_from_stream( in, depth ); }; + return arrayFromStreamBase( in, get_value ); } template @@ -623,8 +625,9 @@ namespace fc { namespace json_relaxed } template - variant variant_from_stream( T& in ) + variant variant_from_stream( T& in, uint32_t depth ) { + FC_ASSERT( depth < MAX_RECURSION_DEPTH, "Too many nested items in JSON string!" ); skip_white_space(in); signed char c = in.peek(); switch( c ) @@ -632,9 +635,9 @@ namespace fc { namespace json_relaxed case '"': return json_relaxed::stringFromStream( in ); case '{': - return json_relaxed::objectFromStream( in ); + return json_relaxed::objectFromStream( in, depth + 1 ); case '[': - return json_relaxed::arrayFromStream( in ); + return json_relaxed::arrayFromStream( in, depth + 1 ); case '-': case '+': case '.': diff --git a/src/io/json.cpp b/src/io/json.cpp index d3296c9b4..5d23e7081 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -14,15 +14,15 @@ namespace fc { // forward declarations of provided functions - template variant variant_from_stream( T& in ); + template variant variant_from_stream( T& in, uint32_t depth = 0 ); template char parseEscape( T& in ); template fc::string stringFromStream( T& in ); template bool skip_white_space( T& in ); template fc::string stringFromToken( T& in ); template variant_object objectFromStreamBase( T& in, std::function& get_key, std::function& get_value ); - template variant_object objectFromStream( T& in ); + template variant_object objectFromStream( T& in, uint32_t depth ); template variants arrayFromStreamBase( T& in, std::function& get_value ); - template variants arrayFromStream( T& in ); + template variants arrayFromStream( T& in, uint32_t depth ); template variant number_from_stream( T& in ); template variant token_from_stream( T& in ); void escape_string( const string& str, ostream& os ); @@ -32,6 +32,8 @@ namespace fc fc::string pretty_print( const fc::string& v, uint8_t indent ); } +#define MAX_RECURSION_DEPTH 200 + #include namespace fc @@ -169,7 +171,7 @@ namespace fc } template - variant_object objectFromStreamBase( T& in, std::string (*get_key)(T&), variant (*get_value)(T&) ) + variant_object objectFromStreamBase( T& in, std::function& get_key, std::function& get_value ) { mutable_variant_object obj; try @@ -218,14 +220,15 @@ namespace fc } template - variant_object objectFromStream( T& in ) + variant_object objectFromStream( T& in, uint32_t depth ) { - return objectFromStreamBase( in, []( T& in ){ return stringFromStream( in ); }, - []( T& in ){ return variant_from_stream( in ); } ); + std::function get_key = []( T& in ){ return stringFromStream( in ); }; + std::function get_value = [depth]( T& in ){ return variant_from_stream( in, depth ); }; + return objectFromStreamBase( in, get_key, get_value ); } template - variants arrayFromStreamBase( T& in, variant (*get_value)(T&) ) + variants arrayFromStreamBase( T& in, std::function& get_value ) { variants ar; try @@ -255,9 +258,10 @@ namespace fc } template - variants arrayFromStream( T& in ) + variants arrayFromStream( T& in, uint32_t depth ) { - return arrayFromStreamBase( in, []( T& in ){ return variant_from_stream( in ); } ); + std::function get_value = [depth]( T& in ){ return variant_from_stream( in, depth ); }; + return arrayFromStreamBase( in, get_value ); } template @@ -401,8 +405,10 @@ namespace fc template - variant variant_from_stream( T& in ) + variant variant_from_stream( T& in, uint32_t depth ) { + if( depth > MAX_RECURSION_DEPTH ) + FC_THROW_EXCEPTION( parse_error_exception, "Too many nested items in JSON input!" ); skip_white_space(in); signed char c = in.peek(); switch( c ) @@ -410,9 +416,9 @@ namespace fc case '"': return stringFromStream( in ); case '{': - return objectFromStream( in ); + return objectFromStream( in, depth + 1 ); case '[': - return arrayFromStream( in ); + return arrayFromStream( in, depth + 1 ); case '-': case '.': case '0': @@ -441,30 +447,8 @@ namespace fc } } - - /** the purpose of this check is to verify that we will not get a stack overflow in the recursive descent parser */ - void check_string_depth( const string& utf8_str ) - { - int32_t open_object = 0; - int32_t open_array = 0; - for( auto c : utf8_str ) - { - switch( c ) - { - case '{': open_object++; break; - case '}': open_object--; break; - case '[': open_array++; break; - case ']': open_array--; break; - default: break; - } - FC_ASSERT( open_object < 100 && open_array < 100, "object graph too deep", ("object depth",open_object)("array depth", open_array) ); - } - } - variant json::from_string( const std::string& utf8_str, parse_type ptype ) { try { - check_string_depth( utf8_str ); - fc::istream_ptr in( new fc::stringstream( utf8_str ) ); fc::buffered_istream bin( in ); return from_stream( bin, ptype ); @@ -474,10 +458,9 @@ namespace fc { variants result; try { - check_string_depth( utf8_str ); fc::stringstream in( utf8_str ); while( true ) - result.push_back(json_relaxed::variant_from_stream( in )); + result.push_back(json_relaxed::variant_from_stream( in, 0 )); } catch ( const fc::eof_exception& ) { return result; } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) From 2bacd5fda8232a302fcd65e31912bf8a23e997b9 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 7 Mar 2018 15:26:30 +0100 Subject: [PATCH 17/23] Added broken_nul_parser to preserve previous behaviour --- include/fc/io/json.hpp | 3 ++- src/io/json.cpp | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/fc/io/json.hpp b/include/fc/io/json.hpp index 10b5314b2..d7c1b0f72 100644 --- a/include/fc/io/json.hpp +++ b/include/fc/io/json.hpp @@ -21,8 +21,9 @@ namespace fc #ifdef WITH_EXOTIC_JSON_PARSERS strict_parser = 1, relaxed_parser = 2, - legacy_parser_with_string_doubles = 3 + legacy_parser_with_string_doubles = 3, #endif + broken_nul_parser = 4 }; enum output_formatting { diff --git a/src/io/json.cpp b/src/io/json.cpp index 5d23e7081..6230fd198 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -441,6 +441,9 @@ namespace fc case EOF: FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" ); case 0: + if( parser_type == fc::json::broken_nul_parser ) + return variant(); + [[fallthrough]]; default: FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", ("c", c)("s", stringFromToken(in)) ); From 66ed9fc3dcf5cef4ed49ecb18a8ec3a8aba60773 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 7 Mar 2018 22:41:45 +0100 Subject: [PATCH 18/23] Minor fixes --- src/io/json.cpp | 16 +++++++++++----- tests/io/json_tests.cpp | 12 ++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/io/json.cpp b/src/io/json.cpp index 6230fd198..606fa8517 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -32,6 +32,12 @@ namespace fc fc::string pretty_print( const fc::string& v, uint8_t indent ); } +#if __cplusplus > 201402L +#define FALLTHROUGH [[fallthrough]]; +#else +#define FALLTHROUGH +#endif + #define MAX_RECURSION_DEPTH 200 #include @@ -290,7 +296,7 @@ namespace fc if (dot) FC_THROW_EXCEPTION(parse_error_exception, "Can't parse a number with two decimal places"); dot = true; - [[fallthrough]]; + FALLTHROUGH case '0': case '1': case '2': @@ -320,7 +326,7 @@ namespace fc { // read error ends the loop } fc::string str = ss.str(); - if (str == "-." || str == ".") // check the obviously wrong things we could have encountered + if (str == "-." || str == "." || str == "-") // check the obviously wrong things we could have encountered FC_THROW_EXCEPTION(parse_error_exception, "Can't parse token \"${token}\" as a JSON numeric constant", ("token", str)); if( dot ) return @@ -443,7 +449,7 @@ namespace fc case 0: if( parser_type == fc::json::broken_nul_parser ) return variant(); - [[fallthrough]]; + FALLTHROUGH default: FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", ("c", c)("s", stringFromToken(in)) ); @@ -591,7 +597,7 @@ namespace fc return; case variant::int64_type: if( format == json::stringify_large_ints_and_doubles && - v.as_int64() > 0xffffffff ) + ( v.as_int64() > 0xffffffff || v.as_int64() < -int64_t(0xffffffff) ) ) os << '"'< Date: Thu, 8 Mar 2018 18:54:49 +0100 Subject: [PATCH 19/23] Make broken_nul_parser usable --- src/io/json.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/io/json.cpp b/src/io/json.cpp index 606fa8517..9553ff330 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -772,6 +772,8 @@ namespace fc case relaxed_parser: return json_relaxed::variant_from_stream( in ); #endif + case broken_nul_parser: + return variant_from_stream( in ); default: FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); } From f9802f686007e7efeaa88cba31f647ca96f1ee0c Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Thu, 8 Mar 2018 22:12:28 +0100 Subject: [PATCH 20/23] Added max_depth parameter to all to_/from_ methods --- include/fc/io/json.hpp | 48 +++++++++-------- include/fc/io/json_relaxed.hpp | 19 +++---- src/io/json.cpp | 99 +++++++++++++++++----------------- tests/io/json_tests.cpp | 11 ++++ 4 files changed, 96 insertions(+), 81 deletions(-) diff --git a/include/fc/io/json.hpp b/include/fc/io/json.hpp index d7c1b0f72..21d07c457 100644 --- a/include/fc/io/json.hpp +++ b/include/fc/io/json.hpp @@ -2,6 +2,8 @@ #include #include +#define DEFAULT_MAX_RECURSION_DEPTH 200 + namespace fc { class ostream; @@ -33,52 +35,54 @@ namespace fc #endif }; - static ostream& to_stream( ostream& out, const fc::string&); - static ostream& to_stream( ostream& out, const variant& v, output_formatting format = stringify_large_ints_and_doubles ); - static ostream& to_stream( ostream& out, const variants& v, output_formatting format = stringify_large_ints_and_doubles ); - static ostream& to_stream( ostream& out, const variant_object& v, output_formatting format = stringify_large_ints_and_doubles ); + static ostream& to_stream( ostream& out, const fc::string& ); + static ostream& to_stream( ostream& out, const variant& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); + static ostream& to_stream( ostream& out, const variants& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); + static ostream& to_stream( ostream& out, const variant_object& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); - static variant from_stream( buffered_istream& in, parse_type ptype = legacy_parser ); + static variant from_stream( buffered_istream& in, parse_type ptype = legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); - static variant from_string( const string& utf8_str, parse_type ptype = legacy_parser ); - static variants variants_from_string( const string& utf8_str, parse_type ptype = legacy_parser ); - static string to_string( const variant& v, output_formatting format = stringify_large_ints_and_doubles ); - static string to_pretty_string( const variant& v, output_formatting format = stringify_large_ints_and_doubles ); + static variant from_string( const string& utf8_str, parse_type ptype = legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); + static variants variants_from_string( const string& utf8_str, parse_type ptype = legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); + static string to_string( const variant& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); + static string to_pretty_string( const variant& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); - static bool is_valid( const std::string& json_str, parse_type ptype = legacy_parser ); + static bool is_valid( const std::string& json_str, parse_type ptype = legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); template - static void save_to_file( const T& v, const fc::path& fi, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles ) + static void save_to_file( const T& v, const fc::path& fi, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ) { - save_to_file( variant(v), fi, pretty, format ); + save_to_file( variant(v), fi, pretty, format, max_depth ); } - static void save_to_file( const variant& v, const fc::path& fi, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles ); - static variant from_file( const fc::path& p, parse_type ptype = legacy_parser ); + static void save_to_file( const variant& v, const fc::path& fi, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); + static variant from_file( const fc::path& p, parse_type ptype = legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); template - static T from_file( const fc::path& p, parse_type ptype = legacy_parser ) + static T from_file( const fc::path& p, parse_type ptype = legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ) { - return json::from_file(p, ptype).as(); + return json::from_file(p, ptype, max_depth).as(); } template - static string to_string( const T& v, output_formatting format = stringify_large_ints_and_doubles ) + static string to_string( const T& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ) { - return to_string( variant(v), format ); + return to_string( variant(v), format, max_depth ); } template - static string to_pretty_string( const T& v, output_formatting format = stringify_large_ints_and_doubles ) + static string to_pretty_string( const T& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ) { - return to_pretty_string( variant(v), format ); + return to_pretty_string( variant(v), format, max_depth ); } template - static void save_to_file( const T& v, const std::string& p, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles ) + static void save_to_file( const T& v, const std::string& p, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ) { - save_to_file( variant(v), fc::path(p), pretty ); + save_to_file( variant(v), fc::path(p), pretty, format, max_depth ); } }; } // fc + +#undef DEFAULT_MAX_RECURSION_DEPTH diff --git a/include/fc/io/json_relaxed.hpp b/include/fc/io/json_relaxed.hpp index a9196c232..555b21a86 100644 --- a/include/fc/io/json_relaxed.hpp +++ b/include/fc/io/json_relaxed.hpp @@ -21,7 +21,7 @@ namespace fc { namespace json_relaxed { template - variant variant_from_stream( T& in, uint32_t depth ); + variant variant_from_stream( T& in, uint32_t max_depth ); template fc::string tokenFromStream( T& in ) @@ -569,17 +569,17 @@ namespace fc { namespace json_relaxed } FC_CAPTURE_AND_RETHROW( (token) ) } template - variant_object objectFromStream( T& in, uint32_t depth ) + variant_object objectFromStream( T& in, uint32_t max_depth ) { std::function get_key = []( T& in ){ return json_relaxed::stringFromStream( in ); }; - std::function get_value = [depth]( T& in ){ return json_relaxed::variant_from_stream( in, depth ); }; + std::function get_value = [max_depth]( T& in ){ return json_relaxed::variant_from_stream( in, max_depth ); }; return objectFromStreamBase( in, get_key, get_value ); } template - variants arrayFromStream( T& in, uint32_t depth ) + variants arrayFromStream( T& in, uint32_t max_depth ) { - std::function get_value = [depth]( T& in ){ return json_relaxed::variant_from_stream( in, depth ); }; + std::function get_value = [max_depth]( T& in ){ return json_relaxed::variant_from_stream( in, max_depth ); }; return arrayFromStreamBase( in, get_value ); } @@ -625,9 +625,10 @@ namespace fc { namespace json_relaxed } template - variant variant_from_stream( T& in, uint32_t depth ) + variant variant_from_stream( T& in, uint32_t max_depth ) { - FC_ASSERT( depth < MAX_RECURSION_DEPTH, "Too many nested items in JSON string!" ); + if( max_depth == 0 ) + FC_THROW_EXCEPTION( parse_error_exception, "Too many nested items in JSON input!" ); skip_white_space(in); signed char c = in.peek(); switch( c ) @@ -635,9 +636,9 @@ namespace fc { namespace json_relaxed case '"': return json_relaxed::stringFromStream( in ); case '{': - return json_relaxed::objectFromStream( in, depth + 1 ); + return json_relaxed::objectFromStream( in, max_depth - 1 ); case '[': - return json_relaxed::arrayFromStream( in, depth + 1 ); + return json_relaxed::arrayFromStream( in, max_depth - 1 ); case '-': case '+': case '.': diff --git a/src/io/json.cpp b/src/io/json.cpp index 9553ff330..a0f484eea 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -14,21 +14,21 @@ namespace fc { // forward declarations of provided functions - template variant variant_from_stream( T& in, uint32_t depth = 0 ); + template variant variant_from_stream( T& in, uint32_t max_depth ); template char parseEscape( T& in ); template fc::string stringFromStream( T& in ); template bool skip_white_space( T& in ); template fc::string stringFromToken( T& in ); template variant_object objectFromStreamBase( T& in, std::function& get_key, std::function& get_value ); - template variant_object objectFromStream( T& in, uint32_t depth ); + template variant_object objectFromStream( T& in, uint32_t max_depth ); template variants arrayFromStreamBase( T& in, std::function& get_value ); - template variants arrayFromStream( T& in, uint32_t depth ); + template variants arrayFromStream( T& in, uint32_t max_depth ); template variant number_from_stream( T& in ); template variant token_from_stream( T& in ); void escape_string( const string& str, ostream& os ); - template void to_stream( T& os, const variants& a, json::output_formatting format ); - template void to_stream( T& os, const variant_object& o, json::output_formatting format ); - template void to_stream( T& os, const variant& v, json::output_formatting format ); + template void to_stream( T& os, const variants& a, json::output_formatting format, uint32_t max_depth ); + template void to_stream( T& os, const variant_object& o, json::output_formatting format, uint32_t max_depth ); + template void to_stream( T& os, const variant& v, json::output_formatting format, uint32_t max_depth ); fc::string pretty_print( const fc::string& v, uint8_t indent ); } @@ -38,8 +38,6 @@ namespace fc #define FALLTHROUGH #endif -#define MAX_RECURSION_DEPTH 200 - #include namespace fc @@ -226,10 +224,10 @@ namespace fc } template - variant_object objectFromStream( T& in, uint32_t depth ) + variant_object objectFromStream( T& in, uint32_t max_depth ) { std::function get_key = []( T& in ){ return stringFromStream( in ); }; - std::function get_value = [depth]( T& in ){ return variant_from_stream( in, depth ); }; + std::function get_value = [max_depth]( T& in ){ return variant_from_stream( in, max_depth ); }; return objectFromStreamBase( in, get_key, get_value ); } @@ -264,9 +262,9 @@ namespace fc } template - variants arrayFromStream( T& in, uint32_t depth ) + variants arrayFromStream( T& in, uint32_t max_depth ) { - std::function get_value = [depth]( T& in ){ return variant_from_stream( in, depth ); }; + std::function get_value = [max_depth]( T& in ){ return variant_from_stream( in, max_depth ); }; return arrayFromStreamBase( in, get_value ); } @@ -411,9 +409,9 @@ namespace fc template - variant variant_from_stream( T& in, uint32_t depth ) + variant variant_from_stream( T& in, uint32_t max_depth ) { - if( depth > MAX_RECURSION_DEPTH ) + if( max_depth == 0 ) FC_THROW_EXCEPTION( parse_error_exception, "Too many nested items in JSON input!" ); skip_white_space(in); signed char c = in.peek(); @@ -422,9 +420,9 @@ namespace fc case '"': return stringFromStream( in ); case '{': - return objectFromStream( in, depth + 1 ); + return objectFromStream( in, max_depth - 1 ); case '[': - return arrayFromStream( in, depth + 1 ); + return arrayFromStream( in, max_depth - 1 ); case '-': case '.': case '0': @@ -456,20 +454,20 @@ namespace fc } } - variant json::from_string( const std::string& utf8_str, parse_type ptype ) + variant json::from_string( const std::string& utf8_str, parse_type ptype, uint32_t max_depth ) { try { fc::istream_ptr in( new fc::stringstream( utf8_str ) ); fc::buffered_istream bin( in ); - return from_stream( bin, ptype ); + return from_stream( bin, ptype, max_depth ); } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) } - variants json::variants_from_string( const std::string& utf8_str, parse_type ptype ) + variants json::variants_from_string( const std::string& utf8_str, parse_type ptype, uint32_t max_depth ) { variants result; try { fc::stringstream in( utf8_str ); while( true ) - result.push_back(json_relaxed::variant_from_stream( in, 0 )); + result.push_back(json_relaxed::variant_from_stream( in, max_depth )); } catch ( const fc::eof_exception& ) { return result; } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) @@ -555,14 +553,14 @@ namespace fc } template - void to_stream( T& os, const variants& a, json::output_formatting format ) + void to_stream( T& os, const variants& a, json::output_formatting format, uint32_t max_depth ) { os << '['; auto itr = a.begin(); while( itr != a.end() ) { - to_stream( os, *itr, format ); + to_stream( os, *itr, format, max_depth ); ++itr; if( itr != a.end() ) os << ','; @@ -570,7 +568,7 @@ namespace fc os << ']'; } template - void to_stream( T& os, const variant_object& o, json::output_formatting format ) + void to_stream( T& os, const variant_object& o, json::output_formatting format, uint32_t max_depth ) { os << '{'; auto itr = o.begin(); @@ -579,7 +577,7 @@ namespace fc { escape_string( itr->key(), os ); os << ':'; - to_stream( os, itr->value(), format ); + to_stream( os, itr->value(), format, max_depth ); ++itr; if( itr != o.end() ) os << ','; @@ -588,8 +586,9 @@ namespace fc } template - void to_stream( T& os, const variant& v, json::output_formatting format ) + void to_stream( T& os, const variant& v, json::output_formatting format, uint32_t max_depth ) { + FC_ASSERT( max_depth > 0, "Too many nested objects!" ); switch( v.get_type() ) { case variant::null_type: @@ -625,20 +624,20 @@ namespace fc escape_string( v.as_string(), os ); return; case variant::array_type: - to_stream( os, v.get_array(), format ); + to_stream( os, v.get_array(), format, max_depth - 1 ); return; case variant::object_type: - to_stream(os, v.get_object(), format ); + to_stream(os, v.get_object(), format, max_depth - 1 ); return; default: FC_THROW_EXCEPTION( fc::invalid_arg_exception, "Unsupported variant type: " + v.get_type() ); } } - fc::string json::to_string( const variant& v, output_formatting format /* = stringify_large_ints_and_doubles */ ) + fc::string json::to_string( const variant& v, output_formatting format, uint32_t max_depth ) { fc::stringstream ss; - fc::to_stream( ss, v, format ); + fc::to_stream( ss, v, format, max_depth ); return ss.str(); } @@ -733,74 +732,74 @@ namespace fc - fc::string json::to_pretty_string( const variant& v, output_formatting format /* = stringify_large_ints_and_doubles */ ) + fc::string json::to_pretty_string( const variant& v, output_formatting format, uint32_t max_depth ) { - return pretty_print(to_string(v, format), 2); + return pretty_print(to_string(v, format, max_depth), 2); } - void json::save_to_file( const variant& v, const fc::path& fi, bool pretty, output_formatting format /* = stringify_large_ints_and_doubles */ ) + void json::save_to_file( const variant& v, const fc::path& fi, bool pretty, output_formatting format, uint32_t max_depth ) { if( pretty ) { - auto str = json::to_pretty_string( v, format ); + auto str = json::to_pretty_string( v, format, max_depth ); fc::ofstream o(fi); o.write( str.c_str(), str.size() ); } else { fc::ofstream o(fi); - fc::to_stream( o, v, format ); + fc::to_stream( o, v, format, max_depth ); } } - variant json::from_file( const fc::path& p, parse_type ptype ) + variant json::from_file( const fc::path& p, parse_type ptype, uint32_t max_depth ) { fc::istream_ptr in( new fc::ifstream( p ) ); fc::buffered_istream bin( in ); - return from_stream( bin, ptype ); + return from_stream( bin, ptype, max_depth ); } - variant json::from_stream( buffered_istream& in, parse_type ptype ) + variant json::from_stream( buffered_istream& in, parse_type ptype, uint32_t max_depth ) { switch( ptype ) { case legacy_parser: - return variant_from_stream( in ); + return variant_from_stream( in, max_depth ); #ifdef WITH_EXOTIC_JSON_PARSERS case legacy_parser_with_string_doubles: - return variant_from_stream( in ); + return variant_from_stream( in, max_depth ); case strict_parser: - return json_relaxed::variant_from_stream( in ); + return json_relaxed::variant_from_stream( in, max_depth ); case relaxed_parser: - return json_relaxed::variant_from_stream( in ); + return json_relaxed::variant_from_stream( in, max_depth ); #endif case broken_nul_parser: - return variant_from_stream( in ); + return variant_from_stream( in, max_depth ); default: FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); } } - ostream& json::to_stream( ostream& out, const variant& v, output_formatting format /* = stringify_large_ints_and_doubles */ ) + ostream& json::to_stream( ostream& out, const variant& v, output_formatting format, uint32_t max_depth ) { - fc::to_stream( out, v, format ); + fc::to_stream( out, v, format, max_depth ); return out; } - ostream& json::to_stream( ostream& out, const variants& v, output_formatting format /* = stringify_large_ints_and_doubles */ ) + ostream& json::to_stream( ostream& out, const variants& v, output_formatting format, uint32_t max_depth ) { - fc::to_stream( out, v, format ); + fc::to_stream( out, v, format, max_depth ); return out; } - ostream& json::to_stream( ostream& out, const variant_object& v, output_formatting format /* = stringify_large_ints_and_doubles */ ) + ostream& json::to_stream( ostream& out, const variant_object& v, output_formatting format, uint32_t max_depth ) { - fc::to_stream( out, v, format ); + fc::to_stream( out, v, format, max_depth ); return out; } - bool json::is_valid( const std::string& utf8_str, parse_type ptype ) + bool json::is_valid( const std::string& utf8_str, parse_type ptype, uint32_t max_depth ) { if( utf8_str.size() == 0 ) return false; fc::istream_ptr in( new fc::stringstream( utf8_str ) ); fc::buffered_istream bin( in ); - from_stream( bin, ptype ); + from_stream( bin, ptype, max_depth ); try { bin.peek(); } catch ( const eof_exception& e ) { return true; } return false; } diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp index 9be97fe30..e5421f2ae 100644 --- a/tests/io/json_tests.cpp +++ b/tests/io/json_tests.cpp @@ -321,4 +321,15 @@ BOOST_AUTO_TEST_CASE(precision_test) BOOST_CHECK_EQUAL( "0.5", half ); } +BOOST_AUTO_TEST_CASE(recursion_test) +{ + std::string ten_levels = "[[[[[[[[[[]]]]]]]]]]"; + fc::variant nested = fc::json::from_string( ten_levels ); + BOOST_CHECK_THROW( fc::json::from_string( ten_levels, fc::json::legacy_parser, 9 ), fc::parse_error_exception ); + + std::string back = fc::json::to_string( nested ); + BOOST_CHECK_EQUAL( ten_levels, back ); + BOOST_CHECK_THROW( fc::json::to_string( nested, fc::json::stringify_large_ints_and_doubles, 9 ), fc::assert_exception ); +} + BOOST_AUTO_TEST_SUITE_END() From 1331485c08d5e28110bd1f136590cca1fe91c223 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Thu, 8 Mar 2018 22:25:52 +0100 Subject: [PATCH 21/23] Stringify numbers >MAXINT or #include #include +#include #include #include #include @@ -596,7 +597,7 @@ namespace fc return; case variant::int64_type: if( format == json::stringify_large_ints_and_doubles && - ( v.as_int64() > 0xffffffff || v.as_int64() < -int64_t(0xffffffff) ) ) + ( v.as_int64() > INT32_MAX || v.as_int64() < INT32_MIN ) ) os << '"'< Date: Fri, 9 Mar 2018 18:47:24 +0100 Subject: [PATCH 22/23] Added test case for exception in exception handling --- tests/io/json_tests.cpp | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp index e5421f2ae..42af936da 100644 --- a/tests/io/json_tests.cpp +++ b/tests/io/json_tests.cpp @@ -332,4 +332,45 @@ BOOST_AUTO_TEST_CASE(recursion_test) BOOST_CHECK_THROW( fc::json::to_string( nested, fc::json::stringify_large_ints_and_doubles, 9 ), fc::assert_exception ); } +BOOST_AUTO_TEST_CASE(rethrow_test) +{ + fc::variants biggie; + for( int i = 0; i < 250; i++ ) + { + fc::variant tmp( std::move(biggie) ); + biggie.reserve(1); + biggie.push_back( std::move(tmp) ); + } + + auto test_r = [&biggie](){ + try { + FC_THROW_EXCEPTION( fc::unknown_host_exception, "WTF?" ); + } FC_RETHROW_EXCEPTIONS( warn, "Argh! ${biggie}", ("biggie",biggie) ) }; + BOOST_CHECK_THROW( test_r(), fc::unknown_host_exception ); + + auto test_lr = [&biggie](){ + try { + FC_THROW_EXCEPTION( fc::unknown_host_exception, "WTF?" ); + } FC_LOG_AND_RETHROW() }; + BOOST_CHECK_THROW( test_lr(), fc::unknown_host_exception ); + + auto test_clr = [&biggie](){ + try { + FC_THROW_EXCEPTION( fc::unknown_host_exception, "WTF?" ); + } FC_CAPTURE_LOG_AND_RETHROW( (biggie) ) }; + BOOST_CHECK_THROW( test_clr(), fc::unknown_host_exception ); + + auto test_cl = [&biggie](){ + try { + FC_THROW_EXCEPTION( fc::unknown_host_exception, "WTF?" ); + } FC_CAPTURE_AND_LOG( (biggie) ) }; + test_cl(); + + auto test_cr = [&biggie](){ + try { + FC_THROW_EXCEPTION( fc::unknown_host_exception, "WTF?" ); + } FC_CAPTURE_AND_RETHROW( (biggie) ) }; + BOOST_CHECK_THROW( test_cr(), fc::unknown_host_exception ); +} + BOOST_AUTO_TEST_SUITE_END() From 527daab6b73410a5bc88a28d3e0ede46a28c8219 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Fri, 9 Mar 2018 18:49:21 +0100 Subject: [PATCH 23/23] Moved format_string from variant.cpp to string.cpp (it is declared in string.hpp), added handling of recursion errors --- src/exception.cpp | 23 +++++++++++---- src/log/gelf_appender.cpp | 11 +++++-- src/string.cpp | 56 ++++++++++++++++++++++++++++++++++++ src/variant.cpp | 60 --------------------------------------- 4 files changed, 83 insertions(+), 67 deletions(-) diff --git a/src/exception.cpp b/src/exception.cpp index 0e0b9562a..e173be4f3 100644 --- a/src/exception.cpp +++ b/src/exception.cpp @@ -162,7 +162,14 @@ namespace fc for( auto itr = my->_elog.begin(); itr != my->_elog.end(); ) { ss << itr->get_message() <<"\n"; //fc::format_string( itr->get_format(), itr->get_data() ) <<"\n"; - ss << " " << json::to_string( itr->get_data() )<<"\n"; + try + { + ss << " " << json::to_string( itr->get_data() )<<"\n"; + } + catch( const fc::assert_exception& e ) + { + ss << "ERROR: Failed to convert log data to string!\n"; + } ss << " " << itr->get_context().to_string(); ++itr; if( itr != my->_elog.end() ) ss<<"\n"; @@ -256,10 +263,16 @@ namespace fc ("source_lineno", lineno) ("expr", expr) ; - std::cout - << "FC_ASSERT triggered: " - << fc::json::to_string( assert_trip_info ) << "\n"; - return; + try + { + std::cout + << "FC_ASSERT triggered: " + << fc::json::to_string( assert_trip_info ) << "\n"; + } + catch( const fc::assert_exception& e ) + { // this should never happen. assert_trip_info is flat. + std::cout << "ERROR: Failed to convert info to string?!\n"; + } } bool enable_record_assert_trip = false; diff --git a/src/log/gelf_appender.cpp b/src/log/gelf_appender.cpp index b88387c69..2f14d78be 100644 --- a/src/log/gelf_appender.cpp +++ b/src/log/gelf_appender.cpp @@ -128,8 +128,15 @@ namespace fc if (!context.get_task_name().empty()) gelf_message["_task_name"] = context.get_task_name(); - string gelf_message_as_string = json::to_string(gelf_message); - //unsigned uncompressed_size = gelf_message_as_string.size(); + string gelf_message_as_string; + try + { + gelf_message_as_string = json::to_string(gelf_message); + } + catch( const fc::assert_exception& e ) + { + gelf_message_as_string = "{\"level\":3,\"short_message\":\"ERROR while generating log message\"}"; + } gelf_message_as_string = zlib_compress(gelf_message_as_string); // graylog2 expects the zlib header to be 0x78 0x9c diff --git a/src/string.cpp b/src/string.cpp index 84edb8884..e04f43961 100644 --- a/src/string.cpp +++ b/src/string.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include @@ -256,6 +258,60 @@ namespace fc { } } + string format_string( const string& format, const variant_object& args ) + { + stringstream ss; + size_t prev = 0; + auto next = format.find( '$' ); + while( prev < format.size() ) + { + ss << format.substr( prev, next == string::npos ? string::npos : next - prev ); + + // if we got to the end, return it. + if( next == size_t(string::npos) || next == format.size() ) + return ss.str(); + + // if we are not at the end, then update the start + prev = next + 1; + + if( format[prev] == '{' ) + { + // if the next char is a open, then find close + next = format.find( '}', prev ); + // if we found close... + if( next != string::npos ) + { + // the key is between prev and next + string key = format.substr( prev+1, (next-prev-1) ); + + auto val = args.find( key ); + if( val != args.end() ) + { + if( val->value().is_object() || val->value().is_array() ) + { + try + { + ss << json::to_string( val->value() ); + } + catch( const fc::assert_exception& e ) + { + ss << "[\"ERROR_WHILE_CONVERTING_VALUE_TO_STRING\"]"; + } + } + else + ss << val->value().as_string(); + } + else + ss << "${"<& vo ) // vo = std::vector( b64.c_str(), b64.c_str() + b64.size() ); } -string format_string( const string& format, const variant_object& args ) -{ - stringstream ss; - size_t prev = 0; - auto next = format.find( '$' ); - while( prev != size_t(string::npos) && prev < size_t(format.size()) ) - { - ss << format.substr( prev, size_t(next-prev) ); - - // if we got to the end, return it. - if( next == size_t(string::npos) ) - return ss.str(); - - // if we are not at the end, then update the start - prev = next + 1; - - if( format[prev] == '{' ) - { - // if the next char is a open, then find close - next = format.find( '}', prev ); - // if we found close... - if( next != size_t(string::npos) ) - { - // the key is between prev and next - string key = format.substr( prev+1, (next-prev-1) ); - - auto val = args.find( key ); - if( val != args.end() ) - { - if( val->value().is_object() || val->value().is_array() ) - { - ss << json::to_string( val->value() ); - } - else - { - ss << val->value().as_string(); - } - } - else - { - ss << "${"<