From b76f5506d7da8efcb05cab9c4e84f5774c0436c8 Mon Sep 17 00:00:00 2001 From: Niels Date: Sun, 14 Aug 2016 14:18:25 +0200 Subject: [PATCH 01/18] added a parser for continguous containers --- src/json.hpp | 94 +++++++++++++++++++++++----------- src/json.hpp.re2c | 94 +++++++++++++++++++++++----------- test/src/unit-class_parser.cpp | 48 +++++++++++++++-- 3 files changed, 169 insertions(+), 67 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 5ab8ec264e..752493d024 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -3960,7 +3960,7 @@ class basic_json @return Iterator following the last removed element. If the iterator @a pos refers to the last element, the `end()` iterator is returned. - @tparam InteratorType an @ref iterator or @ref const_iterator + @tparam IteratorType an @ref iterator or @ref const_iterator @post Invalidates iterators and references at or after the point of the erase, including the `end()` iterator. @@ -3982,7 +3982,7 @@ class basic_json @liveexample{The example shows the result of `erase()` for different JSON types.,erase__IteratorType} - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in the given range @sa @ref erase(const typename object_t::key_type&) -- removes the element from an object at the given key @@ -3991,13 +3991,13 @@ class basic_json @since version 1.0.0 */ - template ::value or - std::is_same::value + std::is_same::value or + std::is_same::value , int>::type = 0> - InteratorType erase(InteratorType pos) + IteratorType erase(IteratorType pos) { // make sure iterator fits the current value if (this != pos.m_object) @@ -4005,7 +4005,7 @@ class basic_json throw std::domain_error("iterator does not fit current value"); } - InteratorType result = end(); + IteratorType result = end(); switch (m_type) { @@ -4069,7 +4069,7 @@ class basic_json @return Iterator following the last removed element. If the iterator @a second refers to the last element, the `end()` iterator is returned. - @tparam InteratorType an @ref iterator or @ref const_iterator + @tparam IteratorType an @ref iterator or @ref const_iterator @post Invalidates iterators and references at or after the point of the erase, including the `end()` iterator. @@ -4092,7 +4092,7 @@ class basic_json @liveexample{The example shows the result of `erase()` for different JSON types.,erase__IteratorType_IteratorType} - @sa @ref erase(InteratorType) -- removes the element at a given position + @sa @ref erase(IteratorType) -- removes the element at a given position @sa @ref erase(const typename object_t::key_type&) -- removes the element from an object at the given key @sa @ref erase(const size_type) -- removes the element from an array at @@ -4100,13 +4100,13 @@ class basic_json @since version 1.0.0 */ - template ::value or - std::is_same::value + std::is_same::value or + std::is_same::value , int>::type = 0> - InteratorType erase(InteratorType first, InteratorType last) + IteratorType erase(IteratorType first, IteratorType last) { // make sure iterator fits the current value if (this != first.m_object or this != last.m_object) @@ -4114,7 +4114,7 @@ class basic_json throw std::domain_error("iterators do not fit current value"); } - InteratorType result = end(); + IteratorType result = end(); switch (m_type) { @@ -4186,8 +4186,8 @@ class basic_json @liveexample{The example shows the effect of `erase()`.,erase__key_type} - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in the given range @sa @ref erase(const size_type) -- removes the element from an array at the given index @@ -4223,8 +4223,8 @@ class basic_json @liveexample{The example shows the effect of `erase()`.,erase__size_type} - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in the given range @sa @ref erase(const typename object_t::key_type&) -- removes the element from an object at the given key @@ -7504,21 +7504,29 @@ class basic_json /// the char type to use in the lexer using lexer_char_t = unsigned char; - /// constructor with a given buffer - explicit lexer(const string_t& s) noexcept - : m_stream(nullptr), m_buffer(s) + /// a lexer from a buffer with given length + lexer(const lexer_char_t* buff, const size_t len) noexcept + : m_stream(nullptr), m_buffer(), m_content(buff) { - m_content = reinterpret_cast(m_buffer.c_str()); assert(m_content != nullptr); m_start = m_cursor = m_content; - m_limit = m_content + s.size(); + m_limit = m_content + len; } - /// constructor with a given stream - explicit lexer(std::istream* s) noexcept - : m_stream(s), m_buffer() + /// a lexer from a string literal + explicit lexer(const typename string_t::value_type* buff) noexcept + : m_stream(nullptr), m_buffer(), + m_content(reinterpret_cast(buff)) + { + assert(m_content != nullptr); + m_start = m_cursor = m_content; + m_limit = m_content + strlen(buff); + } + + /// a lexer from an input stream + explicit lexer(std::istream& s) + : m_stream(&s), m_buffer() { - assert(m_stream != nullptr); std::getline(*m_stream, m_buffer); m_content = reinterpret_cast(m_buffer.c_str()); assert(m_content != nullptr); @@ -8853,17 +8861,41 @@ class basic_json class parser { public: - /// constructor for strings + /// a parser reading from a string literal + parser(const typename string_t::value_type* buff, parser_callback_t cb = nullptr) noexcept + : callback(cb), m_lexer(buff) + { + // read first token + get_token(); + } + + /// a parser reading from a string container parser(const string_t& s, const parser_callback_t cb = nullptr) noexcept - : callback(cb), m_lexer(s) + : callback(cb), + m_lexer(reinterpret_cast(s.c_str()), s.size()) { // read first token get_token(); } /// a parser reading from an input stream - parser(std::istream& _is, const parser_callback_t cb = nullptr) noexcept - : callback(cb), m_lexer(&_is) + parser(std::istream& is, const parser_callback_t cb = nullptr) noexcept + : callback(cb), m_lexer(is) + { + // read first token + get_token(); + } + + /// a parser reading from a container with continguous storage + template ::iterator_category, std::random_access_iterator_tag>::value + , int>::type + = 0> + parser(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr) noexcept + : callback(cb), + m_lexer(reinterpret_cast(&(*first)), + static_cast(std::distance(first, last))) { // read first token get_token(); diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index 0d8ffa155e..b8f646af10 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -3960,7 +3960,7 @@ class basic_json @return Iterator following the last removed element. If the iterator @a pos refers to the last element, the `end()` iterator is returned. - @tparam InteratorType an @ref iterator or @ref const_iterator + @tparam IteratorType an @ref iterator or @ref const_iterator @post Invalidates iterators and references at or after the point of the erase, including the `end()` iterator. @@ -3982,7 +3982,7 @@ class basic_json @liveexample{The example shows the result of `erase()` for different JSON types.,erase__IteratorType} - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in the given range @sa @ref erase(const typename object_t::key_type&) -- removes the element from an object at the given key @@ -3991,13 +3991,13 @@ class basic_json @since version 1.0.0 */ - template ::value or - std::is_same::value + std::is_same::value or + std::is_same::value , int>::type = 0> - InteratorType erase(InteratorType pos) + IteratorType erase(IteratorType pos) { // make sure iterator fits the current value if (this != pos.m_object) @@ -4005,7 +4005,7 @@ class basic_json throw std::domain_error("iterator does not fit current value"); } - InteratorType result = end(); + IteratorType result = end(); switch (m_type) { @@ -4069,7 +4069,7 @@ class basic_json @return Iterator following the last removed element. If the iterator @a second refers to the last element, the `end()` iterator is returned. - @tparam InteratorType an @ref iterator or @ref const_iterator + @tparam IteratorType an @ref iterator or @ref const_iterator @post Invalidates iterators and references at or after the point of the erase, including the `end()` iterator. @@ -4092,7 +4092,7 @@ class basic_json @liveexample{The example shows the result of `erase()` for different JSON types.,erase__IteratorType_IteratorType} - @sa @ref erase(InteratorType) -- removes the element at a given position + @sa @ref erase(IteratorType) -- removes the element at a given position @sa @ref erase(const typename object_t::key_type&) -- removes the element from an object at the given key @sa @ref erase(const size_type) -- removes the element from an array at @@ -4100,13 +4100,13 @@ class basic_json @since version 1.0.0 */ - template ::value or - std::is_same::value + std::is_same::value or + std::is_same::value , int>::type = 0> - InteratorType erase(InteratorType first, InteratorType last) + IteratorType erase(IteratorType first, IteratorType last) { // make sure iterator fits the current value if (this != first.m_object or this != last.m_object) @@ -4114,7 +4114,7 @@ class basic_json throw std::domain_error("iterators do not fit current value"); } - InteratorType result = end(); + IteratorType result = end(); switch (m_type) { @@ -4186,8 +4186,8 @@ class basic_json @liveexample{The example shows the effect of `erase()`.,erase__key_type} - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in the given range @sa @ref erase(const size_type) -- removes the element from an array at the given index @@ -4223,8 +4223,8 @@ class basic_json @liveexample{The example shows the effect of `erase()`.,erase__size_type} - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in the given range @sa @ref erase(const typename object_t::key_type&) -- removes the element from an object at the given key @@ -7504,21 +7504,29 @@ class basic_json /// the char type to use in the lexer using lexer_char_t = unsigned char; - /// constructor with a given buffer - explicit lexer(const string_t& s) noexcept - : m_stream(nullptr), m_buffer(s) + /// a lexer from a buffer with given length + lexer(const lexer_char_t* buff, const size_t len) noexcept + : m_stream(nullptr), m_buffer(), m_content(buff) { - m_content = reinterpret_cast(m_buffer.c_str()); assert(m_content != nullptr); m_start = m_cursor = m_content; - m_limit = m_content + s.size(); + m_limit = m_content + len; } - /// constructor with a given stream - explicit lexer(std::istream* s) noexcept - : m_stream(s), m_buffer() + /// a lexer from a string literal + explicit lexer(const typename string_t::value_type* buff) noexcept + : m_stream(nullptr), m_buffer(), + m_content(reinterpret_cast(buff)) + { + assert(m_content != nullptr); + m_start = m_cursor = m_content; + m_limit = m_content + strlen(buff); + } + + /// a lexer from an input stream + explicit lexer(std::istream& s) + : m_stream(&s), m_buffer() { - assert(m_stream != nullptr); std::getline(*m_stream, m_buffer); m_content = reinterpret_cast(m_buffer.c_str()); assert(m_content != nullptr); @@ -8150,17 +8158,41 @@ class basic_json class parser { public: - /// constructor for strings + /// a parser reading from a string literal + parser(const typename string_t::value_type* buff, parser_callback_t cb = nullptr) noexcept + : callback(cb), m_lexer(buff) + { + // read first token + get_token(); + } + + /// a parser reading from a string container parser(const string_t& s, const parser_callback_t cb = nullptr) noexcept - : callback(cb), m_lexer(s) + : callback(cb), + m_lexer(reinterpret_cast(s.c_str()), s.size()) { // read first token get_token(); } /// a parser reading from an input stream - parser(std::istream& _is, const parser_callback_t cb = nullptr) noexcept - : callback(cb), m_lexer(&_is) + parser(std::istream& is, const parser_callback_t cb = nullptr) noexcept + : callback(cb), m_lexer(is) + { + // read first token + get_token(); + } + + /// a parser reading from a container with continguous storage + template ::iterator_category, std::random_access_iterator_tag>::value + , int>::type + = 0> + parser(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr) noexcept + : callback(cb), + m_lexer(reinterpret_cast(&(*first)), + static_cast(std::distance(first, last))) { // read first token get_token(); diff --git a/test/src/unit-class_parser.cpp b/test/src/unit-class_parser.cpp index fe00550346..c59186effa 100644 --- a/test/src/unit-class_parser.cpp +++ b/test/src/unit-class_parser.cpp @@ -32,6 +32,8 @@ SOFTWARE. #include "json.hpp" using nlohmann::json; +#include + TEST_CASE("parser class") { SECTION("parse") @@ -743,11 +745,47 @@ TEST_CASE("parser class") } } - SECTION("copy constructor") + SECTION("constructing from continguous containers") { - json::string_t* s = new json::string_t("[1,2,3,4]"); - json::parser p(*s); - delete s; - CHECK(p.parse() == json({1, 2, 3, 4})); + SECTION("from std::vector") + { + std::vector v = {'t', 'r', 'u', 'e'}; + CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); + } + + SECTION("from std::array") + { + std::array v { {'t', 'r', 'u', 'e'} }; + CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); + } + + SECTION("from array") + { + uint8_t v[] = {'t', 'r', 'u', 'e'}; + CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); + } + + SECTION("from char literal") + { + CHECK (json::parser("true").parse() == json(true)); + } + + SECTION("from std::string") + { + std::string v = {'t', 'r', 'u', 'e'}; + CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); + } + + SECTION("from std::initializer_list") + { + std::initializer_list v = {'t', 'r', 'u', 'e', '\0'}; + CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); + } + + SECTION("from std::valarray") + { + std::valarray v = {'t', 'r', 'u', 'e'}; + CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); + } } } From 01386b3977bc474c3c9febd7c596863bed9b57ef Mon Sep 17 00:00:00 2001 From: Niels Date: Sun, 14 Aug 2016 17:23:15 +0200 Subject: [PATCH 02/18] cleanup --- src/json.hpp | 127 ++++++++++++++++++--------------- src/json.hpp.re2c | 107 ++++++++++++++------------- test/src/unit-class_parser.cpp | 2 +- 3 files changed, 127 insertions(+), 109 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 752493d024..8342b0e1fd 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -37,6 +37,7 @@ SOFTWARE. #include #include #include +#include #include #include #include @@ -7506,7 +7507,7 @@ class basic_json /// a lexer from a buffer with given length lexer(const lexer_char_t* buff, const size_t len) noexcept - : m_stream(nullptr), m_buffer(), m_content(buff) + : m_content(buff) { assert(m_content != nullptr); m_start = m_cursor = m_content; @@ -7515,29 +7516,19 @@ class basic_json /// a lexer from a string literal explicit lexer(const typename string_t::value_type* buff) noexcept - : m_stream(nullptr), m_buffer(), - m_content(reinterpret_cast(buff)) - { - assert(m_content != nullptr); - m_start = m_cursor = m_content; - m_limit = m_content + strlen(buff); - } + : lexer(reinterpret_cast(buff), strlen(buff)) + {} /// a lexer from an input stream explicit lexer(std::istream& s) - : m_stream(&s), m_buffer() + : m_stream(&s), m_line_buffer() { - std::getline(*m_stream, m_buffer); - m_content = reinterpret_cast(m_buffer.c_str()); - assert(m_content != nullptr); - m_start = m_cursor = m_content; - m_limit = m_content + m_buffer.size(); + // fill buffer + fill_line_buffer(); } - /// default constructor - lexer() = default; - // switch off unwanted functions + lexer() = delete; lexer(const lexer&) = delete; lexer operator=(const lexer&) = delete; @@ -7690,7 +7681,7 @@ class basic_json infinite sequence of whitespace or byte-order-marks. This contradicts the assumption of finite input, q.e.d. */ - token_type scan() noexcept + token_type scan() { while (true) { @@ -7742,7 +7733,7 @@ class basic_json }; if ((m_limit - m_cursor) < 5) { - yyfill(); // LCOV_EXCL_LINE; + fill_line_buffer(); // LCOV_EXCL_LINE; } yych = *m_cursor; if (yybm[0 + yych] & 32) @@ -7876,7 +7867,7 @@ class basic_json ++m_cursor; if (m_limit <= m_cursor) { - yyfill(); // LCOV_EXCL_LINE; + fill_line_buffer(); // LCOV_EXCL_LINE; } yych = *m_cursor; if (yybm[0 + yych] & 32) @@ -7946,7 +7937,7 @@ class basic_json m_marker = ++m_cursor; if ((m_limit - m_cursor) < 3) { - yyfill(); // LCOV_EXCL_LINE; + fill_line_buffer(); // LCOV_EXCL_LINE; } yych = *m_cursor; if (yybm[0 + yych] & 64) @@ -8039,7 +8030,7 @@ class basic_json ++m_cursor; if (m_limit <= m_cursor) { - yyfill(); // LCOV_EXCL_LINE; + fill_line_buffer(); // LCOV_EXCL_LINE; } yych = *m_cursor; basic_json_parser_32: @@ -8076,7 +8067,7 @@ class basic_json ++m_cursor; if (m_limit <= m_cursor) { - yyfill(); // LCOV_EXCL_LINE; + fill_line_buffer(); // LCOV_EXCL_LINE; } yych = *m_cursor; if (yych <= 'e') @@ -8220,7 +8211,7 @@ class basic_json ++m_cursor; if (m_limit <= m_cursor) { - yyfill(); // LCOV_EXCL_LINE; + fill_line_buffer(); // LCOV_EXCL_LINE; } yych = *m_cursor; if (yych <= '@') @@ -8256,7 +8247,7 @@ class basic_json m_marker = ++m_cursor; if ((m_limit - m_cursor) < 3) { - yyfill(); // LCOV_EXCL_LINE; + fill_line_buffer(); // LCOV_EXCL_LINE; } yych = *m_cursor; if (yych <= 'D') @@ -8297,7 +8288,7 @@ class basic_json ++m_cursor; if (m_limit <= m_cursor) { - yyfill(); // LCOV_EXCL_LINE; + fill_line_buffer(); // LCOV_EXCL_LINE; } yych = *m_cursor; if (yych <= '/') @@ -8339,7 +8330,7 @@ class basic_json ++m_cursor; if (m_limit <= m_cursor) { - yyfill(); // LCOV_EXCL_LINE; + fill_line_buffer(); // LCOV_EXCL_LINE; } yych = *m_cursor; if (yych <= '@') @@ -8393,7 +8384,7 @@ class basic_json ++m_cursor; if (m_limit <= m_cursor) { - yyfill(); // LCOV_EXCL_LINE; + fill_line_buffer(); // LCOV_EXCL_LINE; } yych = *m_cursor; if (yych <= '@') @@ -8434,7 +8425,7 @@ class basic_json ++m_cursor; if (m_limit <= m_cursor) { - yyfill(); // LCOV_EXCL_LINE; + fill_line_buffer(); // LCOV_EXCL_LINE; } yych = *m_cursor; if (yych <= '@') @@ -8472,30 +8463,57 @@ class basic_json return last_token_type; } - /// append data from the stream to the internal buffer - void yyfill() noexcept + /*! + @brief append data from the stream to the line buffer + + This function is called by the scan() function when the end of the + buffer (`m_limit`) is reached and the `m_cursor` pointer cannot be + incremented without leaving the limits of the line buffer. Note re2c + decides when to call this function. + + @pre + p p p p p p u u u u u x . . . . . . + ^ ^ ^ ^ + m_content m_start | m_limit + m_cursor + + @post + u u u u u x x x x x x x . . . . . . + ^ ^ ^ + | m_cursor m_limit + m_start + m_content + */ + void fill_line_buffer() { + // no stream is used or end of file is reached if (m_stream == nullptr or not * m_stream) { return; } + // number of processed characters (p) const auto offset_start = m_start - m_content; + // offset for m_marker wrt. to m_start const auto offset_marker = m_marker - m_start; + // number of unprocessed characters (u) const auto offset_cursor = m_cursor - m_start; - m_buffer.erase(0, static_cast(offset_start)); + // delete processed characters from line buffer + m_line_buffer.erase(0, static_cast(offset_start)); + // read next line from input stream std::string line; - assert(m_stream != nullptr); std::getline(*m_stream, line); - m_buffer += "\n" + line; // add line with newline symbol + // add line with newline symbol to the line buffer + m_line_buffer += "\n" + line; - m_content = reinterpret_cast(m_buffer.c_str()); + // set pointers + m_content = reinterpret_cast(m_line_buffer.c_str()); assert(m_content != nullptr); m_start = m_content; m_marker = m_start + offset_marker; m_cursor = m_start + offset_cursor; - m_limit = m_start + m_buffer.size() - 1; + m_limit = m_start + m_line_buffer.size() - 1; } /// return string representation of last read token @@ -8837,8 +8855,8 @@ class basic_json private: /// optional input stream std::istream* m_stream = nullptr; - /// the buffer - string_t m_buffer; + /// line buffer buffer for m_stream + string_t m_line_buffer {}; /// the buffer pointer const lexer_char_t* m_content = nullptr; /// pointer to the beginning of the current symbol @@ -8862,29 +8880,20 @@ class basic_json { public: /// a parser reading from a string literal - parser(const typename string_t::value_type* buff, parser_callback_t cb = nullptr) noexcept + parser(const typename string_t::value_type* buff, parser_callback_t cb = nullptr) : callback(cb), m_lexer(buff) - { - // read first token - get_token(); - } + {} /// a parser reading from a string container - parser(const string_t& s, const parser_callback_t cb = nullptr) noexcept + parser(const string_t& s, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(reinterpret_cast(s.c_str()), s.size()) - { - // read first token - get_token(); - } + {} /// a parser reading from an input stream - parser(std::istream& is, const parser_callback_t cb = nullptr) noexcept + parser(std::istream& is, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(is) - { - // read first token - get_token(); - } + {} /// a parser reading from a container with continguous storage template ::iterator_category, std::random_access_iterator_tag>::value , int>::type = 0> - parser(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr) noexcept + parser(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(reinterpret_cast(&(*first)), static_cast(std::distance(first, last))) - { - // read first token - get_token(); - } + {} /// public parser interface basic_json parse() { + // read first token + get_token(); + basic_json result = parse_internal(true); result.assert_invariant(); @@ -9108,7 +9117,7 @@ class basic_json } /// get next token from lexer - typename lexer::token_type get_token() noexcept + typename lexer::token_type get_token() { last_token = m_lexer.scan(); return last_token; diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index b8f646af10..7ff4e1faa9 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -37,6 +37,7 @@ SOFTWARE. #include #include #include +#include #include #include #include @@ -7506,7 +7507,7 @@ class basic_json /// a lexer from a buffer with given length lexer(const lexer_char_t* buff, const size_t len) noexcept - : m_stream(nullptr), m_buffer(), m_content(buff) + : m_content(buff) { assert(m_content != nullptr); m_start = m_cursor = m_content; @@ -7515,29 +7516,19 @@ class basic_json /// a lexer from a string literal explicit lexer(const typename string_t::value_type* buff) noexcept - : m_stream(nullptr), m_buffer(), - m_content(reinterpret_cast(buff)) - { - assert(m_content != nullptr); - m_start = m_cursor = m_content; - m_limit = m_content + strlen(buff); - } + : lexer(reinterpret_cast(buff), strlen(buff)) + {} /// a lexer from an input stream explicit lexer(std::istream& s) - : m_stream(&s), m_buffer() + : m_stream(&s), m_line_buffer() { - std::getline(*m_stream, m_buffer); - m_content = reinterpret_cast(m_buffer.c_str()); - assert(m_content != nullptr); - m_start = m_cursor = m_content; - m_limit = m_content + m_buffer.size(); + // fill buffer + fill_line_buffer(); } - /// default constructor - lexer() = default; - // switch off unwanted functions + lexer() = delete; lexer(const lexer&) = delete; lexer operator=(const lexer&) = delete; @@ -7690,7 +7681,7 @@ class basic_json infinite sequence of whitespace or byte-order-marks. This contradicts the assumption of finite input, q.e.d. */ - token_type scan() noexcept + token_type scan() { while (true) { @@ -7706,7 +7697,7 @@ class basic_json re2c:define:YYCURSOR = m_cursor; re2c:define:YYLIMIT = m_limit; re2c:define:YYMARKER = m_marker; - re2c:define:YYFILL = "yyfill(); // LCOV_EXCL_LINE"; + re2c:define:YYFILL = "fill_line_buffer(); // LCOV_EXCL_LINE"; re2c:yyfill:parameter = 0; re2c:indent:string = " "; re2c:indent:top = 1; @@ -7769,30 +7760,57 @@ class basic_json return last_token_type; } - /// append data from the stream to the internal buffer - void yyfill() noexcept + /*! + @brief append data from the stream to the line buffer + + This function is called by the scan() function when the end of the + buffer (`m_limit`) is reached and the `m_cursor` pointer cannot be + incremented without leaving the limits of the line buffer. Note re2c + decides when to call this function. + + @pre + p p p p p p u u u u u x . . . . . . + ^ ^ ^ ^ + m_content m_start | m_limit + m_cursor + + @post + u u u u u x x x x x x x . . . . . . + ^ ^ ^ + | m_cursor m_limit + m_start + m_content + */ + void fill_line_buffer() { + // no stream is used or end of file is reached if (m_stream == nullptr or not * m_stream) { return; } + // number of processed characters (p) const auto offset_start = m_start - m_content; + // offset for m_marker wrt. to m_start const auto offset_marker = m_marker - m_start; + // number of unprocessed characters (u) const auto offset_cursor = m_cursor - m_start; - m_buffer.erase(0, static_cast(offset_start)); + // delete processed characters from line buffer + m_line_buffer.erase(0, static_cast(offset_start)); + // read next line from input stream std::string line; - assert(m_stream != nullptr); std::getline(*m_stream, line); - m_buffer += "\n" + line; // add line with newline symbol + // add line with newline symbol to the line buffer + m_line_buffer += "\n" + line; - m_content = reinterpret_cast(m_buffer.c_str()); + // set pointers + m_content = reinterpret_cast(m_line_buffer.c_str()); assert(m_content != nullptr); m_start = m_content; m_marker = m_start + offset_marker; m_cursor = m_start + offset_cursor; - m_limit = m_start + m_buffer.size() - 1; + m_limit = m_start + m_line_buffer.size() - 1; } /// return string representation of last read token @@ -8134,8 +8152,8 @@ class basic_json private: /// optional input stream std::istream* m_stream = nullptr; - /// the buffer - string_t m_buffer; + /// line buffer buffer for m_stream + string_t m_line_buffer {}; /// the buffer pointer const lexer_char_t* m_content = nullptr; /// pointer to the beginning of the current symbol @@ -8159,29 +8177,20 @@ class basic_json { public: /// a parser reading from a string literal - parser(const typename string_t::value_type* buff, parser_callback_t cb = nullptr) noexcept + parser(const typename string_t::value_type* buff, parser_callback_t cb = nullptr) : callback(cb), m_lexer(buff) - { - // read first token - get_token(); - } + {} /// a parser reading from a string container - parser(const string_t& s, const parser_callback_t cb = nullptr) noexcept + parser(const string_t& s, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(reinterpret_cast(s.c_str()), s.size()) - { - // read first token - get_token(); - } + {} /// a parser reading from an input stream - parser(std::istream& is, const parser_callback_t cb = nullptr) noexcept + parser(std::istream& is, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(is) - { - // read first token - get_token(); - } + {} /// a parser reading from a container with continguous storage template ::iterator_category, std::random_access_iterator_tag>::value , int>::type = 0> - parser(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr) noexcept + parser(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(reinterpret_cast(&(*first)), static_cast(std::distance(first, last))) - { - // read first token - get_token(); - } + {} /// public parser interface basic_json parse() { + // read first token + get_token(); + basic_json result = parse_internal(true); result.assert_invariant(); @@ -8405,7 +8414,7 @@ class basic_json } /// get next token from lexer - typename lexer::token_type get_token() noexcept + typename lexer::token_type get_token() { last_token = m_lexer.scan(); return last_token; diff --git a/test/src/unit-class_parser.cpp b/test/src/unit-class_parser.cpp index c59186effa..35a76c101e 100644 --- a/test/src/unit-class_parser.cpp +++ b/test/src/unit-class_parser.cpp @@ -784,7 +784,7 @@ TEST_CASE("parser class") SECTION("from std::valarray") { - std::valarray v = {'t', 'r', 'u', 'e'}; + std::valarray v = {'t', 'r', 'u', 'e', '\0'}; CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); } } From c9e5d56c9ce209a03fd6685ab187ffcca0263956 Mon Sep 17 00:00:00 2001 From: Niels Date: Sun, 14 Aug 2016 17:30:53 +0200 Subject: [PATCH 03/18] fix for MSVC --- test/src/unit-class_parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/unit-class_parser.cpp b/test/src/unit-class_parser.cpp index 35a76c101e..c7429dd41f 100644 --- a/test/src/unit-class_parser.cpp +++ b/test/src/unit-class_parser.cpp @@ -749,7 +749,7 @@ TEST_CASE("parser class") { SECTION("from std::vector") { - std::vector v = {'t', 'r', 'u', 'e'}; + std::vector v = {'t', 'r', 'u', 'e', '\0'}; CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); } From 7b42c973bdf512e69a6dec0add21b083ac660dad Mon Sep 17 00:00:00 2001 From: Niels Date: Sun, 14 Aug 2016 17:34:58 +0200 Subject: [PATCH 04/18] fix for MSVC --- test/src/unit-class_parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/unit-class_parser.cpp b/test/src/unit-class_parser.cpp index c7429dd41f..1a4b0a7c61 100644 --- a/test/src/unit-class_parser.cpp +++ b/test/src/unit-class_parser.cpp @@ -755,7 +755,7 @@ TEST_CASE("parser class") SECTION("from std::array") { - std::array v { {'t', 'r', 'u', 'e'} }; + std::array v { {'t', 'r', 'u', 'e', '\0'} }; CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); } From 4871e3941575cec382895e171831146d78a68975 Mon Sep 17 00:00:00 2001 From: Niels Date: Sun, 14 Aug 2016 17:52:21 +0200 Subject: [PATCH 05/18] spell fix --- src/json.hpp | 2 +- src/json.hpp.re2c | 2 +- test/src/unit-class_parser.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 8342b0e1fd..90bd097fb8 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -8895,7 +8895,7 @@ class basic_json : callback(cb), m_lexer(is) {} - /// a parser reading from a container with continguous storage + /// a parser reading from a container with contiguous storage template ::iterator_category, std::random_access_iterator_tag>::value diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index 7ff4e1faa9..8a9a6c1f63 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -8192,7 +8192,7 @@ class basic_json : callback(cb), m_lexer(is) {} - /// a parser reading from a container with continguous storage + /// a parser reading from a container with contiguous storage template ::iterator_category, std::random_access_iterator_tag>::value diff --git a/test/src/unit-class_parser.cpp b/test/src/unit-class_parser.cpp index 1a4b0a7c61..863cec5206 100644 --- a/test/src/unit-class_parser.cpp +++ b/test/src/unit-class_parser.cpp @@ -745,7 +745,7 @@ TEST_CASE("parser class") } } - SECTION("constructing from continguous containers") + SECTION("constructing from contiguous containers") { SECTION("from std::vector") { From 92ee1d56ebc2df0e9a4b16c089c266d622f372d1 Mon Sep 17 00:00:00 2001 From: Niels Date: Sun, 14 Aug 2016 21:59:41 +0200 Subject: [PATCH 06/18] cleanup --- src/json.hpp | 9 ++-- src/json.hpp.re2c | 9 ++-- test/src/unit-class_lexer.cpp | 84 +++++++++++++++++++++++------------ 3 files changed, 62 insertions(+), 40 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 90bd097fb8..a968515812 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -7514,11 +7514,6 @@ class basic_json m_limit = m_content + len; } - /// a lexer from a string literal - explicit lexer(const typename string_t::value_type* buff) noexcept - : lexer(reinterpret_cast(buff), strlen(buff)) - {} - /// a lexer from an input stream explicit lexer(std::istream& s) : m_stream(&s), m_line_buffer() @@ -8881,7 +8876,9 @@ class basic_json public: /// a parser reading from a string literal parser(const typename string_t::value_type* buff, parser_callback_t cb = nullptr) - : callback(cb), m_lexer(buff) + : callback(cb), + m_lexer(reinterpret_cast(buff), + strlen(buff)) {} /// a parser reading from a string container diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index 8a9a6c1f63..82b6a123f8 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -7514,11 +7514,6 @@ class basic_json m_limit = m_content + len; } - /// a lexer from a string literal - explicit lexer(const typename string_t::value_type* buff) noexcept - : lexer(reinterpret_cast(buff), strlen(buff)) - {} - /// a lexer from an input stream explicit lexer(std::istream& s) : m_stream(&s), m_line_buffer() @@ -8178,7 +8173,9 @@ class basic_json public: /// a parser reading from a string literal parser(const typename string_t::value_type* buff, parser_callback_t cb = nullptr) - : callback(cb), m_lexer(buff) + : callback(cb), + m_lexer(reinterpret_cast(buff), + strlen(buff)) {} /// a parser reading from a string container diff --git a/test/src/unit-class_lexer.cpp b/test/src/unit-class_lexer.cpp index 708a8cbfa7..7cece21c68 100644 --- a/test/src/unit-class_lexer.cpp +++ b/test/src/unit-class_lexer.cpp @@ -38,43 +38,67 @@ TEST_CASE("lexer class") { SECTION("structural characters") { - CHECK(json::lexer("[").scan() == json::lexer::token_type::begin_array); - CHECK(json::lexer("]").scan() == json::lexer::token_type::end_array); - CHECK(json::lexer("{").scan() == json::lexer::token_type::begin_object); - CHECK(json::lexer("}").scan() == json::lexer::token_type::end_object); - CHECK(json::lexer(",").scan() == json::lexer::token_type::value_separator); - CHECK(json::lexer(":").scan() == json::lexer::token_type::name_separator); + CHECK(json::lexer(reinterpret_cast("["), + 1).scan() == json::lexer::token_type::begin_array); + CHECK(json::lexer(reinterpret_cast("]"), + 1).scan() == json::lexer::token_type::end_array); + CHECK(json::lexer(reinterpret_cast("{"), + 1).scan() == json::lexer::token_type::begin_object); + CHECK(json::lexer(reinterpret_cast("}"), + 1).scan() == json::lexer::token_type::end_object); + CHECK(json::lexer(reinterpret_cast(","), + 1).scan() == json::lexer::token_type::value_separator); + CHECK(json::lexer(reinterpret_cast(":"), + 1).scan() == json::lexer::token_type::name_separator); } SECTION("literal names") { - CHECK(json::lexer("null").scan() == json::lexer::token_type::literal_null); - CHECK(json::lexer("true").scan() == json::lexer::token_type::literal_true); - CHECK(json::lexer("false").scan() == json::lexer::token_type::literal_false); + CHECK(json::lexer(reinterpret_cast("null"), + 4).scan() == json::lexer::token_type::literal_null); + CHECK(json::lexer(reinterpret_cast("true"), + 4).scan() == json::lexer::token_type::literal_true); + CHECK(json::lexer(reinterpret_cast("false"), + 5).scan() == json::lexer::token_type::literal_false); } SECTION("numbers") { - CHECK(json::lexer("0").scan() == json::lexer::token_type::value_number); - CHECK(json::lexer("1").scan() == json::lexer::token_type::value_number); - CHECK(json::lexer("2").scan() == json::lexer::token_type::value_number); - CHECK(json::lexer("3").scan() == json::lexer::token_type::value_number); - CHECK(json::lexer("4").scan() == json::lexer::token_type::value_number); - CHECK(json::lexer("5").scan() == json::lexer::token_type::value_number); - CHECK(json::lexer("6").scan() == json::lexer::token_type::value_number); - CHECK(json::lexer("7").scan() == json::lexer::token_type::value_number); - CHECK(json::lexer("8").scan() == json::lexer::token_type::value_number); - CHECK(json::lexer("9").scan() == json::lexer::token_type::value_number); + CHECK(json::lexer(reinterpret_cast("0"), + 1).scan() == json::lexer::token_type::value_number); + CHECK(json::lexer(reinterpret_cast("1"), + 1).scan() == json::lexer::token_type::value_number); + CHECK(json::lexer(reinterpret_cast("2"), + 1).scan() == json::lexer::token_type::value_number); + CHECK(json::lexer(reinterpret_cast("3"), + 1).scan() == json::lexer::token_type::value_number); + CHECK(json::lexer(reinterpret_cast("4"), + 1).scan() == json::lexer::token_type::value_number); + CHECK(json::lexer(reinterpret_cast("5"), + 1).scan() == json::lexer::token_type::value_number); + CHECK(json::lexer(reinterpret_cast("6"), + 1).scan() == json::lexer::token_type::value_number); + CHECK(json::lexer(reinterpret_cast("7"), + 1).scan() == json::lexer::token_type::value_number); + CHECK(json::lexer(reinterpret_cast("8"), + 1).scan() == json::lexer::token_type::value_number); + CHECK(json::lexer(reinterpret_cast("9"), + 1).scan() == json::lexer::token_type::value_number); } SECTION("whitespace") { // result is end_of_input, because not token is following - CHECK(json::lexer(" ").scan() == json::lexer::token_type::end_of_input); - CHECK(json::lexer("\t").scan() == json::lexer::token_type::end_of_input); - CHECK(json::lexer("\n").scan() == json::lexer::token_type::end_of_input); - CHECK(json::lexer("\r").scan() == json::lexer::token_type::end_of_input); - CHECK(json::lexer(" \t\n\r\n\t ").scan() == json::lexer::token_type::end_of_input); + CHECK(json::lexer(reinterpret_cast(" "), + 1).scan() == json::lexer::token_type::end_of_input); + CHECK(json::lexer(reinterpret_cast("\t"), + 1).scan() == json::lexer::token_type::end_of_input); + CHECK(json::lexer(reinterpret_cast("\n"), + 1).scan() == json::lexer::token_type::end_of_input); + CHECK(json::lexer(reinterpret_cast("\r"), + 1).scan() == json::lexer::token_type::end_of_input); + CHECK(json::lexer(reinterpret_cast(" \t\n\r\n\t "), + 7).scan() == json::lexer::token_type::end_of_input); } } @@ -100,7 +124,11 @@ TEST_CASE("lexer class") { for (int c = 1; c < 128; ++c) { - auto s = std::string(1, c); + // create string from the ASCII code + const auto s = std::string(1, c); + // store scan() result + const auto res = json::lexer(reinterpret_cast(s.c_str()), + 1).scan(); switch (c) { @@ -122,7 +150,7 @@ TEST_CASE("lexer class") case ('8'): case ('9'): { - CHECK(json::lexer(s.c_str()).scan() != json::lexer::token_type::parse_error); + CHECK(res != json::lexer::token_type::parse_error); break; } @@ -132,14 +160,14 @@ TEST_CASE("lexer class") case ('\n'): case ('\r'): { - CHECK(json::lexer(s.c_str()).scan() == json::lexer::token_type::end_of_input); + CHECK(res == json::lexer::token_type::end_of_input); break; } // anything else is not expected default: { - CHECK(json::lexer(s.c_str()).scan() == json::lexer::token_type::parse_error); + CHECK(res == json::lexer::token_type::parse_error); break; } } From dfc2c1abe56a74a498ddac72f107ce2e60b78674 Mon Sep 17 00:00:00 2001 From: Niels Date: Sun, 14 Aug 2016 23:38:20 +0200 Subject: [PATCH 07/18] added assertion for contiguous memory --- src/json.hpp | 8 +++++++- src/json.hpp.re2c | 8 +++++++- test/src/unit-class_parser.cpp | 14 +++++++------- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index a968515812..c2542682d1 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -8902,7 +8902,13 @@ class basic_json : callback(cb), m_lexer(reinterpret_cast(&(*first)), static_cast(std::distance(first, last))) - {} + { + int i = 0; + assert(std::accumulate(first, last, true, [&i, &first](bool res, decltype(*first) val) + { + return res and (val == *(std::next(std::addressof(*first), i++))); + })); + } /// public parser interface basic_json parse() diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index 82b6a123f8..20eb6e5898 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -8199,7 +8199,13 @@ class basic_json : callback(cb), m_lexer(reinterpret_cast(&(*first)), static_cast(std::distance(first, last))) - {} + { + int i = 0; + assert(std::accumulate(first, last, true, [&i, &first](bool res, decltype(*first) val) + { + return res and (val == *(std::next(std::addressof(*first), i++))); + })); + } /// public parser interface basic_json parse() diff --git a/test/src/unit-class_parser.cpp b/test/src/unit-class_parser.cpp index 863cec5206..b6d6a6157f 100644 --- a/test/src/unit-class_parser.cpp +++ b/test/src/unit-class_parser.cpp @@ -750,42 +750,42 @@ TEST_CASE("parser class") SECTION("from std::vector") { std::vector v = {'t', 'r', 'u', 'e', '\0'}; - CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); + CHECK(json::parser(std::begin(v), std::end(v)).parse() == json(true)); } SECTION("from std::array") { std::array v { {'t', 'r', 'u', 'e', '\0'} }; - CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); + CHECK(json::parser(std::begin(v), std::end(v)).parse() == json(true)); } SECTION("from array") { uint8_t v[] = {'t', 'r', 'u', 'e'}; - CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); + CHECK(json::parser(std::begin(v), std::end(v)).parse() == json(true)); } SECTION("from char literal") { - CHECK (json::parser("true").parse() == json(true)); + CHECK(json::parser("true").parse() == json(true)); } SECTION("from std::string") { std::string v = {'t', 'r', 'u', 'e'}; - CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); + CHECK(json::parser(std::begin(v), std::end(v)).parse() == json(true)); } SECTION("from std::initializer_list") { std::initializer_list v = {'t', 'r', 'u', 'e', '\0'}; - CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); + CHECK(json::parser(std::begin(v), std::end(v)).parse() == json(true)); } SECTION("from std::valarray") { std::valarray v = {'t', 'r', 'u', 'e', '\0'}; - CHECK (json::parser(std::begin(v), std::end(v)).parse() == json(true)); + CHECK(json::parser(std::begin(v), std::end(v)).parse() == json(true)); } } } From 5e67f7af01440b6f91e4aae5bfc792202f6cec8f Mon Sep 17 00:00:00 2001 From: Niels Date: Mon, 15 Aug 2016 22:44:14 +0200 Subject: [PATCH 08/18] added a first version of a parser for #290 --- src/json.hpp | 93 +++++++++++++++++++++++++++---- src/json.hpp.re2c | 93 +++++++++++++++++++++++++++---- test/src/unit-deserialization.cpp | 50 ++++++++++++++++- 3 files changed, 215 insertions(+), 21 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 03c162533b..6bafd00720 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -5962,6 +5962,16 @@ class basic_json return parser(s, cb).parse(); } + /*! + @brief deserialize from string literal + @copydoc parse(const string_t&, const parser_callback_t) + */ + static basic_json parse(const typename string_t::value_type* s, + const parser_callback_t cb = nullptr) + { + return parser(s, cb).parse(); + } + /*! @brief deserialize from stream @@ -6001,6 +6011,75 @@ class basic_json return parser(i, cb).parse(); } + /*! + @brief deserialize from a container with contiguous storage + + This function reads from a nonempty iterator range of a container with + contiguous storage of 1-byte values. Compatible container types include + `std::vector`, `std::string`, `std::array`, `std::valarray`, and + `std::initializer_list`. Furthermore, C-style arrays can be used with + `std::begin()`/`std::end()`. User-defined containers can be used as long + as they implement random-access iterators and a contiguous storage. + + @pre The iterator range is contiguous. Violating this precondition yields + undefined behavior. **This precondition is enforced with an assertion.** + @pre Each element in the range has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with an assertion.** + @pre The iterator range is nonempty. Violating this precondition yields + undefined behavior. **This precondition is enforced with an assertion.** + + @warning There is no way to enforce the preconditions at compile-time. If + the function is called with noncompliant iterators, the behavior + is undefined and will most liekely yield segmentation violation. + + @param[in] first begin of the range to parse (included) + @param[in] last end of the range to parse (excluded) + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @todo Example and references. + + @since version 2.0.3 + */ + template ::iterator_category, std::random_access_iterator_tag>::value + , int>::type + = 0> + static basic_json parse(IteratorType first, IteratorType last, + const parser_callback_t cb = nullptr) + { + // assertion to check that the iterator range is indeed contiguous, + // see http://stackoverflow.com/a/35008842/266378 for more discussion + assert(std::accumulate(first, last, std::make_pair(true, 0), + [&first](std::pair res, decltype(*first) val) + { + res.first &= (val == *(std::next(std::addressof(*first), res.second++))); + return res; + }).first); + + // assertion to check that each element is 1 byte long + assert(std::all_of(first, last, [](decltype(*first) val) + { + return sizeof(val) == 1; + })); + + // assertion that the iterator range is not empty + assert(std::distance(first, last) > 0); + + return parser(first, last, cb).parse(); + } + /*! @brief deserialize from stream @@ -8875,10 +8954,10 @@ class basic_json { public: /// a parser reading from a string literal - parser(const typename string_t::value_type* buff, parser_callback_t cb = nullptr) + parser(const typename string_t::value_type* buff, + const parser_callback_t cb = nullptr) : callback(cb), - m_lexer(reinterpret_cast(buff), - strlen(buff)) + m_lexer(reinterpret_cast(buff), strlen(buff)) {} /// a parser reading from a string container @@ -8902,13 +8981,7 @@ class basic_json : callback(cb), m_lexer(reinterpret_cast(&(*first)), static_cast(std::distance(first, last))) - { - int i = 0; - assert(std::accumulate(first, last, true, [&i, &first](bool res, decltype(*first) val) - { - return res and (val == *(std::next(std::addressof(*first), i++))); - })); - } + {} /// public parser interface basic_json parse() diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index 58724e9110..87f7aabcf6 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -5962,6 +5962,16 @@ class basic_json return parser(s, cb).parse(); } + /*! + @brief deserialize from string literal + @copydoc parse(const string_t&, const parser_callback_t) + */ + static basic_json parse(const typename string_t::value_type* s, + const parser_callback_t cb = nullptr) + { + return parser(s, cb).parse(); + } + /*! @brief deserialize from stream @@ -6001,6 +6011,75 @@ class basic_json return parser(i, cb).parse(); } + /*! + @brief deserialize from a container with contiguous storage + + This function reads from a nonempty iterator range of a container with + contiguous storage of 1-byte values. Compatible container types include + `std::vector`, `std::string`, `std::array`, `std::valarray`, and + `std::initializer_list`. Furthermore, C-style arrays can be used with + `std::begin()`/`std::end()`. User-defined containers can be used as long + as they implement random-access iterators and a contiguous storage. + + @pre The iterator range is contiguous. Violating this precondition yields + undefined behavior. **This precondition is enforced with an assertion.** + @pre Each element in the range has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with an assertion.** + @pre The iterator range is nonempty. Violating this precondition yields + undefined behavior. **This precondition is enforced with an assertion.** + + @warning There is no way to enforce the preconditions at compile-time. If + the function is called with noncompliant iterators, the behavior + is undefined and will most liekely yield segmentation violation. + + @param[in] first begin of the range to parse (included) + @param[in] last end of the range to parse (excluded) + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @todo Example and references. + + @since version 2.0.3 + */ + template ::iterator_category, std::random_access_iterator_tag>::value + , int>::type + = 0> + static basic_json parse(IteratorType first, IteratorType last, + const parser_callback_t cb = nullptr) + { + // assertion to check that the iterator range is indeed contiguous, + // see http://stackoverflow.com/a/35008842/266378 for more discussion + assert(std::accumulate(first, last, std::make_pair(true, 0), + [&first](std::pair res, decltype(*first) val) + { + res.first &= (val == *(std::next(std::addressof(*first), res.second++))); + return res; + }).first); + + // assertion to check that each element is 1 byte long + assert(std::all_of(first, last, [](decltype(*first) val) + { + return sizeof(val) == 1; + })); + + // assertion that the iterator range is not empty + assert(std::distance(first, last) > 0); + + return parser(first, last, cb).parse(); + } + /*! @brief deserialize from stream @@ -8172,10 +8251,10 @@ class basic_json { public: /// a parser reading from a string literal - parser(const typename string_t::value_type* buff, parser_callback_t cb = nullptr) + parser(const typename string_t::value_type* buff, + const parser_callback_t cb = nullptr) : callback(cb), - m_lexer(reinterpret_cast(buff), - strlen(buff)) + m_lexer(reinterpret_cast(buff), strlen(buff)) {} /// a parser reading from a string container @@ -8199,13 +8278,7 @@ class basic_json : callback(cb), m_lexer(reinterpret_cast(&(*first)), static_cast(std::distance(first, last))) - { - int i = 0; - assert(std::accumulate(first, last, true, [&i, &first](bool res, decltype(*first) val) - { - return res and (val == *(std::next(std::addressof(*first), i++))); - })); - } + {} /// public parser interface basic_json parse() diff --git a/test/src/unit-deserialization.cpp b/test/src/unit-deserialization.cpp index 7818857460..0e5e212293 100644 --- a/test/src/unit-deserialization.cpp +++ b/test/src/unit-deserialization.cpp @@ -31,6 +31,8 @@ SOFTWARE. #include "json.hpp" using nlohmann::json; +#include + TEST_CASE("deserialization") { SECTION("stream") @@ -41,13 +43,20 @@ TEST_CASE("deserialization") CHECK(j == json({"foo", 1, 2, 3, false, {{"one", 1}}})); } - SECTION("string") + SECTION("string literal") { auto s = "[\"foo\",1,2,3,false,{\"one\":1}]"; json j = json::parse(s); CHECK(j == json({"foo", 1, 2, 3, false, {{"one", 1}}})); } + SECTION("string_t") + { + json::string_t s = "[\"foo\",1,2,3,false,{\"one\":1}]"; + json j = json::parse(s); + CHECK(j == json({"foo", 1, 2, 3, false, {{"one", 1}}})); + } + SECTION("operator<<") { std::stringstream ss; @@ -70,4 +79,43 @@ TEST_CASE("deserialization") { CHECK("[\"foo\",1,2,3,false,{\"one\":1}]"_json == json({"foo", 1, 2, 3, false, {{"one", 1}}})); } + + SECTION("contiguous containers") + { + SECTION("from std::vector") + { + std::vector v = {'t', 'r', 'u', 'e', '\0'}; + CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + } + + SECTION("from std::array") + { + std::array v { {'t', 'r', 'u', 'e', '\0'} }; + CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + } + + SECTION("from array") + { + uint8_t v[] = {'t', 'r', 'u', 'e'}; + CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + } + + SECTION("from std::string") + { + std::string v = {'t', 'r', 'u', 'e'}; + CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + } + + SECTION("from std::initializer_list") + { + std::initializer_list v = {'t', 'r', 'u', 'e', '\0'}; + CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + } + + SECTION("from std::valarray") + { + std::valarray v = {'t', 'r', 'u', 'e', '\0'}; + CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + } + } } From a78eaa27b5bfdc000ef97391f6664cb317e5cbc9 Mon Sep 17 00:00:00 2001 From: Niels Date: Wed, 17 Aug 2016 21:31:59 +0200 Subject: [PATCH 09/18] fixed unit tests --- test/src/unit-class_parser.cpp | 2 +- test/src/unit-deserialization.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/src/unit-class_parser.cpp b/test/src/unit-class_parser.cpp index b6d6a6157f..775952b926 100644 --- a/test/src/unit-class_parser.cpp +++ b/test/src/unit-class_parser.cpp @@ -761,7 +761,7 @@ TEST_CASE("parser class") SECTION("from array") { - uint8_t v[] = {'t', 'r', 'u', 'e'}; + uint8_t v[] = {'t', 'r', 'u', 'e', '\0'}; CHECK(json::parser(std::begin(v), std::end(v)).parse() == json(true)); } diff --git a/test/src/unit-deserialization.cpp b/test/src/unit-deserialization.cpp index 0e5e212293..965d1d0752 100644 --- a/test/src/unit-deserialization.cpp +++ b/test/src/unit-deserialization.cpp @@ -96,7 +96,7 @@ TEST_CASE("deserialization") SECTION("from array") { - uint8_t v[] = {'t', 'r', 'u', 'e'}; + uint8_t v[] = {'t', 'r', 'u', 'e', '\0'}; CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); } From eef80590037f0aa4d1d100ad4495378082954849 Mon Sep 17 00:00:00 2001 From: Niels Date: Sat, 20 Aug 2016 20:29:33 +0200 Subject: [PATCH 10/18] allowing parsing from contiguous containers --- src/json.hpp | 51 +++++++++++---- src/json.hpp.re2c | 51 +++++++++++---- test/src/unit-deserialization.cpp | 102 +++++++++++++++++++++--------- 3 files changed, 148 insertions(+), 56 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index f10e5960a3..8a1c9a7770 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -5956,11 +5956,13 @@ class basic_json @since version 1.0.0 */ + /* static basic_json parse(const string_t& s, const parser_callback_t cb = nullptr) { return parser(s, cb).parse(); } + */ /*! @brief deserialize from string literal @@ -6012,10 +6014,10 @@ class basic_json } /*! - @brief deserialize from a container with contiguous storage + @brief deserialize from a iterator range with contiguous storage - This function reads from a nonempty iterator range of a container with - contiguous storage of 1-byte values. Compatible container types include + This function reads from an iterator range of a container with contiguous + storage of 1-byte values. Compatible container types include `std::vector`, `std::string`, `std::array`, `std::valarray`, and `std::initializer_list`. Furthermore, C-style arrays can be used with `std::begin()`/`std::end()`. User-defined containers can be used as long @@ -6025,9 +6027,7 @@ class basic_json undefined behavior. **This precondition is enforced with an assertion.** @pre Each element in the range has a size of 1 byte. Violating this precondition yields undefined behavior. **This precondition is enforced - with an assertion.** - @pre The iterator range is nonempty. Violating this precondition yields - undefined behavior. **This precondition is enforced with an assertion.** + with a static assertion.** @warning There is no way to enforce the preconditions at compile-time. If the function is called with noncompliant iterators, the behavior @@ -6053,7 +6053,9 @@ class basic_json */ template ::iterator_category, std::random_access_iterator_tag>::value + std::is_base_of< + std::random_access_iterator_tag, + typename std::iterator_traits::iterator_category>::value , int>::type = 0> static basic_json parse(IteratorType first, IteratorType last, @@ -6069,17 +6071,40 @@ class basic_json }).first); // assertion to check that each element is 1 byte long - assert(std::all_of(first, last, [](decltype(*first) val) - { - return sizeof(val) == 1; - })); + static_assert(sizeof(typename std::iterator_traits::value_type) == 1, + "each element in the iterator range must have the size of 1 byte"); - // assertion that the iterator range is not empty - assert(std::distance(first, last) > 0); + // if iterator range is empty, create a parser with an empty string + // to generate "unexpected EOF" error message + if (std::distance(first, last) <= 0) + { + return parser("").parse(); + } return parser(first, last, cb).parse(); } + template()))>::iterator_category>::value + , int>::type = 0> + static basic_json parse(const ContiguousContainer& c, + const parser_callback_t cb = nullptr) + { + // delegate the call to the iterator-range parse overload + return parse(std::begin(c), std::end(c), cb); + } + + template + static basic_json parse(T (&array)[N], + const parser_callback_t cb = nullptr) + { + // delegate the call to the iterator-range parse overload + return parse(std::begin(array), std::end(array), cb); + } + /*! @brief deserialize from stream diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index b1f5a6334c..cb51f557ca 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -5956,11 +5956,13 @@ class basic_json @since version 1.0.0 */ + /* static basic_json parse(const string_t& s, const parser_callback_t cb = nullptr) { return parser(s, cb).parse(); } + */ /*! @brief deserialize from string literal @@ -6012,10 +6014,10 @@ class basic_json } /*! - @brief deserialize from a container with contiguous storage + @brief deserialize from a iterator range with contiguous storage - This function reads from a nonempty iterator range of a container with - contiguous storage of 1-byte values. Compatible container types include + This function reads from an iterator range of a container with contiguous + storage of 1-byte values. Compatible container types include `std::vector`, `std::string`, `std::array`, `std::valarray`, and `std::initializer_list`. Furthermore, C-style arrays can be used with `std::begin()`/`std::end()`. User-defined containers can be used as long @@ -6025,9 +6027,7 @@ class basic_json undefined behavior. **This precondition is enforced with an assertion.** @pre Each element in the range has a size of 1 byte. Violating this precondition yields undefined behavior. **This precondition is enforced - with an assertion.** - @pre The iterator range is nonempty. Violating this precondition yields - undefined behavior. **This precondition is enforced with an assertion.** + with a static assertion.** @warning There is no way to enforce the preconditions at compile-time. If the function is called with noncompliant iterators, the behavior @@ -6053,7 +6053,9 @@ class basic_json */ template ::iterator_category, std::random_access_iterator_tag>::value + std::is_base_of< + std::random_access_iterator_tag, + typename std::iterator_traits::iterator_category>::value , int>::type = 0> static basic_json parse(IteratorType first, IteratorType last, @@ -6069,17 +6071,40 @@ class basic_json }).first); // assertion to check that each element is 1 byte long - assert(std::all_of(first, last, [](decltype(*first) val) - { - return sizeof(val) == 1; - })); + static_assert(sizeof(typename std::iterator_traits::value_type) == 1, + "each element in the iterator range must have the size of 1 byte"); - // assertion that the iterator range is not empty - assert(std::distance(first, last) > 0); + // if iterator range is empty, create a parser with an empty string + // to generate "unexpected EOF" error message + if (std::distance(first, last) <= 0) + { + return parser("").parse(); + } return parser(first, last, cb).parse(); } + template()))>::iterator_category>::value + , int>::type = 0> + static basic_json parse(const ContiguousContainer& c, + const parser_callback_t cb = nullptr) + { + // delegate the call to the iterator-range parse overload + return parse(std::begin(c), std::end(c), cb); + } + + template + static basic_json parse(T (&array)[N], + const parser_callback_t cb = nullptr) + { + // delegate the call to the iterator-range parse overload + return parse(std::begin(array), std::end(array), cb); + } + /*! @brief deserialize from stream diff --git a/test/src/unit-deserialization.cpp b/test/src/unit-deserialization.cpp index 965d1d0752..b681e3df36 100644 --- a/test/src/unit-deserialization.cpp +++ b/test/src/unit-deserialization.cpp @@ -82,40 +82,82 @@ TEST_CASE("deserialization") SECTION("contiguous containers") { - SECTION("from std::vector") + SECTION("directly") { - std::vector v = {'t', 'r', 'u', 'e', '\0'}; - CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + SECTION("from std::vector") + { + std::vector v = {'t', 'r', 'u', 'e', '\0'}; + CHECK(json::parse(v) == json(true)); + } + + SECTION("from std::array") + { + std::array v { {'t', 'r', 'u', 'e', '\0'} }; + CHECK(json::parse(v) == json(true)); + } + + SECTION("from array") + { + uint8_t v[] = {'t', 'r', 'u', 'e', '\0'}; + CHECK(json::parse(v) == json(true)); + } + + SECTION("from std::string") + { + std::string v = {'t', 'r', 'u', 'e'}; + CHECK(json::parse(v) == json(true)); + } + + SECTION("from std::initializer_list") + { + std::initializer_list v = {'t', 'r', 'u', 'e', '\0'}; + CHECK(json::parse(v) == json(true)); + } } - SECTION("from std::array") + SECTION("via iterator range") { - std::array v { {'t', 'r', 'u', 'e', '\0'} }; - CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); - } - - SECTION("from array") - { - uint8_t v[] = {'t', 'r', 'u', 'e', '\0'}; - CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); - } - - SECTION("from std::string") - { - std::string v = {'t', 'r', 'u', 'e'}; - CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); - } - - SECTION("from std::initializer_list") - { - std::initializer_list v = {'t', 'r', 'u', 'e', '\0'}; - CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); - } - - SECTION("from std::valarray") - { - std::valarray v = {'t', 'r', 'u', 'e', '\0'}; - CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + SECTION("from std::vector") + { + std::vector v = {'t', 'r', 'u', 'e', '\0'}; + CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + } + + SECTION("from std::array") + { + std::array v { {'t', 'r', 'u', 'e', '\0'} }; + CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + } + + SECTION("from array") + { + uint8_t v[] = {'t', 'r', 'u', 'e', '\0'}; + CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + } + + SECTION("from std::string") + { + std::string v = {'t', 'r', 'u', 'e'}; + CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + } + + SECTION("from std::initializer_list") + { + std::initializer_list v = {'t', 'r', 'u', 'e', '\0'}; + CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + } + + SECTION("from std::valarray") + { + std::valarray v = {'t', 'r', 'u', 'e', '\0'}; + CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); + } + + SECTION("with empty range") + { + std::vector v; + CHECK_THROWS_AS(json::parse(std::begin(v), std::end(v)), std::invalid_argument); + } } } } From b4571360df0294f7e824aced8e13f062f3414493 Mon Sep 17 00:00:00 2001 From: Niels Date: Sun, 21 Aug 2016 12:35:40 +0200 Subject: [PATCH 11/18] more on #290 --- README.md | 2 +- .../parse__array__parser_callback_t.cpp | 28 ++++ .../parse__array__parser_callback_t.link | 1 + .../parse__array__parser_callback_t.output | 20 +++ ...contiguouscontainer__parser_callback_t.cpp | 13 ++ ...ontiguouscontainer__parser_callback_t.link | 1 + ...tiguouscontainer__parser_callback_t.output | 6 + ...parse__iteratortype__parser_callback_t.cpp | 13 ++ ...arse__iteratortype__parser_callback_t.link | 1 + ...se__iteratortype__parser_callback_t.output | 6 + .../parse__string__parser_callback_t.cpp | 4 +- .../parse__string__parser_callback_t.link | 2 +- src/json.hpp | 133 ++++++++++++------ src/json.hpp.re2c | 133 ++++++++++++------ test/src/unit-class_parser.cpp | 32 ++--- test/src/unit-deserialization.cpp | 6 + 16 files changed, 299 insertions(+), 102 deletions(-) create mode 100644 doc/examples/parse__array__parser_callback_t.cpp create mode 100644 doc/examples/parse__array__parser_callback_t.link create mode 100644 doc/examples/parse__array__parser_callback_t.output create mode 100644 doc/examples/parse__contiguouscontainer__parser_callback_t.cpp create mode 100644 doc/examples/parse__contiguouscontainer__parser_callback_t.link create mode 100644 doc/examples/parse__contiguouscontainer__parser_callback_t.output create mode 100644 doc/examples/parse__iteratortype__parser_callback_t.cpp create mode 100644 doc/examples/parse__iteratortype__parser_callback_t.link create mode 100644 doc/examples/parse__iteratortype__parser_callback_t.output diff --git a/README.md b/README.md index 3015e860ac..9893fedd24 100644 --- a/README.md +++ b/README.md @@ -511,7 +511,7 @@ To compile and run the tests, you need to execute $ make check =============================================================================== -All tests passed (8905099 assertions in 32 test cases) +All tests passed (8905119 assertions in 32 test cases) ``` For more information, have a look at the file [.travis.yml](https://github.com/nlohmann/json/blob/master/.travis.yml). diff --git a/doc/examples/parse__array__parser_callback_t.cpp b/doc/examples/parse__array__parser_callback_t.cpp new file mode 100644 index 0000000000..8e086d2003 --- /dev/null +++ b/doc/examples/parse__array__parser_callback_t.cpp @@ -0,0 +1,28 @@ +#include + +using json = nlohmann::json; + +int main() +{ + // a JSON text + char text[] = R"( + { + "Image": { + "Width": 800, + "Height": 600, + "Title": "View from 15th Floor", + "Thumbnail": { + "Url": "http://www.example.com/image/481989943", + "Height": 125, + "Width": 100 + }, + "Animated" : false, + "IDs": [116, 943, 234, 38793] + } + } + )"; + + // parse and serialize JSON + json j_complete = json::parse(text); + std::cout << std::setw(4) << j_complete << "\n\n"; +} diff --git a/doc/examples/parse__array__parser_callback_t.link b/doc/examples/parse__array__parser_callback_t.link new file mode 100644 index 0000000000..a1d3cd3498 --- /dev/null +++ b/doc/examples/parse__array__parser_callback_t.link @@ -0,0 +1 @@ +online \ No newline at end of file diff --git a/doc/examples/parse__array__parser_callback_t.output b/doc/examples/parse__array__parser_callback_t.output new file mode 100644 index 0000000000..62bb85863e --- /dev/null +++ b/doc/examples/parse__array__parser_callback_t.output @@ -0,0 +1,20 @@ +{ + "Image": { + "Animated": false, + "Height": 600, + "IDs": [ + 116, + 943, + 234, + 38793 + ], + "Thumbnail": { + "Height": 125, + "Url": "http://www.example.com/image/481989943", + "Width": 100 + }, + "Title": "View from 15th Floor", + "Width": 800 + } +} + diff --git a/doc/examples/parse__contiguouscontainer__parser_callback_t.cpp b/doc/examples/parse__contiguouscontainer__parser_callback_t.cpp new file mode 100644 index 0000000000..5a339079fa --- /dev/null +++ b/doc/examples/parse__contiguouscontainer__parser_callback_t.cpp @@ -0,0 +1,13 @@ +#include + +using json = nlohmann::json; + +int main() +{ + // a JSON text given as std::vector + std::vector text = {'[', '1', ',', '2', ',', '3', ']', '\0'}; + + // parse and serialize JSON + json j_complete = json::parse(text); + std::cout << std::setw(4) << j_complete << "\n\n"; +} diff --git a/doc/examples/parse__contiguouscontainer__parser_callback_t.link b/doc/examples/parse__contiguouscontainer__parser_callback_t.link new file mode 100644 index 0000000000..57d6dc3a94 --- /dev/null +++ b/doc/examples/parse__contiguouscontainer__parser_callback_t.link @@ -0,0 +1 @@ +online \ No newline at end of file diff --git a/doc/examples/parse__contiguouscontainer__parser_callback_t.output b/doc/examples/parse__contiguouscontainer__parser_callback_t.output new file mode 100644 index 0000000000..74633e8086 --- /dev/null +++ b/doc/examples/parse__contiguouscontainer__parser_callback_t.output @@ -0,0 +1,6 @@ +[ + 1, + 2, + 3 +] + diff --git a/doc/examples/parse__iteratortype__parser_callback_t.cpp b/doc/examples/parse__iteratortype__parser_callback_t.cpp new file mode 100644 index 0000000000..3f723c5faa --- /dev/null +++ b/doc/examples/parse__iteratortype__parser_callback_t.cpp @@ -0,0 +1,13 @@ +#include + +using json = nlohmann::json; + +int main() +{ + // a JSON text given as std::vector + std::vector text = {'[', '1', ',', '2', ',', '3', ']', '\0'}; + + // parse and serialize JSON + json j_complete = json::parse(text.begin(), text.end()); + std::cout << std::setw(4) << j_complete << "\n\n"; +} diff --git a/doc/examples/parse__iteratortype__parser_callback_t.link b/doc/examples/parse__iteratortype__parser_callback_t.link new file mode 100644 index 0000000000..63f58fe645 --- /dev/null +++ b/doc/examples/parse__iteratortype__parser_callback_t.link @@ -0,0 +1 @@ +online \ No newline at end of file diff --git a/doc/examples/parse__iteratortype__parser_callback_t.output b/doc/examples/parse__iteratortype__parser_callback_t.output new file mode 100644 index 0000000000..74633e8086 --- /dev/null +++ b/doc/examples/parse__iteratortype__parser_callback_t.output @@ -0,0 +1,6 @@ +[ + 1, + 2, + 3 +] + diff --git a/doc/examples/parse__string__parser_callback_t.cpp b/doc/examples/parse__string__parser_callback_t.cpp index 62982ca6f2..0a4f3b5392 100644 --- a/doc/examples/parse__string__parser_callback_t.cpp +++ b/doc/examples/parse__string__parser_callback_t.cpp @@ -5,7 +5,7 @@ using json = nlohmann::json; int main() { // a JSON text - std::string text = R"( + auto text = R"( { "Image": { "Width": 800, @@ -44,4 +44,4 @@ int main() // parse (with callback) and serialize JSON json j_filtered = json::parse(text, cb); std::cout << std::setw(4) << j_filtered << '\n'; -} \ No newline at end of file +} diff --git a/doc/examples/parse__string__parser_callback_t.link b/doc/examples/parse__string__parser_callback_t.link index 1ad3b7199c..292046b6b9 100644 --- a/doc/examples/parse__string__parser_callback_t.link +++ b/doc/examples/parse__string__parser_callback_t.link @@ -1 +1 @@ -online \ No newline at end of file +online \ No newline at end of file diff --git a/src/json.hpp b/src/json.hpp index 8a1c9a7770..0675a9399b 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -951,7 +951,7 @@ class basic_json With a parser callback function, the result of parsing a JSON text can be influenced. When passed to @ref parse(std::istream&, const - parser_callback_t) or @ref parse(const string_t&, const parser_callback_t), + parser_callback_t) or @ref parse(const char*, const parser_callback_t), it is called on certain events (passed as @ref parse_event_t via parameter @a event) with a set recursion depth @a depth and context JSON value @a parsed. The return value of the callback function is a boolean @@ -994,7 +994,7 @@ class basic_json skipped completely or replaced by an empty discarded object. @sa @ref parse(std::istream&, parser_callback_t) or - @ref parse(const string_t&, parser_callback_t) for examples + @ref parse(const char*, parser_callback_t) for examples @since version 1.0.0 */ @@ -5933,9 +5933,9 @@ class basic_json /// @{ /*! - @brief deserialize from string + @brief deserialize from string literal - @param[in] s string to read a serialized JSON value from + @param[in] s string literal to read a serialized JSON value from @param[in] cb a parser callback function of type @ref parser_callback_t which is used to control the deserialization by filtering unwanted values (optional) @@ -5947,6 +5947,8 @@ class basic_json @a cb has a super-linear complexity. @note A UTF-8 byte order mark is silently ignored. + @note String containers like `std::string` or @ref string_t can be parsed + with @ref parse(const ContiguousContainer&, const parser_callback_t) @liveexample{The example below demonstrates the `parse()` function with and without callback function.,parse__string__parser_callback_t} @@ -5954,24 +5956,47 @@ class basic_json @sa @ref parse(std::istream&, const parser_callback_t) for a version that reads from an input stream - @since version 1.0.0 + @since version 1.0.0 (originally for @ref string_t) */ - /* - static basic_json parse(const string_t& s, + static basic_json parse(const char* s, const parser_callback_t cb = nullptr) { return parser(s, cb).parse(); } - */ /*! - @brief deserialize from string literal - @copydoc parse(const string_t&, const parser_callback_t) + @brief deserialize from an array + + This function reads from an array of 1-byte values. + + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @param[in] array array to read from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from an array.,parse__array__parser_callback_t} + + @since version 2.0.3 */ - static basic_json parse(const typename string_t::value_type* s, + template + static basic_json parse(T (&array)[N], const parser_callback_t cb = nullptr) { - return parser(s, cb).parse(); + // delegate the call to the iterator-range parse overload + return parse(std::begin(array), std::end(array), cb); } /*! @@ -5993,7 +6018,7 @@ class basic_json @liveexample{The example below demonstrates the `parse()` function with and without callback function.,parse__istream__parser_callback_t} - @sa @ref parse(const string_t&, const parser_callback_t) for a version + @sa @ref parse(const char*, const parser_callback_t) for a version that reads from a string @since version 1.0.0 @@ -6014,7 +6039,7 @@ class basic_json } /*! - @brief deserialize from a iterator range with contiguous storage + @brief deserialize from an iterator range with contiguous storage This function reads from an iterator range of a container with contiguous storage of 1-byte values. Compatible container types include @@ -6029,13 +6054,14 @@ class basic_json precondition yields undefined behavior. **This precondition is enforced with a static assertion.** - @warning There is no way to enforce the preconditions at compile-time. If - the function is called with noncompliant iterators, the behavior - is undefined and will most liekely yield segmentation violation. + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with noncompliant iterators and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. - @param[in] first begin of the range to parse (included) - @param[in] last end of the range to parse (excluded) - @param[in] cb a parser callback function of type @ref parser_callback_t + @param[in] first begin of the range to parse (included) + @param[in] last end of the range to parse (excluded) + @param[in] cb a parser callback function of type @ref parser_callback_t which is used to control the deserialization by filtering unwanted values (optional) @@ -6047,7 +6073,8 @@ class basic_json @note A UTF-8 byte order mark is silently ignored. - @todo Example and references. + @liveexample{The example below demonstrates the `parse()` function reading + from an iterator range.,parse__iteratortype__parser_callback_t} @since version 2.0.3 */ @@ -6084,6 +6111,45 @@ class basic_json return parser(first, last, cb).parse(); } + /*! + @brief deserialize from a container with contiguous storage + + This function reads from a container with contiguous storage of 1-byte + values. Compatible container types include `std::vector`, `std::string`, + `std::array`, and `std::initializer_list`. User-defined containers can be + used as long as they implement random-access iterators and a contiguous + storage. + + @pre The container storage is contiguous. Violating this precondition + yields undefined behavior. **This precondition is enforced with an + assertion.** + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with a noncompliant container and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. + + @param[in] c container to read from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from a contiguous container.,parse__contiguouscontainer__parser_callback_t} + + @since version 2.0.3 + */ template - static basic_json parse(T (&array)[N], - const parser_callback_t cb = nullptr) - { - // delegate the call to the iterator-range parse overload - return parse(std::begin(array), std::end(array), cb); - } - /*! @brief deserialize from stream @@ -6158,7 +6216,7 @@ class basic_json Returns the type name as string to be used in error messages - usually to indicate that a function was called on a wrong JSON type. - @return basically a string representation of a the @ref m_type member + @return basically a string representation of a the @a m_type member @complexity Constant. @@ -7626,7 +7684,7 @@ class basic_json fill_line_buffer(); } - // switch off unwanted functions + // switch off unwanted functions (due to pointer members) lexer() = delete; lexer(const lexer&) = delete; lexer operator=(const lexer&) = delete; @@ -8979,24 +9037,17 @@ class basic_json { public: /// a parser reading from a string literal - parser(const typename string_t::value_type* buff, - const parser_callback_t cb = nullptr) + parser(const char* buff, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(reinterpret_cast(buff), strlen(buff)) {} - /// a parser reading from a string container - parser(const string_t& s, const parser_callback_t cb = nullptr) - : callback(cb), - m_lexer(reinterpret_cast(s.c_str()), s.size()) - {} - /// a parser reading from an input stream parser(std::istream& is, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(is) {} - /// a parser reading from a container with contiguous storage + /// a parser reading from an iterator range with contiguous storage template ::iterator_category, std::random_access_iterator_tag>::value @@ -10560,7 +10611,7 @@ if no parse error occurred. */ inline nlohmann::json operator "" _json(const char* s, std::size_t) { - return nlohmann::json::parse(reinterpret_cast(s)); + return nlohmann::json::parse(s); } /*! diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index cb51f557ca..f56c5d3fac 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -951,7 +951,7 @@ class basic_json With a parser callback function, the result of parsing a JSON text can be influenced. When passed to @ref parse(std::istream&, const - parser_callback_t) or @ref parse(const string_t&, const parser_callback_t), + parser_callback_t) or @ref parse(const char*, const parser_callback_t), it is called on certain events (passed as @ref parse_event_t via parameter @a event) with a set recursion depth @a depth and context JSON value @a parsed. The return value of the callback function is a boolean @@ -994,7 +994,7 @@ class basic_json skipped completely or replaced by an empty discarded object. @sa @ref parse(std::istream&, parser_callback_t) or - @ref parse(const string_t&, parser_callback_t) for examples + @ref parse(const char*, parser_callback_t) for examples @since version 1.0.0 */ @@ -5933,9 +5933,9 @@ class basic_json /// @{ /*! - @brief deserialize from string + @brief deserialize from string literal - @param[in] s string to read a serialized JSON value from + @param[in] s string literal to read a serialized JSON value from @param[in] cb a parser callback function of type @ref parser_callback_t which is used to control the deserialization by filtering unwanted values (optional) @@ -5947,6 +5947,8 @@ class basic_json @a cb has a super-linear complexity. @note A UTF-8 byte order mark is silently ignored. + @note String containers like `std::string` or @ref string_t can be parsed + with @ref parse(const ContiguousContainer&, const parser_callback_t) @liveexample{The example below demonstrates the `parse()` function with and without callback function.,parse__string__parser_callback_t} @@ -5954,24 +5956,47 @@ class basic_json @sa @ref parse(std::istream&, const parser_callback_t) for a version that reads from an input stream - @since version 1.0.0 + @since version 1.0.0 (originally for @ref string_t) */ - /* - static basic_json parse(const string_t& s, + static basic_json parse(const char* s, const parser_callback_t cb = nullptr) { return parser(s, cb).parse(); } - */ /*! - @brief deserialize from string literal - @copydoc parse(const string_t&, const parser_callback_t) + @brief deserialize from an array + + This function reads from an array of 1-byte values. + + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @param[in] array array to read from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from an array.,parse__array__parser_callback_t} + + @since version 2.0.3 */ - static basic_json parse(const typename string_t::value_type* s, + template + static basic_json parse(T (&array)[N], const parser_callback_t cb = nullptr) { - return parser(s, cb).parse(); + // delegate the call to the iterator-range parse overload + return parse(std::begin(array), std::end(array), cb); } /*! @@ -5993,7 +6018,7 @@ class basic_json @liveexample{The example below demonstrates the `parse()` function with and without callback function.,parse__istream__parser_callback_t} - @sa @ref parse(const string_t&, const parser_callback_t) for a version + @sa @ref parse(const char*, const parser_callback_t) for a version that reads from a string @since version 1.0.0 @@ -6014,7 +6039,7 @@ class basic_json } /*! - @brief deserialize from a iterator range with contiguous storage + @brief deserialize from an iterator range with contiguous storage This function reads from an iterator range of a container with contiguous storage of 1-byte values. Compatible container types include @@ -6029,13 +6054,14 @@ class basic_json precondition yields undefined behavior. **This precondition is enforced with a static assertion.** - @warning There is no way to enforce the preconditions at compile-time. If - the function is called with noncompliant iterators, the behavior - is undefined and will most liekely yield segmentation violation. + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with noncompliant iterators and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. - @param[in] first begin of the range to parse (included) - @param[in] last end of the range to parse (excluded) - @param[in] cb a parser callback function of type @ref parser_callback_t + @param[in] first begin of the range to parse (included) + @param[in] last end of the range to parse (excluded) + @param[in] cb a parser callback function of type @ref parser_callback_t which is used to control the deserialization by filtering unwanted values (optional) @@ -6047,7 +6073,8 @@ class basic_json @note A UTF-8 byte order mark is silently ignored. - @todo Example and references. + @liveexample{The example below demonstrates the `parse()` function reading + from an iterator range.,parse__iteratortype__parser_callback_t} @since version 2.0.3 */ @@ -6084,6 +6111,45 @@ class basic_json return parser(first, last, cb).parse(); } + /*! + @brief deserialize from a container with contiguous storage + + This function reads from a container with contiguous storage of 1-byte + values. Compatible container types include `std::vector`, `std::string`, + `std::array`, and `std::initializer_list`. User-defined containers can be + used as long as they implement random-access iterators and a contiguous + storage. + + @pre The container storage is contiguous. Violating this precondition + yields undefined behavior. **This precondition is enforced with an + assertion.** + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with a noncompliant container and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. + + @param[in] c container to read from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from a contiguous container.,parse__contiguouscontainer__parser_callback_t} + + @since version 2.0.3 + */ template - static basic_json parse(T (&array)[N], - const parser_callback_t cb = nullptr) - { - // delegate the call to the iterator-range parse overload - return parse(std::begin(array), std::end(array), cb); - } - /*! @brief deserialize from stream @@ -6158,7 +6216,7 @@ class basic_json Returns the type name as string to be used in error messages - usually to indicate that a function was called on a wrong JSON type. - @return basically a string representation of a the @ref m_type member + @return basically a string representation of a the @a m_type member @complexity Constant. @@ -7626,7 +7684,7 @@ class basic_json fill_line_buffer(); } - // switch off unwanted functions + // switch off unwanted functions (due to pointer members) lexer() = delete; lexer(const lexer&) = delete; lexer operator=(const lexer&) = delete; @@ -8276,24 +8334,17 @@ class basic_json { public: /// a parser reading from a string literal - parser(const typename string_t::value_type* buff, - const parser_callback_t cb = nullptr) + parser(const char* buff, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(reinterpret_cast(buff), strlen(buff)) {} - /// a parser reading from a string container - parser(const string_t& s, const parser_callback_t cb = nullptr) - : callback(cb), - m_lexer(reinterpret_cast(s.c_str()), s.size()) - {} - /// a parser reading from an input stream parser(std::istream& is, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(is) {} - /// a parser reading from a container with contiguous storage + /// a parser reading from an iterator range with contiguous storage template ::iterator_category, std::random_access_iterator_tag>::value @@ -9857,7 +9908,7 @@ if no parse error occurred. */ inline nlohmann::json operator "" _json(const char* s, std::size_t) { - return nlohmann::json::parse(reinterpret_cast(s)); + return nlohmann::json::parse(s); } /*! diff --git a/test/src/unit-class_parser.cpp b/test/src/unit-class_parser.cpp index 775952b926..5cae3071c0 100644 --- a/test/src/unit-class_parser.cpp +++ b/test/src/unit-class_parser.cpp @@ -477,7 +477,7 @@ TEST_CASE("parser class") case ('r'): case ('t'): { - CHECK_NOTHROW(json::parser(s).parse()); + CHECK_NOTHROW(json::parser(s.c_str()).parse()); break; } @@ -490,8 +490,8 @@ TEST_CASE("parser class") // any other combination of backslash and character is invalid default: { - CHECK_THROWS_AS(json::parser(s).parse(), std::invalid_argument); - CHECK_THROWS_WITH(json::parser(s).parse(), "parse error - unexpected '\"'"); + CHECK_THROWS_AS(json::parser(s.c_str()).parse(), std::invalid_argument); + CHECK_THROWS_WITH(json::parser(s.c_str()).parse(), "parse error - unexpected '\"'"); break; } } @@ -549,22 +549,22 @@ TEST_CASE("parser class") if (valid(c)) { - CHECK_NOTHROW(json::parser(s1).parse()); - CHECK_NOTHROW(json::parser(s2).parse()); - CHECK_NOTHROW(json::parser(s3).parse()); - CHECK_NOTHROW(json::parser(s4).parse()); + CHECK_NOTHROW(json::parser(s1.c_str()).parse()); + CHECK_NOTHROW(json::parser(s2.c_str()).parse()); + CHECK_NOTHROW(json::parser(s3.c_str()).parse()); + CHECK_NOTHROW(json::parser(s4.c_str()).parse()); } else { - CHECK_THROWS_AS(json::parser(s1).parse(), std::invalid_argument); - CHECK_THROWS_AS(json::parser(s2).parse(), std::invalid_argument); - CHECK_THROWS_AS(json::parser(s3).parse(), std::invalid_argument); - CHECK_THROWS_AS(json::parser(s4).parse(), std::invalid_argument); - - CHECK_THROWS_WITH(json::parser(s1).parse(), "parse error - unexpected '\"'"); - CHECK_THROWS_WITH(json::parser(s2).parse(), "parse error - unexpected '\"'"); - CHECK_THROWS_WITH(json::parser(s3).parse(), "parse error - unexpected '\"'"); - CHECK_THROWS_WITH(json::parser(s4).parse(), "parse error - unexpected '\"'"); + CHECK_THROWS_AS(json::parser(s1.c_str()).parse(), std::invalid_argument); + CHECK_THROWS_AS(json::parser(s2.c_str()).parse(), std::invalid_argument); + CHECK_THROWS_AS(json::parser(s3.c_str()).parse(), std::invalid_argument); + CHECK_THROWS_AS(json::parser(s4.c_str()).parse(), std::invalid_argument); + + CHECK_THROWS_WITH(json::parser(s1.c_str()).parse(), "parse error - unexpected '\"'"); + CHECK_THROWS_WITH(json::parser(s2.c_str()).parse(), "parse error - unexpected '\"'"); + CHECK_THROWS_WITH(json::parser(s3.c_str()).parse(), "parse error - unexpected '\"'"); + CHECK_THROWS_WITH(json::parser(s4.c_str()).parse(), "parse error - unexpected '\"'"); } } } diff --git a/test/src/unit-deserialization.cpp b/test/src/unit-deserialization.cpp index b681e3df36..6e2c78130b 100644 --- a/test/src/unit-deserialization.cpp +++ b/test/src/unit-deserialization.cpp @@ -113,6 +113,12 @@ TEST_CASE("deserialization") std::initializer_list v = {'t', 'r', 'u', 'e', '\0'}; CHECK(json::parse(v) == json(true)); } + + SECTION("empty container") + { + std::vector v; + CHECK_THROWS_AS(json::parse(v), std::invalid_argument); + } } SECTION("via iterator range") From a79d634ccbbbccfacf5ada18d5fe2a1ce6ab338a Mon Sep 17 00:00:00 2001 From: Niels Date: Mon, 22 Aug 2016 21:40:07 +0200 Subject: [PATCH 12/18] integrated proposals for #290 --- .../parse__array__parser_callback_t.cpp | 28 -- .../parse__array__parser_callback_t.link | 1 - .../parse__array__parser_callback_t.output | 20 -- src/json.hpp | 291 ++++++------------ src/json.hpp.re2c | 291 ++++++------------ 5 files changed, 200 insertions(+), 431 deletions(-) delete mode 100644 doc/examples/parse__array__parser_callback_t.cpp delete mode 100644 doc/examples/parse__array__parser_callback_t.link delete mode 100644 doc/examples/parse__array__parser_callback_t.output diff --git a/doc/examples/parse__array__parser_callback_t.cpp b/doc/examples/parse__array__parser_callback_t.cpp deleted file mode 100644 index 8e086d2003..0000000000 --- a/doc/examples/parse__array__parser_callback_t.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include - -using json = nlohmann::json; - -int main() -{ - // a JSON text - char text[] = R"( - { - "Image": { - "Width": 800, - "Height": 600, - "Title": "View from 15th Floor", - "Thumbnail": { - "Url": "http://www.example.com/image/481989943", - "Height": 125, - "Width": 100 - }, - "Animated" : false, - "IDs": [116, 943, 234, 38793] - } - } - )"; - - // parse and serialize JSON - json j_complete = json::parse(text); - std::cout << std::setw(4) << j_complete << "\n\n"; -} diff --git a/doc/examples/parse__array__parser_callback_t.link b/doc/examples/parse__array__parser_callback_t.link deleted file mode 100644 index a1d3cd3498..0000000000 --- a/doc/examples/parse__array__parser_callback_t.link +++ /dev/null @@ -1 +0,0 @@ -online \ No newline at end of file diff --git a/doc/examples/parse__array__parser_callback_t.output b/doc/examples/parse__array__parser_callback_t.output deleted file mode 100644 index 62bb85863e..0000000000 --- a/doc/examples/parse__array__parser_callback_t.output +++ /dev/null @@ -1,20 +0,0 @@ -{ - "Image": { - "Animated": false, - "Height": 600, - "IDs": [ - 116, - 943, - 234, - 38793 - ], - "Thumbnail": { - "Height": 125, - "Url": "http://www.example.com/image/481989943", - "Width": 100 - }, - "Title": "View from 15th Floor", - "Width": 800 - } -} - diff --git a/src/json.hpp b/src/json.hpp index 5a35b16d76..ac5c78b31a 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -1165,11 +1165,9 @@ class basic_json @since version 1.0.0 */ - template ::value and - std::is_constructible::value, int>::type - = 0> + template::value and + std::is_constructible::value, int>::type = 0> basic_json(const CompatibleObjectType& val) : m_type(value_t::object) { @@ -1230,16 +1228,14 @@ class basic_json @since version 1.0.0 */ - template ::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - std::is_constructible::value, int>::type - = 0> + template::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + std::is_constructible::value, int>::type = 0> basic_json(const CompatibleArrayType& val) : m_type(value_t::array) { @@ -1325,10 +1321,8 @@ class basic_json @since version 1.0.0 */ - template ::value, int>::type - = 0> + template::value, int>::type = 0> basic_json(const CompatibleStringType& val) : basic_json(string_t(val)) { @@ -1378,12 +1372,9 @@ class basic_json @since version 1.0.0 */ - template::value) - and std::is_same::value - , int>::type - = 0> + template::value) and + std::is_same::value, int>::type = 0> basic_json(const number_integer_t val) noexcept : m_type(value_t::number_integer), m_value(val) { @@ -1447,13 +1438,11 @@ class basic_json @since version 1.0.0 */ - template::value and std::numeric_limits::is_integer and std::numeric_limits::is_signed, - CompatibleNumberIntegerType>::type - = 0> + CompatibleNumberIntegerType>::type = 0> basic_json(const CompatibleNumberIntegerType val) noexcept : m_type(value_t::number_integer), m_value(static_cast(val)) @@ -1478,12 +1467,9 @@ class basic_json @since version 2.0.0 */ - template::value) - and std::is_same::value - , int>::type - = 0> + template::value) and + std::is_same::value, int>::type = 0> basic_json(const number_unsigned_t val) noexcept : m_type(value_t::number_unsigned), m_value(val) { @@ -1510,13 +1496,11 @@ class basic_json @since version 2.0.0 */ - template ::value and - std::numeric_limits::is_integer and - not std::numeric_limits::is_signed, - CompatibleNumberUnsignedType>::type - = 0> + template::value and + std::numeric_limits::is_integer and + not std::numeric_limits::is_signed, + CompatibleNumberUnsignedType>::type = 0> basic_json(const CompatibleNumberUnsignedType val) noexcept : m_type(value_t::number_unsigned), m_value(static_cast(val)) @@ -1592,11 +1576,9 @@ class basic_json @since version 1.0.0 */ - template::value and - std::is_floating_point::value>::type - > + std::is_floating_point::value>::type> basic_json(const CompatibleNumberFloatType val) noexcept : basic_json(number_float_t(val)) { @@ -1863,12 +1845,9 @@ class basic_json @since version 1.0.0 */ - template ::value or - std::is_same::value - , int>::type - = 0> + template::value or + std::is_same::value, int>::type = 0> basic_json(InputIT first, InputIT last) { assert(first.m_object != nullptr); @@ -2616,11 +2595,9 @@ class basic_json ////////////////// /// get an object (explicit) - template ::value and - std::is_convertible::value - , int>::type = 0> + template::value and + std::is_convertible::value, int>::type = 0> T get_impl(T*) const { if (is_object()) @@ -2647,14 +2624,12 @@ class basic_json } /// get an array (explicit) - template ::value and - not std::is_same::value and - not std::is_arithmetic::value and - not std::is_convertible::value and - not has_mapped_type::value - , int>::type = 0> + template::value and + not std::is_same::value and + not std::is_arithmetic::value and + not std::is_convertible::value and + not has_mapped_type::value, int>::type = 0> T get_impl(T*) const { if (is_array()) @@ -2674,11 +2649,9 @@ class basic_json } /// get an array (explicit) - template ::value and - not std::is_same::value - , int>::type = 0> + template::value and + not std::is_same::value, int>::type = 0> std::vector get_impl(std::vector*) const { if (is_array()) @@ -2699,11 +2672,9 @@ class basic_json } /// get an array (explicit) - template ::value and - not has_mapped_type::value - , int>::type = 0> + template::value and + not has_mapped_type::value, int>::type = 0> T get_impl(T*) const { if (is_array()) @@ -2730,10 +2701,8 @@ class basic_json } /// get a string (explicit) - template ::value - , int>::type = 0> + template::value, int>::type = 0> T get_impl(T*) const { if (is_string()) @@ -2747,10 +2716,8 @@ class basic_json } /// get a number (explicit) - template::value - , int>::type = 0> + template::value, int>::type = 0> T get_impl(T*) const { switch (m_type) @@ -2939,10 +2906,8 @@ class basic_json @since version 1.0.0 */ - template::value - , int>::type = 0> + template::value, int>::type = 0> ValueType get() const { return get_impl(static_cast(nullptr)); @@ -2975,10 +2940,8 @@ class basic_json @since version 1.0.0 */ - template::value - , int>::type = 0> + template::value, int>::type = 0> PointerType get() noexcept { // delegate the call to get_ptr @@ -2989,10 +2952,8 @@ class basic_json @brief get a pointer value (explicit) @copydoc get() */ - template::value - , int>::type = 0> + template::value, int>::type = 0> constexpr const PointerType get() const noexcept { // delegate the call to get_ptr @@ -3025,10 +2986,8 @@ class basic_json @since version 1.0.0 */ - template::value - , int>::type = 0> + template::value, int>::type = 0> PointerType get_ptr() noexcept { // get the type of the PointerType (remove pointer and const) @@ -3054,11 +3013,9 @@ class basic_json @brief get a pointer value (implicit) @copydoc get_ptr() */ - template::value - and std::is_const::type>::value - , int>::type = 0> + template::value and + std::is_const::type>::value, int>::type = 0> constexpr const PointerType get_ptr() const noexcept { // get the type of the PointerType (remove pointer and const) @@ -3106,10 +3063,8 @@ class basic_json @since version 1.1.0 */ - template::value - , int>::type = 0> + template::value, int>::type = 0> ReferenceType get_ref() { // delegate call to get_ref_impl @@ -3120,11 +3075,9 @@ class basic_json @brief get a reference value (implicit) @copydoc get_ref() */ - template::value - and std::is_const::type>::value - , int>::type = 0> + template::value and + std::is_const::type>::value, int>::type = 0> ReferenceType get_ref() const { // delegate call to get_ref_impl @@ -3159,10 +3112,9 @@ class basic_json @since version 1.0.0 */ - template < typename ValueType, typename - std::enable_if < - not std::is_pointer::value - and not std::is_same::value + template < typename ValueType, typename std::enable_if < + not std::is_pointer::value and + not std::is_same::value #ifndef _MSC_VER // Fix for issue #167 operator<< abiguity under VS2015 and not std::is_same>::value #endif @@ -3752,10 +3704,8 @@ class basic_json @since version 1.0.0 */ - template ::value - , int>::type = 0> + template::value, int>::type = 0> ValueType value(const typename object_t::key_type& key, ValueType default_value) const { // at only works for objects @@ -3828,10 +3778,8 @@ class basic_json @since version 2.0.2 */ - template ::value - , int>::type = 0> + template::value, int>::type = 0> ValueType value(const json_pointer& ptr, ValueType default_value) const { // at only works for objects @@ -3992,12 +3940,10 @@ class basic_json @since version 1.0.0 */ - template ::value or - std::is_same::value - , int>::type - = 0> + template::value or + std::is_same::value, int>::type + = 0> IteratorType erase(IteratorType pos) { // make sure iterator fits the current value @@ -4101,12 +4047,10 @@ class basic_json @since version 1.0.0 */ - template ::value or - std::is_same::value - , int>::type - = 0> + template::value or + std::is_same::value, int>::type + = 0> IteratorType erase(IteratorType first, IteratorType last) { // make sure iterator fits the current value @@ -5935,6 +5879,7 @@ class basic_json /*! @brief deserialize from string literal + @tparam CharT character/literal type with size of 1 byte @param[in] s string literal to read a serialized JSON value from @param[in] cb a parser callback function of type @ref parser_callback_t which is used to control the deserialization by filtering unwanted values @@ -5958,45 +5903,12 @@ class basic_json @since version 1.0.0 (originally for @ref string_t) */ - static basic_json parse(const char* s, + template::value and sizeof(CharT) == 1, int>::type = 0> + static basic_json parse(const CharT* s, const parser_callback_t cb = nullptr) { - return parser(s, cb).parse(); - } - - /*! - @brief deserialize from an array - - This function reads from an array of 1-byte values. - - @pre Each element of the container has a size of 1 byte. Violating this - precondition yields undefined behavior. **This precondition is enforced - with a static assertion.** - - @param[in] array array to read from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @return result of the deserialization - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below demonstrates the `parse()` function reading - from an array.,parse__array__parser_callback_t} - - @since version 2.0.3 - */ - template - static basic_json parse(T (&array)[N], - const parser_callback_t cb = nullptr) - { - // delegate the call to the iterator-range parse overload - return parse(std::begin(array), std::end(array), cb); + return parser(reinterpret_cast(s), cb).parse(); } /*! @@ -6059,6 +5971,7 @@ class basic_json assertions switched off, the behavior is undefined and will most likely yield segmentation violation. + @tparam IteratorType iterator of container with contiguous storage @param[in] first begin of the range to parse (included) @param[in] last end of the range to parse (excluded) @param[in] cb a parser callback function of type @ref parser_callback_t @@ -6078,13 +5991,10 @@ class basic_json @since version 2.0.3 */ - template ::iterator_category>::value - , int>::type - = 0> + template::iterator_category>::value, int>::type = 0> static basic_json parse(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr) { @@ -6132,6 +6042,7 @@ class basic_json assertions switched off, the behavior is undefined and will most likely yield segmentation violation. + @tparam ContiguousContainer container type with contiguous storage @param[in] c container to read from @param[in] cb a parser callback function of type @ref parser_callback_t which is used to control the deserialization by filtering unwanted values @@ -6150,11 +6061,10 @@ class basic_json @since version 2.0.3 */ - template()))>::iterator_category>::value + typename std::iterator_traits()))>::iterator_category>::value , int>::type = 0> static basic_json parse(const ContiguousContainer& c, const parser_callback_t cb = nullptr) @@ -9048,11 +8958,10 @@ class basic_json {} /// a parser reading from an iterator range with contiguous storage - template ::iterator_category, std::random_access_iterator_tag>::value - , int>::type - = 0> + template::iterator_category, std::random_access_iterator_tag>::value + , int>::type + = 0> parser(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(reinterpret_cast(&(*first)), @@ -10569,7 +10478,7 @@ namespace std @since version 1.0.0 */ -template <> +template<> inline void swap(nlohmann::json& j1, nlohmann::json& j2) noexcept( is_nothrow_move_constructible::value and @@ -10580,7 +10489,7 @@ inline void swap(nlohmann::json& j1, } /// hash value for JSON objects -template <> +template<> struct hash { /*! diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index a5b8a14f5a..dffb11aff2 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -1165,11 +1165,9 @@ class basic_json @since version 1.0.0 */ - template ::value and - std::is_constructible::value, int>::type - = 0> + template::value and + std::is_constructible::value, int>::type = 0> basic_json(const CompatibleObjectType& val) : m_type(value_t::object) { @@ -1230,16 +1228,14 @@ class basic_json @since version 1.0.0 */ - template ::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - std::is_constructible::value, int>::type - = 0> + template::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + std::is_constructible::value, int>::type = 0> basic_json(const CompatibleArrayType& val) : m_type(value_t::array) { @@ -1325,10 +1321,8 @@ class basic_json @since version 1.0.0 */ - template ::value, int>::type - = 0> + template::value, int>::type = 0> basic_json(const CompatibleStringType& val) : basic_json(string_t(val)) { @@ -1378,12 +1372,9 @@ class basic_json @since version 1.0.0 */ - template::value) - and std::is_same::value - , int>::type - = 0> + template::value) and + std::is_same::value, int>::type = 0> basic_json(const number_integer_t val) noexcept : m_type(value_t::number_integer), m_value(val) { @@ -1447,13 +1438,11 @@ class basic_json @since version 1.0.0 */ - template::value and std::numeric_limits::is_integer and std::numeric_limits::is_signed, - CompatibleNumberIntegerType>::type - = 0> + CompatibleNumberIntegerType>::type = 0> basic_json(const CompatibleNumberIntegerType val) noexcept : m_type(value_t::number_integer), m_value(static_cast(val)) @@ -1478,12 +1467,9 @@ class basic_json @since version 2.0.0 */ - template::value) - and std::is_same::value - , int>::type - = 0> + template::value) and + std::is_same::value, int>::type = 0> basic_json(const number_unsigned_t val) noexcept : m_type(value_t::number_unsigned), m_value(val) { @@ -1510,13 +1496,11 @@ class basic_json @since version 2.0.0 */ - template ::value and - std::numeric_limits::is_integer and - not std::numeric_limits::is_signed, - CompatibleNumberUnsignedType>::type - = 0> + template::value and + std::numeric_limits::is_integer and + not std::numeric_limits::is_signed, + CompatibleNumberUnsignedType>::type = 0> basic_json(const CompatibleNumberUnsignedType val) noexcept : m_type(value_t::number_unsigned), m_value(static_cast(val)) @@ -1592,11 +1576,9 @@ class basic_json @since version 1.0.0 */ - template::value and - std::is_floating_point::value>::type - > + std::is_floating_point::value>::type> basic_json(const CompatibleNumberFloatType val) noexcept : basic_json(number_float_t(val)) { @@ -1863,12 +1845,9 @@ class basic_json @since version 1.0.0 */ - template ::value or - std::is_same::value - , int>::type - = 0> + template::value or + std::is_same::value, int>::type = 0> basic_json(InputIT first, InputIT last) { assert(first.m_object != nullptr); @@ -2616,11 +2595,9 @@ class basic_json ////////////////// /// get an object (explicit) - template ::value and - std::is_convertible::value - , int>::type = 0> + template::value and + std::is_convertible::value, int>::type = 0> T get_impl(T*) const { if (is_object()) @@ -2647,14 +2624,12 @@ class basic_json } /// get an array (explicit) - template ::value and - not std::is_same::value and - not std::is_arithmetic::value and - not std::is_convertible::value and - not has_mapped_type::value - , int>::type = 0> + template::value and + not std::is_same::value and + not std::is_arithmetic::value and + not std::is_convertible::value and + not has_mapped_type::value, int>::type = 0> T get_impl(T*) const { if (is_array()) @@ -2674,11 +2649,9 @@ class basic_json } /// get an array (explicit) - template ::value and - not std::is_same::value - , int>::type = 0> + template::value and + not std::is_same::value, int>::type = 0> std::vector get_impl(std::vector*) const { if (is_array()) @@ -2699,11 +2672,9 @@ class basic_json } /// get an array (explicit) - template ::value and - not has_mapped_type::value - , int>::type = 0> + template::value and + not has_mapped_type::value, int>::type = 0> T get_impl(T*) const { if (is_array()) @@ -2730,10 +2701,8 @@ class basic_json } /// get a string (explicit) - template ::value - , int>::type = 0> + template::value, int>::type = 0> T get_impl(T*) const { if (is_string()) @@ -2747,10 +2716,8 @@ class basic_json } /// get a number (explicit) - template::value - , int>::type = 0> + template::value, int>::type = 0> T get_impl(T*) const { switch (m_type) @@ -2939,10 +2906,8 @@ class basic_json @since version 1.0.0 */ - template::value - , int>::type = 0> + template::value, int>::type = 0> ValueType get() const { return get_impl(static_cast(nullptr)); @@ -2975,10 +2940,8 @@ class basic_json @since version 1.0.0 */ - template::value - , int>::type = 0> + template::value, int>::type = 0> PointerType get() noexcept { // delegate the call to get_ptr @@ -2989,10 +2952,8 @@ class basic_json @brief get a pointer value (explicit) @copydoc get() */ - template::value - , int>::type = 0> + template::value, int>::type = 0> constexpr const PointerType get() const noexcept { // delegate the call to get_ptr @@ -3025,10 +2986,8 @@ class basic_json @since version 1.0.0 */ - template::value - , int>::type = 0> + template::value, int>::type = 0> PointerType get_ptr() noexcept { // get the type of the PointerType (remove pointer and const) @@ -3054,11 +3013,9 @@ class basic_json @brief get a pointer value (implicit) @copydoc get_ptr() */ - template::value - and std::is_const::type>::value - , int>::type = 0> + template::value and + std::is_const::type>::value, int>::type = 0> constexpr const PointerType get_ptr() const noexcept { // get the type of the PointerType (remove pointer and const) @@ -3106,10 +3063,8 @@ class basic_json @since version 1.1.0 */ - template::value - , int>::type = 0> + template::value, int>::type = 0> ReferenceType get_ref() { // delegate call to get_ref_impl @@ -3120,11 +3075,9 @@ class basic_json @brief get a reference value (implicit) @copydoc get_ref() */ - template::value - and std::is_const::type>::value - , int>::type = 0> + template::value and + std::is_const::type>::value, int>::type = 0> ReferenceType get_ref() const { // delegate call to get_ref_impl @@ -3159,10 +3112,9 @@ class basic_json @since version 1.0.0 */ - template < typename ValueType, typename - std::enable_if < - not std::is_pointer::value - and not std::is_same::value + template < typename ValueType, typename std::enable_if < + not std::is_pointer::value and + not std::is_same::value #ifndef _MSC_VER // Fix for issue #167 operator<< abiguity under VS2015 and not std::is_same>::value #endif @@ -3752,10 +3704,8 @@ class basic_json @since version 1.0.0 */ - template ::value - , int>::type = 0> + template::value, int>::type = 0> ValueType value(const typename object_t::key_type& key, ValueType default_value) const { // at only works for objects @@ -3828,10 +3778,8 @@ class basic_json @since version 2.0.2 */ - template ::value - , int>::type = 0> + template::value, int>::type = 0> ValueType value(const json_pointer& ptr, ValueType default_value) const { // at only works for objects @@ -3992,12 +3940,10 @@ class basic_json @since version 1.0.0 */ - template ::value or - std::is_same::value - , int>::type - = 0> + template::value or + std::is_same::value, int>::type + = 0> IteratorType erase(IteratorType pos) { // make sure iterator fits the current value @@ -4101,12 +4047,10 @@ class basic_json @since version 1.0.0 */ - template ::value or - std::is_same::value - , int>::type - = 0> + template::value or + std::is_same::value, int>::type + = 0> IteratorType erase(IteratorType first, IteratorType last) { // make sure iterator fits the current value @@ -5935,6 +5879,7 @@ class basic_json /*! @brief deserialize from string literal + @tparam CharT character/literal type with size of 1 byte @param[in] s string literal to read a serialized JSON value from @param[in] cb a parser callback function of type @ref parser_callback_t which is used to control the deserialization by filtering unwanted values @@ -5958,45 +5903,12 @@ class basic_json @since version 1.0.0 (originally for @ref string_t) */ - static basic_json parse(const char* s, + template::value and sizeof(CharT) == 1, int>::type = 0> + static basic_json parse(const CharT* s, const parser_callback_t cb = nullptr) { - return parser(s, cb).parse(); - } - - /*! - @brief deserialize from an array - - This function reads from an array of 1-byte values. - - @pre Each element of the container has a size of 1 byte. Violating this - precondition yields undefined behavior. **This precondition is enforced - with a static assertion.** - - @param[in] array array to read from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @return result of the deserialization - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below demonstrates the `parse()` function reading - from an array.,parse__array__parser_callback_t} - - @since version 2.0.3 - */ - template - static basic_json parse(T (&array)[N], - const parser_callback_t cb = nullptr) - { - // delegate the call to the iterator-range parse overload - return parse(std::begin(array), std::end(array), cb); + return parser(reinterpret_cast(s), cb).parse(); } /*! @@ -6059,6 +5971,7 @@ class basic_json assertions switched off, the behavior is undefined and will most likely yield segmentation violation. + @tparam IteratorType iterator of container with contiguous storage @param[in] first begin of the range to parse (included) @param[in] last end of the range to parse (excluded) @param[in] cb a parser callback function of type @ref parser_callback_t @@ -6078,13 +5991,10 @@ class basic_json @since version 2.0.3 */ - template ::iterator_category>::value - , int>::type - = 0> + template::iterator_category>::value, int>::type = 0> static basic_json parse(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr) { @@ -6132,6 +6042,7 @@ class basic_json assertions switched off, the behavior is undefined and will most likely yield segmentation violation. + @tparam ContiguousContainer container type with contiguous storage @param[in] c container to read from @param[in] cb a parser callback function of type @ref parser_callback_t which is used to control the deserialization by filtering unwanted values @@ -6150,11 +6061,10 @@ class basic_json @since version 2.0.3 */ - template()))>::iterator_category>::value + typename std::iterator_traits()))>::iterator_category>::value , int>::type = 0> static basic_json parse(const ContiguousContainer& c, const parser_callback_t cb = nullptr) @@ -8345,11 +8255,10 @@ class basic_json {} /// a parser reading from an iterator range with contiguous storage - template ::iterator_category, std::random_access_iterator_tag>::value - , int>::type - = 0> + template::iterator_category, std::random_access_iterator_tag>::value + , int>::type + = 0> parser(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(reinterpret_cast(&(*first)), @@ -9866,7 +9775,7 @@ namespace std @since version 1.0.0 */ -template <> +template<> inline void swap(nlohmann::json& j1, nlohmann::json& j2) noexcept( is_nothrow_move_constructible::value and @@ -9877,7 +9786,7 @@ inline void swap(nlohmann::json& j1, } /// hash value for JSON objects -template <> +template<> struct hash { /*! From 1d66ab9f7a5898267ebecefe5c2ba4ee0ffec613 Mon Sep 17 00:00:00 2001 From: Niels Date: Tue, 23 Aug 2016 22:38:05 +0200 Subject: [PATCH 13/18] fixed lexer issue which required null byte at the end of contiguous storage containers #290 --- src/json.hpp | 49 +++++++++++++++++++++---------- src/json.hpp.re2c | 49 +++++++++++++++++++++---------- test/src/unit-class_parser.cpp | 10 +++---- test/src/unit-deserialization.cpp | 18 ++++++------ 4 files changed, 82 insertions(+), 44 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index ac5c78b31a..0e25f746e0 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -8538,6 +8538,13 @@ class basic_json incremented without leaving the limits of the line buffer. Note re2c decides when to call this function. + If the lexer reads from contiguous storage, there is no trailing null + byte. Therefore, this function must make sure to add these padding + null bytes. + + If the lexer reads from an input stream, this function reads the next + line of the input. + @pre p p p p p p u u u u u x . . . . . . ^ ^ ^ ^ @@ -8553,26 +8560,38 @@ class basic_json */ void fill_line_buffer() { - // no stream is used or end of file is reached - if (m_stream == nullptr or not * m_stream) - { - return; - } - // number of processed characters (p) const auto offset_start = m_start - m_content; // offset for m_marker wrt. to m_start - const auto offset_marker = m_marker - m_start; + const auto offset_marker = (m_marker == nullptr) ? 0 : m_marker - m_start; // number of unprocessed characters (u) const auto offset_cursor = m_cursor - m_start; - // delete processed characters from line buffer - m_line_buffer.erase(0, static_cast(offset_start)); - // read next line from input stream - std::string line; - std::getline(*m_stream, line); - // add line with newline symbol to the line buffer - m_line_buffer += "\n" + line; + // no stream is used or end of file is reached + if (m_stream == nullptr or not * m_stream) + { + // copy unprocessed characters to line buffer + m_line_buffer.clear(); + for (m_cursor = m_start; m_cursor != m_limit; ++m_cursor) + { + m_line_buffer.append(1, static_cast(*m_cursor)); + } + + // append 5 characters (size of longest keyword "false") to + // make sure that there is sufficient space between m_cursor + // and m_limit + m_line_buffer.append(5, '\0'); + } + else + { + // delete processed characters from line buffer + m_line_buffer.erase(0, static_cast(offset_start)); + // read next line from input stream + std::string line; + std::getline(*m_stream, line); + // add line with newline symbol to the line buffer + m_line_buffer += "\n" + line; + } // set pointers m_content = reinterpret_cast(m_line_buffer.c_str()); @@ -8580,7 +8599,7 @@ class basic_json m_start = m_content; m_marker = m_start + offset_marker; m_cursor = m_start + offset_cursor; - m_limit = m_start + m_line_buffer.size() - 1; + m_limit = m_start + m_line_buffer.size(); } /// return string representation of last read token diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index dffb11aff2..089517bb1e 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -7835,6 +7835,13 @@ class basic_json incremented without leaving the limits of the line buffer. Note re2c decides when to call this function. + If the lexer reads from contiguous storage, there is no trailing null + byte. Therefore, this function must make sure to add these padding + null bytes. + + If the lexer reads from an input stream, this function reads the next + line of the input. + @pre p p p p p p u u u u u x . . . . . . ^ ^ ^ ^ @@ -7850,26 +7857,38 @@ class basic_json */ void fill_line_buffer() { - // no stream is used or end of file is reached - if (m_stream == nullptr or not * m_stream) - { - return; - } - // number of processed characters (p) const auto offset_start = m_start - m_content; // offset for m_marker wrt. to m_start - const auto offset_marker = m_marker - m_start; + const auto offset_marker = (m_marker == nullptr) ? 0 : m_marker - m_start; // number of unprocessed characters (u) const auto offset_cursor = m_cursor - m_start; - // delete processed characters from line buffer - m_line_buffer.erase(0, static_cast(offset_start)); - // read next line from input stream - std::string line; - std::getline(*m_stream, line); - // add line with newline symbol to the line buffer - m_line_buffer += "\n" + line; + // no stream is used or end of file is reached + if (m_stream == nullptr or not * m_stream) + { + // copy unprocessed characters to line buffer + m_line_buffer.clear(); + for (m_cursor = m_start; m_cursor != m_limit; ++m_cursor) + { + m_line_buffer.append(1, static_cast(*m_cursor)); + } + + // append 5 characters (size of longest keyword "false") to + // make sure that there is sufficient space between m_cursor + // and m_limit + m_line_buffer.append(5, '\0'); + } + else + { + // delete processed characters from line buffer + m_line_buffer.erase(0, static_cast(offset_start)); + // read next line from input stream + std::string line; + std::getline(*m_stream, line); + // add line with newline symbol to the line buffer + m_line_buffer += "\n" + line; + } // set pointers m_content = reinterpret_cast(m_line_buffer.c_str()); @@ -7877,7 +7896,7 @@ class basic_json m_start = m_content; m_marker = m_start + offset_marker; m_cursor = m_start + offset_cursor; - m_limit = m_start + m_line_buffer.size() - 1; + m_limit = m_start + m_line_buffer.size(); } /// return string representation of last read token diff --git a/test/src/unit-class_parser.cpp b/test/src/unit-class_parser.cpp index 32a6ac8f30..6fcf947de6 100644 --- a/test/src/unit-class_parser.cpp +++ b/test/src/unit-class_parser.cpp @@ -761,19 +761,19 @@ TEST_CASE("parser class") { SECTION("from std::vector") { - std::vector v = {'t', 'r', 'u', 'e', '\0'}; + std::vector v = {'t', 'r', 'u', 'e'}; CHECK(json::parser(std::begin(v), std::end(v)).parse() == json(true)); } SECTION("from std::array") { - std::array v { {'t', 'r', 'u', 'e', '\0'} }; + std::array v { {'t', 'r', 'u', 'e'} }; CHECK(json::parser(std::begin(v), std::end(v)).parse() == json(true)); } SECTION("from array") { - uint8_t v[] = {'t', 'r', 'u', 'e', '\0'}; + uint8_t v[] = {'t', 'r', 'u', 'e'}; CHECK(json::parser(std::begin(v), std::end(v)).parse() == json(true)); } @@ -790,13 +790,13 @@ TEST_CASE("parser class") SECTION("from std::initializer_list") { - std::initializer_list v = {'t', 'r', 'u', 'e', '\0'}; + std::initializer_list v = {'t', 'r', 'u', 'e'}; CHECK(json::parser(std::begin(v), std::end(v)).parse() == json(true)); } SECTION("from std::valarray") { - std::valarray v = {'t', 'r', 'u', 'e', '\0'}; + std::valarray v = {'t', 'r', 'u', 'e'}; CHECK(json::parser(std::begin(v), std::end(v)).parse() == json(true)); } } diff --git a/test/src/unit-deserialization.cpp b/test/src/unit-deserialization.cpp index 6e2c78130b..dcd7c2721d 100644 --- a/test/src/unit-deserialization.cpp +++ b/test/src/unit-deserialization.cpp @@ -86,19 +86,19 @@ TEST_CASE("deserialization") { SECTION("from std::vector") { - std::vector v = {'t', 'r', 'u', 'e', '\0'}; + std::vector v = {'t', 'r', 'u', 'e'}; CHECK(json::parse(v) == json(true)); } SECTION("from std::array") { - std::array v { {'t', 'r', 'u', 'e', '\0'} }; + std::array v { {'t', 'r', 'u', 'e'} }; CHECK(json::parse(v) == json(true)); } SECTION("from array") { - uint8_t v[] = {'t', 'r', 'u', 'e', '\0'}; + uint8_t v[] = {'t', 'r', 'u', 'e'}; CHECK(json::parse(v) == json(true)); } @@ -110,7 +110,7 @@ TEST_CASE("deserialization") SECTION("from std::initializer_list") { - std::initializer_list v = {'t', 'r', 'u', 'e', '\0'}; + std::initializer_list v = {'t', 'r', 'u', 'e'}; CHECK(json::parse(v) == json(true)); } @@ -125,19 +125,19 @@ TEST_CASE("deserialization") { SECTION("from std::vector") { - std::vector v = {'t', 'r', 'u', 'e', '\0'}; + std::vector v = {'t', 'r', 'u', 'e'}; CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); } SECTION("from std::array") { - std::array v { {'t', 'r', 'u', 'e', '\0'} }; + std::array v { {'t', 'r', 'u', 'e'} }; CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); } SECTION("from array") { - uint8_t v[] = {'t', 'r', 'u', 'e', '\0'}; + uint8_t v[] = {'t', 'r', 'u', 'e'}; CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); } @@ -149,13 +149,13 @@ TEST_CASE("deserialization") SECTION("from std::initializer_list") { - std::initializer_list v = {'t', 'r', 'u', 'e', '\0'}; + std::initializer_list v = {'t', 'r', 'u', 'e'}; CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); } SECTION("from std::valarray") { - std::valarray v = {'t', 'r', 'u', 'e', '\0'}; + std::valarray v = {'t', 'r', 'u', 'e'}; CHECK(json::parse(std::begin(v), std::end(v)) == json(true)); } From 8b833c452a8ef13487968681518beacfe2204ce7 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 24 Aug 2016 07:12:56 +0200 Subject: [PATCH 14/18] experiment: changed order of parse functions --- src/json.hpp | 70 ++++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 0e25f746e0..3c1898a8a2 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -5876,41 +5876,6 @@ class basic_json /// @name deserialization /// @{ - /*! - @brief deserialize from string literal - - @tparam CharT character/literal type with size of 1 byte - @param[in] s string literal to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @return result of the deserialization - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - @note String containers like `std::string` or @ref string_t can be parsed - with @ref parse(const ContiguousContainer&, const parser_callback_t) - - @liveexample{The example below demonstrates the `parse()` function with - and without callback function.,parse__string__parser_callback_t} - - @sa @ref parse(std::istream&, const parser_callback_t) for a version that - reads from an input stream - - @since version 1.0.0 (originally for @ref string_t) - */ - template::value and sizeof(CharT) == 1, int>::type = 0> - static basic_json parse(const CharT* s, - const parser_callback_t cb = nullptr) - { - return parser(reinterpret_cast(s), cb).parse(); - } - /*! @brief deserialize from stream @@ -6073,6 +6038,41 @@ class basic_json return parse(std::begin(c), std::end(c), cb); } + /*! + @brief deserialize from string literal + + @tparam CharT character/literal type with size of 1 byte + @param[in] s string literal to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + @note String containers like `std::string` or @ref string_t can be parsed + with @ref parse(const ContiguousContainer&, const parser_callback_t) + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__string__parser_callback_t} + + @sa @ref parse(std::istream&, const parser_callback_t) for a version that + reads from an input stream + + @since version 1.0.0 (originally for @ref string_t) + */ + template::value and sizeof(CharT) == 1, int>::type = 0> + static basic_json parse(const CharT* s, + const parser_callback_t cb = nullptr) + { + return parser(reinterpret_cast(s), cb).parse(); + } + /*! @brief deserialize from stream From 04c6c886ebfbff0a05dbd4db2048ab5a317dc15f Mon Sep 17 00:00:00 2001 From: Niels Date: Wed, 24 Aug 2016 21:02:33 +0200 Subject: [PATCH 15/18] another experiment --- .../parse__array__parser_callback_t.cpp | 28 +++++ .../parse__array__parser_callback_t.link | 1 + .../parse__array__parser_callback_t.output | 20 ++++ src/json.hpp | 108 ++++++++++++------ src/json.hpp.re2c | 44 ++++++- 5 files changed, 163 insertions(+), 38 deletions(-) create mode 100644 doc/examples/parse__array__parser_callback_t.cpp create mode 100644 doc/examples/parse__array__parser_callback_t.link create mode 100644 doc/examples/parse__array__parser_callback_t.output diff --git a/doc/examples/parse__array__parser_callback_t.cpp b/doc/examples/parse__array__parser_callback_t.cpp new file mode 100644 index 0000000000..8e086d2003 --- /dev/null +++ b/doc/examples/parse__array__parser_callback_t.cpp @@ -0,0 +1,28 @@ +#include + +using json = nlohmann::json; + +int main() +{ + // a JSON text + char text[] = R"( + { + "Image": { + "Width": 800, + "Height": 600, + "Title": "View from 15th Floor", + "Thumbnail": { + "Url": "http://www.example.com/image/481989943", + "Height": 125, + "Width": 100 + }, + "Animated" : false, + "IDs": [116, 943, 234, 38793] + } + } + )"; + + // parse and serialize JSON + json j_complete = json::parse(text); + std::cout << std::setw(4) << j_complete << "\n\n"; +} diff --git a/doc/examples/parse__array__parser_callback_t.link b/doc/examples/parse__array__parser_callback_t.link new file mode 100644 index 0000000000..3389916f5e --- /dev/null +++ b/doc/examples/parse__array__parser_callback_t.link @@ -0,0 +1 @@ +online \ No newline at end of file diff --git a/doc/examples/parse__array__parser_callback_t.output b/doc/examples/parse__array__parser_callback_t.output new file mode 100644 index 0000000000..62bb85863e --- /dev/null +++ b/doc/examples/parse__array__parser_callback_t.output @@ -0,0 +1,20 @@ +{ + "Image": { + "Animated": false, + "Height": 600, + "IDs": [ + 116, + 943, + 234, + 38793 + ], + "Thumbnail": { + "Height": 125, + "Url": "http://www.example.com/image/481989943", + "Width": 100 + }, + "Title": "View from 15th Floor", + "Width": 800 + } +} + diff --git a/src/json.hpp b/src/json.hpp index 3c1898a8a2..3fdcbaa1bd 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -5876,6 +5876,78 @@ class basic_json /// @name deserialization /// @{ + /*! + @brief deserialize from an array + + This function reads from an array of 1-byte values. + + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @param[in] array array to read from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from an array.,parse__array__parser_callback_t} + + @since version 2.0.3 + */ + template + static basic_json parse(T (&array)[N], + const parser_callback_t cb = nullptr) + { + // delegate the call to the iterator-range parse overload + return parse(std::begin(array), std::end(array), cb); + } + + /*! + @brief deserialize from string literal + + @tparam CharT character/literal type with size of 1 byte + @param[in] s string literal to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + @note String containers like `std::string` or @ref string_t can be parsed + with @ref parse(const ContiguousContainer&, const parser_callback_t) + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__string__parser_callback_t} + + @sa @ref parse(std::istream&, const parser_callback_t) for a version that + reads from an input stream + + @since version 1.0.0 (originally for @ref string_t) + */ + template::value and + std::is_integral::type>::value and + sizeof(std::remove_pointer) == 1, int>::type = 0> + static basic_json parse(const CharPT s, + const parser_callback_t cb = nullptr) + { + return parser(reinterpret_cast(s), cb).parse(); + } + /*! @brief deserialize from stream @@ -6027,6 +6099,7 @@ class basic_json @since version 2.0.3 */ template::value and std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits()))>::iterator_category>::value @@ -6038,41 +6111,6 @@ class basic_json return parse(std::begin(c), std::end(c), cb); } - /*! - @brief deserialize from string literal - - @tparam CharT character/literal type with size of 1 byte - @param[in] s string literal to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @return result of the deserialization - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - @note String containers like `std::string` or @ref string_t can be parsed - with @ref parse(const ContiguousContainer&, const parser_callback_t) - - @liveexample{The example below demonstrates the `parse()` function with - and without callback function.,parse__string__parser_callback_t} - - @sa @ref parse(std::istream&, const parser_callback_t) for a version that - reads from an input stream - - @since version 1.0.0 (originally for @ref string_t) - */ - template::value and sizeof(CharT) == 1, int>::type = 0> - static basic_json parse(const CharT* s, - const parser_callback_t cb = nullptr) - { - return parser(reinterpret_cast(s), cb).parse(); - } - /*! @brief deserialize from stream diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index 089517bb1e..5107e05640 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -5876,6 +5876,41 @@ class basic_json /// @name deserialization /// @{ + /*! + @brief deserialize from an array + + This function reads from an array of 1-byte values. + + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @param[in] array array to read from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from an array.,parse__array__parser_callback_t} + + @since version 2.0.3 + */ + template + static basic_json parse(T (&array)[N], + const parser_callback_t cb = nullptr) + { + // delegate the call to the iterator-range parse overload + return parse(std::begin(array), std::end(array), cb); + } + /*! @brief deserialize from string literal @@ -5903,9 +5938,11 @@ class basic_json @since version 1.0.0 (originally for @ref string_t) */ - template::value and sizeof(CharT) == 1, int>::type = 0> - static basic_json parse(const CharT* s, + template::value and + std::is_integral::type>::value and + sizeof(std::remove_pointer) == 1, int>::type = 0> + static basic_json parse(const CharPT s, const parser_callback_t cb = nullptr) { return parser(reinterpret_cast(s), cb).parse(); @@ -6062,6 +6099,7 @@ class basic_json @since version 2.0.3 */ template::value and std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits()))>::iterator_category>::value From 6e6e1c9b1fec37b3f400ce347195666701d3910b Mon Sep 17 00:00:00 2001 From: Niels Date: Tue, 30 Aug 2016 18:26:07 +0200 Subject: [PATCH 16/18] implemented idea from #290 --- src/json.hpp | 2 +- src/json.hpp.re2c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 0aa45af52d..8ac262e69c 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -5942,7 +5942,7 @@ class basic_json template::value and std::is_integral::type>::value and - sizeof(std::remove_pointer) == 1, int>::type = 0> + sizeof(typename std::remove_pointer::type) == 1, int>::type = 0> static basic_json parse(const CharPT s, const parser_callback_t cb = nullptr) { diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index dbc5dec3b4..3b67645807 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -5942,7 +5942,7 @@ class basic_json template::value and std::is_integral::type>::value and - sizeof(std::remove_pointer) == 1, int>::type = 0> + sizeof(typename std::remove_pointer::type) == 1, int>::type = 0> static basic_json parse(const CharPT s, const parser_callback_t cb = nullptr) { From 58cbf4b3ef23d913528140fc0b1b861464f0dfb5 Mon Sep 17 00:00:00 2001 From: Niels Date: Wed, 31 Aug 2016 18:23:46 +0200 Subject: [PATCH 17/18] added another test case --- test/src/unit-deserialization.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/src/unit-deserialization.cpp b/test/src/unit-deserialization.cpp index 49ae7b4be0..d94632133c 100644 --- a/test/src/unit-deserialization.cpp +++ b/test/src/unit-deserialization.cpp @@ -148,6 +148,18 @@ TEST_CASE("deserialization") CHECK(json::parse(v) == json(true)); } + SECTION("from chars") + { + uint8_t *v = new uint8_t[5]; + v[0] = 't'; + v[1] = 'r'; + v[2] = 'u'; + v[3] = 'e'; + v[4] = '\0'; + CHECK(json::parse(v) == json(true)); + delete[] v; + } + SECTION("from std::string") { std::string v = {'t', 'r', 'u', 'e'}; From 740b66f225457b2e2ef5d059c61f6db7e153fc14 Mon Sep 17 00:00:00 2001 From: Niels Date: Wed, 31 Aug 2016 20:23:21 +0200 Subject: [PATCH 18/18] cleanup --- .github/CONTRIBUTING.md | 10 ++-------- README.md | 4 +++- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 2b41608105..04a7cc1cc5 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -41,16 +41,10 @@ There are currently two files which need to be edited: 2. [`test/src/unit.cpp`](https://github.com/nlohmann/json/blob/master/test/unit.cpp) - This contains the [Catch](https://github.com/philsquared/Catch) unit tests which currently cover [100 %](https://coveralls.io/github/nlohmann/json) of the library's code. - If you add or change a feature, please also add a unit test to this file. The unit tests can be compiled with + If you add or change a feature, please also add a unit test to this file. The unit tests can be compiled and executed with ```sh - make - ``` - - and can be executed with - - ```sh - ./json_unit + make check ``` The test cases are also executed with several different compilers on [Travis](https://travis-ci.org/nlohmann/json) once you open a pull request. diff --git a/README.md b/README.md index c145340806..5b0c965274 100644 --- a/README.md +++ b/README.md @@ -492,6 +492,8 @@ I deeply appreciate the help of the following people. - [Mário Feroldi](https://github.com/thelostt) fixed a small typo. - [duncanwerner](https://github.com/duncanwerner) found a really embarrassing performance regression in the 2.0.0 release. - [Damien](https://github.com/dtoma) fixed one of the last conversion warnings. +- [Thomas Braun](https://github.com/t-b) fixed a warning in a test case. +- [Théo DELRIEU](https://github.com/theodelrieu) patiently and constructively oversaw the long way toward [iterator-range parsing](https://github.com/nlohmann/json/issues/290). Thanks a lot for helping out! @@ -510,7 +512,7 @@ To compile and run the tests, you need to execute $ make check =============================================================================== -All tests passed (8905119 assertions in 32 test cases) +All tests passed (8905154 assertions in 35 test cases) ``` For more information, have a look at the file [.travis.yml](https://github.com/nlohmann/json/blob/master/.travis.yml).