From 1f53a53f549d427e0acdf43b7999cda0621b60ed Mon Sep 17 00:00:00 2001 From: Derek Bailey Date: Thu, 26 Jan 2023 12:35:22 -0800 Subject: [PATCH 01/62] First working hack of adding 64-bit. Don't judge :) --- include/flatbuffers/base.h | 2 + include/flatbuffers/buffer.h | 10 +- include/flatbuffers/flatbuffer_builder.h | 83 ++++++++++------- include/flatbuffers/idl.h | 3 + include/flatbuffers/vector_downward.h | 44 +++++---- src/annotated_binary_text_gen.cpp | 8 +- src/binary_annotator.cpp | 48 +++++++--- src/binary_annotator.h | 2 + src/idl_gen_cpp.cpp | 8 +- src/idl_parser.cpp | 13 ++- tests/64bit/run.sh | 5 + tests/64bit/test_64bit.afb | 25 +++++ tests/64bit/test_64bit.bin | Bin 0 -> 40 bytes tests/64bit/test_64bit.fbs | 6 ++ tests/64bit/test_64bit.json | 4 + tests/64bit/test_64bit_generated.h | 112 +++++++++++++++++++++++ tests/test_builder.cpp | 13 +-- 17 files changed, 313 insertions(+), 73 deletions(-) create mode 100755 tests/64bit/run.sh create mode 100644 tests/64bit/test_64bit.afb create mode 100644 tests/64bit/test_64bit.bin create mode 100644 tests/64bit/test_64bit.fbs create mode 100644 tests/64bit/test_64bit.json create mode 100644 tests/64bit/test_64bit_generated.h diff --git a/include/flatbuffers/base.h b/include/flatbuffers/base.h index ae3508b4994..8004b9f72c5 100644 --- a/include/flatbuffers/base.h +++ b/include/flatbuffers/base.h @@ -324,6 +324,8 @@ namespace flatbuffers { // offset values between 32bit and 64bit systems. typedef uint32_t uoffset_t; +typedef uint64_t uoffset64_t; + // Signed offsets for references that can go in both directions. typedef int32_t soffset_t; diff --git a/include/flatbuffers/buffer.h b/include/flatbuffers/buffer.h index e26a153c3fb..e9e2257a987 100644 --- a/include/flatbuffers/buffer.h +++ b/include/flatbuffers/buffer.h @@ -33,6 +33,14 @@ template struct Offset { bool IsNull() const { return !o; } }; +template struct Offset64 { + uoffset64_t o; + Offset64() : o(0) {} + Offset64(uoffset64_t _o) : o(_o) {} + Offset64 Union() const { return Offset64(o); } + bool IsNull() const { return !o; } +}; + inline void EndianCheck() { int endiantest = 1; // If this fails, see FLATBUFFERS_LITTLEENDIAN above. @@ -42,7 +50,7 @@ inline void EndianCheck() { } template FLATBUFFERS_CONSTEXPR size_t AlignOf() { - // clang-format off +// clang-format off #ifdef _MSC_VER return __alignof(T); #else diff --git a/include/flatbuffers/flatbuffer_builder.h b/include/flatbuffers/flatbuffer_builder.h index caf9a3d1566..9f72e914180 100644 --- a/include/flatbuffers/flatbuffer_builder.h +++ b/include/flatbuffers/flatbuffer_builder.h @@ -37,6 +37,12 @@ namespace flatbuffers { +// namespace internal { +// template class FlatBufferBuilder_; +// } + + + // Converts a Field ID to a virtual table offset. inline voffset_t FieldIndexToOffset(voffset_t field_id) { // Should correspond to what EndTable() below builds up. @@ -59,18 +65,19 @@ T *data(std::vector &v) { return v.empty() ? reinterpret_cast(&t) : &v.front(); } + /// @addtogroup flatbuffers_cpp_api /// @{ -/// @class FlatBufferBuilder +/// @class FlatBufferBuilder_ /// @brief Helper class to hold data needed in creation of a FlatBuffer. /// To serialize data, you typically call one of the `Create*()` functions in /// the generated code, which in turn call a sequence of `StartTable`/ /// `PushElement`/`AddElement`/`EndTable`, or the builtin `CreateString`/ /// `CreateVector` functions. Do this is depth-first order to build up a tree to /// the root. `Finish()` wraps up the buffer ready for transport. -class FlatBufferBuilder { +template class FlatBufferBuilder_ { public: - /// @brief Default constructor for FlatBufferBuilder. + /// @brief Default constructor for FlatBufferBuilder_. /// @param[in] initial_size The initial size of the buffer, in bytes. Defaults /// to `1024`. /// @param[in] allocator An `Allocator` to use. If null will use @@ -81,7 +88,7 @@ class FlatBufferBuilder { /// minimum alignment upon reallocation. Only needed if you intend to store /// types with custom alignment AND you wish to read the buffer in-place /// directly after creation. - explicit FlatBufferBuilder( + explicit FlatBufferBuilder_( size_t initial_size = 1024, Allocator *allocator = nullptr, bool own_allocator = false, size_t buffer_minalign = AlignOf()) @@ -97,8 +104,8 @@ class FlatBufferBuilder { EndianCheck(); } - /// @brief Move constructor for FlatBufferBuilder. - FlatBufferBuilder(FlatBufferBuilder &&other) noexcept + /// @brief Move constructor for FlatBufferBuilder_. + FlatBufferBuilder_(FlatBufferBuilder_ &&other) noexcept : buf_(1024, nullptr, false, AlignOf()), num_field_loc(0), max_voffset_(0), @@ -115,15 +122,15 @@ class FlatBufferBuilder { Swap(other); } - /// @brief Move assignment operator for FlatBufferBuilder. - FlatBufferBuilder &operator=(FlatBufferBuilder &&other) noexcept { + /// @brief Move assignment operator for FlatBufferBuilder_. + FlatBufferBuilder_ &operator=(FlatBufferBuilder_ &&other) noexcept { // Move construct a temporary and swap idiom - FlatBufferBuilder temp(std::move(other)); + FlatBufferBuilder_ temp(std::move(other)); Swap(temp); return *this; } - void Swap(FlatBufferBuilder &other) { + void Swap(FlatBufferBuilder_ &other) { using std::swap; buf_.swap(other.buf_); swap(num_field_loc, other.num_field_loc); @@ -136,7 +143,7 @@ class FlatBufferBuilder { swap(string_pool, other.string_pool); } - ~FlatBufferBuilder() { + ~FlatBufferBuilder_() { if (string_pool) delete string_pool; } @@ -145,7 +152,7 @@ class FlatBufferBuilder { buf_.reset(); // deallocate buffer } - /// @brief Reset all the state in this FlatBufferBuilder so it can be reused + /// @brief Reset all the state in this FlatBufferBuilder_ so it can be reused /// to construct another buffer. void Clear() { ClearOffsets(); @@ -181,7 +188,7 @@ class FlatBufferBuilder { uint8_t *GetCurrentBufferPointer() const { return buf_.data(); } /// @brief Get the released pointer to the serialized buffer. - /// @warning Do NOT attempt to use this FlatBufferBuilder afterwards! + /// @warning Do NOT attempt to use this FlatBufferBuilder_ afterwards! /// @return A `FlatBuffer` that owns the buffer and its allocator and /// behaves similar to a `unique_ptr` with a deleter. FLATBUFFERS_ATTRIBUTE([[deprecated("use Release() instead")]]) @@ -225,7 +232,7 @@ class FlatBufferBuilder { void Finished() const { // If you get this assert, you're attempting to get access a buffer // which hasn't been finished yet. Be sure to call - // FlatBufferBuilder::Finish with your root table. + // FlatBufferBuilder_::Finish with your root table. // If you really need to access an unfinished buffer, call // GetCurrentBufferPointer instead. FLATBUFFERS_ASSERT(finished); @@ -307,6 +314,11 @@ class FlatBufferBuilder { AddElement(field, ReferTo(off.o), static_cast(0)); } + template void AddOffset(voffset_t field, Offset64 off) { + if (off.IsNull()) return; // Don't store. + AddElement(field, ReferTo(off.o), static_cast(0)); + } + template void AddStruct(voffset_t field, const T *structptr) { if (!structptr) return; // Default, don't store. Align(AlignOf()); @@ -321,13 +333,13 @@ class FlatBufferBuilder { // Offsets initially are relative to the end of the buffer (downwards). // This function converts them to be relative to the current location // in the buffer (when stored here), pointing upwards. - uoffset_t ReferTo(uoffset_t off) { + template T ReferTo(T off) { // Align to ensure GetSize() below is correct. - Align(sizeof(uoffset_t)); + Align(sizeof(T)); // Offset must refer to something already in buffer. - const uoffset_t size = GetSize(); + const T size = GetSize(); FLATBUFFERS_ASSERT(off && off <= size); - return size - off + static_cast(sizeof(uoffset_t)); + return size - off + static_cast(sizeof(T)); } void NotNested() { @@ -346,7 +358,7 @@ class FlatBufferBuilder { // From generated code (or from the parser), we call StartTable/EndTable // with a sequence of AddElement calls in between. - uoffset_t StartTable() { + size_t StartTable() { NotNested(); nested = true; return GetSize(); @@ -426,7 +438,13 @@ class FlatBufferBuilder { // This checks a required field has been set in a given table that has // just been constructed. - template void Required(Offset table, voffset_t field); + template void Required(Offset table, voffset_t field) { + auto table_ptr = reinterpret_cast(buf_.data_at(table.o)); + bool ok = table_ptr->GetOptionalFieldOffset(field) != 0; + // If this fails, the caller will show what field needs to be set. + FLATBUFFERS_ASSERT(ok); + (void)ok; + } uoffset_t StartStruct(size_t alignment) { Align(alignment); @@ -689,6 +707,11 @@ class FlatBufferBuilder { return CreateVector(data(v), v.size()); } + template> + Offset64> CreateVector64(const std::vector &v) { + return CreateVector(data(v), v.size()); + } + // vector may be implemented using a bit-set, so we can't access it as // an array. Instead, read elements manually. // Background: https://isocpp.org/blog/2012/11/on-vectorbool @@ -1109,7 +1132,7 @@ class FlatBufferBuilder { Finish(root.o, file_identifier, true); } - void SwapBufAllocator(FlatBufferBuilder &other) { + void SwapBufAllocator(FlatBufferBuilder_ &other) { buf_.swap_allocator(other.buf_); } @@ -1119,8 +1142,8 @@ class FlatBufferBuilder { protected: // You shouldn't really be copying instances of this class. - FlatBufferBuilder(const FlatBufferBuilder &); - FlatBufferBuilder &operator=(const FlatBufferBuilder &); + FlatBufferBuilder_(const FlatBufferBuilder_ &); + FlatBufferBuilder_ &operator=(const FlatBufferBuilder_ &); void Finish(uoffset_t root, const char *file_identifier, bool size_prefix) { NotNested(); @@ -1144,7 +1167,7 @@ class FlatBufferBuilder { voffset_t id; }; - vector_downward buf_; + internal::vector_downward_ buf_; // Accumulating offsets of table members while it is being built. // We store these in the scratch pad of buf_, after the vtable offsets. @@ -1198,6 +1221,9 @@ class FlatBufferBuilder { }; /// @} +// Hack to support C++11 with default template parameters. +using FlatBufferBuilder = FlatBufferBuilder_; + /// Helpers to get a typed pointer to objects that are currently being built. /// @warning Creating new objects will lead to reallocations and invalidates /// the pointer! @@ -1212,15 +1238,6 @@ const T *GetTemporaryPointer(FlatBufferBuilder &fbb, Offset offset) { return GetMutableTemporaryPointer(fbb, offset); } -template -void FlatBufferBuilder::Required(Offset table, voffset_t field) { - auto table_ptr = reinterpret_cast(buf_.data_at(table.o)); - bool ok = table_ptr->GetOptionalFieldOffset(field) != 0; - // If this fails, the caller will show what field needs to be set. - FLATBUFFERS_ASSERT(ok); - (void)ok; -} - } // namespace flatbuffers #endif // FLATBUFFERS_VECTOR_DOWNWARD_H_ diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index eb385fab840..3417deb60be 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -326,6 +326,7 @@ struct FieldDef : public Definition { shared(false), native_inline(false), flexbuffer(false), + offset64(false), presence(kDefault), nested_flatbuffer(nullptr), padding(0), @@ -352,6 +353,7 @@ struct FieldDef : public Definition { bool native_inline; // Field will be defined inline (instead of as a pointer) // for native tables if field is a struct. bool flexbuffer; // This field contains FlexBuffer data. + bool offset64; enum Presence { // Field must always be present. @@ -952,6 +954,7 @@ class Parser : public ParserState { known_attributes_["native_default"] = true; known_attributes_["flexbuffer"] = true; known_attributes_["private"] = true; + known_attributes_["offset64"] = true; } // Copying is not allowed diff --git a/include/flatbuffers/vector_downward.h b/include/flatbuffers/vector_downward.h index e0aed840b02..4fa24215679 100644 --- a/include/flatbuffers/vector_downward.h +++ b/include/flatbuffers/vector_downward.h @@ -25,16 +25,24 @@ namespace flatbuffers { +namespace internal { +template class vector_downward_; +} + +// Hack to support C++11 with default template parameters. +using vector_downward = internal::vector_downward_<>; + +namespace internal { // This is a minimal replication of std::vector functionality, // except growing from higher to lower addresses. i.e. push_back() inserts data // in the lowest address in the vector. // Since this vector leaves the lower part unused, we support a "scratch-pad" // that can be stored there for temporary data, to share the allocated space. // Essentially, this supports 2 std::vectors in a single buffer. -class vector_downward { +template class vector_downward_ { public: - explicit vector_downward(size_t initial_size, Allocator *allocator, - bool own_allocator, size_t buffer_minalign) + explicit vector_downward_(size_t initial_size, Allocator *allocator, + bool own_allocator, size_t buffer_minalign) : allocator_(allocator), own_allocator_(own_allocator), initial_size_(initial_size), @@ -45,7 +53,7 @@ class vector_downward { cur_(nullptr), scratch_(nullptr) {} - vector_downward(vector_downward &&other) noexcept + vector_downward_(vector_downward_ &&other) noexcept // clang-format on : allocator_(other.allocator_), own_allocator_(other.own_allocator_), @@ -66,14 +74,14 @@ class vector_downward { other.scratch_ = nullptr; } - vector_downward &operator=(vector_downward &&other) noexcept { + vector_downward_ &operator=(vector_downward_ &&other) noexcept { // Move construct a temporary and swap idiom - vector_downward temp(std::move(other)); + vector_downward_ temp(std::move(other)); swap(temp); return *this; } - ~vector_downward() { + ~vector_downward_() { clear_buffer(); clear_allocator(); } @@ -147,7 +155,7 @@ class vector_downward { if (len) { ensure_space(len); cur_ -= len; - size_ += static_cast(len); + size_ += static_cast(len); } return cur_; } @@ -155,11 +163,9 @@ class vector_downward { // Returns nullptr if using the DefaultAllocator. Allocator *get_custom_allocator() { return allocator_; } - inline uoffset_t size() const { return size_; } + inline SizeT size() const { return size_; } - uoffset_t scratch_size() const { - return static_cast(scratch_ - buf_); - } + SizeT scratch_size() const { return static_cast(scratch_ - buf_); } size_t capacity() const { return reserved_; } @@ -211,12 +217,12 @@ class vector_downward { void pop(size_t bytes_to_remove) { cur_ += bytes_to_remove; - size_ -= static_cast(bytes_to_remove); + size_ -= bytes_to_remove; } void scratch_pop(size_t bytes_to_remove) { scratch_ -= bytes_to_remove; } - void swap(vector_downward &other) { + void swap(vector_downward_ &other) { using std::swap; swap(allocator_, other.allocator_); swap(own_allocator_, other.own_allocator_); @@ -229,7 +235,7 @@ class vector_downward { swap(scratch_, other.scratch_); } - void swap_allocator(vector_downward &other) { + void swap_allocator(vector_downward_ &other) { using std::swap; swap(allocator_, other.allocator_); swap(own_allocator_, other.own_allocator_); @@ -237,15 +243,16 @@ class vector_downward { private: // You shouldn't really be copying instances of this class. - FLATBUFFERS_DELETE_FUNC(vector_downward(const vector_downward &)); - FLATBUFFERS_DELETE_FUNC(vector_downward &operator=(const vector_downward &)); + FLATBUFFERS_DELETE_FUNC(vector_downward_(const vector_downward_ &)); + FLATBUFFERS_DELETE_FUNC( + vector_downward_ &operator=(const vector_downward_ &)); Allocator *allocator_; bool own_allocator_; size_t initial_size_; size_t buffer_minalign_; size_t reserved_; - uoffset_t size_; + SizeT size_; uint8_t *buf_; uint8_t *cur_; // Points at location between empty (below) and used (above). uint8_t *scratch_; // Points to the end of the scratchpad in use. @@ -268,6 +275,7 @@ class vector_downward { } }; +} // namespace internal } // namespace flatbuffers #endif // FLATBUFFERS_VECTOR_DOWNWARD_H_ diff --git a/src/annotated_binary_text_gen.cpp b/src/annotated_binary_text_gen.cpp index 1d44803e53a..b51ac69d11b 100644 --- a/src/annotated_binary_text_gen.cpp +++ b/src/annotated_binary_text_gen.cpp @@ -1,4 +1,5 @@ #include "annotated_binary_text_gen.h" +#include #include #include @@ -44,7 +45,9 @@ static std::string ToString(const BinarySectionType type) { } static bool IsOffset(const BinaryRegionType type) { - return type == BinaryRegionType::UOffset || type == BinaryRegionType::SOffset; + return type == BinaryRegionType::UOffset || + type == BinaryRegionType::SOffset || + type == BinaryRegionType::UOffset64; } template std::string ToString(T value) { @@ -119,6 +122,9 @@ static std::string ToValueString(const BinaryRegion ®ion, case BinaryRegionType::UType: return ToValueString(region, binary); // Handle Offsets separately, incase they add additional details. + case BinaryRegionType::UOffset64: + s += ToValueString(region, binary); + break; case BinaryRegionType::UOffset: s += ToValueString(region, binary); break; diff --git a/src/binary_annotator.cpp b/src/binary_annotator.cpp index c5fa42f7297..8dd21ee4f5b 100644 --- a/src/binary_annotator.cpp +++ b/src/binary_annotator.cpp @@ -1,6 +1,9 @@ #include "binary_annotator.h" +#include + #include +#include #include #include #include @@ -656,7 +659,30 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset, } // Read the offset - const auto offset_from_field = ReadScalar(field_offset); + const bool offset64 = true; + // field->attributes()->LookupByKey("offset64") != nullptr; + + uint64_t offset = 0; + bool has_offset_from = false; + uint64_t length = 0; + BinaryRegionType type = BinaryRegionType::UOffset; + + if (offset64) { + length = sizeof(uint64_t); + type = BinaryRegionType::UOffset64; + const auto offset_from_field = ReadScalar(field_offset); + if ((has_offset_from = offset_from_field.has_value())) { + offset = offset_from_field.value(); + std::cout << "reading 64 bit offset: " << offset << std::endl; + } + } else { + length = sizeof(uint32_t); + const auto offset_from_field = ReadScalar(field_offset); + if ((has_offset_from = offset_from_field.has_value())) { + offset = offset_from_field.value(); + } + } + // const auto offset_from_field = ReadScalar(field_offset); uint64_t offset_of_next_item = 0; BinaryRegionComment offset_field_comment; offset_field_comment.type = BinaryRegionCommentType::TableOffsetField; @@ -666,7 +692,7 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset, // Validate any field that isn't inline (i.e., non-structs). if (!IsInlineField(field)) { - if (!offset_from_field.has_value()) { + if (!has_offset_from) { const uint64_t remaining = RemainingBytes(field_offset); SetError(offset_field_comment, @@ -678,14 +704,14 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset, continue; } - offset_of_next_item = field_offset + offset_from_field.value(); + offset_of_next_item = field_offset + offset; if (!IsValidOffset(offset_of_next_item)) { SetError(offset_field_comment, BinaryRegionStatus::ERROR_OFFSET_OUT_OF_BINARY); - regions.push_back(MakeBinaryRegion( - field_offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, - offset_of_next_item, offset_field_comment)); + regions.push_back( + MakeBinaryRegion(field_offset, length, type, 0, + offset_of_next_item, offset_field_comment)); continue; } } @@ -703,7 +729,7 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset, offset_field_comment.default_value = "(table)"; regions.push_back(MakeBinaryRegion( - field_offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + field_offset, length, type, 0, offset_of_next_item, offset_field_comment)); BuildTable(offset_of_next_item, BinarySectionType::Table, @@ -714,7 +740,7 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset, case reflection::BaseType::String: { offset_field_comment.default_value = "(string)"; regions.push_back(MakeBinaryRegion( - field_offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + field_offset, length, type, 0, offset_of_next_item, offset_field_comment)); BuildString(offset_of_next_item, table, field); } break; @@ -722,7 +748,7 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset, case reflection::BaseType::Vector: { offset_field_comment.default_value = "(vector)"; regions.push_back(MakeBinaryRegion( - field_offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + field_offset, length, type, 0, offset_of_next_item, offset_field_comment)); BuildVector(offset_of_next_item, table, field, table_offset, vtable->fields); @@ -768,8 +794,8 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset, offset_field_comment.default_value = "(union of type `" + enum_type + "`)"; - regions.push_back(MakeBinaryRegion(field_offset, sizeof(uint32_t), - BinaryRegionType::UOffset, 0, + regions.push_back(MakeBinaryRegion(field_offset, length, + type, 0, union_offset, offset_field_comment)); } break; diff --git a/src/binary_annotator.h b/src/binary_annotator.h index fd0a9af0c7d..b52dcc95e7d 100644 --- a/src/binary_annotator.h +++ b/src/binary_annotator.h @@ -48,6 +48,7 @@ enum class BinaryRegionType { Float = 15, Double = 16, UType = 17, + UOffset64 = 18, }; template @@ -216,6 +217,7 @@ inline static BinaryRegionType GetRegionType(reflection::BaseType base_type) { inline static std::string ToString(const BinaryRegionType type) { switch (type) { case BinaryRegionType::UOffset: return "UOffset32"; + case BinaryRegionType::UOffset64: return "UOffset64"; case BinaryRegionType::SOffset: return "SOffset32"; case BinaryRegionType::VOffset: return "VOffset16"; case BinaryRegionType::Bool: return "bool"; diff --git a/src/idl_gen_cpp.cpp b/src/idl_gen_cpp.cpp index 922be05accf..f1a461733bb 100644 --- a/src/idl_gen_cpp.cpp +++ b/src/idl_gen_cpp.cpp @@ -766,12 +766,15 @@ class CppGenerator : public BaseGenerator { // Return a C++ type for any type (scalar/pointer) specifically for // building a flatbuffer. std::string GenTypeWire(const Type &type, const char *postfix, - bool user_facing_type) const { + bool user_facing_type, bool _64_bit_offset = false) const { if (IsScalar(type.base_type)) { return GenTypeBasic(type, user_facing_type) + postfix; } else if (IsStruct(type)) { return "const " + GenTypePointer(type) + " *"; } else { + if(_64_bit_offset) { + return "::flatbuffers::Offset64<" + GenTypePointer(type) + ">" + postfix; + } return "::flatbuffers::Offset<" + GenTypePointer(type) + ">" + postfix; } } @@ -2896,8 +2899,9 @@ class CppGenerator : public BaseGenerator { // void add_name(type name) { // fbb_.AddElement(offset, name, default); // } + const bool is_offset_64 = field.attributes.Lookup("offset64"); code_.SetValue("FIELD_NAME", Name(field)); - code_.SetValue("FIELD_TYPE", GenTypeWire(field.value.type, " ", true)); + code_.SetValue("FIELD_TYPE", GenTypeWire(field.value.type, " ", true, is_offset_64)); code_.SetValue("ADD_OFFSET", Name(struct_def) + "::" + offset); code_.SetValue("ADD_NAME", name); code_.SetValue("ADD_VALUE", value); diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 36497eef248..5b30c4cfa98 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -19,8 +19,10 @@ #include #include #include +#include #include "flatbuffers/base.h" +#include "flatbuffers/buffer.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" @@ -1036,6 +1038,8 @@ CheckedError Parser::ParseField(StructDef &struct_def) { } } + field->offset64 = field->attributes.Lookup("offset64") != nullptr; + // For historical convenience reasons, string keys are assumed required. // Scalars are kDefault unless otherwise specified. // Nonscalars are kOptional unless required; @@ -1529,6 +1533,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, CTYPE val, valdef; \ ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \ + std::cout << "adding element: " << field->name << " val: " << field_value.offset << " 64-bit: " << field->offset64 << std::endl; \ builder_.AddElement(field_value.offset, val, valdef); \ } \ } \ @@ -1543,7 +1548,13 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, } else { \ CTYPE val; \ ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ - builder_.AddOffset(field_value.offset, val); \ + std::cout << "adding offset: " << field->name << " val: " << val.o << " 64-bit: " << field->offset64 << std::endl; \ + if(field->offset64) { \ + Offset64 offset(val.o); \ + builder_.AddOffset(field_value.offset, offset); \ + } else { \ + builder_.AddOffset(field_value.offset, val); \ + }\ } \ break; FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD) diff --git a/tests/64bit/run.sh b/tests/64bit/run.sh new file mode 100755 index 00000000000..9691b872adf --- /dev/null +++ b/tests/64bit/run.sh @@ -0,0 +1,5 @@ +./flatc --cpp -o tests/64bit tests/64bit/test_64bit.fbs + +./flatc -b -o tests/64bit tests/64bit/test_64bit.fbs tests/64bit/test_64bit.json + +./flatc -o tests/64bit --annotate tests/64bit/test_64bit.fbs -- tests/64bit/test_64bit.bin diff --git a/tests/64bit/test_64bit.afb b/tests/64bit/test_64bit.afb new file mode 100644 index 00000000000..2d04a471863 --- /dev/null +++ b/tests/64bit/test_64bit.afb @@ -0,0 +1,25 @@ +// Annotated Flatbuffer Binary +// +// Schema file: tests/64bit/test_64bit.fbs +// Binary file: tests/64bit/test_64bit.bin + +header: + +0x00 | 0C 00 00 00 | UOffset32 | 0x0000000C (12) Loc: +0x0C | offset to root table `RootTable` + +vtable (RootTable): + +0x04 | 08 00 | uint16_t | 0x0008 (8) | size of this vtable + +0x06 | 14 00 | uint16_t | 0x0014 (20) | size of referring table + +0x08 | 04 00 | VOffset16 | 0x0004 (4) | offset to field `big_vector` (id: 0) + +0x0A | 10 00 | VOffset16 | 0x0010 (16) | offset to field `a` (id: 1) + +root_table (RootTable): + +0x0C | 08 00 00 00 | SOffset32 | 0x00000008 (8) Loc: +0x04 | offset to vtable + +0x10 | 10 00 00 00 00 00 00 00 | UOffset64 | 0x0000000000000010 (16) Loc: +0x20 | offset to field `big_vector` (vector) + +0x18 | 00 00 00 00 | uint8_t[4] | .... | padding + +0x1C | D2 04 00 00 | uint32_t | 0x000004D2 (1234) | table field `a` (Int) + +vector (RootTable.big_vector): + +0x20 | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of vector (# items) + +0x24 | 01 | uint8_t | 0x01 (1) | value[0] + +0x25 | 02 | uint8_t | 0x02 (2) | value[1] + +0x26 | 03 | uint8_t | 0x03 (3) | value[2] diff --git a/tests/64bit/test_64bit.bin b/tests/64bit/test_64bit.bin new file mode 100644 index 0000000000000000000000000000000000000000..b0308bffe3413613c5872f91926cd93e6ee2b030 GIT binary patch literal 40 icmd;KU|`^25Mf|p5CGB)KoS8iu`n<&1G$V$%nSe&ya6-- literal 0 HcmV?d00001 diff --git a/tests/64bit/test_64bit.fbs b/tests/64bit/test_64bit.fbs new file mode 100644 index 00000000000..7aac9315f67 --- /dev/null +++ b/tests/64bit/test_64bit.fbs @@ -0,0 +1,6 @@ +table RootTable { + big_vector:[ubyte] (offset64); + a:int; +} + +root_type RootTable; \ No newline at end of file diff --git a/tests/64bit/test_64bit.json b/tests/64bit/test_64bit.json new file mode 100644 index 00000000000..e017120571c --- /dev/null +++ b/tests/64bit/test_64bit.json @@ -0,0 +1,4 @@ +{ + "big_vector": [1, 2, 3], + "a": 1234 +} \ No newline at end of file diff --git a/tests/64bit/test_64bit_generated.h b/tests/64bit/test_64bit_generated.h new file mode 100644 index 00000000000..95c456892aa --- /dev/null +++ b/tests/64bit/test_64bit_generated.h @@ -0,0 +1,112 @@ +// automatically generated by the FlatBuffers compiler, do not modify + + +#ifndef FLATBUFFERS_GENERATED_TEST64BIT_H_ +#define FLATBUFFERS_GENERATED_TEST64BIT_H_ + +#include "flatbuffers/flatbuffers.h" + +// Ensure the included flatbuffers.h is the same version as when this file was +// generated, otherwise it may not be compatible. +static_assert(FLATBUFFERS_VERSION_MAJOR == 23 && + FLATBUFFERS_VERSION_MINOR == 1 && + FLATBUFFERS_VERSION_REVISION == 21, + "Non-compatible flatbuffers version included"); + +struct RootTable; +struct RootTableBuilder; + +struct RootTable FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { + typedef RootTableBuilder Builder; + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_BIG_VECTOR = 4, + VT_A = 6 + }; + const ::flatbuffers::Vector *big_vector() const { + return GetPointer *>(VT_BIG_VECTOR); + } + int32_t a() const { + return GetField(VT_A, 0); + } + bool Verify(::flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyOffset(verifier, VT_BIG_VECTOR) && + verifier.VerifyVector(big_vector()) && + VerifyField(verifier, VT_A, 4) && + verifier.EndTable(); + } +}; + +struct RootTableBuilder { + typedef RootTable Table; + ::flatbuffers::FlatBufferBuilder &fbb_; + ::flatbuffers::uoffset_t start_; + void add_big_vector(::flatbuffers::Offset64<::flatbuffers::Vector> big_vector) { + fbb_.AddOffset(RootTable::VT_BIG_VECTOR, big_vector); + } + void add_a(int32_t a) { + fbb_.AddElement(RootTable::VT_A, a, 0); + } + explicit RootTableBuilder(::flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + ::flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = ::flatbuffers::Offset(end); + return o; + } +}; + +inline ::flatbuffers::Offset CreateRootTable( + ::flatbuffers::FlatBufferBuilder &_fbb, + ::flatbuffers::Offset<::flatbuffers::Vector> big_vector = 0, + int32_t a = 0) { + RootTableBuilder builder_(_fbb); + builder_.add_a(a); + builder_.add_big_vector(big_vector); + return builder_.Finish(); +} + +inline ::flatbuffers::Offset CreateRootTableDirect( + ::flatbuffers::FlatBufferBuilder &_fbb, + const std::vector *big_vector = nullptr, + int32_t a = 0) { + auto big_vector__ = big_vector ? _fbb.CreateVector(*big_vector) : 0; + return CreateRootTable( + _fbb, + big_vector__, + a); +} + +inline const RootTable *GetRootTable(const void *buf) { + return ::flatbuffers::GetRoot(buf); +} + +inline const RootTable *GetSizePrefixedRootTable(const void *buf) { + return ::flatbuffers::GetSizePrefixedRoot(buf); +} + +inline bool VerifyRootTableBuffer( + ::flatbuffers::Verifier &verifier) { + return verifier.VerifyBuffer(nullptr); +} + +inline bool VerifySizePrefixedRootTableBuffer( + ::flatbuffers::Verifier &verifier) { + return verifier.VerifySizePrefixedBuffer(nullptr); +} + +inline void FinishRootTableBuffer( + ::flatbuffers::FlatBufferBuilder &fbb, + ::flatbuffers::Offset root) { + fbb.Finish(root); +} + +inline void FinishSizePrefixedRootTableBuffer( + ::flatbuffers::FlatBufferBuilder &fbb, + ::flatbuffers::Offset root) { + fbb.FinishSizePrefixed(root); +} + +#endif // FLATBUFFERS_GENERATED_TEST64BIT_H_ diff --git a/tests/test_builder.cpp b/tests/test_builder.cpp index 047a380c06e..9d9c0e0eace 100644 --- a/tests/test_builder.cpp +++ b/tests/test_builder.cpp @@ -1,5 +1,6 @@ #include "test_builder.h" +#include "flatbuffers/base.h" #include "flatbuffers/stl_emulation.h" #include "monster_test_generated.h" @@ -14,13 +15,13 @@ class TestHeapBuilder : public flatbuffers::FlatBufferBuilder { public: TestHeapBuilder() - : flatbuffers::FlatBufferBuilder(2048, new OwnedAllocator(), true) {} + : FlatBufferBuilder_(2048, new OwnedAllocator(), true) {} TestHeapBuilder(TestHeapBuilder &&other) - : FlatBufferBuilder(std::move(other)) {} + : FlatBufferBuilder_(std::move(other)) {} TestHeapBuilder &operator=(TestHeapBuilder &&other) { - FlatBufferBuilder::operator=(std::move(other)); + FlatBufferBuilder_::operator=(std::move(other)); return *this; } }; @@ -38,10 +39,10 @@ struct GrpcLikeMessageBuilder : private AllocatorMember, public: GrpcLikeMessageBuilder() - : flatbuffers::FlatBufferBuilder(1024, &member_allocator_, false) {} + : FlatBufferBuilder_(1024, &member_allocator_, false) {} GrpcLikeMessageBuilder(GrpcLikeMessageBuilder &&other) - : FlatBufferBuilder(1024, &member_allocator_, false) { + : FlatBufferBuilder_(1024, &member_allocator_, false) { // Default construct and swap idiom. Swap(other); } @@ -55,7 +56,7 @@ struct GrpcLikeMessageBuilder : private AllocatorMember, void Swap(GrpcLikeMessageBuilder &other) { // No need to swap member_allocator_ because it's stateless. - FlatBufferBuilder::Swap(other); + FlatBufferBuilder_::Swap(other); // After swapping the FlatBufferBuilder, we swap back the allocator, which // restores the original allocator back in place. This is necessary because // MessageBuilder's allocator is its own member (SliceAllocatorMember). The From 0033e1550c6c2e7a7f762621f4d8c62fd156b699 Mon Sep 17 00:00:00 2001 From: Derek Bailey Date: Thu, 26 Jan 2023 13:07:46 -0800 Subject: [PATCH 02/62] Made vector_downward work on 64 bit types --- include/flatbuffers/flatbuffer_builder.h | 46 ++++++--------- include/flatbuffers/vector_downward.h | 73 +++++++++++------------- tests/test_builder.cpp | 12 ++-- 3 files changed, 56 insertions(+), 75 deletions(-) diff --git a/include/flatbuffers/flatbuffer_builder.h b/include/flatbuffers/flatbuffer_builder.h index 9f72e914180..ddc6e7d6971 100644 --- a/include/flatbuffers/flatbuffer_builder.h +++ b/include/flatbuffers/flatbuffer_builder.h @@ -37,12 +37,6 @@ namespace flatbuffers { -// namespace internal { -// template class FlatBufferBuilder_; -// } - - - // Converts a Field ID to a virtual table offset. inline voffset_t FieldIndexToOffset(voffset_t field_id) { // Should correspond to what EndTable() below builds up. @@ -65,19 +59,18 @@ T *data(std::vector &v) { return v.empty() ? reinterpret_cast(&t) : &v.front(); } - /// @addtogroup flatbuffers_cpp_api /// @{ -/// @class FlatBufferBuilder_ +/// @class FlatBufferBuilder /// @brief Helper class to hold data needed in creation of a FlatBuffer. /// To serialize data, you typically call one of the `Create*()` functions in /// the generated code, which in turn call a sequence of `StartTable`/ /// `PushElement`/`AddElement`/`EndTable`, or the builtin `CreateString`/ /// `CreateVector` functions. Do this is depth-first order to build up a tree to /// the root. `Finish()` wraps up the buffer ready for transport. -template class FlatBufferBuilder_ { +class FlatBufferBuilder { public: - /// @brief Default constructor for FlatBufferBuilder_. + /// @brief Default constructor for FlatBufferBuilder. /// @param[in] initial_size The initial size of the buffer, in bytes. Defaults /// to `1024`. /// @param[in] allocator An `Allocator` to use. If null will use @@ -88,7 +81,7 @@ template class FlatBufferBuilder_ { /// minimum alignment upon reallocation. Only needed if you intend to store /// types with custom alignment AND you wish to read the buffer in-place /// directly after creation. - explicit FlatBufferBuilder_( + explicit FlatBufferBuilder( size_t initial_size = 1024, Allocator *allocator = nullptr, bool own_allocator = false, size_t buffer_minalign = AlignOf()) @@ -104,8 +97,8 @@ template class FlatBufferBuilder_ { EndianCheck(); } - /// @brief Move constructor for FlatBufferBuilder_. - FlatBufferBuilder_(FlatBufferBuilder_ &&other) noexcept + /// @brief Move constructor for FlatBufferBuilder. + FlatBufferBuilder(FlatBufferBuilder &&other) noexcept : buf_(1024, nullptr, false, AlignOf()), num_field_loc(0), max_voffset_(0), @@ -122,15 +115,15 @@ template class FlatBufferBuilder_ { Swap(other); } - /// @brief Move assignment operator for FlatBufferBuilder_. - FlatBufferBuilder_ &operator=(FlatBufferBuilder_ &&other) noexcept { + /// @brief Move assignment operator for FlatBufferBuilder. + FlatBufferBuilder &operator=(FlatBufferBuilder &&other) noexcept { // Move construct a temporary and swap idiom - FlatBufferBuilder_ temp(std::move(other)); + FlatBufferBuilder temp(std::move(other)); Swap(temp); return *this; } - void Swap(FlatBufferBuilder_ &other) { + void Swap(FlatBufferBuilder &other) { using std::swap; buf_.swap(other.buf_); swap(num_field_loc, other.num_field_loc); @@ -143,7 +136,7 @@ template class FlatBufferBuilder_ { swap(string_pool, other.string_pool); } - ~FlatBufferBuilder_() { + ~FlatBufferBuilder() { if (string_pool) delete string_pool; } @@ -152,7 +145,7 @@ template class FlatBufferBuilder_ { buf_.reset(); // deallocate buffer } - /// @brief Reset all the state in this FlatBufferBuilder_ so it can be reused + /// @brief Reset all the state in this FlatBufferBuilder so it can be reused /// to construct another buffer. void Clear() { ClearOffsets(); @@ -188,7 +181,7 @@ template class FlatBufferBuilder_ { uint8_t *GetCurrentBufferPointer() const { return buf_.data(); } /// @brief Get the released pointer to the serialized buffer. - /// @warning Do NOT attempt to use this FlatBufferBuilder_ afterwards! + /// @warning Do NOT attempt to use this FlatBufferBuilder afterwards! /// @return A `FlatBuffer` that owns the buffer and its allocator and /// behaves similar to a `unique_ptr` with a deleter. FLATBUFFERS_ATTRIBUTE([[deprecated("use Release() instead")]]) @@ -232,7 +225,7 @@ template class FlatBufferBuilder_ { void Finished() const { // If you get this assert, you're attempting to get access a buffer // which hasn't been finished yet. Be sure to call - // FlatBufferBuilder_::Finish with your root table. + // FlatBufferBuilder::Finish with your root table. // If you really need to access an unfinished buffer, call // GetCurrentBufferPointer instead. FLATBUFFERS_ASSERT(finished); @@ -1132,7 +1125,7 @@ template class FlatBufferBuilder_ { Finish(root.o, file_identifier, true); } - void SwapBufAllocator(FlatBufferBuilder_ &other) { + void SwapBufAllocator(FlatBufferBuilder &other) { buf_.swap_allocator(other.buf_); } @@ -1142,8 +1135,8 @@ template class FlatBufferBuilder_ { protected: // You shouldn't really be copying instances of this class. - FlatBufferBuilder_(const FlatBufferBuilder_ &); - FlatBufferBuilder_ &operator=(const FlatBufferBuilder_ &); + FlatBufferBuilder(const FlatBufferBuilder &); + FlatBufferBuilder &operator=(const FlatBufferBuilder &); void Finish(uoffset_t root, const char *file_identifier, bool size_prefix) { NotNested(); @@ -1167,7 +1160,7 @@ template class FlatBufferBuilder_ { voffset_t id; }; - internal::vector_downward_ buf_; + vector_downward buf_; // Accumulating offsets of table members while it is being built. // We store these in the scratch pad of buf_, after the vtable offsets. @@ -1221,9 +1214,6 @@ template class FlatBufferBuilder_ { }; /// @} -// Hack to support C++11 with default template parameters. -using FlatBufferBuilder = FlatBufferBuilder_; - /// Helpers to get a typed pointer to objects that are currently being built. /// @warning Creating new objects will lead to reallocations and invalidates /// the pointer! diff --git a/include/flatbuffers/vector_downward.h b/include/flatbuffers/vector_downward.h index 4fa24215679..bf8b1b19dd7 100644 --- a/include/flatbuffers/vector_downward.h +++ b/include/flatbuffers/vector_downward.h @@ -25,24 +25,16 @@ namespace flatbuffers { -namespace internal { -template class vector_downward_; -} - -// Hack to support C++11 with default template parameters. -using vector_downward = internal::vector_downward_<>; - -namespace internal { // This is a minimal replication of std::vector functionality, // except growing from higher to lower addresses. i.e. push_back() inserts data // in the lowest address in the vector. // Since this vector leaves the lower part unused, we support a "scratch-pad" // that can be stored there for temporary data, to share the allocated space. // Essentially, this supports 2 std::vectors in a single buffer. -template class vector_downward_ { +class vector_downward { public: - explicit vector_downward_(size_t initial_size, Allocator *allocator, - bool own_allocator, size_t buffer_minalign) + explicit vector_downward(uint64_t initial_size, Allocator *allocator, + bool own_allocator, uint64_t buffer_minalign) : allocator_(allocator), own_allocator_(own_allocator), initial_size_(initial_size), @@ -53,7 +45,7 @@ template class vector_downward_ { cur_(nullptr), scratch_(nullptr) {} - vector_downward_(vector_downward_ &&other) noexcept + vector_downward(vector_downward &&other) noexcept // clang-format on : allocator_(other.allocator_), own_allocator_(other.own_allocator_), @@ -74,14 +66,14 @@ template class vector_downward_ { other.scratch_ = nullptr; } - vector_downward_ &operator=(vector_downward_ &&other) noexcept { + vector_downward &operator=(vector_downward &&other) noexcept { // Move construct a temporary and swap idiom - vector_downward_ temp(std::move(other)); + vector_downward temp(std::move(other)); swap(temp); return *this; } - ~vector_downward_() { + ~vector_downward() { clear_buffer(); clear_allocator(); } @@ -116,10 +108,10 @@ template class vector_downward_ { } // Relinquish the pointer to the caller. - uint8_t *release_raw(size_t &allocated_bytes, size_t &offset) { + uint8_t *release_raw(uint64_t &allocated_bytes, uint64_t &offset) { auto *buf = buf_; allocated_bytes = reserved_; - offset = static_cast(cur_ - buf_); + offset = static_cast(cur_ - buf_); // release_raw only relinquishes the buffer ownership. // Does not deallocate or reset the allocator. Destructor will do that. @@ -142,20 +134,20 @@ template class vector_downward_ { return fb; } - size_t ensure_space(size_t len) { + uint64_t ensure_space(uint64_t len) { FLATBUFFERS_ASSERT(cur_ >= scratch_ && scratch_ >= buf_); - if (len > static_cast(cur_ - scratch_)) { reallocate(len); } + if (len > static_cast(cur_ - scratch_)) { reallocate(len); } // Beyond this, signed offsets may not have enough range: // (FlatBuffers > 2GB not supported). FLATBUFFERS_ASSERT(size() < FLATBUFFERS_MAX_BUFFER_SIZE); return len; } - inline uint8_t *make_space(size_t len) { + inline uint8_t *make_space(uint64_t len) { if (len) { ensure_space(len); cur_ -= len; - size_ += static_cast(len); + size_ += len; } return cur_; } @@ -163,11 +155,11 @@ template class vector_downward_ { // Returns nullptr if using the DefaultAllocator. Allocator *get_custom_allocator() { return allocator_; } - inline SizeT size() const { return size_; } + inline uint64_t size() const { return size_; } - SizeT scratch_size() const { return static_cast(scratch_ - buf_); } + uint64_t scratch_size() const { return static_cast(scratch_ - buf_); } - size_t capacity() const { return reserved_; } + uint64_t capacity() const { return reserved_; } uint8_t *data() const { FLATBUFFERS_ASSERT(cur_); @@ -184,9 +176,9 @@ template class vector_downward_ { return scratch_; } - uint8_t *data_at(size_t offset) const { return buf_ + reserved_ - offset; } + uint8_t *data_at(uint64_t offset) const { return buf_ + reserved_ - offset; } - void push(const uint8_t *bytes, size_t num) { + void push(const uint8_t *bytes, uint64_t num) { if (num > 0) { memcpy(make_space(num), bytes, num); } } @@ -204,25 +196,25 @@ template class vector_downward_ { // fill() is most frequently called with small byte counts (<= 4), // which is why we're using loops rather than calling memset. - void fill(size_t zero_pad_bytes) { + void fill(uint64_t zero_pad_bytes) { make_space(zero_pad_bytes); - for (size_t i = 0; i < zero_pad_bytes; i++) cur_[i] = 0; + for (uint64_t i = 0; i < zero_pad_bytes; i++) cur_[i] = 0; } // Version for when we know the size is larger. // Precondition: zero_pad_bytes > 0 - void fill_big(size_t zero_pad_bytes) { + void fill_big(uint64_t zero_pad_bytes) { memset(make_space(zero_pad_bytes), 0, zero_pad_bytes); } - void pop(size_t bytes_to_remove) { + void pop(uint64_t bytes_to_remove) { cur_ += bytes_to_remove; size_ -= bytes_to_remove; } - void scratch_pop(size_t bytes_to_remove) { scratch_ -= bytes_to_remove; } + void scratch_pop(uint64_t bytes_to_remove) { scratch_ -= bytes_to_remove; } - void swap(vector_downward_ &other) { + void swap(vector_downward &other) { using std::swap; swap(allocator_, other.allocator_); swap(own_allocator_, other.own_allocator_); @@ -235,7 +227,7 @@ template class vector_downward_ { swap(scratch_, other.scratch_); } - void swap_allocator(vector_downward_ &other) { + void swap_allocator(vector_downward &other) { using std::swap; swap(allocator_, other.allocator_); swap(own_allocator_, other.own_allocator_); @@ -243,21 +235,21 @@ template class vector_downward_ { private: // You shouldn't really be copying instances of this class. - FLATBUFFERS_DELETE_FUNC(vector_downward_(const vector_downward_ &)); + FLATBUFFERS_DELETE_FUNC(vector_downward(const vector_downward &)); FLATBUFFERS_DELETE_FUNC( - vector_downward_ &operator=(const vector_downward_ &)); + vector_downward &operator=(const vector_downward &)); Allocator *allocator_; bool own_allocator_; - size_t initial_size_; - size_t buffer_minalign_; - size_t reserved_; - SizeT size_; + uint64_t initial_size_; + uint64_t buffer_minalign_; + uint64_t reserved_; + uint64_t size_; uint8_t *buf_; uint8_t *cur_; // Points at location between empty (below) and used (above). uint8_t *scratch_; // Points to the end of the scratchpad in use. - void reallocate(size_t len) { + void reallocate(uint64_t len) { auto old_reserved = reserved_; auto old_size = size(); auto old_scratch_size = scratch_size(); @@ -275,7 +267,6 @@ template class vector_downward_ { } }; -} // namespace internal } // namespace flatbuffers #endif // FLATBUFFERS_VECTOR_DOWNWARD_H_ diff --git a/tests/test_builder.cpp b/tests/test_builder.cpp index 9d9c0e0eace..f998a4bc607 100644 --- a/tests/test_builder.cpp +++ b/tests/test_builder.cpp @@ -15,13 +15,13 @@ class TestHeapBuilder : public flatbuffers::FlatBufferBuilder { public: TestHeapBuilder() - : FlatBufferBuilder_(2048, new OwnedAllocator(), true) {} + : FlatBufferBuilder(2048, new OwnedAllocator(), true) {} TestHeapBuilder(TestHeapBuilder &&other) - : FlatBufferBuilder_(std::move(other)) {} + : FlatBufferBuilder(std::move(other)) {} TestHeapBuilder &operator=(TestHeapBuilder &&other) { - FlatBufferBuilder_::operator=(std::move(other)); + FlatBufferBuilder::operator=(std::move(other)); return *this; } }; @@ -39,10 +39,10 @@ struct GrpcLikeMessageBuilder : private AllocatorMember, public: GrpcLikeMessageBuilder() - : FlatBufferBuilder_(1024, &member_allocator_, false) {} + : FlatBufferBuilder(1024, &member_allocator_, false) {} GrpcLikeMessageBuilder(GrpcLikeMessageBuilder &&other) - : FlatBufferBuilder_(1024, &member_allocator_, false) { + : FlatBufferBuilder(1024, &member_allocator_, false) { // Default construct and swap idiom. Swap(other); } @@ -56,7 +56,7 @@ struct GrpcLikeMessageBuilder : private AllocatorMember, void Swap(GrpcLikeMessageBuilder &other) { // No need to swap member_allocator_ because it's stateless. - FlatBufferBuilder_::Swap(other); + FlatBufferBuilder::Swap(other); // After swapping the FlatBufferBuilder, we swap back the allocator, which // restores the original allocator back in place. This is necessary because // MessageBuilder's allocator is its own member (SliceAllocatorMember). The From a91e67d8a83c46b4c65ebeb0909e96c1cbc3919f Mon Sep 17 00:00:00 2001 From: Derek Bailey Date: Thu, 26 Jan 2023 14:50:43 -0800 Subject: [PATCH 03/62] vector_downward uses size_t, added offset64 to reflection --- include/flatbuffers/flatbuffer_builder.h | 2 +- include/flatbuffers/reflection_generated.h | 21 ++++++-- include/flatbuffers/vector_downward.h | 58 +++++++++++++--------- python/flatbuffers/reflection/Field.py | 23 ++++++--- reflection/reflection.fbs | 2 + src/binary_annotator.cpp | 9 +--- src/idl_parser.cpp | 3 +- 7 files changed, 74 insertions(+), 44 deletions(-) diff --git a/include/flatbuffers/flatbuffer_builder.h b/include/flatbuffers/flatbuffer_builder.h index ddc6e7d6971..7f6df17aeb8 100644 --- a/include/flatbuffers/flatbuffer_builder.h +++ b/include/flatbuffers/flatbuffer_builder.h @@ -157,7 +157,7 @@ class FlatBufferBuilder { } /// @brief The current size of the serialized buffer, counting from the end. - /// @return Returns an `uoffset_t` with the current size of the buffer. + /// @return Returns an `size_t` with the current size of the buffer. uoffset_t GetSize() const { return buf_.size(); } /// @brief Get the serialized buffer (after you call `Finish()`). diff --git a/include/flatbuffers/reflection_generated.h b/include/flatbuffers/reflection_generated.h index f236916f474..091f2ce0439 100644 --- a/include/flatbuffers/reflection_generated.h +++ b/include/flatbuffers/reflection_generated.h @@ -601,7 +601,8 @@ struct Field FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { VT_ATTRIBUTES = 22, VT_DOCUMENTATION = 24, VT_OPTIONAL = 26, - VT_PADDING = 28 + VT_PADDING = 28, + VT_OFFSET64 = 30 }; const ::flatbuffers::String *name() const { return GetPointer(VT_NAME); @@ -649,6 +650,10 @@ struct Field FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { uint16_t padding() const { return GetField(VT_PADDING, 0); } + /// If the field uses 64-bit offsets. + bool offset64() const { + return GetField(VT_OFFSET64, 0) != 0; + } bool Verify(::flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyOffsetRequired(verifier, VT_NAME) && @@ -670,6 +675,7 @@ struct Field FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { verifier.VerifyVectorOfStrings(documentation()) && VerifyField(verifier, VT_OPTIONAL, 1) && VerifyField(verifier, VT_PADDING, 2) && + VerifyField(verifier, VT_OFFSET64, 1) && verifier.EndTable(); } }; @@ -717,6 +723,9 @@ struct FieldBuilder { void add_padding(uint16_t padding) { fbb_.AddElement(Field::VT_PADDING, padding, 0); } + void add_offset64(bool offset64) { + fbb_.AddElement(Field::VT_OFFSET64, static_cast(offset64), 0); + } explicit FieldBuilder(::flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); @@ -744,7 +753,8 @@ inline ::flatbuffers::Offset CreateField( ::flatbuffers::Offset<::flatbuffers::Vector<::flatbuffers::Offset>> attributes = 0, ::flatbuffers::Offset<::flatbuffers::Vector<::flatbuffers::Offset<::flatbuffers::String>>> documentation = 0, bool optional = false, - uint16_t padding = 0) { + uint16_t padding = 0, + bool offset64 = false) { FieldBuilder builder_(_fbb); builder_.add_default_real(default_real); builder_.add_default_integer(default_integer); @@ -755,6 +765,7 @@ inline ::flatbuffers::Offset CreateField( builder_.add_padding(padding); builder_.add_offset(offset); builder_.add_id(id); + builder_.add_offset64(offset64); builder_.add_optional(optional); builder_.add_key(key); builder_.add_required(required); @@ -776,7 +787,8 @@ inline ::flatbuffers::Offset CreateFieldDirect( std::vector<::flatbuffers::Offset> *attributes = nullptr, const std::vector<::flatbuffers::Offset<::flatbuffers::String>> *documentation = nullptr, bool optional = false, - uint16_t padding = 0) { + uint16_t padding = 0, + bool offset64 = false) { auto name__ = name ? _fbb.CreateString(name) : 0; auto attributes__ = attributes ? _fbb.CreateVectorOfSortedTables(attributes) : 0; auto documentation__ = documentation ? _fbb.CreateVector<::flatbuffers::Offset<::flatbuffers::String>>(*documentation) : 0; @@ -794,7 +806,8 @@ inline ::flatbuffers::Offset CreateFieldDirect( attributes__, documentation__, optional, - padding); + padding, + offset64); } struct Object FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { diff --git a/include/flatbuffers/vector_downward.h b/include/flatbuffers/vector_downward.h index bf8b1b19dd7..a312aa5f9f0 100644 --- a/include/flatbuffers/vector_downward.h +++ b/include/flatbuffers/vector_downward.h @@ -18,6 +18,7 @@ #define FLATBUFFERS_VECTOR_DOWNWARD_H_ #include +#include #include "flatbuffers/base.h" #include "flatbuffers/default_allocator.h" @@ -33,8 +34,8 @@ namespace flatbuffers { // Essentially, this supports 2 std::vectors in a single buffer. class vector_downward { public: - explicit vector_downward(uint64_t initial_size, Allocator *allocator, - bool own_allocator, uint64_t buffer_minalign) + explicit vector_downward(size_t initial_size, Allocator *allocator, + bool own_allocator, size_t buffer_minalign) : allocator_(allocator), own_allocator_(own_allocator), initial_size_(initial_size), @@ -108,10 +109,10 @@ class vector_downward { } // Relinquish the pointer to the caller. - uint8_t *release_raw(uint64_t &allocated_bytes, uint64_t &offset) { + uint8_t *release_raw(size_t &allocated_bytes, size_t &offset) { auto *buf = buf_; allocated_bytes = reserved_; - offset = static_cast(cur_ - buf_); + offset = vector_downward::offset(); // release_raw only relinquishes the buffer ownership. // Does not deallocate or reset the allocator. Destructor will do that. @@ -134,16 +135,18 @@ class vector_downward { return fb; } - uint64_t ensure_space(uint64_t len) { + size_t ensure_space(size_t len) { FLATBUFFERS_ASSERT(cur_ >= scratch_ && scratch_ >= buf_); - if (len > static_cast(cur_ - scratch_)) { reallocate(len); } + // If the length is larger than the unused part of the buffer, we need to + // grow. + if (len > unused_buffer_size()) { reallocate(len); } // Beyond this, signed offsets may not have enough range: // (FlatBuffers > 2GB not supported). FLATBUFFERS_ASSERT(size() < FLATBUFFERS_MAX_BUFFER_SIZE); return len; } - inline uint8_t *make_space(uint64_t len) { + inline uint8_t *make_space(size_t len) { if (len) { ensure_space(len); cur_ -= len; @@ -155,11 +158,19 @@ class vector_downward { // Returns nullptr if using the DefaultAllocator. Allocator *get_custom_allocator() { return allocator_; } - inline uint64_t size() const { return size_; } + // The current offset into the buffer. + size_t offset() const { return cur_ - buf_; } - uint64_t scratch_size() const { return static_cast(scratch_ - buf_); } + // The total size of the vector (both the buffer and scratch parts). + inline size_t size() const { return size_; } - uint64_t capacity() const { return reserved_; } + // The size of the buffer part of the vector that is currently unused. + size_t unused_buffer_size() const { return cur_ - scratch_; } + + // The size of the scratch part of the vector. + size_t scratch_size() const { return scratch_ - buf_; } + + size_t capacity() const { return reserved_; } uint8_t *data() const { FLATBUFFERS_ASSERT(cur_); @@ -176,9 +187,9 @@ class vector_downward { return scratch_; } - uint8_t *data_at(uint64_t offset) const { return buf_ + reserved_ - offset; } + uint8_t *data_at(size_t offset) const { return buf_ + reserved_ - offset; } - void push(const uint8_t *bytes, uint64_t num) { + void push(const uint8_t *bytes, size_t num) { if (num > 0) { memcpy(make_space(num), bytes, num); } } @@ -196,23 +207,23 @@ class vector_downward { // fill() is most frequently called with small byte counts (<= 4), // which is why we're using loops rather than calling memset. - void fill(uint64_t zero_pad_bytes) { + void fill(size_t zero_pad_bytes) { make_space(zero_pad_bytes); - for (uint64_t i = 0; i < zero_pad_bytes; i++) cur_[i] = 0; + for (size_t i = 0; i < zero_pad_bytes; i++) cur_[i] = 0; } // Version for when we know the size is larger. // Precondition: zero_pad_bytes > 0 - void fill_big(uint64_t zero_pad_bytes) { + void fill_big(size_t zero_pad_bytes) { memset(make_space(zero_pad_bytes), 0, zero_pad_bytes); } - void pop(uint64_t bytes_to_remove) { + void pop(size_t bytes_to_remove) { cur_ += bytes_to_remove; size_ -= bytes_to_remove; } - void scratch_pop(uint64_t bytes_to_remove) { scratch_ -= bytes_to_remove; } + void scratch_pop(size_t bytes_to_remove) { scratch_ -= bytes_to_remove; } void swap(vector_downward &other) { using std::swap; @@ -236,20 +247,19 @@ class vector_downward { private: // You shouldn't really be copying instances of this class. FLATBUFFERS_DELETE_FUNC(vector_downward(const vector_downward &)); - FLATBUFFERS_DELETE_FUNC( - vector_downward &operator=(const vector_downward &)); + FLATBUFFERS_DELETE_FUNC(vector_downward &operator=(const vector_downward &)); Allocator *allocator_; bool own_allocator_; - uint64_t initial_size_; - uint64_t buffer_minalign_; - uint64_t reserved_; - uint64_t size_; + size_t initial_size_; + size_t buffer_minalign_; + size_t reserved_; + size_t size_; uint8_t *buf_; uint8_t *cur_; // Points at location between empty (below) and used (above). uint8_t *scratch_; // Points to the end of the scratchpad in use. - void reallocate(uint64_t len) { + void reallocate(size_t len) { auto old_reserved = reserved_; auto old_size = size(); auto old_scratch_size = scratch_size(); diff --git a/python/flatbuffers/reflection/Field.py b/python/flatbuffers/reflection/Field.py index 36ceb2bfec7..2f2526a52e6 100644 --- a/python/flatbuffers/reflection/Field.py +++ b/python/flatbuffers/reflection/Field.py @@ -155,8 +155,16 @@ def Padding(self): return self._tab.Get(flatbuffers.number_types.Uint16Flags, o + self._tab.Pos) return 0 -def FieldStart(builder): - builder.StartObject(13) + # If the field uses 64-bit offsets. + # Field + def Offset64(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(30)) + if o != 0: + return bool(self._tab.Get(flatbuffers.number_types.BoolFlags, o + self._tab.Pos)) + return False + +def FieldStart(builder): + builder.StartObject(14) def Start(builder): FieldStart(builder) @@ -248,11 +256,12 @@ def AddOptional(builder: flatbuffers.Builder, optional: bool): def FieldAddPadding(builder, padding): builder.PrependUint16Slot(12, padding, 0) -def AddPadding(builder: flatbuffers.Builder, padding: int): - FieldAddPadding(builder, padding) - -def FieldEnd(builder): +def AddPadding(builder, padding): + return FieldAddPadding(builder, padding) +def FieldAddOffset64(builder, offset64): builder.PrependBoolSlot(13, offset64, 0) +def AddOffset64(builder, offset64): + return FieldAddOffset64(builder, offset64) +def FieldEnd(builder): return builder.EndObject() - def End(builder): return FieldEnd(builder) diff --git a/reflection/reflection.fbs b/reflection/reflection.fbs index b71a184bab0..790178d2d87 100644 --- a/reflection/reflection.fbs +++ b/reflection/reflection.fbs @@ -85,6 +85,8 @@ table Field { optional:bool = false; /// Number of padding octets to always add after this field. Structs only. padding:uint16 = 0; + /// If the field uses 64-bit offsets. + offset64:bool = false; } table Object { // Used for both tables and structs. diff --git a/src/binary_annotator.cpp b/src/binary_annotator.cpp index 8dd21ee4f5b..c6b4d554f59 100644 --- a/src/binary_annotator.cpp +++ b/src/binary_annotator.cpp @@ -659,24 +659,19 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset, } // Read the offset - const bool offset64 = true; - // field->attributes()->LookupByKey("offset64") != nullptr; - uint64_t offset = 0; bool has_offset_from = false; - uint64_t length = 0; + uint64_t length = sizeof(uint32_t); BinaryRegionType type = BinaryRegionType::UOffset; - if (offset64) { + if (field->offset64()) { length = sizeof(uint64_t); type = BinaryRegionType::UOffset64; const auto offset_from_field = ReadScalar(field_offset); if ((has_offset_from = offset_from_field.has_value())) { offset = offset_from_field.value(); - std::cout << "reading 64 bit offset: " << offset << std::endl; } } else { - length = sizeof(uint32_t); const auto offset_from_field = ReadScalar(field_offset); if ((has_offset_from = offset_from_field.has_value())) { offset = offset_from_field.value(); diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 5b30c4cfa98..5426b11fd90 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -3933,7 +3933,7 @@ Offset FieldDef::Serialize(FlatBufferBuilder *builder, IsInteger(value.type.base_type) ? StringToInt(value.constant.c_str()) : 0, // result may be platform-dependent if underlying is float (not double) IsFloat(value.type.base_type) ? d : 0.0, deprecated, IsRequired(), key, - attr__, docs__, IsOptional(), static_cast(padding)); + attr__, docs__, IsOptional(), static_cast(padding), offset64); // TODO: value.constant is almost always "0", we could save quite a bit of // space by sharing it. Same for common values of value.type. } @@ -3951,6 +3951,7 @@ bool FieldDef::Deserialize(Parser &parser, const reflection::Field *field) { presence = FieldDef::MakeFieldPresence(field->optional(), field->required()); padding = field->padding(); key = field->key(); + offset64 = field->offset64(); if (!DeserializeAttributes(parser, field->attributes())) return false; // TODO: this should probably be handled by a separate attribute if (attributes.Lookup("flexbuffer")) { From 8a4b28360d670bb857910b0b86111aee0a62bc98 Mon Sep 17 00:00:00 2001 From: Derek Bailey Date: Thu, 26 Jan 2023 16:39:52 -0800 Subject: [PATCH 04/62] cleaned up adding offset64 in parser --- include/flatbuffers/buffer.h | 2 +- include/flatbuffers/idl.h | 2 +- src/idl_parser.cpp | 42 ++++++++++++++++++++++-------------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/include/flatbuffers/buffer.h b/include/flatbuffers/buffer.h index e9e2257a987..ee88d80d85a 100644 --- a/include/flatbuffers/buffer.h +++ b/include/flatbuffers/buffer.h @@ -36,7 +36,7 @@ template struct Offset { template struct Offset64 { uoffset64_t o; Offset64() : o(0) {} - Offset64(uoffset64_t _o) : o(_o) {} + explicit Offset64(const uoffset64_t offset) : o(offset) {} Offset64 Union() const { return Offset64(o); } bool IsNull() const { return !o; } }; diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 3417deb60be..d00bf9d498f 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -353,7 +353,7 @@ struct FieldDef : public Definition { bool native_inline; // Field will be defined inline (instead of as a pointer) // for native tables if field is a struct. bool flexbuffer; // This field contains FlexBuffer data. - bool offset64; + bool offset64; // If the field uses 64-bit offsets. enum Presence { // Field must always be present. diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 5426b11fd90..f69e6b9371a 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -16,10 +16,10 @@ #include #include +#include #include #include #include -#include #include "flatbuffers/base.h" #include "flatbuffers/buffer.h" @@ -126,6 +126,14 @@ CheckedError atot>(const char *s, Parser &parser, return NoError(); } +template<> +CheckedError atot>(const char *s, Parser &parser, + Offset64 *val) { + (void)parser; + *val = Offset64(atoi(s)); + return NoError(); +} + template static T *LookupTableByName(const SymbolTable &table, const std::string &name, @@ -959,11 +967,11 @@ CheckedError Parser::ParseField(StructDef &struct_def) { ECHECK(AddField(struct_def, name, type, &field)); if (typefield) { - // We preserve the relation between the typefield - // and field, so we can easily map it in the code - // generators. - typefield->sibling_union_field = field; - field->sibling_union_field = typefield; + // We preserve the relation between the typefield + // and field, so we can easily map it in the code + // generators. + typefield->sibling_union_field = field; + field->sibling_union_field = typefield; } if (token_ == '=') { @@ -1038,6 +1046,7 @@ CheckedError Parser::ParseField(StructDef &struct_def) { } } + // Record that this field uses 64-bit offsets. field->offset64 = field->attributes.Lookup("offset64") != nullptr; // For historical convenience reasons, string keys are assumed required. @@ -1062,7 +1071,8 @@ CheckedError Parser::ParseField(StructDef &struct_def) { if (field->key) { if (struct_def.has_key) return Error("only one field may be set as 'key'"); struct_def.has_key = true; - auto is_valid = IsScalar(type.base_type) || IsString(type) || IsStruct(type); + auto is_valid = + IsScalar(type.base_type) || IsString(type) || IsStruct(type); if (IsArray(type)) { is_valid |= IsScalar(type.VectorType().base_type) || IsStruct(type.VectorType()); @@ -1514,7 +1524,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, if (!struct_def.sortbysize || size == SizeOf(field_value.type.base_type)) { switch (field_value.type.base_type) { -// clang-format off + // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \ case BASE_TYPE_ ## ENUM: \ builder_.Pad(field->padding); \ @@ -1533,7 +1543,6 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, CTYPE val, valdef; \ ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \ - std::cout << "adding element: " << field->name << " val: " << field_value.offset << " 64-bit: " << field->offset64 << std::endl; \ builder_.AddElement(field_value.offset, val, valdef); \ } \ } \ @@ -1546,15 +1555,16 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, if (IsStruct(field->value.type)) { \ SerializeStruct(*field->value.type.struct_def, field_value); \ } else { \ - CTYPE val; \ - ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ - std::cout << "adding offset: " << field->name << " val: " << val.o << " 64-bit: " << field->offset64 << std::endl; \ + /* Special case for fields that use 64-bit addressing */ \ if(field->offset64) { \ - Offset64 offset(val.o); \ + Offset64 offset; \ + ECHECK(atot(field_value.constant.c_str(), *this, &offset)); \ builder_.AddOffset(field_value.offset, offset); \ } else { \ + CTYPE val; \ + ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ builder_.AddOffset(field_value.offset, val); \ - }\ + } \ } \ break; FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD) @@ -1569,7 +1579,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, } } } - } + } // namespace flatbuffers for (size_t i = 0; i < fieldn_outer; i++) field_stack_.pop_back(); if (struct_def.fixed) { @@ -1650,7 +1660,7 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue, // start at the back, since we're building the data backwards. auto &val = field_stack_.back().first; switch (val.type.base_type) { -// clang-format off + // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE,...) \ case BASE_TYPE_ ## ENUM: \ if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \ From 00a5c351891a1865698b0e06b55124f84b953f73 Mon Sep 17 00:00:00 2001 From: Derek Bailey Date: Fri, 27 Jan 2023 10:40:33 -0800 Subject: [PATCH 05/62] Add C++ testing skeleton for 64-bit --- CMakeLists.txt | 2 ++ tests/64bit/offset64_test.cpp | 11 +++++++++++ tests/64bit/offset64_test.h | 12 ++++++++++++ tests/test.cpp | 2 ++ 4 files changed, 27 insertions(+) create mode 100644 tests/64bit/offset64_test.cpp create mode 100644 tests/64bit/offset64_test.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e0b3248aed0..00d3d6b6adb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,6 +234,8 @@ set(FlatBuffers_Tests_SRCS tests/native_type_test_impl.cpp tests/alignment_test.h tests/alignment_test.cpp + tests/64bit/offset64_test.h + tests/64bit/offset64_test.cpp include/flatbuffers/code_generators.h src/code_generators.cpp ) diff --git a/tests/64bit/offset64_test.cpp b/tests/64bit/offset64_test.cpp new file mode 100644 index 00000000000..cbfa30b52db --- /dev/null +++ b/tests/64bit/offset64_test.cpp @@ -0,0 +1,11 @@ +#include "offset64_test.h" + +#include "test_assert.h" + +namespace flatbuffers { +namespace tests { + +void Offset64Test() { TEST_ASSERT(true); } + +} // namespace tests +} // namespace flatbuffers diff --git a/tests/64bit/offset64_test.h b/tests/64bit/offset64_test.h new file mode 100644 index 00000000000..cd6e202c054 --- /dev/null +++ b/tests/64bit/offset64_test.h @@ -0,0 +1,12 @@ +#ifndef TESTS_64BIT_OFFSET64_TEST_H +#define TESTS_64BIT_OFFSET64_TEST_H + +namespace flatbuffers { +namespace tests { + +void Offset64Test(); + +} // namespace tests +} // namespace flatbuffers + +#endif // TESTS_64BIT_OFFSET64_TEST_H diff --git a/tests/test.cpp b/tests/test.cpp index 1faf33e4234..b31fc4e9804 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -42,6 +42,7 @@ #if !defined(_MSC_VER) || _MSC_VER >= 1700 # include "arrays_test_generated.h" #endif +#include "64bit/offset64_test.h" #include "flexbuffers_test.h" #include "is_quiet_nan.h" @@ -1651,6 +1652,7 @@ int FlatBufferTests(const std::string &tests_data_path) { NestedStructKeyInStructTest(); FixedSizedStructArrayKeyInStructTest(); EmbeddedSchemaAccess(); + Offset64Test(); return 0; } } // namespace From b08d3edfcf9a2063a6b1ee64fcec58b3bd63d5db Mon Sep 17 00:00:00 2001 From: Derek Bailey Date: Fri, 27 Jan 2023 12:40:42 -0800 Subject: [PATCH 06/62] working test for CreateVector64 --- include/flatbuffers/buffer.h | 2 +- include/flatbuffers/flatbuffer_builder.h | 16 ++-- src/idl_gen_cpp.cpp | 46 ++++++---- tests/64bit/offset64_test.cpp | 22 ++++- tests/64bit/run.sh | 2 - tests/64bit/test_64bit_generated.h | 107 ++++++++++++++++++++++- 6 files changed, 164 insertions(+), 31 deletions(-) diff --git a/include/flatbuffers/buffer.h b/include/flatbuffers/buffer.h index ee88d80d85a..20925498206 100644 --- a/include/flatbuffers/buffer.h +++ b/include/flatbuffers/buffer.h @@ -36,7 +36,7 @@ template struct Offset { template struct Offset64 { uoffset64_t o; Offset64() : o(0) {} - explicit Offset64(const uoffset64_t offset) : o(offset) {} + Offset64(const uoffset64_t offset) : o(offset) {} Offset64 Union() const { return Offset64(o); } bool IsNull() const { return !o; } }; diff --git a/include/flatbuffers/flatbuffer_builder.h b/include/flatbuffers/flatbuffer_builder.h index 7f6df17aeb8..55cdecb9af4 100644 --- a/include/flatbuffers/flatbuffer_builder.h +++ b/include/flatbuffers/flatbuffer_builder.h @@ -24,6 +24,7 @@ #include "flatbuffers/allocator.h" #include "flatbuffers/array.h" #include "flatbuffers/base.h" +#include "flatbuffers/buffer.h" #include "flatbuffers/buffer_ref.h" #include "flatbuffers/default_allocator.h" #include "flatbuffers/detached_buffer.h" @@ -432,11 +433,11 @@ class FlatBufferBuilder { // This checks a required field has been set in a given table that has // just been constructed. template void Required(Offset table, voffset_t field) { - auto table_ptr = reinterpret_cast(buf_.data_at(table.o)); + auto table_ptr = reinterpret_cast(buf_.data_at(table.o)); bool ok = table_ptr->GetOptionalFieldOffset(field) != 0; // If this fails, the caller will show what field needs to be set. FLATBUFFERS_ASSERT(ok); - (void)ok; + (void)ok; } uoffset_t StartStruct(size_t alignment) { @@ -637,14 +638,15 @@ class FlatBufferBuilder { /// @param[in] v A pointer to the array of type `T` to serialize into the /// buffer as a `vector`. /// @param[in] len The number of elements to serialize. - /// @return Returns a typed `Offset` into the serialized data indicating + /// @return Returns a typed `TOffset` into the serialized data indicating /// where the vector is stored. - template Offset> CreateVector(const T *v, size_t len) { + template>> + OffsetT CreateVector(const T *v, size_t len) { // If this assert hits, you're specifying a template argument that is // causing the wrong overload to be selected, remove it. AssertScalarT(); StartVector(len); - if (len == 0) { return Offset>(EndVector(len)); } + if (len == 0) { return OffsetT(EndVector(len)); } // clang-format off #if FLATBUFFERS_LITTLEENDIAN PushBytes(reinterpret_cast(v), len * sizeof(T)); @@ -658,7 +660,7 @@ class FlatBufferBuilder { } #endif // clang-format on - return Offset>(EndVector(len)); + return OffsetT(EndVector(len)); } /// @brief Serialize an array like object into a FlatBuffer `vector`. @@ -702,7 +704,7 @@ class FlatBufferBuilder { template> Offset64> CreateVector64(const std::vector &v) { - return CreateVector(data(v), v.size()); + return CreateVector>>(data(v), v.size()); } // vector may be implemented using a bit-set, so we can't access it as diff --git a/src/idl_gen_cpp.cpp b/src/idl_gen_cpp.cpp index f1a461733bb..e315983a1e9 100644 --- a/src/idl_gen_cpp.cpp +++ b/src/idl_gen_cpp.cpp @@ -766,16 +766,15 @@ class CppGenerator : public BaseGenerator { // Return a C++ type for any type (scalar/pointer) specifically for // building a flatbuffer. std::string GenTypeWire(const Type &type, const char *postfix, - bool user_facing_type, bool _64_bit_offset = false) const { + bool user_facing_type, + bool _64_bit_offset = false) const { if (IsScalar(type.base_type)) { return GenTypeBasic(type, user_facing_type) + postfix; } else if (IsStruct(type)) { return "const " + GenTypePointer(type) + " *"; } else { - if(_64_bit_offset) { - return "::flatbuffers::Offset64<" + GenTypePointer(type) + ">" + postfix; - } - return "::flatbuffers::Offset<" + GenTypePointer(type) + ">" + postfix; + return "::flatbuffers::Offset" + std::string(_64_bit_offset ? "64" : "") + + "<" + GenTypePointer(type) + ">" + postfix; } } @@ -1794,7 +1793,8 @@ class CppGenerator : public BaseGenerator { if (IsStruct(vtype)) { type = WrapInNameSpace(*vtype.struct_def); } else { - type = GenTypeWire(vtype, "", VectorElementUserFacing(vtype)); + type = GenTypeWire(vtype, "", VectorElementUserFacing(vtype), + field.offset64); } if (TypeHasKey(vtype)) { code_.SetValue("PARAM_TYPE", "std::vector<" + type + "> *"); @@ -1808,7 +1808,8 @@ class CppGenerator : public BaseGenerator { if (field.IsScalarOptional()) code_.SetValue("PARAM_TYPE", GenOptionalDecl(type) + " "); else - code_.SetValue("PARAM_TYPE", GenTypeWire(type, " ", true)); + code_.SetValue("PARAM_TYPE", + GenTypeWire(type, " ", true, field.offset64)); } code_ += "{{PRE}}{{PARAM_TYPE}}{{PARAM_NAME}} = {{PARAM_VALUE}}\\"; } @@ -2636,7 +2637,7 @@ class CppGenerator : public BaseGenerator { auto offset_str = GenFieldOffsetName(field); if (is_scalar) { - const auto wire_type = GenTypeWire(type, "", false); + const auto wire_type = GenTypeWire(type, "", false, field.offset64); code_.SetValue("SET_FN", "SetField<" + wire_type + ">"); code_.SetValue("OFFSET_NAME", offset_str); code_.SetValue("FIELD_TYPE", GenTypeBasic(type, true)); @@ -2862,9 +2863,9 @@ class CppGenerator : public BaseGenerator { // Generate code to do force_align for the vector. if (align > 1) { const auto vtype = field.value.type.VectorType(); - const std::string &type = IsStruct(vtype) - ? WrapInNameSpace(*vtype.struct_def) - : GenTypeWire(vtype, "", false); + const std::string & type = IsStruct(vtype) + ? WrapInNameSpace(*vtype.struct_def) + : GenTypeWire(vtype, "", false, field.offset64); return "_fbb.ForceVectorAlignment(" + field_size + ", sizeof(" + type + "), " + std::to_string(static_cast(align)) + ");"; } @@ -2899,14 +2900,15 @@ class CppGenerator : public BaseGenerator { // void add_name(type name) { // fbb_.AddElement(offset, name, default); // } - const bool is_offset_64 = field.attributes.Lookup("offset64"); code_.SetValue("FIELD_NAME", Name(field)); - code_.SetValue("FIELD_TYPE", GenTypeWire(field.value.type, " ", true, is_offset_64)); + code_.SetValue("FIELD_TYPE", + GenTypeWire(field.value.type, " ", true, field.offset64)); code_.SetValue("ADD_OFFSET", Name(struct_def) + "::" + offset); code_.SetValue("ADD_NAME", name); code_.SetValue("ADD_VALUE", value); if (is_scalar) { - const auto type = GenTypeWire(field.value.type, "", false); + const auto type = + GenTypeWire(field.value.type, "", false, field.offset64); code_.SetValue("ADD_FN", "AddElement<" + type + ">"); } else if (IsStruct(field.value.type)) { code_.SetValue("ADD_FN", "AddStruct"); @@ -3031,9 +3033,14 @@ class CppGenerator : public BaseGenerator { const auto type = WrapInNameSpace(*vtype.struct_def); code_ += "_fbb.CreateVectorOfSortedTables<" + type + ">\\"; } else { - const auto type = - GenTypeWire(vtype, "", VectorElementUserFacing(vtype)); - code_ += "_fbb.CreateVector<" + type + ">\\"; + const auto type = GenTypeWire( + vtype, "", VectorElementUserFacing(vtype), field->offset64); + + // If the field uses 64-bit addressing, create a 64-bit vector. + code_.SetValue("64OFFSET", field->offset64 ? "64" : ""); + code_.SetValue("TYPE", type); + + code_ += "_fbb.CreateVector{{64OFFSET}}<{{TYPE}}>\\"; } code_ += has_key ? "({{FIELD_NAME}}) : 0;" : "(*{{FIELD_NAME}}) : 0;"; @@ -3417,7 +3424,10 @@ class CppGenerator : public BaseGenerator { code += "(__va->_" + value + "[i]" + GenPtrGet(field) + ")) : 0"; code += "; }, &_va )"; } else { - code += "_fbb.CreateVector(" + value + ")"; + // If the field uses 64-bit addressing, create a 64-bit vector. + code += "_fbb.CreateVector" + + std::string(field.offset64 ? "64" : "") + "(" + value + + ")"; } break; } diff --git a/tests/64bit/offset64_test.cpp b/tests/64bit/offset64_test.cpp index cbfa30b52db..c9763619a88 100644 --- a/tests/64bit/offset64_test.cpp +++ b/tests/64bit/offset64_test.cpp @@ -1,11 +1,31 @@ #include "offset64_test.h" +#include + +#include "flatbuffers/flatbuffers.h" +#include "test_64bit_generated.h" #include "test_assert.h" namespace flatbuffers { namespace tests { -void Offset64Test() { TEST_ASSERT(true); } +void Offset64Test() { + flatbuffers::FlatBufferBuilder fbb; + + std::vector data; + data.resize(10); + + const Offset64> big_vector_offset = fbb.CreateVector64(data); + + RootTableBuilder root_table_builder(fbb); + root_table_builder.add_big_vector(big_vector_offset); + const Offset root_table_offset = root_table_builder.Finish(); + + fbb.Finish(root_table_offset); + + flatbuffers::Verifier verifier(fbb.GetBufferPointer(), fbb.GetSize()); + TEST_EQ(VerifyRootTableBuffer(verifier), true); +} } // namespace tests } // namespace flatbuffers diff --git a/tests/64bit/run.sh b/tests/64bit/run.sh index 9691b872adf..5b93973f0de 100755 --- a/tests/64bit/run.sh +++ b/tests/64bit/run.sh @@ -1,5 +1,3 @@ -./flatc --cpp -o tests/64bit tests/64bit/test_64bit.fbs - ./flatc -b -o tests/64bit tests/64bit/test_64bit.fbs tests/64bit/test_64bit.json ./flatc -o tests/64bit --annotate tests/64bit/test_64bit.fbs -- tests/64bit/test_64bit.bin diff --git a/tests/64bit/test_64bit_generated.h b/tests/64bit/test_64bit_generated.h index 95c456892aa..d3502f2a0a8 100644 --- a/tests/64bit/test_64bit_generated.h +++ b/tests/64bit/test_64bit_generated.h @@ -15,9 +15,25 @@ static_assert(FLATBUFFERS_VERSION_MAJOR == 23 && struct RootTable; struct RootTableBuilder; +struct RootTableT; + +bool operator==(const RootTableT &lhs, const RootTableT &rhs); +bool operator!=(const RootTableT &lhs, const RootTableT &rhs); + +inline const ::flatbuffers::TypeTable *RootTableTypeTable(); + +struct RootTableT : public ::flatbuffers::NativeTable { + typedef RootTable TableType; + std::vector big_vector{}; + int32_t a = 0; +}; struct RootTable FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { + typedef RootTableT NativeTableType; typedef RootTableBuilder Builder; + static const ::flatbuffers::TypeTable *MiniReflectTypeTable() { + return RootTableTypeTable(); + } enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_BIG_VECTOR = 4, VT_A = 6 @@ -25,9 +41,15 @@ struct RootTable FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { const ::flatbuffers::Vector *big_vector() const { return GetPointer *>(VT_BIG_VECTOR); } + ::flatbuffers::Vector *mutable_big_vector() { + return GetPointer<::flatbuffers::Vector *>(VT_BIG_VECTOR); + } int32_t a() const { return GetField(VT_A, 0); } + bool mutate_a(int32_t _a = 0) { + return SetField(VT_A, _a, 0); + } bool Verify(::flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_BIG_VECTOR) && @@ -35,6 +57,9 @@ struct RootTable FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { VerifyField(verifier, VT_A, 4) && verifier.EndTable(); } + RootTableT *UnPack(const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(RootTableT *_o, const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; + static ::flatbuffers::Offset Pack(::flatbuffers::FlatBufferBuilder &_fbb, const RootTableT* _o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); }; struct RootTableBuilder { @@ -60,7 +85,7 @@ struct RootTableBuilder { inline ::flatbuffers::Offset CreateRootTable( ::flatbuffers::FlatBufferBuilder &_fbb, - ::flatbuffers::Offset<::flatbuffers::Vector> big_vector = 0, + ::flatbuffers::Offset64<::flatbuffers::Vector> big_vector = 0, int32_t a = 0) { RootTableBuilder builder_(_fbb); builder_.add_a(a); @@ -72,13 +97,71 @@ inline ::flatbuffers::Offset CreateRootTableDirect( ::flatbuffers::FlatBufferBuilder &_fbb, const std::vector *big_vector = nullptr, int32_t a = 0) { - auto big_vector__ = big_vector ? _fbb.CreateVector(*big_vector) : 0; + auto big_vector__ = big_vector ? _fbb.CreateVector64(*big_vector) : 0; return CreateRootTable( _fbb, big_vector__, a); } +::flatbuffers::Offset CreateRootTable(::flatbuffers::FlatBufferBuilder &_fbb, const RootTableT *_o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); + + +inline bool operator==(const RootTableT &lhs, const RootTableT &rhs) { + return + (lhs.big_vector == rhs.big_vector) && + (lhs.a == rhs.a); +} + +inline bool operator!=(const RootTableT &lhs, const RootTableT &rhs) { + return !(lhs == rhs); +} + + +inline RootTableT *RootTable::UnPack(const ::flatbuffers::resolver_function_t *_resolver) const { + auto _o = std::unique_ptr(new RootTableT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void RootTable::UnPackTo(RootTableT *_o, const ::flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = big_vector(); if (_e) { _o->big_vector.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->big_vector.begin()); } } + { auto _e = a(); _o->a = _e; } +} + +inline ::flatbuffers::Offset RootTable::Pack(::flatbuffers::FlatBufferBuilder &_fbb, const RootTableT* _o, const ::flatbuffers::rehasher_function_t *_rehasher) { + return CreateRootTable(_fbb, _o, _rehasher); +} + +inline ::flatbuffers::Offset CreateRootTable(::flatbuffers::FlatBufferBuilder &_fbb, const RootTableT *_o, const ::flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { ::flatbuffers::FlatBufferBuilder *__fbb; const RootTableT* __o; const ::flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _big_vector = _o->big_vector.size() ? _fbb.CreateVector64(_o->big_vector) : 0; + auto _a = _o->a; + return CreateRootTable( + _fbb, + _big_vector, + _a); +} + +inline const ::flatbuffers::TypeTable *RootTableTypeTable() { + static const ::flatbuffers::TypeCode type_codes[] = { + { ::flatbuffers::ET_UCHAR, 1, -1 }, + { ::flatbuffers::ET_INT, 0, -1 } + }; + static const char * const names[] = { + "big_vector", + "a" + }; + static const ::flatbuffers::TypeTable tt = { + ::flatbuffers::ST_TABLE, 2, type_codes, nullptr, nullptr, nullptr, names + }; + return &tt; +} + inline const RootTable *GetRootTable(const void *buf) { return ::flatbuffers::GetRoot(buf); } @@ -87,6 +170,14 @@ inline const RootTable *GetSizePrefixedRootTable(const void *buf) { return ::flatbuffers::GetSizePrefixedRoot(buf); } +inline RootTable *GetMutableRootTable(void *buf) { + return ::flatbuffers::GetMutableRoot(buf); +} + +inline RootTable *GetMutableSizePrefixedRootTable(void *buf) { + return ::flatbuffers::GetMutableSizePrefixedRoot(buf); +} + inline bool VerifyRootTableBuffer( ::flatbuffers::Verifier &verifier) { return verifier.VerifyBuffer(nullptr); @@ -109,4 +200,16 @@ inline void FinishSizePrefixedRootTableBuffer( fbb.FinishSizePrefixed(root); } +inline flatbuffers::unique_ptr UnPackRootTable( + const void *buf, + const ::flatbuffers::resolver_function_t *res = nullptr) { + return flatbuffers::unique_ptr(GetRootTable(buf)->UnPack(res)); +} + +inline flatbuffers::unique_ptr UnPackSizePrefixedRootTable( + const void *buf, + const ::flatbuffers::resolver_function_t *res = nullptr) { + return flatbuffers::unique_ptr(GetSizePrefixedRootTable(buf)->UnPack(res)); +} + #endif // FLATBUFFERS_GENERATED_TEST64BIT_H_ From e396b340e5fad66a8d77f21dabd0bcdfae4225b4 Mon Sep 17 00:00:00 2001 From: Derek Bailey Date: Fri, 27 Jan 2023 15:32:29 -0800 Subject: [PATCH 07/62] working >2 GiB buffers --- include/flatbuffers/base.h | 6 ++-- include/flatbuffers/vector.h | 40 +++++++++++++------------- include/flatbuffers/vector_downward.h | 9 +++--- include/flatbuffers/verifier.h | 6 ++-- tests/64bit/offset64_test.cpp | 41 ++++++++++++++++++++------- 5 files changed, 64 insertions(+), 38 deletions(-) diff --git a/include/flatbuffers/base.h b/include/flatbuffers/base.h index 8004b9f72c5..74ac9c8ca27 100644 --- a/include/flatbuffers/base.h +++ b/include/flatbuffers/base.h @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -323,11 +324,11 @@ namespace flatbuffers { // Also, using a consistent offset type maintains compatibility of serialized // offset values between 32bit and 64bit systems. typedef uint32_t uoffset_t; - typedef uint64_t uoffset64_t; // Signed offsets for references that can go in both directions. typedef int32_t soffset_t; +typedef int64_t soffset64_t; // Offset/index used in v-tables, can be changed to uint8_t in // format forks to save a bit of space if desired. @@ -336,7 +337,8 @@ typedef uint16_t voffset_t; typedef uintmax_t largest_scalar_t; // In 32bits, this evaluates to 2GB - 1 -#define FLATBUFFERS_MAX_BUFFER_SIZE ((1ULL << (sizeof(::flatbuffers::soffset_t) * 8 - 1)) - 1) +#define FLATBUFFERS_MAX_BUFFER_SIZE std::numeric_limits<::flatbuffers::soffset_t>::max() +#define FLATBUFFERS_MAX_64_BUFFER_SIZE std::numeric_limits<::flatbuffers::soffset64_t>::max() // The minimum size buffer that can be a valid flatbuffer. // Includes the offset to the root table (uoffset_t), the offset to the vtable diff --git a/include/flatbuffers/vector.h b/include/flatbuffers/vector.h index 9cb6a2da89c..c75049cd4aa 100644 --- a/include/flatbuffers/vector.h +++ b/include/flatbuffers/vector.h @@ -27,7 +27,7 @@ struct String; // An STL compatible iterator implementation for Vector below, effectively // calling Get() for every element. -template +template struct VectorIterator { typedef std::random_access_iterator_tag iterator_category; typedef IT value_type; @@ -35,7 +35,7 @@ struct VectorIterator { typedef IT *pointer; typedef IT &reference; - VectorIterator(Data data, uoffset_t i) + VectorIterator(Data data, SizeT i) : data_(data + IndirectHelper::element_stride * i) {} VectorIterator(const VectorIterator &other) : data_(other.data_) {} VectorIterator() : data_(nullptr) {} @@ -85,12 +85,12 @@ struct VectorIterator { return temp; } - VectorIterator operator+(const uoffset_t &offset) const { + VectorIterator operator+(const SizeT &offset) const { return VectorIterator(data_ + offset * IndirectHelper::element_stride, 0); } - VectorIterator &operator+=(const uoffset_t &offset) { + VectorIterator &operator+=(const SizeT &offset) { data_ += offset * IndirectHelper::element_stride; return *this; } @@ -106,12 +106,12 @@ struct VectorIterator { return temp; } - VectorIterator operator-(const uoffset_t &offset) const { + VectorIterator operator-(const SizeT &offset) const { return VectorIterator(data_ - offset * IndirectHelper::element_stride, 0); } - VectorIterator &operator-=(const uoffset_t &offset) { + VectorIterator &operator-=(const SizeT &offset) { data_ -= offset * IndirectHelper::element_stride; return *this; } @@ -145,7 +145,7 @@ struct VectorReverseIterator : public std::reverse_iterator { // This is used as a helper type for accessing vectors. // Vector::data() assumes the vector elements start after the length field. -template class Vector { +template class Vector { public: typedef VectorIterator::mutable_return_type> iterator; @@ -160,39 +160,39 @@ template class Vector { static FLATBUFFERS_CONSTEXPR bool is_span_observable = scalar_tag::value && (FLATBUFFERS_LITTLEENDIAN || sizeof(T) == 1); - uoffset_t size() const { return EndianScalar(length_); } + SizeT size() const { return EndianScalar(length_); } // Deprecated: use size(). Here for backwards compatibility. FLATBUFFERS_ATTRIBUTE([[deprecated("use size() instead")]]) - uoffset_t Length() const { return size(); } + SizeT Length() const { return size(); } typedef typename IndirectHelper::return_type return_type; typedef typename IndirectHelper::mutable_return_type mutable_return_type; typedef return_type value_type; - return_type Get(uoffset_t i) const { + return_type Get(SizeT i) const { FLATBUFFERS_ASSERT(i < size()); return IndirectHelper::Read(Data(), i); } - return_type operator[](uoffset_t i) const { return Get(i); } + return_type operator[](SizeT i) const { return Get(i); } // If this is a Vector of enums, T will be its storage type, not the enum // type. This function makes it convenient to retrieve value with enum // type E. - template E GetEnum(uoffset_t i) const { + template E GetEnum(SizeT i) const { return static_cast(Get(i)); } // If this a vector of unions, this does the cast for you. There's no check // to make sure this is the right type! - template const U *GetAs(uoffset_t i) const { + template const U *GetAs(SizeT i) const { return reinterpret_cast(Get(i)); } // If this a vector of unions, this does the cast for you. There's no check // to make sure this is actually a string! - const String *GetAsString(uoffset_t i) const { + const String *GetAsString(SizeT i) const { return reinterpret_cast(Get(i)); } @@ -226,7 +226,7 @@ template class Vector { // Change elements if you have a non-const pointer to this object. // Scalars only. See reflection.h, and the documentation. - void Mutate(uoffset_t i, const T &val) { + void Mutate(SizeT i, const T &val) { FLATBUFFERS_ASSERT(i < size()); WriteScalar(data() + i, val); } @@ -234,15 +234,15 @@ template class Vector { // Change an element of a vector of tables (or strings). // "val" points to the new table/string, as you can obtain from // e.g. reflection::AddFlatBuffer(). - void MutateOffset(uoffset_t i, const uint8_t *val) { + void MutateOffset(SizeT i, const uint8_t *val) { FLATBUFFERS_ASSERT(i < size()); - static_assert(sizeof(T) == sizeof(uoffset_t), "Unrelated types"); + static_assert(sizeof(T) == sizeof(SizeT), "Unrelated types"); WriteScalar(data() + i, - static_cast(val - (Data() + i * sizeof(uoffset_t)))); + static_cast(val - (Data() + i * sizeof(SizeT)))); } // Get a mutable pointer to tables/strings inside this vector. - mutable_return_type GetMutableObject(uoffset_t i) const { + mutable_return_type GetMutableObject(SizeT i) const { FLATBUFFERS_ASSERT(i < size()); return const_cast(IndirectHelper::Read(Data(), i)); } @@ -280,7 +280,7 @@ template class Vector { // try to construct these manually. Vector(); - uoffset_t length_; + SizeT length_; private: // This class is a pointer. Copying will therefore create an invalid object. diff --git a/include/flatbuffers/vector_downward.h b/include/flatbuffers/vector_downward.h index a312aa5f9f0..8b63b3cadf4 100644 --- a/include/flatbuffers/vector_downward.h +++ b/include/flatbuffers/vector_downward.h @@ -17,9 +17,10 @@ #ifndef FLATBUFFERS_VECTOR_DOWNWARD_H_ #define FLATBUFFERS_VECTOR_DOWNWARD_H_ -#include #include +#include + #include "flatbuffers/base.h" #include "flatbuffers/default_allocator.h" #include "flatbuffers/detached_buffer.h" @@ -140,9 +141,9 @@ class vector_downward { // If the length is larger than the unused part of the buffer, we need to // grow. if (len > unused_buffer_size()) { reallocate(len); } - // Beyond this, signed offsets may not have enough range: - // (FlatBuffers > 2GB not supported). - FLATBUFFERS_ASSERT(size() < FLATBUFFERS_MAX_BUFFER_SIZE); + // TODO(dbaileychess): Should this default to 32-bit max size with an opt + // in? + FLATBUFFERS_ASSERT(size() < FLATBUFFERS_MAX_64_BUFFER_SIZE); return len; } diff --git a/include/flatbuffers/verifier.h b/include/flatbuffers/verifier.h index 87d3f54a5db..3e1a27890b4 100644 --- a/include/flatbuffers/verifier.h +++ b/include/flatbuffers/verifier.h @@ -34,12 +34,14 @@ class Verifier FLATBUFFERS_FINAL_CLASS { bool check_alignment = true; // If true, run verifier on nested flatbuffers bool check_nested_flatbuffers = true; + // The maximum size of a buffer. + size_t max_size = FLATBUFFERS_MAX_BUFFER_SIZE; }; explicit Verifier(const uint8_t *const buf, const size_t buf_len, const Options &opts) : buf_(buf), size_(buf_len), opts_(opts) { - FLATBUFFERS_ASSERT(size_ < FLATBUFFERS_MAX_BUFFER_SIZE); + FLATBUFFERS_ASSERT(size_ < opts.max_size); } // Deprecated API, please construct with Verifier::Options. @@ -142,7 +144,7 @@ class Verifier FLATBUFFERS_FINAL_CLASS { // Check the whole array. If this is a string, the byte past the array must // be 0. const auto size = ReadScalar(vec); - const auto max_elems = FLATBUFFERS_MAX_BUFFER_SIZE / elem_size; + const auto max_elems = opts_.max_size / elem_size; if (!Check(size < max_elems)) return false; // Protect against byte_size overflowing. const auto byte_size = sizeof(size) + elem_size * size; diff --git a/tests/64bit/offset64_test.cpp b/tests/64bit/offset64_test.cpp index c9763619a88..7e4c45971ad 100644 --- a/tests/64bit/offset64_test.cpp +++ b/tests/64bit/offset64_test.cpp @@ -1,7 +1,10 @@ #include "offset64_test.h" -#include +#include +#include +#include +#include "flatbuffers/base.h" #include "flatbuffers/flatbuffers.h" #include "test_64bit_generated.h" #include "test_assert.h" @@ -12,19 +15,37 @@ namespace tests { void Offset64Test() { flatbuffers::FlatBufferBuilder fbb; - std::vector data; - data.resize(10); + size_t vector_size = 3LL << 3; // change to 30 to get 3 GiB. - const Offset64> big_vector_offset = fbb.CreateVector64(data); + { + std::vector data; + data.resize(vector_size); - RootTableBuilder root_table_builder(fbb); - root_table_builder.add_big_vector(big_vector_offset); - const Offset root_table_offset = root_table_builder.Finish(); + const Offset64> big_vector_offset = + fbb.CreateVector64(data); - fbb.Finish(root_table_offset); + RootTableBuilder root_table_builder(fbb); + root_table_builder.add_big_vector(big_vector_offset); + const Offset root_table_offset = root_table_builder.Finish(); - flatbuffers::Verifier verifier(fbb.GetBufferPointer(), fbb.GetSize()); - TEST_EQ(VerifyRootTableBuffer(verifier), true); + fbb.Finish(root_table_offset); + + + Verifier::Options options; + // Allow the verifier to verify 64-bit buffers. + options.max_size = FLATBUFFERS_MAX_64_BUFFER_SIZE; + + Verifier verifier(fbb.GetBufferPointer(), fbb.GetSize(), options); + + TEST_EQ(VerifyRootTableBuffer(verifier), true); + } + + { + const RootTable *root_table = GetRootTable(fbb.GetBufferPointer()); + + // Expect the big vector to be properly sized. + TEST_EQ(root_table->big_vector()->size(), vector_size); + } } } // namespace tests From 44b9b915f0d018d022f5e096c77495a584634c8d Mon Sep 17 00:00:00 2001 From: Derek Bailey Date: Fri, 27 Jan 2023 16:27:41 -0800 Subject: [PATCH 08/62] support for large strings --- include/flatbuffers/flatbuffer_builder.h | 38 +++++++++++--------- src/idl_gen_cpp.cpp | 14 ++++++-- tests/64bit/offset64_test.cpp | 14 +++++--- tests/64bit/test_64bit.afb | 42 ++++++++++++++++------- tests/64bit/test_64bit.bin | Bin 40 -> 112 bytes tests/64bit/test_64bit.fbs | 1 + tests/64bit/test_64bit.json | 3 +- tests/64bit/test_64bit_generated.h | 42 ++++++++++++++++++----- 8 files changed, 109 insertions(+), 45 deletions(-) diff --git a/include/flatbuffers/flatbuffer_builder.h b/include/flatbuffers/flatbuffer_builder.h index 55cdecb9af4..8ecf0bdc0ff 100644 --- a/include/flatbuffers/flatbuffer_builder.h +++ b/include/flatbuffers/flatbuffer_builder.h @@ -470,43 +470,47 @@ class FlatBufferBuilder { /// @param[in] str A const char pointer to the data to be stored as a string. /// @param[in] len The number of bytes that should be stored from `str`. /// @return Returns the offset in the buffer where the string starts. - Offset CreateString(const char *str, size_t len) { + template> + OffsetT CreateString(const char *str, size_t len) { NotNested(); PreAlign(len + 1); // Always 0-terminated. buf_.fill(1); PushBytes(reinterpret_cast(str), len); PushElement(static_cast(len)); - return Offset(GetSize()); + return OffsetT(GetSize()); } /// @brief Store a string in the buffer, which is null-terminated. /// @param[in] str A const char pointer to a C-string to add to the buffer. /// @return Returns the offset in the buffer where the string starts. - Offset CreateString(const char *str) { - return CreateString(str, strlen(str)); + template> + OffsetT CreateString(const char *str) { + return CreateString(str, strlen(str)); } /// @brief Store a string in the buffer, which is null-terminated. /// @param[in] str A char pointer to a C-string to add to the buffer. /// @return Returns the offset in the buffer where the string starts. - Offset CreateString(char *str) { - return CreateString(str, strlen(str)); + template> OffsetT CreateString(char *str) { + return CreateString(str, strlen(str)); } /// @brief Store a string in the buffer, which can contain any binary data. /// @param[in] str A const reference to a std::string to store in the buffer. /// @return Returns the offset in the buffer where the string starts. - Offset CreateString(const std::string &str) { - return CreateString(str.c_str(), str.length()); + template> + OffsetT CreateString(const std::string &str) { + return CreateString(str.c_str(), str.length()); } - // clang-format off +// clang-format off #ifdef FLATBUFFERS_HAS_STRING_VIEW /// @brief Store a string in the buffer, which can contain any binary data. /// @param[in] str A const string_view to copy in to the buffer. /// @return Returns the offset in the buffer where the string starts. - Offset CreateString(flatbuffers::string_view str) { - return CreateString(str.data(), str.size()); + template> + OffsetT CreateString(flatbuffers::string_view str) { + return CreateString(str.data(), str.size()); } #endif // FLATBUFFERS_HAS_STRING_VIEW // clang-format on @@ -514,16 +518,18 @@ class FlatBufferBuilder { /// @brief Store a string in the buffer, which can contain any binary data. /// @param[in] str A const pointer to a `String` struct to add to the buffer. /// @return Returns the offset in the buffer where the string starts - Offset CreateString(const String *str) { - return str ? CreateString(str->c_str(), str->size()) : 0; + template> + OffsetT CreateString(const String *str) { + return str ? CreateString(str->c_str(), str->size()) : 0; } /// @brief Store a string in the buffer, which can contain any binary data. /// @param[in] str A const reference to a std::string like type with support /// of T::c_str() and T::length() to store in the buffer. /// @return Returns the offset in the buffer where the string starts. - template Offset CreateString(const T &str) { - return CreateString(str.c_str(), str.length()); + template> + OffsetT CreateString(const T &str) { + return CreateString(str.c_str(), str.length()); } /// @brief Store a string in the buffer, which can contain any binary data. @@ -647,7 +653,7 @@ class FlatBufferBuilder { AssertScalarT(); StartVector(len); if (len == 0) { return OffsetT(EndVector(len)); } - // clang-format off +// clang-format off #if FLATBUFFERS_LITTLEENDIAN PushBytes(reinterpret_cast(v), len * sizeof(T)); #else diff --git a/src/idl_gen_cpp.cpp b/src/idl_gen_cpp.cpp index e315983a1e9..af4a8e76ea2 100644 --- a/src/idl_gen_cpp.cpp +++ b/src/idl_gen_cpp.cpp @@ -3008,7 +3008,12 @@ class CppGenerator : public BaseGenerator { code_.SetValue("FIELD_NAME", Name(*field)); if (IsString(field->value.type)) { if (!field->shared) { - code_.SetValue("CREATE_STRING", "CreateString"); + code_.SetValue( + "CREATE_STRING", + "CreateString" + std::string(field->offset64 + ? "<::flatbuffers::Offset64<" + "::flatbuffers::String>>" + : "")); } else { code_.SetValue("CREATE_STRING", "CreateSharedString"); } @@ -3293,7 +3298,12 @@ class CppGenerator : public BaseGenerator { // _fbb.CreateSharedString(_o->field) case BASE_TYPE_STRING: { if (!field.shared) { - code += "_fbb.CreateString("; + code += "_fbb.CreateString" + + std::string(field.offset64 + ? ">" + : "") + + "("; } else { code += "_fbb.CreateSharedString("; } diff --git a/tests/64bit/offset64_test.cpp b/tests/64bit/offset64_test.cpp index 7e4c45971ad..e3cff52e014 100644 --- a/tests/64bit/offset64_test.cpp +++ b/tests/64bit/offset64_test.cpp @@ -1,8 +1,9 @@ #include "offset64_test.h" #include -#include + #include +#include #include "flatbuffers/base.h" #include "flatbuffers/flatbuffers.h" @@ -15,22 +16,25 @@ namespace tests { void Offset64Test() { flatbuffers::FlatBufferBuilder fbb; - size_t vector_size = 3LL << 3; // change to 30 to get 3 GiB. + size_t vector_size = 3LL << 3; // change to 30 to get 3 GiB. { std::vector data; - data.resize(vector_size); + data.resize(vector_size); const Offset64> big_vector_offset = fbb.CreateVector64(data); + const Offset64 big_string_offset = + fbb.CreateString>("some big string"); + RootTableBuilder root_table_builder(fbb); root_table_builder.add_big_vector(big_vector_offset); + root_table_builder.add_big_string(big_string_offset); const Offset root_table_offset = root_table_builder.Finish(); fbb.Finish(root_table_offset); - Verifier::Options options; // Allow the verifier to verify 64-bit buffers. options.max_size = FLATBUFFERS_MAX_64_BUFFER_SIZE; @@ -45,6 +49,8 @@ void Offset64Test() { // Expect the big vector to be properly sized. TEST_EQ(root_table->big_vector()->size(), vector_size); + + TEST_EQ_STR(root_table->big_string()->c_str(), "some big string"); } } diff --git a/tests/64bit/test_64bit.afb b/tests/64bit/test_64bit.afb index 2d04a471863..5f2be6d6ecf 100644 --- a/tests/64bit/test_64bit.afb +++ b/tests/64bit/test_64bit.afb @@ -4,22 +4,38 @@ // Binary file: tests/64bit/test_64bit.bin header: - +0x00 | 0C 00 00 00 | UOffset32 | 0x0000000C (12) Loc: +0x0C | offset to root table `RootTable` + +0x00 | 14 00 00 00 | UOffset32 | 0x00000014 (20) Loc: +0x14 | offset to root table `RootTable` + +padding: + +0x04 | 00 00 00 00 00 00 | uint8_t[6] | ...... | padding vtable (RootTable): - +0x04 | 08 00 | uint16_t | 0x0008 (8) | size of this vtable - +0x06 | 14 00 | uint16_t | 0x0014 (20) | size of referring table - +0x08 | 04 00 | VOffset16 | 0x0004 (4) | offset to field `big_vector` (id: 0) - +0x0A | 10 00 | VOffset16 | 0x0010 (16) | offset to field `a` (id: 1) + +0x0A | 0A 00 | uint16_t | 0x000A (10) | size of this vtable + +0x0C | 20 00 | uint16_t | 0x0020 (32) | size of referring table + +0x0E | 04 00 | VOffset16 | 0x0004 (4) | offset to field `big_vector` (id: 0) + +0x10 | 10 00 | VOffset16 | 0x0010 (16) | offset to field `a` (id: 1) + +0x12 | 14 00 | VOffset16 | 0x0014 (20) | offset to field `big_string` (id: 2) root_table (RootTable): - +0x0C | 08 00 00 00 | SOffset32 | 0x00000008 (8) Loc: +0x04 | offset to vtable - +0x10 | 10 00 00 00 00 00 00 00 | UOffset64 | 0x0000000000000010 (16) Loc: +0x20 | offset to field `big_vector` (vector) - +0x18 | 00 00 00 00 | uint8_t[4] | .... | padding - +0x1C | D2 04 00 00 | uint32_t | 0x000004D2 (1234) | table field `a` (Int) + +0x14 | 0A 00 00 00 | SOffset32 | 0x0000000A (10) Loc: +0x0A | offset to vtable + +0x18 | 50 00 00 00 00 00 00 00 | UOffset64 | 0x0000000000000050 (80) Loc: +0x68 | offset to field `big_vector` (vector) + +0x20 | 00 00 00 00 | uint8_t[4] | .... | padding + +0x24 | D2 04 00 00 | uint32_t | 0x000004D2 (1234) | table field `a` (Int) + +0x28 | 0C 00 00 00 00 00 00 00 | UOffset64 | 0x000000000000000C (12) Loc: +0x34 | offset to field `big_string` (string) + +0x30 | 00 00 00 00 | uint8_t[4] | .... | padding + +string (RootTable.big_string): + +0x34 | 2E 00 00 00 | uint32_t | 0x0000002E (46) | length of string + +0x38 | 74 68 69 73 20 69 73 20 | char[46] | this is | string literal + +0x40 | 61 20 62 69 67 20 73 74 | | a big st + +0x48 | 72 69 6E 67 20 77 68 69 | | ring whi + +0x50 | 63 68 20 68 61 73 20 61 | | ch has a + +0x58 | 20 36 34 2D 62 69 74 20 | | 64-bit + +0x60 | 6F 66 66 73 65 74 | | offset + +0x66 | 00 | char | 0x00 (0) | string terminator vector (RootTable.big_vector): - +0x20 | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of vector (# items) - +0x24 | 01 | uint8_t | 0x01 (1) | value[0] - +0x25 | 02 | uint8_t | 0x02 (2) | value[1] - +0x26 | 03 | uint8_t | 0x03 (3) | value[2] + +0x68 | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of vector (# items) + +0x6C | 01 | uint8_t | 0x01 (1) | value[0] + +0x6D | 02 | uint8_t | 0x02 (2) | value[1] + +0x6E | 03 | uint8_t | 0x03 (3) | value[2] diff --git a/tests/64bit/test_64bit.bin b/tests/64bit/test_64bit.bin index b0308bffe3413613c5872f91926cd93e6ee2b030..e8fdc5a6f697d5ca71e0b7fbd36b0e9612fe8aa8 100644 GIT binary patch literal 112 zcmWe(fB`NB1qK!d0R|BUE}&2VOc+RCVqsw5LE`HH*(Dj7#R@=_sF0MIu25W3l$n>V lP@a*QoS~4B2og3k(M`%MQOHkAD^4w8U| big_vector{}; int32_t a = 0; + std::string big_string{}; }; struct RootTable FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { @@ -36,7 +37,8 @@ struct RootTable FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { } enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_BIG_VECTOR = 4, - VT_A = 6 + VT_A = 6, + VT_BIG_STRING = 8 }; const ::flatbuffers::Vector *big_vector() const { return GetPointer *>(VT_BIG_VECTOR); @@ -50,11 +52,19 @@ struct RootTable FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { bool mutate_a(int32_t _a = 0) { return SetField(VT_A, _a, 0); } + const ::flatbuffers::String *big_string() const { + return GetPointer(VT_BIG_STRING); + } + ::flatbuffers::String *mutable_big_string() { + return GetPointer<::flatbuffers::String *>(VT_BIG_STRING); + } bool Verify(::flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_BIG_VECTOR) && verifier.VerifyVector(big_vector()) && VerifyField(verifier, VT_A, 4) && + VerifyOffset(verifier, VT_BIG_STRING) && + verifier.VerifyString(big_string()) && verifier.EndTable(); } RootTableT *UnPack(const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; @@ -72,6 +82,9 @@ struct RootTableBuilder { void add_a(int32_t a) { fbb_.AddElement(RootTable::VT_A, a, 0); } + void add_big_string(::flatbuffers::Offset64<::flatbuffers::String> big_string) { + fbb_.AddOffset(RootTable::VT_BIG_STRING, big_string); + } explicit RootTableBuilder(::flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); @@ -86,8 +99,10 @@ struct RootTableBuilder { inline ::flatbuffers::Offset CreateRootTable( ::flatbuffers::FlatBufferBuilder &_fbb, ::flatbuffers::Offset64<::flatbuffers::Vector> big_vector = 0, - int32_t a = 0) { + int32_t a = 0, + ::flatbuffers::Offset64<::flatbuffers::String> big_string = 0) { RootTableBuilder builder_(_fbb); + builder_.add_big_string(big_string); builder_.add_a(a); builder_.add_big_vector(big_vector); return builder_.Finish(); @@ -96,12 +111,15 @@ inline ::flatbuffers::Offset CreateRootTable( inline ::flatbuffers::Offset CreateRootTableDirect( ::flatbuffers::FlatBufferBuilder &_fbb, const std::vector *big_vector = nullptr, - int32_t a = 0) { + int32_t a = 0, + const char *big_string = nullptr) { auto big_vector__ = big_vector ? _fbb.CreateVector64(*big_vector) : 0; + auto big_string__ = big_string ? _fbb.CreateString<::flatbuffers::Offset64<::flatbuffers::String>>(big_string) : 0; return CreateRootTable( _fbb, big_vector__, - a); + a, + big_string__); } ::flatbuffers::Offset CreateRootTable(::flatbuffers::FlatBufferBuilder &_fbb, const RootTableT *_o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); @@ -110,7 +128,8 @@ ::flatbuffers::Offset CreateRootTable(::flatbuffers::FlatBufferBuilde inline bool operator==(const RootTableT &lhs, const RootTableT &rhs) { return (lhs.big_vector == rhs.big_vector) && - (lhs.a == rhs.a); + (lhs.a == rhs.a) && + (lhs.big_string == rhs.big_string); } inline bool operator!=(const RootTableT &lhs, const RootTableT &rhs) { @@ -129,6 +148,7 @@ inline void RootTable::UnPackTo(RootTableT *_o, const ::flatbuffers::resolver_fu (void)_resolver; { auto _e = big_vector(); if (_e) { _o->big_vector.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->big_vector.begin()); } } { auto _e = a(); _o->a = _e; } + { auto _e = big_string(); if (_e) _o->big_string = _e->str(); } } inline ::flatbuffers::Offset RootTable::Pack(::flatbuffers::FlatBufferBuilder &_fbb, const RootTableT* _o, const ::flatbuffers::rehasher_function_t *_rehasher) { @@ -141,23 +161,27 @@ inline ::flatbuffers::Offset CreateRootTable(::flatbuffers::FlatBuffe struct _VectorArgs { ::flatbuffers::FlatBufferBuilder *__fbb; const RootTableT* __o; const ::flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; auto _big_vector = _o->big_vector.size() ? _fbb.CreateVector64(_o->big_vector) : 0; auto _a = _o->a; + auto _big_string = _o->big_string.empty() ? 0 : _fbb.CreateString>(_o->big_string); return CreateRootTable( _fbb, _big_vector, - _a); + _a, + _big_string); } inline const ::flatbuffers::TypeTable *RootTableTypeTable() { static const ::flatbuffers::TypeCode type_codes[] = { { ::flatbuffers::ET_UCHAR, 1, -1 }, - { ::flatbuffers::ET_INT, 0, -1 } + { ::flatbuffers::ET_INT, 0, -1 }, + { ::flatbuffers::ET_STRING, 0, -1 } }; static const char * const names[] = { "big_vector", - "a" + "a", + "big_string" }; static const ::flatbuffers::TypeTable tt = { - ::flatbuffers::ST_TABLE, 2, type_codes, nullptr, nullptr, nullptr, names + ::flatbuffers::ST_TABLE, 3, type_codes, nullptr, nullptr, nullptr, names }; return &tt; } From d02a18703cef1e9951901fbb821e7253f9493b09 Mon Sep 17 00:00:00 2001 From: Derek Bailey Date: Fri, 27 Jan 2023 17:02:19 -0800 Subject: [PATCH 09/62] simplified CreateString<> to just provide the offset type --- include/flatbuffers/flatbuffer_builder.h | 34 +++++++++++++----------- src/idl_gen_cpp.cpp | 6 ++--- tests/64bit/offset64_test.cpp | 2 +- tests/64bit/test_64bit_generated.h | 4 +-- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/include/flatbuffers/flatbuffer_builder.h b/include/flatbuffers/flatbuffer_builder.h index 8ecf0bdc0ff..7f3b052294e 100644 --- a/include/flatbuffers/flatbuffer_builder.h +++ b/include/flatbuffers/flatbuffer_builder.h @@ -470,46 +470,47 @@ class FlatBufferBuilder { /// @param[in] str A const char pointer to the data to be stored as a string. /// @param[in] len The number of bytes that should be stored from `str`. /// @return Returns the offset in the buffer where the string starts. - template> - OffsetT CreateString(const char *str, size_t len) { + template class OffsetT = Offset> + OffsetT CreateString(const char *str, size_t len) { NotNested(); PreAlign(len + 1); // Always 0-terminated. buf_.fill(1); PushBytes(reinterpret_cast(str), len); PushElement(static_cast(len)); - return OffsetT(GetSize()); + return OffsetT(GetSize()); } /// @brief Store a string in the buffer, which is null-terminated. /// @param[in] str A const char pointer to a C-string to add to the buffer. /// @return Returns the offset in the buffer where the string starts. - template> - OffsetT CreateString(const char *str) { + template class OffsetT = Offset> + OffsetT CreateString(const char *str) { return CreateString(str, strlen(str)); } /// @brief Store a string in the buffer, which is null-terminated. /// @param[in] str A char pointer to a C-string to add to the buffer. /// @return Returns the offset in the buffer where the string starts. - template> OffsetT CreateString(char *str) { + template class OffsetT = Offset> + OffsetT CreateString(char *str) { return CreateString(str, strlen(str)); } /// @brief Store a string in the buffer, which can contain any binary data. /// @param[in] str A const reference to a std::string to store in the buffer. /// @return Returns the offset in the buffer where the string starts. - template> - OffsetT CreateString(const std::string &str) { + template class OffsetT = Offset> + OffsetT CreateString(const std::string &str) { return CreateString(str.c_str(), str.length()); } -// clang-format off + // clang-format off #ifdef FLATBUFFERS_HAS_STRING_VIEW /// @brief Store a string in the buffer, which can contain any binary data. /// @param[in] str A const string_view to copy in to the buffer. /// @return Returns the offset in the buffer where the string starts. - template> - OffsetT CreateString(flatbuffers::string_view str) { + template