Skip to content

Commit

Permalink
implement packing and add tests (no float support)
Browse files Browse the repository at this point in the history
  • Loading branch information
@CrustyAuklet
CrustyAuklet committed May 1, 2020
1 parent 0e3a611 commit 42e1e0c
Show file tree
Hide file tree
Showing 2 changed files with 162 additions and 16 deletions.
150 changes: 147 additions & 3 deletions include/bitpacker/bitpacker.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,6 @@ namespace bitpacker {

namespace impl {


struct format_string {
};

Expand Down Expand Up @@ -478,6 +477,7 @@ namespace bitpacker {
#if bitpacker_CPP20_OR_GREATER
// constexpr in c++20 and greater
using std::reverse;
using std::copy;
#else
// copied from cppref but constexpr, we KNOW its trivial types
template < class BidirIt >
Expand All @@ -490,6 +490,15 @@ namespace bitpacker {
++first;
}
}

template<class InputIt, class OutputIt>
constexpr OutputIt copy(InputIt first, InputIt last, OutputIt d_first)
{
while (first != last) {
*d_first++ = *first++;
}
return d_first;
}
#endif

template <typename Fmt, size_t... Items, typename Input>
Expand Down Expand Up @@ -526,8 +535,8 @@ namespace bitpacker {
// to remain binary compatible with bitstruct: bitcount is actual bits.
// Any partial bytes end up in the last byte/char, left aligned.
constexpr unsigned charsize = 8U;
constexpr unsigned full_bytes = UnpackedType::bits / 8;
constexpr unsigned extra_bits = UnpackedType::bits % 8;
constexpr unsigned full_bytes = UnpackedType::bits / charsize;
constexpr unsigned extra_bits = UnpackedType::bits % charsize;
constexpr size_t return_size = bit2byte(UnpackedType::bits);
typename UnpackedType::return_type buff{};

Expand Down Expand Up @@ -606,6 +615,141 @@ namespace bitpacker {
return unpacked;
}

/***************************************************************************************************
* Compile time packing functionality
***************************************************************************************************/

namespace impl {

template <size_t N>
constexpr auto remove_non_padding(const std::array< RawFormatType, N > &types) noexcept
{
std::array< RawFormatType, N > buff{};
size_t insert_idx = 0;
for (const auto& t : types) {
if (isPadding(t.formatChar)) {
buff[insert_idx++] = t;
}
}
return buff;
}

template <typename RepType, typename T>
constexpr RepType convert_for_pack(const T& val)
{
return static_cast<RepType>(val);
}

template <typename PackedType, typename InputType>
constexpr int packElement(span<byte_type> buffer, size_t offset, InputType elem)
{
// TODO: Implement these formats
static_assert(PackedType::format != 'f', "Unpacking Floats not supported yet...");

if constexpr (PackedType::format == 'u' || PackedType::format == 's') {
static_assert(PackedType::bits <= 64, "Integer types must be 64 bits or less");
auto val = convert_for_pack< typename PackedType::rep_type >(elem);
if (PackedType::bit_endian == impl::Endian::little) {
val = impl::reverse_bits< decltype(val), PackedType::bits >(val);
}
pack_into(buffer, offset, PackedType::bits, val);
}
if constexpr (PackedType::format == 'b') {
static_assert(PackedType::bits <= 64, "Boolean types must be 64 bits or less");
pack_into(buffer, offset, PackedType::bits, static_cast<bool>(elem));
}
if constexpr (isPadding(PackedType::format)) {
static_assert(PackedType::bits <= 64, "Padding fields must be 64 bits or less");
if(PackedType::format == 'P') {
constexpr auto val = static_cast< unsigned_type<PackedType::bits> >(-1);
pack_into(buffer, offset, PackedType::bits, val);
}
else {
pack_into(buffer, offset, PackedType::bits, 0U);
}
}
if constexpr (PackedType::format == 'f') {
static_assert(PackedType::bits == 16 || PackedType::bits == 32 || PackedType::bits == 64,
"Expected float size of 16, 32, or 64 bits");
}
if constexpr (isByteType(PackedType::format)) {
// to remain binary compatible with bitstruct: bitcount is actual bits.
// Any partial bytes end up in the last byte/char, left aligned.
constexpr unsigned charsize = 8U;
constexpr unsigned full_bytes = PackedType::bits / charsize;
constexpr unsigned extra_bits = PackedType::bits % charsize;
constexpr unsigned byte_count = bit2byte(PackedType::bits);

if(PackedType::bit_endian == impl::Endian::little) {
constexpr auto size = std::extent_v<decltype(elem)>;
std::array<uint8_t, byte_count> arr{};
bitpacker::impl::copy(&elem[0], &elem[0]+byte_count, arr.begin());

// little endian bitwise in bitstruct means the entire length flipped.
// to simulate this we reverse the order then flip each bytes bit order
bitpacker::impl::reverse(std::begin(arr), std::end(arr));
for(auto &v : arr) {
v = impl::reverse_bits< decltype(v), ByteSize >(v);
}

for(int bits = PackedType::bits, idx = 0; bits > 0; bits -= charsize) {
const auto field_size = bits < charsize ? bits : charsize;
pack_into<uint8_t>(buffer, offset + (idx * charsize), charsize, arr[idx]);
++idx;
}
}
else {
for(int bits = PackedType::bits, idx = 0; bits > 0; bits -= charsize) {
const auto field_size = bits < charsize ? bits : charsize;
pack_into<uint8_t>(buffer, offset + (idx * charsize), charsize, elem[idx]);
++idx;
}
}
}
return 0;
}

template <typename Fmt, size_t... Items>
constexpr auto insert_padding(span<byte_type> buffer, std::index_sequence<Items...>)
{
constexpr auto formats_only_pad = impl::remove_non_padding(impl::get_type_array(Fmt{}));
using FormatTypes = std::tuple< typename impl::FormatType< formats_only_pad[Items].formatChar,
formats_only_pad[Items].count,
formats_only_pad[Items].endian >... >;
int _[] = { 0, packElement< std::tuple_element_t<Items, FormatTypes> >(buffer, formats_only_pad[Items].offset, 0)... };
(void)_; // _ is a dummy for pack expansion
}

template <typename Fmt, size_t... Items, typename... Args>
constexpr auto pack(std::index_sequence<Items...>, Args&&... args)
{
static_assert(sizeof...(args) == sizeof...(Items), "pack expected items for packing != sizeof...(args) passed");
constexpr auto byte_order = impl::get_byte_order(Fmt{});
static_assert(byte_order == impl::Endian::big, "Unpacking little endian byte order not supported yet...");
constexpr auto formats_no_pad = impl::remove_padding(impl::get_type_array(Fmt{}));

using ArrayType = std::array<byte_type, calcbytes(Fmt{})>;
ArrayType output{};

using FormatTypes = std::tuple< typename impl::FormatType< formats_no_pad[Items].formatChar,
formats_no_pad[Items].count,
formats_no_pad[Items].endian >... >;

impl::insert_padding<Fmt>( output, std::make_index_sequence<impl::count_padding(Fmt{})>());
int _[] = { 0, packElement< std::tuple_element_t<Items, FormatTypes> >(output, formats_no_pad[Items].offset, args)... };
(void)_; // _ is a dummy for pack expansion

return output;
}

} // namespace impl

template < typename Fmt, typename... Args >
constexpr auto pack(Fmt, Args&&... args)
{
return impl::pack< Fmt >(std::make_index_sequence< impl::count_non_padding(Fmt{}) >(), std::forward< Args >(args)...);
}

} // namespace bitpacker

#define BP_STRING(s) [] { \
Expand Down
28 changes: 15 additions & 13 deletions tests/python_common.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ auto runPythonPack(Fmt, const Args &... toPack)
// Read the python script output
std::ifstream inputFile(packedBinaryFilePath, std::ios::binary);
inputFile = std::ifstream(packedBinaryFilePath, std::ios::binary);
std::vector< char > outputBuffer(
std::vector< uint8_t > outputBuffer(
(std::istreambuf_iterator< char >(inputFile)),
(std::istreambuf_iterator< char >()));
inputFile.close();
Expand All @@ -169,12 +169,12 @@ auto runPythonPack(Fmt, const Args &... toPack)
return outputBuffer;
}

inline std::string print_data_vector(const std::vector< char > &vec)
inline std::string print_data_vector(const std::vector< uint8_t >& vec)
{
std::stringstream ss;
ss << "{ ";
for (const auto &v : vec) {
ss << "0x" << std::hex << (static_cast< unsigned >(v) & 0xFFU) << ", ";
ss << "0x" << std::hex << (static_cast< unsigned >(v)& 0xFFU) << ", ";
}
std::string retval = ss.str();
return retval.substr(0, retval.size() - 2) + " }";
Expand Down Expand Up @@ -264,24 +264,26 @@ bool compareBitpackerTuples(const T1& lhs, const T2& rhs)
template < typename Fmt, typename... Args >
void testPackAgainstPython(Fmt, const Args &... toPack)
{
//auto packed = bitpacker::pack(Fmt{}, toPack...);
auto pythonPacked = runPythonPack(Fmt{}, toPack...);
auto bp_packed = bitpacker::pack(Fmt{}, toPack...);
auto py_packed = runPythonPack(Fmt{}, toPack...);

//CAPTURE(packed);
INFO("Format String: " << Fmt::value())
INFO("From Python-pack: " << print_data_vector(pythonPacked));
INFO("From BitPacker-pack : " << print_data_vector( std::vector<uint8_t>(bp_packed.begin(), bp_packed.end())) );
INFO("From Python-pack : " << print_data_vector(py_packed));

//REQUIRE(packed.size() == pythonPacked.size());
//REQUIRE(std::equal(packed.begin(), packed.end(), pythonPacked.begin()));
REQUIRE(bp_packed.size() == py_packed.size());
REQUIRE(std::equal(bp_packed.begin(), bp_packed.end(), py_packed.begin(), py_packed.end()));

bitpacker::span< const bitpacker::byte_type > buffer(reinterpret_cast< uint8_t * >(pythonPacked.data()), pythonPacked.size());
auto unpacked = bitpacker::unpack(Fmt{}, buffer);
bitpacker::span< const bitpacker::byte_type > buffer(reinterpret_cast< uint8_t * >(py_packed.data()), py_packed.size());
auto py_unpacked = bitpacker::unpack(Fmt{}, buffer);
auto bp_unpacked = bitpacker::unpack(Fmt{}, bp_packed);

INFO("Unpacked by Bitpacker: " << print_data_tuple(unpacked));
INFO("Python Unpacked by BitPacker: " << print_data_tuple(py_unpacked));
INFO("BitPacker Unpacked by BitPacker: " << print_data_tuple(bp_unpacked));
INFO("Expected (uncast): " << print_data_tuple(std::make_tuple(toPack...)));

// explicitly creating tuple of ReturnTypes will make tests pass if the value is implicitly convertable to the expected return type
// for example: multi-bit bool values
const bool success = compareBitpackerTuples(unpacked, std::make_tuple(toPack...));
const bool success = compareBitpackerTuples(py_unpacked, std::make_tuple(toPack...));
REQUIRE(success);
}

0 comments on commit 42e1e0c

Please sign in to comment.