From 13b25a58154acf0f19ff93ec4cfd253a4baa2615 Mon Sep 17 00:00:00 2001 From: Kai Zhao Date: Thu, 24 Feb 2022 12:26:35 -0500 Subject: [PATCH] bugfix 1. long->int64_t 2.uint->SZ::uint --- include/SZ3/api/impl/SZDispatcher.hpp | 4 +- include/SZ3/api/impl/SZImpl.hpp | 4 +- include/SZ3/api/impl/SZImplOMP.hpp | 4 +- include/SZ3/api/impl/SZInterp.hpp | 8 +- include/SZ3/api/impl/SZLorenzoReg.hpp | 7 +- include/SZ3/encoder/HuffmanEncoder.hpp | 18 +-- include/SZ3/utils/ByteUtil.hpp | 182 ++----------------------- 7 files changed, 35 insertions(+), 192 deletions(-) diff --git a/include/SZ3/api/impl/SZDispatcher.hpp b/include/SZ3/api/impl/SZDispatcher.hpp index 15235034..83e564be 100644 --- a/include/SZ3/api/impl/SZDispatcher.hpp +++ b/include/SZ3/api/impl/SZDispatcher.hpp @@ -9,7 +9,7 @@ #include -template +template char *SZ_compress_dispatcher(SZ::Config &conf, T *data, size_t &outSize) { assert(N == conf.N); @@ -27,7 +27,7 @@ char *SZ_compress_dispatcher(SZ::Config &conf, T *data, size_t &outSize) { } -template +template void SZ_decompress_dispatcher(SZ::Config &conf, char *cmpData, size_t cmpSize, T *decData) { if (conf.cmprAlgo == SZ::ALGO_LORENZO_REG) { SZ_decompress_LorenzoReg(conf, cmpData, cmpSize, decData); diff --git a/include/SZ3/api/impl/SZImpl.hpp b/include/SZ3/api/impl/SZImpl.hpp index 8bccae3d..a0765d34 100644 --- a/include/SZ3/api/impl/SZImpl.hpp +++ b/include/SZ3/api/impl/SZImpl.hpp @@ -6,7 +6,7 @@ #include "SZ3/api/impl/SZImplOMP.hpp" #include -template +template char *SZ_compress_impl(SZ::Config &conf, T *data, size_t &outSize) { #ifndef _OPENMP conf.openmp=false; @@ -19,7 +19,7 @@ char *SZ_compress_impl(SZ::Config &conf, T *data, size_t &outSize) { } -template +template void SZ_decompress_impl(SZ::Config &conf, char *cmpData, size_t cmpSize, T *decData) { #ifndef _OPENMP conf.openmp=false; diff --git a/include/SZ3/api/impl/SZImplOMP.hpp b/include/SZ3/api/impl/SZImplOMP.hpp index e3e102ce..e238f717 100644 --- a/include/SZ3/api/impl/SZImplOMP.hpp +++ b/include/SZ3/api/impl/SZImplOMP.hpp @@ -8,7 +8,7 @@ #include "omp.h" -template +template char *SZ_compress_OMP(SZ::Config &conf, T *data, size_t &outSize) { #ifdef _OPENMP @@ -101,7 +101,7 @@ char *SZ_compress_OMP(SZ::Config &conf, T *data, size_t &outSize) { } -template +template void SZ_decompress_OMP(const SZ::Config &conf, char *cmpData, size_t cmpSize, T *decData) { #ifdef _OPENMP diff --git a/include/SZ3/api/impl/SZInterp.hpp b/include/SZ3/api/impl/SZInterp.hpp index f8f0e8b9..eef284c8 100644 --- a/include/SZ3/api/impl/SZInterp.hpp +++ b/include/SZ3/api/impl/SZInterp.hpp @@ -15,7 +15,7 @@ #include -template +template char *SZ_compress_Interp(SZ::Config &conf, T *data, size_t &outSize) { // std::cout << "****************** Interp Compression ****************" << std::endl; @@ -38,7 +38,7 @@ char *SZ_compress_Interp(SZ::Config &conf, T *data, size_t &outSize) { } -template +template void SZ_decompress_Interp(const SZ::Config &conf, char *cmpData, size_t cmpSize, T *decData) { assert(conf.cmprAlgo == SZ::ALGO_INTERP); SZ::uchar const *cmpDataPos = (SZ::uchar *) cmpData; @@ -50,7 +50,7 @@ void SZ_decompress_Interp(const SZ::Config &conf, char *cmpData, size_t cmpSize, } -template +template double do_not_use_this_interp_compress_block_test(T *data, std::vector dims, size_t num, double eb, int interp_op, int direction_op, int block_size) { @@ -73,7 +73,7 @@ double do_not_use_this_interp_compress_block_test(T *data, std::vector d return compression_ratio; } -template +template char *SZ_compress_Interp_lorenzo(SZ::Config &conf, T *data, size_t &outSize) { assert(conf.cmprAlgo == SZ::ALGO_INTERP_LORENZO); diff --git a/include/SZ3/api/impl/SZLorenzoReg.hpp b/include/SZ3/api/impl/SZLorenzoReg.hpp index f7c1eb65..e9666a92 100644 --- a/include/SZ3/api/impl/SZLorenzoReg.hpp +++ b/include/SZ3/api/impl/SZLorenzoReg.hpp @@ -15,11 +15,12 @@ #include "SZ3/utils/Extraction.hpp" #include "SZ3/utils/QuantOptimizatioin.hpp" #include "SZ3/utils/Config.hpp" +#include "SZ3/def.hpp" #include #include -template +template std::shared_ptr> make_lorenzo_regression_compressor(const SZ::Config &conf, Quantizer quantizer, Encoder encoder, Lossless lossless) { std::vector>> predictors; @@ -73,7 +74,7 @@ make_lorenzo_regression_compressor(const SZ::Config &conf, Quantizer quantizer, } -template +template char *SZ_compress_LorenzoReg(SZ::Config &conf, T *data, size_t &outSize) { assert(N == conf.N); @@ -95,7 +96,7 @@ char *SZ_compress_LorenzoReg(SZ::Config &conf, T *data, size_t &outSize) { } -template +template void SZ_decompress_LorenzoReg(const SZ::Config &conf, char *cmpData, size_t cmpSize, T *decData) { assert(conf.cmprAlgo == SZ::ALGO_LORENZO_REG); diff --git a/include/SZ3/encoder/HuffmanEncoder.hpp b/include/SZ3/encoder/HuffmanEncoder.hpp index b02d4ff7..da0932d8 100644 --- a/include/SZ3/encoder/HuffmanEncoder.hpp +++ b/include/SZ3/encoder/HuffmanEncoder.hpp @@ -110,9 +110,9 @@ namespace SZ { //save the huffman Tree in the compressed data uint save(uchar *&c) { write(offset, c); - intToBytes_bigEndian(c, nodeCount); + int32ToBytes_bigEndian(c, nodeCount); c += sizeof(int); - intToBytes_bigEndian(c, huffmanTree->stateNum / 2); + int32ToBytes_bigEndian(c, huffmanTree->stateNum / 2); c += sizeof(int); uint totalSize = 0;// = convert_HuffTree_to_bytes_anyStates(nodeCount, c); // std::cout << "nodeCount = " << nodeCount << std::endl; @@ -150,13 +150,13 @@ namespace SZ { 1; //it's equal to the number of bytes involved (for *outSize) byteSizep = bitSize / 8; //it's used to move the pointer p for next data if (byteSize <= 8) { - longToBytes_bigEndian(p, (huffmanTree->code[state])[0]); + int64ToBytes_bigEndian(p, (huffmanTree->code[state])[0]); p += byteSizep; } else //byteSize>8 { - longToBytes_bigEndian(p, (huffmanTree->code[state])[0]); + int64ToBytes_bigEndian(p, (huffmanTree->code[state])[0]); p += 8; - longToBytes_bigEndian(p, (huffmanTree->code[state])[1]); + int64ToBytes_bigEndian(p, (huffmanTree->code[state])[1]); p += (byteSizep - 8); } outSize += byteSize; @@ -166,7 +166,7 @@ namespace SZ { if (lackBits < bitSize) { p++; long newCode = (huffmanTree->code[state])[0] << lackBits; - longToBytes_bigEndian(p, newCode); + int64ToBytes_bigEndian(p, newCode); if (bitSize <= 64) { bitSize -= lackBits; @@ -186,7 +186,7 @@ namespace SZ { *p = (*p) | (unsigned char) ((huffmanTree->code[state])[0] >> (64 - lackBits)); p++; newCode = (huffmanTree->code[state])[1] << lackBits; - longToBytes_bigEndian(p, newCode); + int64ToBytes_bigEndian(p, newCode); bitSize -= lackBits; byteSize = bitSize % 8 == 0 ? bitSize / 8 : bitSize / 8 + 1; byteSizep = bitSize / 8; @@ -261,8 +261,8 @@ namespace SZ { //load Huffman tree void load(const uchar *&c, size_t &remaining_length) { read(offset, c, remaining_length); - nodeCount = bytesToInt_bigEndian(c); - int stateNum = bytesToInt_bigEndian(c + sizeof(int)) * 2; + nodeCount = bytesToInt32_bigEndian(c); + int stateNum = bytesToInt32_bigEndian(c + sizeof(int)) * 2; size_t encodeStartIndex; if (nodeCount <= 256) encodeStartIndex = 1 + 3 * nodeCount * sizeof(unsigned char) + nodeCount * sizeof(T); diff --git a/include/SZ3/utils/ByteUtil.hpp b/include/SZ3/utils/ByteUtil.hpp index 44898249..43f977f2 100644 --- a/include/SZ3/utils/ByteUtil.hpp +++ b/include/SZ3/utils/ByteUtil.hpp @@ -51,9 +51,9 @@ namespace SZ { data[2] = tmp; } - inline unsigned short bytesToUInt16_bigEndian(const uchar *bytes) { - int temp = 0; - unsigned short res = 0; + inline int16_t bytesToInt16_bigEndian(unsigned char *bytes) { + int16_t temp = 0; + int16_t res = 0; temp = bytes[0] & 0xff; res |= temp; @@ -65,9 +65,9 @@ namespace SZ { return res; } - inline unsigned int bytesToUInt32_bigEndian(const uchar *bytes) { - unsigned int temp = 0; - unsigned int res = 0; + inline int32_t bytesToInt32_bigEndian(const unsigned char *bytes) { + int32_t temp = 0; + int32_t res = 0; res <<= 8; temp = bytes[0] & 0xff; @@ -88,85 +88,10 @@ namespace SZ { return res; } - inline unsigned long bytesToUInt64_bigEndian(const uchar *b) { - unsigned long temp = 0; - unsigned long res = 0; - res <<= 8; - temp = b[0] & 0xff; - res |= temp; - - res <<= 8; - temp = b[1] & 0xff; - res |= temp; - - res <<= 8; - temp = b[2] & 0xff; - res |= temp; - - res <<= 8; - temp = b[3] & 0xff; - res |= temp; - - res <<= 8; - temp = b[4] & 0xff; - res |= temp; - - res <<= 8; - temp = b[5] & 0xff; - res |= temp; - - res <<= 8; - temp = b[6] & 0xff; - res |= temp; - - res <<= 8; - temp = b[7] & 0xff; - res |= temp; - - return res; - } - - inline short bytesToInt16_bigEndian(unsigned char *bytes) { - int temp = 0; - short res = 0; - - temp = bytes[0] & 0xff; - res |= temp; - - res <<= 8; - temp = bytes[1] & 0xff; - res |= temp; - - return res; - } - - inline int bytesToInt32_bigEndian(unsigned char *bytes) { - int temp = 0; - int res = 0; - - res <<= 8; - temp = bytes[0] & 0xff; - res |= temp; - - res <<= 8; - temp = bytes[1] & 0xff; - res |= temp; - - res <<= 8; - temp = bytes[2] & 0xff; - res |= temp; - - res <<= 8; - temp = bytes[3] & 0xff; - res |= temp; - - return res; - } - - inline long bytesToInt64_bigEndian(const unsigned char *b) { - long temp = 0; - long res = 0; + inline int64_t bytesToInt64_bigEndian(const unsigned char *b) { + int64_t temp = 0; + int64_t res = 0; res <<= 8; temp = b[0] & 0xff; @@ -203,99 +128,22 @@ namespace SZ { return res; } - inline int bytesToInt_bigEndian(const unsigned char *bytes) { - int temp = 0; - int res = 0; - res <<= 8; - temp = bytes[0] & 0xff; - res |= temp; - res <<= 8; - temp = bytes[1] & 0xff; - res |= temp; - - res <<= 8; - temp = bytes[2] & 0xff; - res |= temp; - - res <<= 8; - temp = bytes[3] & 0xff; - res |= temp; - - return res; - } - - inline long bytesToLong_bigEndian(const uchar *b) { - long temp = 0; - long res = 0; - - res <<= 8; - temp = b[0] & 0xff; - res |= temp; - - res <<= 8; - temp = b[1] & 0xff; - res |= temp; - - res <<= 8; - temp = b[2] & 0xff; - res |= temp; - - res <<= 8; - temp = b[3] & 0xff; - res |= temp; - - res <<= 8; - temp = b[4] & 0xff; - res |= temp; - - res <<= 8; - temp = b[5] & 0xff; - res |= temp; - - res <<= 8; - temp = b[6] & 0xff; - res |= temp; - - res <<= 8; - temp = b[7] & 0xff; - res |= temp; - - return res; - } - - inline void longToBytes_bigEndian(unsigned char *b, unsigned long num) { - b[0] = (unsigned char) (num >> 56); - b[1] = (unsigned char) (num >> 48); - b[2] = (unsigned char) (num >> 40); - b[3] = (unsigned char) (num >> 32); - b[4] = (unsigned char) (num >> 24); - b[5] = (unsigned char) (num >> 16); - b[6] = (unsigned char) (num >> 8); - b[7] = (unsigned char) (num); - } - inline void int16ToBytes_bigEndian(unsigned char *b, uint16_t num) { + inline void int16ToBytes_bigEndian(unsigned char *b, int16_t num) { b[0] = (unsigned char) (num >> 8); b[1] = (unsigned char) (num); } - inline void int32ToBytes_bigEndian(unsigned char *b, uint32_t num) { + inline void int32ToBytes_bigEndian(unsigned char *b, int32_t num) { b[0] = (unsigned char) (num >> 24); b[1] = (unsigned char) (num >> 16); b[2] = (unsigned char) (num >> 8); b[3] = (unsigned char) (num); } - inline void intToBytes_bigEndian(unsigned char *b, unsigned int num) { - b[0] = (unsigned char) (num >> 24); - b[1] = (unsigned char) (num >> 16); - b[2] = (unsigned char) (num >> 8); - b[3] = (unsigned char) (num); - } - - inline void int64ToBytes_bigEndian(unsigned char *b, uint64_t num) { + inline void int64ToBytes_bigEndian(unsigned char *b, int64_t num) { b[0] = (unsigned char) (num >> 56); b[1] = (unsigned char) (num >> 48); b[2] = (unsigned char) (num >> 40); @@ -306,12 +154,6 @@ namespace SZ { b[7] = (unsigned char) (num); } - int bytesToInt(const unsigned char *bytes) { - lfloat buf; - memcpy(buf.byte, bytes, 4); - return buf.ivalue; - } - std::string floatToBinary(float f) { lfloat u; u.value = f;