diff --git a/Doxyfile b/Doxyfile index 7777795f8..7c173d45a 100644 --- a/Doxyfile +++ b/Doxyfile @@ -753,7 +753,7 @@ WARN_LOGFILE = # spaces. # Note: If this tag is empty the current directory is searched. -INPUT = libmbcommon libmbbootimg +INPUT = libmbcommon libmbbootimg libmbsparse # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/examples/desparse.cpp b/examples/desparse.cpp index fd4232d71..1e2cbab9c 100644 --- a/examples/desparse.cpp +++ b/examples/desparse.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 Andrew Gunnerson + * Copyright (C) 2016-2017 Andrew Gunnerson * * This file is part of DualBootPatcher * @@ -26,67 +26,6 @@ #include "mbcommon/file/standard.h" #include "mbsparse/sparse.h" -typedef std::unique_ptr ScopedSparseCtx; - -struct Context -{ - std::string path; - mb::StandardFile file; -}; - -bool cbOpen(void *userData) -{ - Context *ctx = static_cast(userData); - if (ctx->file.open(ctx->path, mb::FileOpenMode::READ_ONLY) - != mb::FileStatus::OK) { - fprintf(stderr, "%s: Failed to open: %s\n", - ctx->path.c_str(), ctx->file.error_string().c_str()); - return false; - } - return true; -} - -bool cbClose(void *userData) -{ - Context *ctx = static_cast(userData); - if (ctx->file.close() != mb::FileStatus::OK) { - fprintf(stderr, "%s: Failed to close: %s\n", - ctx->path.c_str(), ctx->file.error_string().c_str()); - return false; - } - return true; -} - -bool cbRead(void *buf, uint64_t size, uint64_t *bytesRead, void *userData) -{ - Context *ctx = static_cast(userData); - size_t total = 0; - while (size > 0) { - size_t partial; - if (ctx->file.read(buf, size, &partial) != mb::FileStatus::OK) { - fprintf(stderr, "%s: Failed to read: %s\n", - ctx->path.c_str(), ctx->file.error_string().c_str()); - return false; - } - size -= partial; - total += partial; - buf = static_cast(buf) + partial; - } - *bytesRead = total; - return true; -} - -bool cbSeek(int64_t offset, int whence, void *userData) -{ - Context *ctx = static_cast(userData); - if (ctx->file.seek(offset, whence, nullptr) != mb::FileStatus::OK) { - fprintf(stderr, "%s: Failed to seek: %s\n", - ctx->path.c_str(), ctx->file.error_string().c_str()); - return false; - } - return true; -} - int main(int argc, char *argv[]) { if (argc != 3) { @@ -94,53 +33,64 @@ int main(int argc, char *argv[]) return EXIT_FAILURE; } - const char *inputFile = argv[1]; - const char *outputFile = argv[2]; + const char *input_path = argv[1]; + const char *output_path = argv[2]; - Context ctx; - ctx.path = inputFile; + mb::StandardFile input_file; + mb::StandardFile output_file; + mb::sparse::SparseFile sparse_file; - ScopedSparseCtx sparseCtx(sparseCtxNew(), &sparseCtxFree); - if (!sparseCtx) { - fprintf(stderr, "Out of memory\n"); + if (input_file.open(input_path, mb::FileOpenMode::READ_ONLY) + != mb::FileStatus::OK) { + fprintf(stderr, "%s: Failed to open for reading: %s\n", + input_path, input_file.error_string().c_str()); return EXIT_FAILURE; } - if (!sparseOpen(sparseCtx.get(), &cbOpen, &cbClose, &cbRead, &cbSeek, - nullptr, &ctx)) { + if (sparse_file.open(&input_file) != mb::FileStatus::OK) { + fprintf(stderr, "%s: %s\n", + input_path, sparse_file.error_string().c_str()); return EXIT_FAILURE; } - mb::StandardFile file; - - if (file.open(outputFile, mb::FileOpenMode::WRITE_ONLY) + if (output_file.open(output_path, mb::FileOpenMode::WRITE_ONLY) != mb::FileStatus::OK) { fprintf(stderr, "%s: Failed to open for writing: %s\n", - outputFile, file.error_string().c_str()); + output_path, output_file.error_string().c_str()); return EXIT_FAILURE; } - uint64_t bytesRead; + size_t n_read; char buf[10240]; - bool ret; - while ((ret = sparseRead(sparseCtx.get(), buf, sizeof(buf), &bytesRead)) - && bytesRead > 0) { + mb::FileStatus ret; + while ((ret = sparse_file.read(buf, sizeof(buf), &n_read)) + == mb::FileStatus::OK && n_read > 0) { char *ptr = buf; - size_t bytesWritten; - while (bytesRead > 0) { - if (file.write(buf, bytesRead, &bytesWritten) + size_t n_written; + + while (n_read > 0) { + if (output_file.write(buf, n_read, &n_written) != mb::FileStatus::OK) { - fprintf(stderr, "%s: Failed to write: %s\n", - outputFile, file.error_string().c_str()); + fprintf(stderr, "%s: Failed to write file: %s\n", + output_path, output_file.error_string().c_str()); return EXIT_FAILURE; } - bytesRead -= bytesWritten; - ptr += bytesWritten; + n_read -= n_written; + ptr += n_written; } } - if (!ret) { + + if (ret != mb::FileStatus::OK) { + fprintf(stderr, "%s: Failed to read file: %s\n", + input_path, sparse_file.error_string().c_str()); + return EXIT_FAILURE; + } + + if (output_file.close() != mb::FileStatus::OK) { + fprintf(stderr, "%s: Failed to close file: %s\n", + output_path, output_file.error_string().c_str()); return EXIT_FAILURE; } - return file.close() == mb::FileStatus::OK ? EXIT_SUCCESS : EXIT_FAILURE; + return EXIT_SUCCESS; } diff --git a/libmbcommon/include/mbcommon/algorithm.h b/libmbcommon/include/mbcommon/algorithm.h new file mode 100644 index 000000000..7b3657978 --- /dev/null +++ b/libmbcommon/include/mbcommon/algorithm.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2017 Andrew Gunnerson + * + * This file is part of DualBootPatcher + * + * DualBootPatcher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * DualBootPatcher is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with DualBootPatcher. If not, see . + */ + +#pragma once + +#include +#include + +namespace mb +{ + +#if defined(__cpp_constexpr) && __cpp_constexpr-0 >= 201304 + +template> +ForwardIt binary_find(ForwardIt first, ForwardIt last, const T &value, + Compare comp={}) +{ + ForwardIt it = std::lower_bound(first, last, value, comp); + return (it != last && !bool(comp(value, *it))) ? it : last; +} + +#else + +template +ForwardIt binary_find(ForwardIt first, ForwardIt last, const T &value) +{ + ForwardIt it = std::lower_bound(first, last, value); + return (it != last && !(value < *it)) ? it : last; +} + +template +ForwardIt binary_find(ForwardIt first, ForwardIt last, const T &value, + Compare comp) +{ + ForwardIt it = std::lower_bound(first, last, value, comp); + return (it != last && !bool(comp(value, *it))) ? it : last; +} + +#endif + +} diff --git a/libmbsparse/CMakeLists.txt b/libmbsparse/CMakeLists.txt index 39c1f2623..9b4573134 100644 --- a/libmbsparse/CMakeLists.txt +++ b/libmbsparse/CMakeLists.txt @@ -9,6 +9,8 @@ set(MBSPARSE_TESTS_SOURCES tests/test_sparse.cpp ) +add_definitions(-DMBSPARSE_BUILD) + set(variants) if(MBP_TARGET_HAS_BUILDS) diff --git a/libmbsparse/include/mbsparse/guard_p.h b/libmbsparse/include/mbsparse/guard_p.h new file mode 100644 index 000000000..0b92a0d8a --- /dev/null +++ b/libmbsparse/include/mbsparse/guard_p.h @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2017 Andrew Gunnerson + * + * This file is part of DualBootPatcher + * + * DualBootPatcher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * DualBootPatcher is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with DualBootPatcher. If not, see . + */ + +#pragma once + +#ifndef MBSPARSE_BUILD +#error libmbsparse private headers cannot be used +#endif diff --git a/libmbsparse/include/mbsparse/sparse.h b/libmbsparse/include/mbsparse/sparse.h index 77737b83a..5c1f1da16 100644 --- a/libmbsparse/include/mbsparse/sparse.h +++ b/libmbsparse/include/mbsparse/sparse.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015-2016 Andrew Gunnerson + * Copyright (C) 2015-2017 Andrew Gunnerson * Copyright (C) 2010 The Android Open Source Project * * This file is part of DualBootPatcher @@ -20,43 +20,48 @@ #pragma once -#include "mbcommon/common.h" -#include "mbsparse/sparse_header.h" - -#ifdef __cplusplus -#include -#else -#include -#include -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -typedef bool (*SparseOpenCb)(void *userData); -typedef bool (*SparseCloseCb)(void *userData); -typedef bool (*SparseReadCb)(void *buf, uint64_t size, uint64_t *bytesRead, - void *userData); -typedef bool (*SparseSeekCb)(int64_t offset, int whence, void *userData); -typedef bool (*SparseSkipCb)(uint64_t offset, void *userData); - -struct SparseCtx; - -MB_EXPORT struct SparseCtx * sparseCtxNew(); -MB_EXPORT bool sparseCtxFree(struct SparseCtx *ctx); - -MB_EXPORT bool sparseOpen(struct SparseCtx *ctx, SparseOpenCb openCb, - SparseCloseCb closeCb, SparseReadCb readCb, - SparseSeekCb seekCb, SparseSkipCb skipCb, - void *userData); -MB_EXPORT bool sparseClose(struct SparseCtx *ctx); -MB_EXPORT bool sparseRead(struct SparseCtx *ctx, void *buf, uint64_t size, - uint64_t *bytesRead); -MB_EXPORT bool sparseSeek(struct SparseCtx *ctx, int64_t offset, int whence); -MB_EXPORT bool sparseTell(struct SparseCtx *ctx, uint64_t *offset); -MB_EXPORT bool sparseSize(struct SparseCtx *ctx, uint64_t *size); - -#ifdef __cplusplus +#include "mbcommon/file.h" + +namespace mb +{ +namespace sparse +{ + +class SparseFilePrivate; +class MB_EXPORT SparseFile : public File +{ + MB_DECLARE_PRIVATE(SparseFile) + +public: + SparseFile(); + SparseFile(File *file); + virtual ~SparseFile(); + + MB_DISABLE_COPY_CONSTRUCT_AND_ASSIGN(SparseFile) + MB_DEFAULT_MOVE_CONSTRUCT_AND_ASSIGN(SparseFile) + + // File open + FileStatus open(File *file); + + // File size + uint64_t size(); + +protected: + /*! \cond INTERNAL */ + SparseFile(SparseFilePrivate *priv); + SparseFile(SparseFilePrivate *priv, File *file); + /*! \endcond */ + + virtual FileStatus on_open() override; + virtual FileStatus on_close() override; + virtual FileStatus on_read(void *buf, size_t size, + size_t *bytes_read) override; + virtual FileStatus on_seek(int64_t offset, int whence, + uint64_t *new_offset) override; + +private: + std::unique_ptr _priv_ptr; +}; + +} } -#endif \ No newline at end of file diff --git a/libmbsparse/include/mbsparse/sparse_header.h b/libmbsparse/include/mbsparse/sparse_header.h deleted file mode 100644 index d6fafd42d..000000000 --- a/libmbsparse/include/mbsparse/sparse_header.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2015-2016 Andrew Gunnerson - * Copyright (C) 2010 The Android Open Source Project - * - * This file is part of DualBootPatcher - * - * DualBootPatcher is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * DualBootPatcher is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with DualBootPatcher. If not, see . - */ - -#pragma once - -#ifdef __cplusplus -#include -#else -#include -#endif - -#define SPARSE_HEADER_MAGIC 0xed26ff3a -#define SPARSE_HEADER_MAJOR_VER 1 - -#define CHUNK_TYPE_RAW 0xCAC1 -#define CHUNK_TYPE_FILL 0xCAC2 -#define CHUNK_TYPE_DONT_CARE 0xCAC3 -#define CHUNK_TYPE_CRC32 0xCAC4 - -struct SparseHeader -{ - uint32_t magic; // 0xed26ff3a - uint16_t major_version; // (0x1) - reject images with higher major versions - uint16_t minor_version; // (0x0) - allow images with higer minor versions - uint16_t file_hdr_sz; // 28 bytes for first revision of the file format - uint16_t chunk_hdr_sz; // 12 bytes for first revision of the file format - uint32_t blk_sz; // block size in bytes, must be a multiple of 4 (4096) - uint32_t total_blks; // total blocks in the non-sparse output image - uint32_t total_chunks; // total chunks in the sparse input image - uint32_t image_checksum; // CRC32 checksum of the original data, counting "don't care" - // as 0. Standard 802.3 polynomial, use a Public Domain - // table implementation -}; - -struct ChunkHeader -{ - uint16_t chunk_type; // 0xCAC1 -> raw; 0xCAC2 -> fill; 0xCAC3 -> don't care - uint16_t reserved1; - uint32_t chunk_sz; // in blocks in output image - uint32_t total_sz; // in bytes of chunk input file including chunk header and data -}; - -/* - * Following a Raw or Fill or CRC32 chunk is data. - * - For a Raw chunk, it's the data in chunk_sz * blk_sz. - * - For a Fill chunk, it's 4 bytes of the fill data. - * - For a CRC32 chunk, it's 4 bytes of CRC32 - */ \ No newline at end of file diff --git a/libmbsparse/include/mbsparse/sparse_p.h b/libmbsparse/include/mbsparse/sparse_p.h new file mode 100644 index 000000000..ad3fac284 --- /dev/null +++ b/libmbsparse/include/mbsparse/sparse_p.h @@ -0,0 +1,161 @@ +/* + * Copyright (C) 2015-2017 Andrew Gunnerson + * Copyright (C) 2010 The Android Open Source Project + * + * This file is part of DualBootPatcher + * + * DualBootPatcher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * DualBootPatcher is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with DualBootPatcher. If not, see . + */ + +#pragma once + +#include "mbsparse/guard_p.h" + +#include + +namespace mb +{ +namespace sparse +{ + +/*! \cond INTERNAL */ + +constexpr uint32_t SPARSE_HEADER_MAGIC = 0xed26ff3a; +constexpr int SPARSE_HEADER_MAJOR_VER = 1; + +constexpr uint16_t CHUNK_TYPE_RAW = 0xCAC1; +constexpr uint16_t CHUNK_TYPE_FILL = 0xCAC2; +constexpr uint16_t CHUNK_TYPE_DONT_CARE = 0xCAC3; +constexpr uint16_t CHUNK_TYPE_CRC32 = 0xCAC4; + +struct SparseHeader +{ + uint32_t magic; // 0xed26ff3a + uint16_t major_version; // (0x1) - reject images with higher major versions + uint16_t minor_version; // (0x0) - allow images with higer minor versions + uint16_t file_hdr_sz; // 28 bytes for first revision of the file format + uint16_t chunk_hdr_sz; // 12 bytes for first revision of the file format + uint32_t blk_sz; // block size in bytes, must be a multiple of 4 (4096) + uint32_t total_blks; // total blocks in the non-sparse output image + uint32_t total_chunks; // total chunks in the sparse input image + uint32_t image_checksum; // CRC32 checksum of the original data, counting "don't care" + // as 0. Standard 802.3 polynomial, use a Public Domain + // table implementation +}; + +struct ChunkHeader +{ + uint16_t chunk_type; // 0xCAC1 -> raw; 0xCAC2 -> fill; 0xCAC3 -> don't care + uint16_t reserved1; + uint32_t chunk_sz; // in blocks in output image + uint32_t total_sz; // in bytes of chunk input file including chunk header and data +}; + +/* + * Following a Raw or Fill or CRC32 chunk is data. + * - For a Raw chunk, it's the data in chunk_sz * blk_sz. + * - For a Fill chunk, it's 4 bytes of the fill data. + * - For a CRC32 chunk, it's 4 bytes of CRC32 + */ + +/*! \brief Minimum information we need from the chunk headers while reading */ +struct ChunkInfo +{ + /*! \brief Same as ChunkHeader::chunk_type */ + uint16_t type; + + /*! \brief Start of byte range in output file that this chunk represents */ + uint64_t begin; + /*! \brief End of byte range in output file that this chunk represents */ + uint64_t end; + + /*! \brief Start of byte range for the entire chunk in the source file */ + uint64_t src_begin; + /*! \brief End of byte range for the entire chunk in the source file */ + uint64_t src_end; + + /*! \brief [CHUNK_TYPE_RAW only] Start of raw bytes in input file */ + uint64_t raw_begin; + /*! \brief [CHUNK_TYPE_RAW only] End of raw bytes in input file */ + uint64_t raw_end; + + /*! \brief [CHUNK_TYPE_FILL only] Filler value for the chunk */ + uint32_t fill_val; +}; + +enum class Seekability +{ + CAN_SEEK, + CAN_SKIP, + CAN_READ, +}; + +class SparseFilePrivate +{ + MB_DECLARE_PUBLIC(SparseFile) + +public: + SparseFilePrivate(SparseFile *sf); + ~SparseFilePrivate() = default; + + void clear(); + + MB_DISABLE_COPY_CONSTRUCT_AND_ASSIGN(SparseFilePrivate) + + FileStatus wread(void *buf, size_t size); + FileStatus wseek(int64_t offset); + FileStatus skip_bytes(uint64_t bytes); + + FileStatus process_sparse_header(const void *preread_data, + size_t preread_size); + + bool process_raw_chunk(const ChunkHeader &chdr, uint64_t tgt_offset, + ChunkInfo &chunk_out); + bool process_fill_chunk(const ChunkHeader &chdr, uint64_t tgt_offset, + ChunkInfo &chunk_out); + bool process_skip_chunk(const ChunkHeader &chdr, uint64_t tgt_offset, + ChunkInfo &chunk_out); + bool process_crc32_chunk(const ChunkHeader &chdr, uint64_t tgt_offset, + ChunkInfo &chunk_out); + bool process_chunk(const ChunkHeader &chdr, uint64_t tgt_offset, + ChunkInfo &chunk_out); + + FileStatus move_to_chunk(uint64_t offset); + + File *file; + Seekability seekability; + + // Expected CRC32 checksum. We currently do *not* validate this. It would + // only work if the entire file was read sequentially anyway. + uint32_t expected_crc32; + // Relative offset in input file + uint64_t cur_src_offset; + // Absolute offset in output file + uint64_t cur_tgt_offset; + // Expected file size + uint64_t file_size; + + SparseHeader shdr; + + std::vector chunks; + decltype(chunks)::iterator chunk; + +private: + SparseFile *_pub_ptr; +}; + +/*! \endcond */ + +} +} diff --git a/libmbsparse/src/sparse.cpp b/libmbsparse/src/sparse.cpp index 109fe9a81..bce3b3c19 100644 --- a/libmbsparse/src/sparse.cpp +++ b/libmbsparse/src/sparse.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015-2016 Andrew Gunnerson + * Copyright (C) 2015-2017 Andrew Gunnerson * Copyright (C) 2010 The Android Open Source Project * * This file is part of DualBootPatcher @@ -18,11 +18,6 @@ * along with DualBootPatcher. If not, see . */ -#ifdef __ANDROID__ -// Android does not support C++11 properly... -#define __STDC_LIMIT_MACROS -#endif - #include "mbsparse/sparse.h" // For std::min() @@ -35,293 +30,339 @@ #include #include +#include "mbcommon/algorithm.h" +#include "mbcommon/endian.h" +#include "mbcommon/file_util.h" #include "mbcommon/string.h" -#include "mblog/logging.h" + +#include "mbsparse/sparse_p.h" // Enable debug logging of headers, offsets, etc.? -#define SPARSE_DEBUG 1 +#define SPARSE_DEBUG 0 // Enable debug logging of operations (warning! very verbose!) #define SPARSE_DEBUG_OPER 0 -// Enable logging of errors -#define SPARSE_ERROR 1 -#if SPARSE_DEBUG -#define DEBUG(...) LOGD(__VA_ARGS__) -#else -#define DEBUG(...) +#if SPARSE_DEBUG || SPARSE_DEBUG_OPER +# include "mblog/logging.h" #endif -#if SPARSE_DEBUG_OPER -#define OPER(...) LOGD(__VA_ARGS__) +#if SPARSE_DEBUG +# define DEBUG(...) LOGD(__VA_ARGS__) #else -#define OPER(...) +# define DEBUG(...) #endif -#if SPARSE_ERROR -#define ERROR(...) LOGE(__VA_ARGS__) +#if SPARSE_DEBUG_OPER +# define OPER(...) LOGD(__VA_ARGS__) #else -#define ERROR(...) +# define OPER(...) #endif -/*! \brief Minimum information we need from the chunk headers while reading */ -struct ChunkInfo +namespace mb { - /*! \brief Same as ChunkHeader::chunk_type */ - uint16_t type; - - /*! \brief Start of byte range in output file that this chunk represents */ - uint64_t begin; - /*! \brief End of byte range in output file that this chunk represents */ - uint64_t end; - - /*! \brief Start of byte range for the entire chunk in the source file */ - uint64_t srcBegin; - /*! \brief End of byte range for the entire chunk in the source file */ - uint64_t srcEnd; - - /*! \brief [CHUNK_TYPE_RAW only] Start of raw bytes in input file */ - uint64_t rawBegin; - /*! \brief [CHUNK_TYPE_RAW only] End of raw bytes in input file */ - uint64_t rawEnd; - - /*! \brief [CHUNK_TYPE_FILL only] Filler value for the chunk */ - uint32_t fillVal; -}; - -struct SparseCtx +namespace sparse { - // Callbacks - SparseOpenCb cbOpen; - SparseCloseCb cbClose; - SparseReadCb cbRead; - SparseSeekCb cbSeek; - SparseSkipCb cbSkip; - void *cbUserData; - void setCallbacks(SparseOpenCb openCb, SparseCloseCb closeCb, - SparseReadCb readCb, SparseSeekCb seekCb, - SparseSkipCb skipCb, void *userData); - void clearCallbacks(); +static void fix_sparse_header_byte_order(SparseHeader &header) +{ + header.magic = mb_le32toh(header.magic); + header.major_version = mb_le16toh(header.major_version); + header.minor_version = mb_le16toh(header.minor_version); + header.file_hdr_sz = mb_le16toh(header.file_hdr_sz); + header.chunk_hdr_sz = mb_le16toh(header.chunk_hdr_sz); + header.blk_sz = mb_le32toh(header.blk_sz); + header.total_blks = mb_le32toh(header.total_blks); + header.total_chunks = mb_le32toh(header.total_chunks); + header.image_checksum = mb_le32toh(header.image_checksum); +} - // Callback wrappers to avoid passing ctx->cbUserdata everywhere - bool open(); - bool close(); - bool read(void *buf, uint64_t size, uint64_t *bytesRead); - bool seek(int64_t offset, int whence); - bool skip(uint64_t offset); +static void fix_chunk_header_byte_order(ChunkHeader &header) +{ + header.chunk_type = mb_le16toh(header.chunk_type); + header.reserved1 = mb_le16toh(header.reserved1); + header.chunk_sz = mb_le32toh(header.chunk_sz); + header.total_sz = mb_le32toh(header.total_sz); +} - bool skipBytes(uint64_t bytes); +#if SPARSE_DEBUG +static void dump_sparse_header(const SparseHeader &header) +{ + DEBUG("Sparse header:"); + DEBUG("- magic: 0x%08" PRIx32, header.magic); + DEBUG("- major_version: 0x%04" PRIx16, header.major_version); + DEBUG("- minor_version: 0x%04" PRIx16, header.minor_version); + DEBUG("- file_hdr_sz: %" PRIu16 " (bytes)", header.file_hdr_sz); + DEBUG("- chunk_hdr_sz: %" PRIu16 " (bytes)", header.chunk_hdr_sz); + DEBUG("- blk_sz: %" PRIu32 " (bytes)", header.blk_sz); + DEBUG("- total_blks: %" PRIu32, header.total_blks); + DEBUG("- total_chunks: %" PRIu32, header.total_chunks); + DEBUG("- image_checksum: 0x%08" PRIu32, header.image_checksum); +} - bool isOpen; +static const char * chunk_type_to_string(uint16_t chunk_type) +{ + switch (chunk_type) { + case CHUNK_TYPE_RAW: + return "Raw"; + case CHUNK_TYPE_FILL: + return "Fill"; + case CHUNK_TYPE_DONT_CARE: + return "Don't care"; + case CHUNK_TYPE_CRC32: + return "CRC32"; + default: + return "Unknown"; + } +} - uint32_t expectedCrc32 = 0; +static void dump_chunk_header(const ChunkHeader &header) +{ + DEBUG("Chunk header:"); + DEBUG("- chunk_type: 0x%04" PRIx16 " (%s)", header.chunk_type, + chunk_type_to_string(header.chunk_type)); + DEBUG("- reserved1: 0x%04" PRIx16, header.reserved1); + DEBUG("- chunk_sz: %" PRIu32 " (blocks)", header.chunk_sz); + DEBUG("- total_sz: %" PRIu32 " (bytes)", header.total_sz); +} +#endif - uint64_t srcOffset = 0; - uint64_t outOffset = 0; - uint64_t fileSize; +/*! \cond INTERNAL */ - SparseHeader shdr; +struct OffsetComp +{ + bool operator()(uint64_t offset, const ChunkInfo &chunk) const + { + return offset < chunk.begin; + } - std::vector chunks; - size_t chunk = 0; + bool operator()(const ChunkInfo &chunk, uint64_t offset) const + { + return chunk.end <= offset; + } }; -void SparseCtx::setCallbacks(SparseOpenCb openCb, SparseCloseCb closeCb, - SparseReadCb readCb, SparseSeekCb seekCb, - SparseSkipCb skipCb, void *userData) +SparseFilePrivate::SparseFilePrivate(SparseFile *sf) + : _pub_ptr(sf) { - cbOpen = openCb; - cbClose = closeCb; - cbRead = readCb; - cbSeek = seekCb; - cbSkip = skipCb; - cbUserData = userData; + clear(); } -void SparseCtx::clearCallbacks() +void SparseFilePrivate::clear() { - cbOpen = nullptr; - cbClose = nullptr; - cbRead = nullptr; - cbSeek = nullptr; - cbSkip = nullptr; - cbUserData = nullptr; + file = nullptr; + expected_crc32 = 0; + cur_src_offset = 0; + cur_tgt_offset = 0; + file_size = 0; + chunks.clear(); + chunk = chunks.end(); } -bool SparseCtx::open() +FileStatus SparseFilePrivate::wread(void *buf, size_t size) { - return cbOpen && cbOpen(cbUserData); -} - -bool SparseCtx::close() -{ - return cbClose && cbClose(cbUserData); -} + MB_PUBLIC(SparseFile); + size_t bytes_read; + + auto ret = file_read_fully(*file, buf, size, &bytes_read); + if (ret != FileStatus::OK) { + pub->set_error(file->error(), "Failed to read file: %s", + file->error_string().c_str()); + return ret; + } -bool SparseCtx::read(void *buf, uint64_t size, uint64_t *bytesRead) -{ - assert(cbRead != nullptr); - if (cbRead && cbRead(buf, size, bytesRead, cbUserData)) { - srcOffset += *bytesRead; - return true; + if (bytes_read != size) { + pub->set_error(file->error(), "Requested %" MB_PRIzu + " bytes, but only read %" MB_PRIzu " bytes", + size, bytes_read); + return FileStatus::FATAL; } - return false; + + cur_src_offset += bytes_read; + return FileStatus::OK; } -bool SparseCtx::seek(int64_t offset, int whence) +FileStatus SparseFilePrivate::wseek(int64_t offset) { - // We should never seek the source file from the end - assert(whence != SEEK_END); - if (cbSeek && whence != SEEK_END && cbSeek(offset, whence, cbUserData)) { - if (whence == SEEK_SET) { - srcOffset = offset; - } else if (whence == SEEK_CUR) { - srcOffset += offset; - } - return true; + MB_PUBLIC(SparseFile); + + auto ret = file->seek(offset, SEEK_CUR, nullptr); + if (ret != FileStatus::OK) { + pub->set_error(file->error(), "Failed to seek file: %s", + file->error_string().c_str()); + return ret; } - return false; -} -bool SparseCtx::skip(uint64_t offset) -{ - return cbSkip && cbSkip(offset, cbUserData); + cur_src_offset += offset; + return FileStatus::OK; } /*! * \brief Skip a certain amount of bytes * - * If a seek callback is available, use it to seek the specified amount of - * bytes. Otherwise, if a skip callback is available, use it to skip the - * specified amount of bytes. If neither callback is available, read the - * specified amount of bytes and discard the data. - * * \param bytes Number of bytes to skip - * \return Whether the specified amount of bytes were successfully skipped + * \return + * * #FileStatus::OK if the bytes are successfully skipped + * * \<= #FileStatus::WARN if a file operation fails */ -bool SparseCtx::skipBytes(uint64_t bytes) +FileStatus SparseFilePrivate::skip_bytes(uint64_t bytes) { - if (cbSeek) { - // If we ran random-read, then simply seek - return seek(bytes, SEEK_CUR); - } else if (cbSkip) { - // Otherwise, if there is a skip function, then use that - return skip(bytes); - } else { - // If neither are available, read the specified number of bytes - char dummy[10240]; - while (bytes > 0) { - uint64_t toRead = std::min(sizeof(dummy), bytes); - uint64_t bytesRead; - if (!read(dummy, toRead, &bytesRead) || bytesRead == 0) { - DEBUG("Read failed or reached EOF when skipping bytes"); - return false; - } - bytes -= bytesRead; - } - return true; - } -} + MB_PUBLIC(SparseFile); -#if SPARSE_DEBUG -static void dumpSparseHeader(SparseHeader *header) -{ - DEBUG("Sparse header:"); - DEBUG("- magic: 0x%08" PRIx32, header->magic); - DEBUG("- major_version: 0x%04" PRIx16, header->major_version); - DEBUG("- minor_version: 0x%04" PRIx16, header->minor_version); - DEBUG("- file_hdr_sz: %" PRIu16 " (bytes)", header->file_hdr_sz); - DEBUG("- chunk_hdr_sz: %" PRIu16 " (bytes)", header->chunk_hdr_sz); - DEBUG("- blk_sz: %" PRIu32 " (bytes)", header->blk_sz); - DEBUG("- total_blks: %" PRIu32, header->total_blks); - DEBUG("- total_chunks: %" PRIu32, header->total_chunks); - DEBUG("- image_checksum: 0x%08" PRIu32, header->image_checksum); -} + if (bytes == 0) { + return FileStatus::OK; + } -static const char * chunkTypeToString(uint16_t chunkType) -{ - switch (chunkType) { - case CHUNK_TYPE_RAW: - return "Raw"; - case CHUNK_TYPE_FILL: - return "Fill"; - case CHUNK_TYPE_DONT_CARE: - return "Don't care"; - case CHUNK_TYPE_CRC32: - return "CRC32"; - default: - return "Unknown"; + switch (seekability) { + case Seekability::CAN_SEEK: + case Seekability::CAN_SKIP: + return wseek(bytes); + case Seekability::CAN_READ: + uint64_t discarded; + auto ret = file_read_discard(*file, bytes, &discarded); + if (ret != FileStatus::OK) { + pub->set_error(file->error(), "Failed to read and discard data: %s", + file->error_string().c_str()); + return ret; + } else if (discarded != bytes) { + pub->set_error(FileError::INTERNAL_ERROR, + "Reached EOF when skipping bytes"); + return FileStatus::FATAL; + } + return FileStatus::OK; } -} -static void dumpChunkHeader(ChunkHeader *header) -{ - DEBUG("Chunk header:"); - DEBUG("- chunk_type: 0x%04" PRIx16 " (%s)", header->chunk_type, - chunkTypeToString(header->chunk_type)); - DEBUG("- reserved1: 0x%04" PRIx16, header->reserved1); - DEBUG("- chunk_sz: %" PRIu32 " (blocks)", header->chunk_sz); - DEBUG("- total_sz: %" PRIu32 " (bytes)", header->total_sz); + // Unreached + return FileStatus::FATAL; } -#endif /*! - * \brief Read the specified number of bytes completely (no partial reads) + * \brief Read and verify sparse header + * + * Thie function will check the following properties: + * - Magic field matches expected value + * - Major version is 1 + * - The sparse header size field is at least the expected size of the sparse + * header structure + * - The chunk header size field is at least the expected size of the chunk + * header structure * - * \param ctx Sparse context - * \param buf Output buffer - * \param size Bytes to read - * \return Whether the specified amount of bytes (no more or less) were - * successfully read + * The value of the minor version field is ignored. + * + * \note This does not process any of the chunk headers. The chunk headers are + * processed on-the-fly when reading the sparse file. + * + * \pre The file position should be at the beginning of the sparse header. + * \post The file position will be advanced to the byte after the sparse header. + * If the sparse header size, as specified by the \a file_hdr_sz field, is + * larger than the expected size, the extra bytes will be skipped as well. + * + * \param preread_data Data already read from the file (for seek checks) + * \param preread_size Size of data already read from the file (must be less + * than `sizeof(SparseHeader)`) + * + * \return + * * #FileStatus::OK if the sparse header is successfully read + * * \<= #FileStatus::WARN if a file operation fails */ -static bool readFully(SparseCtx *ctx, void *buf, std::size_t size) +FileStatus SparseFilePrivate::process_sparse_header(const void *preread_data, + size_t preread_size) { - uint64_t bytesRead; - if (!ctx->read(buf, size, &bytesRead)) { - ERROR("Sparse read callback returned failure"); - return false; + MB_PUBLIC(SparseFile); + FileStatus ret; + + assert(preread_size < sizeof(shdr)); + + // Read header + memcpy(&shdr, preread_data, preread_size); + ret = wread(reinterpret_cast(&shdr) + preread_size, + sizeof(shdr) - preread_size); + if (ret != FileStatus::OK) { + return ret; } - if (bytesRead != size) { - ERROR("Requested %" PRIu64 " bytes, but only read %" PRIu64 " bytes", - (uint64_t) size, bytesRead); - return false; + + fix_sparse_header_byte_order(shdr); + +#if SPARSE_DEBUG + dump_sparse_header(shdr); +#endif + + // Check magic bytes + if (shdr.magic != SPARSE_HEADER_MAGIC) { + pub->set_error(FileError::INTERNAL_ERROR, + "Expected magic to be %08x, but got %08x", + SPARSE_HEADER_MAGIC, shdr.magic); + return FileStatus::FATAL; } - return true; + + // Check major version + if (shdr.major_version != SPARSE_HEADER_MAJOR_VER) { + pub->set_error(FileError::INTERNAL_ERROR, + "Expected major version to be %u, but got %u", + SPARSE_HEADER_MAJOR_VER, shdr.major_version); + return FileStatus::FATAL; + } + + // Check sparse header size field + if (shdr.file_hdr_sz < sizeof(SparseHeader)) { + pub->set_error(FileError::INTERNAL_ERROR, + "Expected sparse header size to be at least %" MB_PRIzu + ", but have %" PRIu32, sizeof(shdr), shdr.file_hdr_sz); + return FileStatus::FATAL; + } + + // Check chunk header size field + if (shdr.chunk_hdr_sz < sizeof(ChunkHeader)) { + pub->set_error(FileError::INTERNAL_ERROR, + "Expected chunk header size to be at least %" MB_PRIzu + ", but have %" PRIu32, sizeof(SparseHeader), + shdr.chunk_hdr_sz); + return FileStatus::FATAL; + } + + // Skip any extra bytes in the file header + ret = skip_bytes(shdr.file_hdr_sz - sizeof(SparseHeader)); + if (ret != FileStatus::OK) { + return ret; + } + + file_size = static_cast(shdr.total_blks) * shdr.blk_sz; + + return FileStatus::OK; } /*! * \brief Read and verify raw chunk header * - * Thie function will check the following properties: - * - (No additional checks) + * \param[in] chdr Chunk header + * \param[in] tgt_offset Offset of the output file + * \param[out] chunk_out ChunkInfo to store chunk info * - * \param ctx Sparse context - * \param chunkHeader Chunk header for the current chunk - * \param outOffset Current offset of the output file * \return Whether the chunk header is valid */ -static bool processRawChunk(SparseCtx *ctx, ChunkHeader *chunkHeader, - uint64_t outOffset) +bool SparseFilePrivate::process_raw_chunk(const ChunkHeader &chdr, + uint64_t tgt_offset, + ChunkInfo &chunk_out) { - uint32_t dataSize = chunkHeader->total_sz - ctx->shdr.chunk_hdr_sz; - uint64_t chunkSize = (uint64_t) chunkHeader->chunk_sz * ctx->shdr.blk_sz; + MB_PUBLIC(SparseFile); + + uint32_t data_size = chdr.total_sz - shdr.chunk_hdr_sz; + uint64_t chunk_size = static_cast(chdr.chunk_sz) * shdr.blk_sz; - if (dataSize != chunkSize) { - ERROR("Number of data blocks (%" PRIu32 ") does not match" - " number of expected blocks (%" PRIu32 ")", - dataSize / ctx->shdr.blk_sz, chunkHeader->chunk_sz); + if (data_size != chunk_size) { + pub->set_error(FileError::INTERNAL_ERROR, + "Number of data blocks (%" PRIu32 ") does not match" + " number of expected blocks (%" PRIu32 ")", + data_size / shdr.blk_sz, chdr.chunk_sz); return false; } - ctx->chunks.emplace_back(); - ChunkInfo &chunk = ctx->chunks.back(); - chunk.type = chunkHeader->chunk_type; - chunk.begin = outOffset; - chunk.end = outOffset + dataSize; - chunk.srcBegin = ctx->srcOffset - ctx->shdr.chunk_hdr_sz; - chunk.srcEnd = ctx->srcOffset + dataSize; - chunk.rawBegin = ctx->srcOffset; - chunk.rawEnd = chunk.rawBegin + dataSize; + chunk_out.type = chdr.chunk_type; + chunk_out.begin = tgt_offset; + chunk_out.end = tgt_offset + data_size; + chunk_out.src_begin = cur_src_offset - shdr.chunk_hdr_sz; + chunk_out.src_end = cur_src_offset + data_size; + chunk_out.raw_begin = cur_src_offset; + chunk_out.raw_end = cur_src_offset + data_size; return true; } @@ -329,43 +370,46 @@ static bool processRawChunk(SparseCtx *ctx, ChunkHeader *chunkHeader, /*! * \brief Read and verify fill chunk header * - * Thie function will check the following properties: - * - Chunk data size is 4 bytes (sizeof(uint32_t)) + * This function will check the following properties: + * * Chunk data size is 4 bytes (`sizeof(uint32_t)`) + * + * \param[in] chdr Chunk header + * \param[in] tgt_offset Offset of the output file + * \param[out] chunk_out ChunkInfo to store chunk info * - * \param ctx Sparse context - * \param chunkHeader Chunk header for the current chunk - * \param outOffset Current offset of the output file * \return Whether the chunk header is valid */ -static bool processFillChunk(SparseCtx *ctx, ChunkHeader *chunkHeader, - uint64_t outOffset) +bool SparseFilePrivate::process_fill_chunk(const ChunkHeader &chdr, + uint64_t tgt_offset, + ChunkInfo &chunk_out) { - uint32_t dataSize = chunkHeader->total_sz - ctx->shdr.chunk_hdr_sz; - uint64_t chunkSize = (uint64_t) chunkHeader->chunk_sz * ctx->shdr.blk_sz; - uint32_t fillVal; + MB_PUBLIC(SparseFile); + + uint32_t data_size = chdr.total_sz - shdr.chunk_hdr_sz; + uint64_t chunk_size = static_cast(chdr.chunk_sz) * shdr.blk_sz; + uint32_t fill_val; - if (dataSize != sizeof(fillVal)) { - ERROR("Data size (%" PRIu32 ") does not match size of 32-bit integer", - dataSize); + if (data_size != sizeof(fill_val)) { + pub->set_error(FileError::INTERNAL_ERROR, + "Data size (%" PRIu32 ") does not match size of 32-bit " + "integer", data_size); return false; } - uint64_t srcBegin = ctx->srcOffset - ctx->shdr.chunk_hdr_sz; + uint64_t src_begin = cur_src_offset - shdr.chunk_hdr_sz; - if (!readFully(ctx, &fillVal, sizeof(fillVal))) { + if (wread(&fill_val, sizeof(fill_val)) != FileStatus::OK) { return false; } - uint64_t srcEnd = ctx->srcOffset; + uint64_t src_end = cur_src_offset; - ctx->chunks.emplace_back(); - ChunkInfo &chunk = ctx->chunks.back(); - chunk.type = chunkHeader->chunk_type; - chunk.begin = outOffset; - chunk.end = outOffset + chunkSize; - chunk.srcBegin = srcBegin; - chunk.srcEnd = srcEnd; - chunk.fillVal = fillVal; + chunk_out.type = chdr.chunk_type; + chunk_out.begin = tgt_offset; + chunk_out.end = tgt_offset + chunk_size; + chunk_out.src_begin = src_begin; + chunk_out.src_end = src_end; + chunk_out.fill_val = fill_val; return true; } @@ -373,32 +417,35 @@ static bool processFillChunk(SparseCtx *ctx, ChunkHeader *chunkHeader, /*! * \brief Read and verify skip chunk header * - * Thie function will check the following properties: - * - Chunk data size is 0 bytes + * This function will check the following properties: + * * Chunk data size is 0 bytes + * + * \param[in] chdr Chunk header + * \param[in] tgt_offset Offset of the output file + * \param[out] chunk_out ChunkInfo to store chunk info * - * \param ctx Sparse context - * \param chunkHeader Chunk header for the current chunk - * \param outOffset Current offset of the output file * \return Whether the chunk header is valid */ -static bool processSkipChunk(SparseCtx *ctx, ChunkHeader *chunkHeader, - uint64_t outOffset) +bool SparseFilePrivate::process_skip_chunk(const ChunkHeader &chdr, + uint64_t tgt_offset, + ChunkInfo &chunk_out) { - uint32_t dataSize = chunkHeader->total_sz - ctx->shdr.chunk_hdr_sz; - uint64_t chunkSize = (uint64_t) chunkHeader->chunk_sz * ctx->shdr.blk_sz; + MB_PUBLIC(SparseFile); + + uint32_t data_size = chdr.total_sz - shdr.chunk_hdr_sz; + uint64_t chunk_size = static_cast(chdr.chunk_sz) * shdr.blk_sz; - if (dataSize != 0) { - ERROR("Data size (%" PRIu32 ") is not 0", dataSize); + if (data_size != 0) { + pub->set_error(FileError::INTERNAL_ERROR, + "Data size (%" PRIu32 ") is not 0", data_size); return false; } - ctx->chunks.emplace_back(); - ChunkInfo &chunk = ctx->chunks.back(); - chunk.type = chunkHeader->chunk_type; - chunk.begin = outOffset; - chunk.end = outOffset + chunkSize; - chunk.srcBegin = ctx->srcOffset - ctx->shdr.chunk_hdr_sz; - chunk.srcEnd = ctx->srcOffset + dataSize; + chunk_out.type = chdr.chunk_type; + chunk_out.begin = tgt_offset; + chunk_out.end = tgt_offset + chunk_size; + chunk_out.src_begin = cur_src_offset - shdr.chunk_hdr_sz; + chunk_out.src_end = cur_src_offset + data_size; return true; } @@ -406,393 +453,419 @@ static bool processSkipChunk(SparseCtx *ctx, ChunkHeader *chunkHeader, /*! * \brief Read and verify CRC32 chunk header * - * Thie function will check the following properties: - * - Chunk data size is 4 bytes (sizeof(uint32_t)) + * This function will check the following properties: + * * Chunk data size is 4 bytes (`sizeof(uint32_t)`) + * + * \param[in] chdr Chunk header + * \param[in] tgt_offset Offset of the output file + * \param[out] chunk_out ChunkInfo to store chunk info * - * \param ctx Sparse context - * \param chunkHeader Chunk header for the current chunk - * \param outOffset Current offset of the output file * \return Whether the chunk header is valid */ -static bool processCrc32Chunk(SparseCtx *ctx, ChunkHeader *chunkHeader, - uint64_t outOffset) +bool SparseFilePrivate::process_crc32_chunk(const ChunkHeader &chdr, + uint64_t tgt_offset, + ChunkInfo &chunk_out) { - (void) outOffset; + MB_PUBLIC(SparseFile); + (void) tgt_offset; - uint32_t dataSize = chunkHeader->total_sz - ctx->shdr.chunk_hdr_sz; - uint64_t chunkSize = (uint64_t) chunkHeader->chunk_sz * ctx->shdr.blk_sz; - uint32_t expectedCrc32; + uint32_t data_size = chdr.total_sz - shdr.chunk_hdr_sz; + uint64_t chunk_size = static_cast(chdr.chunk_sz) * shdr.blk_sz; + uint32_t crc32; - if (chunkSize != 0) { - ERROR("Chunk data size (%" PRIu64 ") is not 0", chunkSize); + if (chunk_size != 0) { + pub->set_error(FileError::INTERNAL_ERROR, + "Chunk data size (%" PRIu64 ") is not 0", chunk_size); return false; } - if (dataSize != sizeof(expectedCrc32)) { - ERROR("Data size (%" PRIu32 ") does not match size of 32-bit integer", - dataSize); + if (data_size != sizeof(crc32)) { + pub->set_error(FileError::INTERNAL_ERROR, + "Data size (%" PRIu32 ") does not match size of 32-bit" + " integer", data_size); return false; } - if (!readFully(ctx, &expectedCrc32, sizeof(expectedCrc32))) { + if (wread(&crc32, sizeof(crc32)) != FileStatus::OK) { return false; } - ctx->expectedCrc32 = expectedCrc32; + expected_crc32 = mb_le32toh(crc32); - ctx->chunks.emplace_back(); - ChunkInfo &chunk = ctx->chunks.back(); - chunk.type = chunkHeader->chunk_type; - chunk.begin = outOffset; - chunk.end = outOffset; - chunk.srcBegin = ctx->srcOffset - ctx->shdr.chunk_hdr_sz; - chunk.srcEnd = ctx->srcOffset + dataSize; + chunk_out.type = chdr.chunk_type; + chunk_out.begin = tgt_offset; + chunk_out.end = tgt_offset; + chunk_out.src_begin = tgt_offset - shdr.chunk_hdr_sz; + chunk_out.src_end = tgt_offset + data_size; return true; } /*! - * \brief Verify chunk header and add to chunk list + * \brief Read and verify chunk header * - * Thie function will check the following properties: - * - Number of chunk data blocks matches expected number of blocks + * This function will check the following properties: + * * Number of chunk data blocks matches expected number of blocks * * \pre The file position should be at the byte immediately after the chunk * header - * \pre The value pointed by \a srcOffset should be set to the file position of - * the source file - * \pre \a outOffset should be set to the beginning of the range in the output + * \pre \p tgt_offset should be set to the beginning of the range in the output * file that this chunk represents * \post If the chunk is a raw chunk, the source file position will be advanced * to the first byte of the raw data. Otherwise, the source file position * will advanced to the byte after the entire chunk. - * \post The value pointed by \a srcOffset will be set to the new source file - * position + * \post \a cur_src_offset will be set to the new source file position + * + * \param[in] chdr Chunk header + * \param[in] tgt_offset Offset of the output file + * \param[out] chunk_out ChunkInfo to store chunk info * - * \param ctx Sparse context - * \param chunkHeader Chunk header for the current chunk - * \param outOffset Current offset of the output file * \return Whether the chunk header is valid */ -static bool processChunk(SparseCtx *ctx, ChunkHeader *chunkHeader, - uint64_t outOffset) +bool SparseFilePrivate::process_chunk(const ChunkHeader &chdr, + uint64_t tgt_offset, + ChunkInfo &chunk_out) { - bool ret; + MB_PUBLIC(SparseFile); - switch (chunkHeader->chunk_type) { + if (chdr.total_sz < shdr.chunk_hdr_sz) { + pub->set_error(FileError::INTERNAL_ERROR, + "Total chunk size (%" PRIu32 ") smaller than chunk " + "header size", chdr.total_sz); + return false; + } + + switch (chdr.chunk_type) { case CHUNK_TYPE_RAW: - ret = processRawChunk(ctx, chunkHeader, outOffset); - break; + return process_raw_chunk(chdr, tgt_offset, chunk_out); case CHUNK_TYPE_FILL: - ret = processFillChunk(ctx, chunkHeader, outOffset); - break; + return process_fill_chunk(chdr, tgt_offset, chunk_out); case CHUNK_TYPE_DONT_CARE: - ret = processSkipChunk(ctx, chunkHeader, outOffset); - break; + return process_skip_chunk(chdr, tgt_offset, chunk_out); case CHUNK_TYPE_CRC32: - ret = processCrc32Chunk(ctx, chunkHeader, outOffset); - break; + return process_crc32_chunk(chdr, tgt_offset, chunk_out); default: - ERROR("Unknown chunk type: %u", chunkHeader->chunk_type); - ret = false; - break; - } - - return ret; -} - -/*! - * \brief Read and verify sparse header - * - * Thie function will check the following properties: - * - Magic field matches expected value - * - Major version is 1 - * - The sparse header size field is at least the expected size of the sparse - * header structure - * - The chunk header size field is at least the expected size of the chunk - * header structure - * - * The value of the minor version field is ignored. - * - * \note The structure is read directly into memory. The host CPU must be - * little-endian and support having the structure layout in memory without - * padding. - * - * \note This does not process any of the chunk headers. The chunk headers are - * processed on-the-fly when reading the archive. - * - * \pre The file position should be at the beginning of the file (or at the - * beginning of the sparse header - * \post The file position will be advanced to the byte after the sparse header. - * If the sparse header size, as specified by the \a file_hdr_sz field, is - * larger than the expected size, the extra bytes will be skipped as well. - * - * \param ctx Sparse context - * \return Whether the sparse header is valid - */ -bool processSparseHeader(SparseCtx *ctx) -{ - // Read header - if (!readFully(ctx, &ctx->shdr, sizeof(SparseHeader))) { - return false; - } - -#if SPARSE_DEBUG - dumpSparseHeader(&ctx->shdr); -#endif - - // Check magic bytes - if (ctx->shdr.magic != SPARSE_HEADER_MAGIC) { - ERROR("Expected magic to be %08x, but got %08x", - SPARSE_HEADER_MAGIC, ctx->shdr.magic); - return false; - } - - // Check major version - if (ctx->shdr.major_version != SPARSE_HEADER_MAJOR_VER) { - ERROR("Expected major version to be %u, but got %u", - SPARSE_HEADER_MAJOR_VER, ctx->shdr.major_version); - return false; - } - - // Check file header size field - if (ctx->shdr.file_hdr_sz < sizeof(SparseHeader)) { - ERROR("Expected file header size to be at least %" MB_PRIzu - ", but have %" PRIu32, - sizeof(ctx->shdr), ctx->shdr.file_hdr_sz); + pub->set_error(FileError::INTERNAL_ERROR, + "Unknown chunk type: %u", chdr.chunk_type); return false; } - - // Check chunk header size field - if (ctx->shdr.chunk_hdr_sz < sizeof(ChunkHeader)) { - ERROR("Expected chunk header size to be at least %" MB_PRIzu - ", but have %" PRIu32, - sizeof(SparseHeader), ctx->shdr.chunk_hdr_sz); - return false; - } - - // Skip any extra bytes in the file header - std::size_t diff = ctx->shdr.file_hdr_sz - sizeof(SparseHeader); - if (!ctx->skipBytes(diff)) { - return false; - } - - ctx->fileSize = (uint64_t) ctx->shdr.total_blks * ctx->shdr.blk_sz; - - return true; } /*! - * \brief Find and move to chunk that is responsible for the specified offset + * \brief Move to chunk that is responsible for the specified offset * - * \warning Always check if the offset exceeds the range of all chunks (EOF) by - * testing: "ctx->chunk == ctx->shdr.total_chunks" + * \note This function only returns a failure code if a file operation fails. To + * check if a matching chunk is found, use `chunk != chunks.end()`. * * \return True unless an error occurs + * * #FileStatus::OK if all file operations succeed + * * \<= #FileStatus::WARN if an error occurs */ -bool tryMoveToChunkForOffset(SparseCtx *ctx, uint64_t offset) +FileStatus SparseFilePrivate::move_to_chunk(uint64_t offset) { - // If were at EOF, move back one so we can search again - if (ctx->shdr.total_chunks != 0 && ctx->chunk == ctx->shdr.total_chunks) { - --ctx->chunk; + MB_PUBLIC(SparseFile); + + // No action needed if the offset is in the current chunk + if (chunk != chunks.end() + && offset >= chunk->begin && offset < chunk->end) { + return FileStatus::OK; } - // Allow backwards seeking. If we read some chunks before, then search - // backwards if offset is less than the beginning of the chunk. - if (ctx->chunk < ctx->chunks.size()) { - while (offset < ctx->chunks[ctx->chunk].begin) { - if (ctx->chunk == 0) { - break; - } - --ctx->chunk; + // If the offset is in the current range of chunks, then do a binary search + // to find the right chunk + if (!chunks.empty() && offset < chunks.back().end) { + chunk = binary_find(chunks.begin(), chunks.end(), offset, OffsetComp()); + if (chunk != chunks.end()) { + return FileStatus::OK; } } - for (; ctx->chunk < ctx->shdr.total_chunks; ++ctx->chunk) { - // If we don't have the chunk yet, then read it - if (ctx->chunk >= ctx->chunks.size()) { - DEBUG("Reading next chunk (#%" MB_PRIzu ")", ctx->chunk); - - // Get starting offset for chunk in source file and starting - // offset for data in the output file - uint64_t srcBegin = ctx->shdr.file_hdr_sz; - uint64_t outBegin = 0; - if (ctx->chunk > 0) { - srcBegin = ctx->chunks[ctx->chunk - 1].srcEnd; - outBegin = ctx->chunks[ctx->chunk - 1].end; - } + // We don't have the chunk, so read until we find it + chunk = chunks.end(); + while (chunks.size() < shdr.total_chunks) { + size_t chunk_num = chunks.size(); + + DEBUG("Reading next chunk (#%" MB_PRIzu ")", chunk_num); + + // Get starting source and output offsets for chunk + uint64_t src_begin; + uint64_t tgt_begin; + + // First chunk starts after the sparse header. Remaining chunks are + // contiguous. + if (chunks.empty()) { + src_begin = shdr.file_hdr_sz; + tgt_begin = 0; + } else { + src_begin = chunks.back().src_end; + tgt_begin = chunks.back().end; + } - // Skip to srcBegin - if (srcBegin < ctx->srcOffset) { - ERROR("- Internal error: srcBegin (%" PRIu64 ")" - " < srcOffset (%" PRIu64 ")", srcBegin, ctx->srcOffset); - return false; - } + // Skip to src_begin + if (src_begin < cur_src_offset) { + pub->set_error(FileError::INTERNAL_ERROR, + "Internal error: src_begin (%" PRIu64 ")" + " < cur_src_offset (%" PRIu64 ")", + src_begin, cur_src_offset); + return FileStatus::FATAL; + } - uint64_t diff = srcBegin - ctx->srcOffset; - if (diff > 0 && !ctx->skipBytes(diff)) { - ERROR("- Failed to skip to chunk #%" MB_PRIzu, ctx->chunk); - return false; - } + FileStatus ret = skip_bytes(src_begin - cur_src_offset); + if (ret != FileStatus::OK) { + pub->set_error(pub->error(), + "Failed to skip to chunk #%" MB_PRIzu ": %s", + chunk_num, pub->error_string().c_str()); + return ret; + } - ChunkHeader chunkHeader; + ChunkHeader chdr; - if (!readFully(ctx, &chunkHeader, sizeof(ChunkHeader))) { - ERROR("- Failed to read chunk header for chunk %" MB_PRIzu, - ctx->chunk); - return false; - } + ret = wread(&chdr, sizeof(chdr)); + if (ret != FileStatus::OK) { + pub->set_error(pub->error(), + "Failed to read chunk header for chunk %" MB_PRIzu + ": %s", chunk_num, pub->error_string().c_str()); + return ret; + } + + fix_chunk_header_byte_order(chdr); #if SPARSE_DEBUG - dumpChunkHeader(&chunkHeader); + dump_chunk_header(chdr); #endif - // Skip any extra bytes in the chunk header. processSparseHeader() - // checks the size to make sure that the value won't underflow - diff = ctx->shdr.chunk_hdr_sz - sizeof(ChunkHeader); - if (!ctx->skipBytes(diff)) { - ERROR("- Failed to skip extra bytes in chunk #%" MB_PRIzu "'s header", - ctx->chunk); - return false; - } + // Skip any extra bytes in the chunk header. process_sparse_header() + // checks the size to make sure that the value won't underflow. + ret = skip_bytes(shdr.chunk_hdr_sz - sizeof(chdr)); + if (ret != FileStatus::OK) { + pub->set_error(pub->error(), + "Failed to skip extra bytes in chunk #%" MB_PRIzu + "'s header: %s", chunk_num, + pub->error_string().c_str()); + return ret; + } - if (!processChunk(ctx, &chunkHeader, outBegin)) { - return false; - } + ChunkInfo chunk_info{}; - OPER("- Chunk #%" MB_PRIzu " covers source range (%" PRIu64 " - %" PRIu64 ")", - ctx->chunk, ctx->chunks[ctx->chunk].srcBegin, - ctx->chunks[ctx->chunk].srcEnd); - OPER("- Chunk #%" MB_PRIzu " covers output range (%" PRIu64 " - %" PRIu64 ")", - ctx->chunk, ctx->chunks[ctx->chunk].begin, - ctx->chunks[ctx->chunk].end); - - // Make sure the chunk does not end after the header-specified file - // size - if (ctx->chunks[ctx->chunk].end > ctx->fileSize) { - ERROR("Chunk #%" MB_PRIzu " ends (%" PRIu64 ") after the file size " - "specified in the sparse header (%" PRIu64 ")", - ctx->chunk, ctx->chunks[ctx->chunk].end, ctx->fileSize); - return false; - } + if (!process_chunk(chdr, tgt_begin, chunk_info)) { + return FileStatus::FATAL; + } - // If we just read the last chunk, make sure it ends at the same - // position as specified in the sparse header - if (ctx->chunk == ctx->shdr.total_chunks - 1 - && ctx->chunks[ctx->chunk].end != ctx->fileSize) { - ERROR("Last chunk does not end (%" PRIu64 ")" - " at position specified by sparse header (%" PRIu64 ")", - ctx->chunks[ctx->chunk].end, ctx->fileSize); - return false; - } + OPER("Chunk #%" MB_PRIzu " covers source range (%" PRIu64 " - %" PRIu64 ")", + chunk_num, chunk_info.src_begin, chunk_info.src_end); + OPER("Chunk #%" MB_PRIzu " covers output range (%" PRIu64 " - %" PRIu64 ")", + chunk_num, chunk_info.begin, chunk_info.end); + + // Make sure the chunk does not end after the header-specified file size + if (chunk_info.end > file_size) { + pub->set_error(FileError::INTERNAL_ERROR, + "Chunk #%" MB_PRIzu " ends (%" PRIu64 ") after the " + "file size specified in the sparse header (%" + PRIu64 ")", chunk_num, chunk_info.end, file_size); + return FileStatus::FATAL; } - if (offset >= ctx->chunks[ctx->chunk].begin - && offset < ctx->chunks[ctx->chunk].end) { - // Found matching chunk. Stop looking + chunks.push_back(std::move(chunk_info)); + + // If we just read the last chunk, make sure it ends at the same + // position as specified in the sparse header + if (chunks.size() == shdr.total_chunks + && chunks.back().end != file_size) { + pub->set_error(FileError::INTERNAL_ERROR, + "Last chunk does not end (%" PRIu64 ") at position" + " specified by sparse header (%" PRIu64 ")", + chunks.back().end, file_size); + return FileStatus::FATAL; + } + + if (offset >= chunks.back().begin && offset < chunks.back().end) { + chunk = chunks.end() - 1; break; } + + // Make sure iterator is not invalidated + chunk = chunks.end(); } - return true; + return FileStatus::OK; } -extern "C" { +/*! \endcond */ -SparseCtx * sparseCtxNew() +/*! + * \class SparseFile + * + * \brief Open Android sparse file image. + */ + +/*! + * \brief Construct unbound SparseFile. + * + * The File handle will not be bound to any file. One of the open functions will + * need to be called to open a file. + */ +SparseFile::SparseFile() + : SparseFile(new SparseFilePrivate(this)) +{ +} + +/*! + * \brief Open sparse file from File handle. + * + * Construct the file handle and open the file. Use is_open() to check if the + * file was successfully opened. + * + * \sa open(File *) + * + * \param file File to open + */ +SparseFile::SparseFile(File *file) + : SparseFile(new SparseFilePrivate(this), file) +{ +} + +/*! \cond INTERNAL */ +SparseFile::SparseFile(SparseFilePrivate *priv) + : _priv_ptr(priv) { - SparseCtx *ctx = new(std::nothrow) SparseCtx(); - if (!ctx) { - return nullptr; - } - ctx->isOpen = false; - return ctx; } -bool sparseCtxFree(SparseCtx *ctx) +SparseFile::SparseFile(SparseFilePrivate *priv, File *file) + : _priv_ptr(priv) { - bool ret = true; - if (ctx->isOpen) { - ret = ctx->close(); + open(file); +} +/*! \endcond */ + +SparseFile::~SparseFile() +{ + close(); +} + +/*! + * \brief Open sparse file from File handle. + * + * \note The SparseFile will *not* take ownership of \p file. The caller must + * ensure that it is properly closed and destroyed when it is no longer + * needed. + * + * \param file File to open + * + * \return + * * #FileStatus::OK if the file is successfully opened + * * \<= #FileStatus::WARN if an error occurs + */ +FileStatus SparseFile::open(File *file) +{ + MB_PRIVATE(SparseFile); + if (priv) { + priv->file = file; } - delete ctx; - return ret; + return File::open(); +} + +/*! + * \brief Get the expected size of the sparse file + * + * \return Size of the sparse file. The return value is undefined if the sparse + * file is not opened. + */ +uint64_t SparseFile::size() +{ + MB_PRIVATE(SparseFile); + return priv->file_size; } /*! * \brief Open sparse file for reading * - * The source, which may not necessarily be a file, is read by calling functions - * provided by the caller. + * The file does not need to support any operation other than reading. If the + * file supports forward skipping or random seeking, then those operations will + * be used to speed up the read process. If the file supports random seeking, + * then the sparse file will also support random reads. + * + * The following test will be run to determine if the file supports forward + * skipping. * - * All callbacks besides \a readCb are optional. The read callback should always - * read the specified number of bytes before EOF is reached. The library will - * treat a short read count as EOF. + * \code{.cpp} + * file.seek(0, SEEK_CUR, nullptr) != FileStatus::UNSUPPORTED; + * \endcode * - * If a seek callback is provided, then the library will allow random reads to - * the sparse file. Otherwise, the sparse file can only be read sequentially and - * \a sparseRead() would not work. A skip callback can also be provided for - * skipping bytes in a backwards-unseekable source. If both a seek callback and - * a skip callback are provided, the seek callback will always be used. If - * neither are provided, then the read callback will be repeatedly called and - * the results discarded. + * The following test will be run (following the forward skipping test) to + * determine if the file supports random seeking. * - * The open and close callbacks are provided for convenience only. If an error - * occurs between the time the open callback is called and this function - * returns, then the close callback will be called. Thus, the underlying source - * will always be "open" if this function returns true and "closed" if this - * function returns false. + * \code{.cpp} + * file.read(buf, 1, &n) == FileStatus::OK; + * file.seek(-1, SEEK_CUR, nullptr) != FileStatus::UNSUPPORTED; + * \endcode * - * \note If a seek callback is provided, the library to seek to the beginning of - * the source. Otherwise, it's up to the caller to ensure that the - * position of the source is at the beginning of the sparse file. + * \note This function will fail if the file handle is not open. * - * \note After the sparse file is closed, another sparse file can be opened - * using the same \a ctx object. + * \pre The caller should position the file at the beginning of the sparse file + * data. SparseFile will use that as the base offset and only perform + * relative seeks. This allows for opening sparse files where the data + * isn't at the beginning of the file. * - * \param ctx Sparse context - * \param openCb Open callback - * \param closeCb Close callback - * \param readCb Read callback - * \param seekCb Seek callback - * \param skipCb Skip callback - * \param userData Caller-supplied pointer to pass to callback functions - * \return Whether the sparse file is opened and the sparse header is determined - * to be valid + * \return + * * FileStatus::OK if the sparse file is successfully opened + * * \<= FileStatus::WARN if a file operation fails or header validation fails */ -bool sparseOpen(SparseCtx *ctx, SparseOpenCb openCb, SparseCloseCb closeCb, - SparseReadCb readCb, SparseSeekCb seekCb, SparseSkipCb skipCb, - void *userData) +FileStatus SparseFile::on_open() { - if (ctx->isOpen) { - return false; + MB_PRIVATE(SparseFile); + + if (!priv->file->is_open()) { + set_error(FileError::INVALID_ARGUMENT, + "Underlying file is not open"); + return FileStatus::FAILED; } - ctx->setCallbacks(openCb, closeCb, readCb, seekCb, skipCb, userData); + unsigned char first_byte; + size_t n; + FileStatus ret; - if (ctx->cbOpen && !ctx->open()) { - ctx->clearCallbacks(); - return false; - } + priv->seekability = Seekability::CAN_READ; - // Goto beginning of file if we can. Otherwise, just assume the file is at - // the beginning - if (ctx->cbSeek && !ctx->seek(0, SEEK_SET)) { - return false; + ret = priv->file->seek(0, SEEK_CUR, nullptr); + if (ret == FileStatus::OK) { + DEBUG("File supports forward skipping"); + priv->seekability = Seekability::CAN_SKIP; + } else if (ret != FileStatus::UNSUPPORTED) { + set_error(priv->file->error(), "%s", + priv->file->error_string().c_str()); + return ret; } - // Process main sparse header - if (!processSparseHeader(ctx)) { - ctx->close(); - ctx->clearCallbacks(); - return false; + ret = priv->file->read(&first_byte, 1, &n); + if (ret != FileStatus::OK) { + set_error(priv->file->error(), "%s", + priv->file->error_string().c_str()); + return ret; + } else if (n != 1) { + set_error(FileError::INTERNAL_ERROR, + "Failed to read first byte of file"); + return FileStatus::FAILED; + } + priv->cur_src_offset += n; + + ret = priv->file->seek(-static_cast(n), SEEK_CUR, nullptr); + if (ret == FileStatus::OK) { + DEBUG("File supports random seeking"); + priv->seekability = Seekability::CAN_SEEK; + priv->cur_src_offset -= n; + n = 0; + } else if (ret != FileStatus::UNSUPPORTED) { + set_error(priv->file->error(), "%s", + priv->file->error_string().c_str()); + return ret; } - // The chunk headers are processed on demand - - ctx->isOpen = true; + ret = priv->process_sparse_header(&first_byte, n); + if (ret != FileStatus::OK) { + return ret; + } - return true; + return FileStatus::OK; } /*! @@ -802,251 +875,219 @@ bool sparseOpen(SparseCtx *ctx, SparseOpenCb openCb, SparseCloseCb closeCb, * sparse file will be closed. The return value is the return value of the * close callback function (if one was provided). * - * \return Whether the sparse file was closed + * \return + * * #FileStatus::OK if no error was encountered when closing the file. + * * \<= #FileStatus::WARN if an error occurs while closing the file */ -bool sparseClose(SparseCtx *ctx) +FileStatus SparseFile::on_close() { - if (!ctx->isOpen) { - return false; - } + MB_PRIVATE(SparseFile); - ctx->isOpen = false; - ctx->srcOffset = 0; - ctx->outOffset = 0; - ctx->expectedCrc32 = 0; - ctx->chunks.clear(); - ctx->chunk = 0; - - bool ret = true; - if (ctx->cbClose) { - ret = ctx->close(); - } + // Reset to allow opening another file + priv->clear(); - ctx->clearCallbacks(); - return ret; + return FileStatus::OK; } /*! * \brief Read sparse file * * This function will read the specified amount of bytes from the source file. - * If this function returns true and \a bytesRead is less than \a size, then - * the end of the sparse file (EOF) has been reached. If any error occurs, the - * function will return false and any further attempts to read or seek the - * sparse file results in undefined behavior as the internal state will be - * invalid. + * If this function returns #FileStatus::OK and \p *bytes_read is less than + * \p size, then the end of the sparse file (EOF) has been reached. If any error + * occurs, the function will return \<= #FileStatus::WARN and any further + * attempts to read or seek the sparse file results in undefined behavior as the + * internal state will be invalid. * * Some examples of failures include: - * - Source reaching EOF before the end of the sparse file is reached - * - Ran out of sparse chunks despite the main sparse header promising more + * * Source reaching EOF before the end of the sparse file is reached + * * Ran out of sparse chunks despite the main sparse header promising more * chunks - * - Chunk header is invalid + * * Chunk header is invalid * - * \param ctx Sparse context * \param buf Buffer to read data into * \param size Number of bytes to read - * \param bytesRead Number of bytes that were read + * \param bytes_read Number of bytes that were read + * * \return Whether the specified number of bytes were successfully read */ -bool sparseRead(SparseCtx *ctx, void *buf, uint64_t size, uint64_t *bytesRead) +FileStatus SparseFile::on_read(void *buf, size_t size, size_t *bytes_read) { - if (!ctx->isOpen) { - return false; - } + MB_PRIVATE(SparseFile); - OPER("read(buf, %" PRId64 ", *bytesRead)", size); + OPER("read(buf, %" MB_PRIzu ", *bytesRead)", size); - uint64_t totalRead = 0; + uint64_t total_read = 0; while (size > 0) { - // If no chunks have been read yet or the current offset exceeds the - // range of the current chunk, then look for the next chunk. - - if (ctx->chunks.empty() - || ctx->chunk == ctx->shdr.total_chunks - || ctx->outOffset >= ctx->chunks[ctx->chunk].end) { - if (!tryMoveToChunkForOffset(ctx, ctx->outOffset)) { - return false; - } - - if (ctx->chunk == ctx->shdr.total_chunks) { - OPER("- Found EOF"); - break; - } + auto ret = priv->move_to_chunk(priv->cur_tgt_offset); + if (ret != FileStatus::OK) { + return ret; + } else if (priv->chunk == priv->chunks.end()) { + OPER("Reached EOF"); + break; } - assert(ctx->outOffset >= ctx->chunks[ctx->chunk].begin - && ctx->outOffset < ctx->chunks[ctx->chunk].end); + assert(priv->cur_tgt_offset >= priv->chunk->begin + && priv->cur_tgt_offset < priv->chunk->end); // Read until the end of the current chunk - uint64_t nRead = 0; - uint64_t toRead = std::min(size, - ctx->chunks[ctx->chunk].end - ctx->outOffset); + uint64_t n_read = 0; + uint64_t to_read = std::min( + size, priv->chunk->end - priv->cur_tgt_offset); - OPER("- Reading %" PRIu64 " bytes from chunk %" MB_PRIzu, - toRead, ctx->chunk); + OPER("Reading %" PRIu64 " bytes from chunk %" MB_PRIzu, + to_read, priv->chunk - priv->chunks.begin()); - switch (ctx->chunks[ctx->chunk].type) { + switch (priv->chunk->type) { case CHUNK_TYPE_RAW: { // Figure out how much to seek in the input data - uint64_t diff = ctx->outOffset - ctx->chunks[ctx->chunk].begin; - OPER("- Raw data is %" PRIu64 " bytes into the raw chunk", diff); - if (ctx->cbSeek && !ctx->seek( - ctx->chunks[ctx->chunk].rawBegin + diff, SEEK_SET)) { - return false; - } - if (!ctx->read(buf, toRead, &nRead)) { - return false; + uint64_t diff = priv->cur_tgt_offset - priv->chunk->begin; + OPER("Raw data is %" PRIu64 " bytes into the raw chunk", diff); + + uint64_t raw_src_offset = priv->chunk->raw_begin + diff; + if (raw_src_offset != priv->cur_src_offset) { + assert(priv->seekability == Seekability::CAN_SEEK); + + int64_t seek_offset; + if (raw_src_offset < priv->cur_src_offset) { + seek_offset = -static_cast( + priv->cur_src_offset - raw_src_offset); + } else { + seek_offset = raw_src_offset - priv->cur_src_offset; + } + + ret = priv->wseek(seek_offset); + if (ret != FileStatus::OK) { + return ret; + } } - if (nRead == 0) { - // EOF - goto done; + + ret = priv->wread(buf, to_read); + if (ret != FileStatus::OK) { + return ret; } + + n_read = to_read; break; } case CHUNK_TYPE_FILL: { - assert(sizeof(ctx->chunks[ctx->chunk].fillVal) == sizeof(uint32_t)); - auto shift = (ctx->outOffset - ctx->chunks[ctx->chunk].begin) + static_assert(sizeof(priv->chunk->fill_val) == sizeof(uint32_t), + "Mismatched fill_val size"); + auto shift = (priv->cur_tgt_offset - priv->chunk->begin) % sizeof(uint32_t); - uint32_t fillVal = ctx->chunks[ctx->chunk].fillVal; - uint32_t shifted = 0; + uint32_t fill_val = mb_htole32(priv->chunk->fill_val); + unsigned char shifted[4]; for (size_t i = 0; i < sizeof(uint32_t); ++i) { - ((char *) &shifted)[i] = - ((char *) &fillVal)[(i + shift) % sizeof(uint32_t)]; - //uint32_t amount = 8 * ((i + shift) % sizeof(uint32_t)); - //shifted |= (fillVal & (0xff << amount)) >> amount << i * 8; + shifted[i] = reinterpret_cast(&fill_val) + [(i + shift) % sizeof(uint32_t)]; } - char *tempBuf = (char *) buf; - while (toRead > 0) { - size_t toWrite = std::min(sizeof(shifted), toRead); - memcpy(tempBuf, &shifted, toWrite); - nRead += toWrite; - toRead -= toWrite; - tempBuf += toWrite; + unsigned char *temp_buf = reinterpret_cast(buf); + while (to_read > 0) { + size_t to_write = std::min(sizeof(shifted), to_read); + memcpy(temp_buf, &shifted, to_write); + n_read += to_write; + to_read -= to_write; + temp_buf += to_write; } break; } case CHUNK_TYPE_DONT_CARE: - memset(buf, 0, toRead); - nRead = toRead; + memset(buf, 0, to_read); + n_read = to_read; break; - case CHUNK_TYPE_CRC32: - assert(false); default: - return false; + assert(false); } - OPER("- Read %" PRIu64 " bytes", nRead); - totalRead += nRead; - ctx->outOffset += nRead; - size -= nRead; - buf = (char *) buf + nRead; + OPER("Read %" PRIu64 " bytes", n_read); + total_read += n_read; + priv->cur_tgt_offset += n_read; + size -= n_read; + buf = reinterpret_cast(buf) + n_read; } -done: - *bytesRead = totalRead; - return true; + *bytes_read = total_read; + return FileStatus::OK; } /*! * \brief Seek sparse file * - * \a whence takes the same \a SEEK_SET, \a SEEK_CUR, and \a SEEK_END values as - * \a lseek() in . + * \p whence takes the same \a SEEK_SET, \a SEEK_CUR, and \a SEEK_END values as + * \a lseek() in `\`. * - * \note If a seek callback was not provided, then this function will always - * return false; + * \note Seeking will only work if the underlying file handle supports seeking. + * + * \param[in] offset Offset to seek + * \param[in] whence \a SEEK_SET, \a SEEK_CUR, or \a SEEK_END + * \param[out] new_offset_out Pointer to store new offset of sparse file + * (can be NULL) * - * \param ctx Sparse context - * \param offset Offset to seek - * \param whence \a SEEK_SET, \a SEEK_CUR, or \a SEEK_END * \return Whether the seeking was successful + * * #FileStatus::OK if seeking is successful + * * \<= #FileStatus::WARN if an error occurs + * * #FileStatus::UNSUPPORTED if seeking is not supported */ -bool sparseSeek(SparseCtx *ctx, int64_t offset, int whence) +FileStatus SparseFile::on_seek(int64_t offset, int whence, + uint64_t *new_offset_out) { - if (!ctx->isOpen) { - return false; - } + MB_PRIVATE(SparseFile); OPER("seek(%" PRId64 ", %d)", offset, whence); - if (!ctx->cbSeek) { - OPER("- Cannot seek because no seek callback is registered"); - return false; + if (priv->seekability != Seekability::CAN_SEEK) { + set_error(FileError::UNSUPPORTED, + "Underlying file does not support seeking"); + return FileStatus::UNSUPPORTED; } - uint64_t newOffset; + uint64_t new_offset; switch (whence) { case SEEK_SET: if (offset < 0) { - OPER("- Tried to seek to negative offset"); - return false; + set_error(FileError::INVALID_ARGUMENT, + "Cannot seek to negative offset"); + return FileStatus::FAILED; } - newOffset = offset; + new_offset = offset; break; case SEEK_CUR: - if ((offset < 0 && (uint64_t) -offset > ctx->outOffset) - || (offset > 0 && ctx->outOffset >= UINT64_MAX - offset)) { - OPER("- Offset overflows uint64_t"); - return false; + if ((offset < 0 && static_cast(-offset) > priv->cur_tgt_offset) + || (offset > 0 && priv->cur_tgt_offset >= UINT64_MAX - offset)) { + set_error(FileError::INVALID_ARGUMENT, "Offset overflows uint64_t"); + return FileStatus::FAILED; } - newOffset = ctx->outOffset + offset; + new_offset = priv->cur_tgt_offset + offset; break; case SEEK_END: - if ((offset < 0 && (uint64_t) -offset > ctx->fileSize) - || (offset > 0 && ctx->fileSize >= UINT64_MAX - offset)) { - OPER("- Offset overflows uint64_t"); - return false; + if ((offset < 0 && static_cast(-offset) > priv->file_size) + || (offset > 0 && priv->file_size >= UINT64_MAX - offset)) { + set_error(FileError::INVALID_ARGUMENT, "Offset overflows uint64_t"); + return FileStatus::FAILED; } - newOffset = ctx->fileSize + offset; + new_offset = priv->file_size + offset; break; default: - OPER("- Invalid seek whence: %d", whence); - return false; + set_error(FileError::INVALID_ARGUMENT, + "Invalid seek whence: %d", whence); + return FileStatus::FAILED; } - if (!tryMoveToChunkForOffset(ctx, newOffset)) { - return false; + FileStatus ret = priv->move_to_chunk(new_offset); + if (ret != FileStatus::OK) { + return ret; } - // May move past EOF, which is okay (mimics lseek behavior), but - // sparseRead() will know to read nothing in that case - ctx->outOffset = newOffset; - return true; -} + // May move past EOF, which is okay (mimics lseek behavior), but read() + priv->cur_tgt_offset = new_offset; -/*! - * \brief Get file pointer position in sparse file - * - * \param ctx Sparse context - * \param offset Output pointer for current position in sparse file - * \return True, unless the file is not open - */ -bool sparseTell(SparseCtx *ctx, uint64_t *offset) -{ - if (!ctx->isOpen) { - return false; + if (new_offset_out) { + *new_offset_out = new_offset; } - *offset = ctx->outOffset; - return true; + return FileStatus::OK; } -/*! - * \brief Get the expected size of the sparse file - * - * \param ctx Sparse context - * \param size Output pointer for the expected file size - * \return True, unless the file is not open - */ -bool sparseSize(SparseCtx *ctx, uint64_t *size) -{ - if (!ctx->isOpen) { - return false; - } - - *size = ctx->fileSize; - return true; } - } diff --git a/libmbsparse/tests/test_sparse.cpp b/libmbsparse/tests/test_sparse.cpp index d784be84a..b6d2f9d1d 100644 --- a/libmbsparse/tests/test_sparse.cpp +++ b/libmbsparse/tests/test_sparse.cpp @@ -21,358 +21,711 @@ #include "mbsparse/sparse.h" -struct SparseTest : testing::Test -{ - SparseCtx *_ctx; - std::vector _data; - size_t _pos = 0; +#include "mbcommon/endian.h" +#include "mbcommon/file/memory.h" - SparseTest() - { - _ctx = sparseCtxNew(); - } +#include "mbsparse/sparse_p.h" - virtual ~SparseTest() - { - sparseCtxFree(_ctx); - } - - static bool cbRead(void *buf, uint64_t size, uint64_t *bytesRead, - void *userData) +class CustomMemoryFile : public mb::MemoryFile +{ +public: + void set_seekability(mb::sparse::Seekability seekability) { - SparseTest *test = static_cast(userData); - if (test->_pos > test->_data.size()) { - *bytesRead = 0; - } else { - uint64_t canRead = std::min( - size, test->_data.size() - test->_pos); - memcpy(buf, test->_data.data() + test->_pos, canRead); - test->_pos += canRead; - *bytesRead = canRead; - } - return true; + _seekability = seekability; } - static bool cbSeek(int64_t offset, int whence, void *userData) +protected: + virtual mb::FileStatus on_seek(int64_t offset, int whence, + uint64_t *new_offset) override { - SparseTest *test = static_cast(userData); - switch (whence) { - case SEEK_SET: - if (offset < 0) { - return false; - } - test->_pos = offset; - break; - case SEEK_CUR: - if ((offset < 0 && (uint64_t) -offset > test->_pos) - || (offset > 0 && test->_pos >= UINT64_MAX - offset)) { - return false; + switch (_seekability) { + case mb::sparse::Seekability::CAN_SEEK: + return mb::MemoryFile::on_seek(offset, whence, new_offset); + case mb::sparse::Seekability::CAN_SKIP: + if (whence == SEEK_CUR && offset >= 0) { + return mb::MemoryFile::on_seek(offset, whence, new_offset); } - test->_pos += offset; break; - case SEEK_END: - if ((offset < 0 && (uint64_t) -offset > test->_data.size()) - || (offset > 0 && test->_data.size() >= UINT64_MAX - offset)) { - return false; - } - test->_pos = test->_data.size() + offset; + case mb::sparse::Seekability::CAN_READ: break; default: - return false; + return mb::FileStatus::FATAL; } - return true; - } - bool sparseOpen() - { - return ::sparseOpen(_ctx, nullptr, nullptr, &cbRead, &cbSeek, nullptr, - this); + return mb::FileStatus::UNSUPPORTED; } - bool sparseOpenNoSeek() - { - return ::sparseOpen(_ctx, nullptr, nullptr, &cbRead, nullptr, nullptr, - this); - } +private: + mb::sparse::Seekability _seekability = mb::sparse::Seekability::CAN_SEEK; +}; - bool sparseClose() - { - return ::sparseClose(_ctx); - } +struct SparseTest : testing::Test +{ + CustomMemoryFile _source_file; + mb::sparse::SparseFile _file; + void *_data = nullptr; + size_t _size = 0; - bool sparseRead(void *buf, uint64_t size, uint64_t *bytesRead) - { - return ::sparseRead(_ctx, buf, size, bytesRead); - } + static constexpr unsigned char expected_valid_data[] = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'a', 'b', 'c', 'd', 'e', 'f', + 0x78, 0x56, 0x34, 0x12, 0x78, 0x56, 0x34, 0x12, 0x78, 0x56, 0x34, 0x12, + 0x78, 0x56, 0x34, 0x12, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; - bool sparseSeek(int64_t offset, int whence) + virtual ~SparseTest() { - return ::sparseSeek(_ctx, offset, whence); + free(_data); } - bool sparseTell(uint64_t *offset) + void SetUp() override { - return ::sparseTell(_ctx, offset); + ASSERT_EQ(_source_file.open(&_data, &_size), mb::FileStatus::OK); } - void buildDataHeaderProperSized() + void build_valid_data() { - SparseHeader hdr; - memset(&hdr, 0, sizeof(SparseHeader)); - hdr.magic = SPARSE_HEADER_MAGIC; - hdr.major_version = SPARSE_HEADER_MAJOR_VER; - hdr.minor_version = 0; - hdr.file_hdr_sz = sizeof(SparseHeader); - hdr.chunk_hdr_sz = sizeof(ChunkHeader); - hdr.blk_sz = 4096; - hdr.total_blks = 1; - hdr.total_chunks = 0; - hdr.image_checksum = 0; - - _data.resize(std::max(hdr.blk_sz, sizeof(SparseHeader))); - memcpy(_data.data(), &hdr, sizeof(SparseHeader)); - } + size_t n; - void buildDataHeaderExtraSized() - { - SparseHeader hdr; - memset(&hdr, 0, sizeof(SparseHeader)); - hdr.magic = SPARSE_HEADER_MAGIC; - hdr.major_version = SPARSE_HEADER_MAJOR_VER; - hdr.minor_version = 0; - hdr.file_hdr_sz = sizeof(SparseHeader) + 100; - hdr.chunk_hdr_sz = sizeof(ChunkHeader) + 100; - hdr.blk_sz = 4096; - hdr.total_blks = 1; - hdr.total_chunks = 0; - hdr.image_checksum = 0; - - _data.resize(std::max(hdr.blk_sz, sizeof(SparseHeader) + 100)); - memcpy(_data.data(), &hdr, sizeof(SparseHeader)); - } + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4; + shdr.total_blks = 12; + shdr.total_chunks = 4; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); - void buildDataHeaderUnderSized() - { - // Build sparse header - SparseHeader hdr; - memset(&hdr, 0, sizeof(SparseHeader)); - hdr.magic = SPARSE_HEADER_MAGIC; - hdr.major_version = SPARSE_HEADER_MAJOR_VER; - hdr.minor_version = 0; - hdr.file_hdr_sz = sizeof(SparseHeader) - 4; - hdr.chunk_hdr_sz = sizeof(ChunkHeader); - hdr.blk_sz = 4096; - hdr.total_blks = 0; - hdr.total_chunks = 0; - hdr.image_checksum = 0; - - _data.resize(sizeof(SparseHeader)); - memcpy(_data.data(), &hdr, sizeof(SparseHeader)); - } + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), + mb::FileStatus::OK); - void buildDataCompleteValid() - { - SparseHeader hdr; - auto const *hdrPtr = reinterpret_cast(&hdr); - - memset(&hdr, 0, sizeof(SparseHeader)); - hdr.magic = SPARSE_HEADER_MAGIC; - hdr.major_version = SPARSE_HEADER_MAJOR_VER; - hdr.minor_version = 0; - hdr.file_hdr_sz = sizeof(SparseHeader); - hdr.chunk_hdr_sz = sizeof(ChunkHeader); - hdr.blk_sz = 4; - hdr.total_blks = 12; - hdr.total_chunks = 4; - hdr.image_checksum = 0; - _data.insert(_data.end(), hdrPtr, hdrPtr + sizeof(SparseHeader)); - - ChunkHeader chdr; - auto const *chdrPtr = reinterpret_cast(&chdr); + mb::sparse::ChunkHeader chdr; // [1/4] Add raw chunk header - memset(&chdr, 0, sizeof(ChunkHeader)); - chdr.chunk_type = CHUNK_TYPE_RAW; + chdr = {}; + chdr.chunk_type = mb::sparse::CHUNK_TYPE_RAW; chdr.chunk_sz = 4; // 16 bytes - chdr.total_sz = hdr.chunk_hdr_sz + chdr.chunk_sz * hdr.blk_sz; - _data.insert(_data.end(), chdrPtr, chdrPtr + sizeof(ChunkHeader)); - // [1/4] Add raw chunk data - const char *hexDigits = "0123456789abcdef"; - _data.insert(_data.end(), hexDigits, hexDigits + 16); + chdr.total_sz = shdr.chunk_hdr_sz + chdr.chunk_sz * shdr.blk_sz; + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr), &n), + mb::FileStatus::OK); + ASSERT_EQ(_source_file.write("0123456789abcdef", 16, &n), + mb::FileStatus::OK); // [2/4] Add fill chunk header - memset(&chdr, 0, sizeof(ChunkHeader)); - chdr.chunk_type = CHUNK_TYPE_FILL; + chdr = {}; + chdr.chunk_type = mb::sparse::CHUNK_TYPE_FILL; chdr.chunk_sz = 4; // 16 bytes - chdr.total_sz = hdr.chunk_hdr_sz + sizeof(uint32_t); - _data.insert(_data.end(), chdrPtr, chdrPtr + sizeof(ChunkHeader)); - // [2/4] Add fill chunk data - uint32_t fillVal = 0x12345678; - auto const *fillValPtr = - reinterpret_cast(&fillVal); - _data.insert(_data.end(), fillValPtr, fillValPtr + sizeof(uint32_t)); + chdr.total_sz = shdr.chunk_hdr_sz + sizeof(uint32_t); + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr), &n), + mb::FileStatus::OK); + + uint32_t fill_val = mb_htole32(0x12345678); + ASSERT_EQ(_source_file.write(&fill_val, sizeof(fill_val), &n), + mb::FileStatus::OK); // [3/4] Add skip chunk header - memset(&chdr, 0, sizeof(ChunkHeader)); - chdr.chunk_type = CHUNK_TYPE_DONT_CARE; + chdr = {}; + chdr.chunk_type = mb::sparse::CHUNK_TYPE_DONT_CARE; chdr.chunk_sz = 4; // 16 bytes - chdr.total_sz = hdr.chunk_hdr_sz; - _data.insert(_data.end(), chdrPtr, chdrPtr + sizeof(ChunkHeader)); + chdr.total_sz = shdr.chunk_hdr_sz; + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr), &n), + mb::FileStatus::OK); // [4/4] Add CRC32 chunk header - memset(&chdr, 0, sizeof(ChunkHeader)); - chdr.chunk_type = CHUNK_TYPE_CRC32; + chdr = {}; + chdr.chunk_type = mb::sparse::CHUNK_TYPE_CRC32; chdr.chunk_sz = 0; - chdr.total_sz = hdr.chunk_hdr_sz + sizeof(uint32_t); - _data.insert(_data.end(), chdrPtr, chdrPtr + sizeof(ChunkHeader)); - // [4/4] Add CRC32 chunk data - uint32_t crc32 = 0; - auto const *crc32Ptr = reinterpret_cast(&crc32); - _data.insert(_data.end(), crc32Ptr, crc32Ptr + sizeof(uint32_t)); + chdr.total_sz = shdr.chunk_hdr_sz + sizeof(uint32_t); + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr), &n), + mb::FileStatus::OK); + ASSERT_EQ(_source_file.write("\x00\x00\x00\x00", 4, &n), + mb::FileStatus::OK); + + // Move back to beginning of the file + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + } + + static void fix_sparse_header_byte_order(mb::sparse::SparseHeader &header) + { + header.magic = mb_htole32(header.magic); + header.major_version = mb_htole16(header.major_version); + header.minor_version = mb_htole16(header.minor_version); + header.file_hdr_sz = mb_htole16(header.file_hdr_sz); + header.chunk_hdr_sz = mb_htole16(header.chunk_hdr_sz); + header.blk_sz = mb_htole32(header.blk_sz); + header.total_blks = mb_htole32(header.total_blks); + header.total_chunks = mb_htole32(header.total_chunks); + header.image_checksum = mb_htole32(header.image_checksum); + } + + static void fix_chunk_header_byte_order(mb::sparse::ChunkHeader &header) + { + header.chunk_type = mb_htole16(header.chunk_type); + header.reserved1 = mb_htole16(header.reserved1); + header.chunk_sz = mb_htole32(header.chunk_sz); + header.total_sz = mb_htole32(header.total_sz); } }; -TEST_F(SparseTest, ReadPerfectlySizedHeader) +constexpr unsigned char SparseTest::expected_valid_data[]; + +TEST_F(SparseTest, CheckOpeningUnopenedFileFails) +{ + ASSERT_EQ(_source_file.close(), mb::FileStatus::OK); + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::FAILED); + ASSERT_NE(_file.error_string().find("not open"), std::string::npos); +} + +TEST_F(SparseTest, CheckSparseHeaderInvalidMagicFatal) +{ + mb::sparse::SparseHeader shdr = {}; + shdr.magic = 0xaabbccdd; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4096; + shdr.total_blks = 1; + shdr.total_chunks = 0; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.truncate(shdr.total_blks * shdr.blk_sz), + mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("magic"), std::string::npos); +} + +TEST_F(SparseTest, CheckSparseHeaderInvalidMajorVersionFatal) +{ + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = 0xaabb; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4096; + shdr.total_blks = 1; + shdr.total_chunks = 0; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.truncate(shdr.total_blks * shdr.blk_sz), + mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("major version"), std::string::npos); +} + +TEST_F(SparseTest, CheckSparseHeaderInvalidMinorVersionOk) +{ + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0xaabb; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4096; + shdr.total_blks = 1; + shdr.total_chunks = 0; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.truncate(shdr.total_blks * shdr.blk_sz), + mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); +} + +TEST_F(SparseTest, CheckSparseHeaderUndersizedSparseHeaderSizeFatal) +{ + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader) - 1; + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4096; + shdr.total_blks = 1; + shdr.total_chunks = 0; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.truncate(shdr.total_blks * shdr.blk_sz), + mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("sparse header"), std::string::npos); +} + +TEST_F(SparseTest, CheckSparseHeaderUndersizedChunkHeaderSizeFatal) +{ + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader) - 1; + shdr.blk_sz = 4096; + shdr.total_blks = 1; + shdr.total_chunks = 0; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.truncate(shdr.total_blks * shdr.blk_sz), + mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("chunk header"), std::string::npos); +} + +TEST_F(SparseTest, CheckInvalidRawChunkFatal) { + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4; + shdr.total_blks = 1; + shdr.total_chunks = 1; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + char buf[1024]; - uint64_t bytesRead; - uint64_t pos; - buildDataHeaderProperSized(); - ASSERT_TRUE(sparseOpen()); - ASSERT_TRUE(sparseRead(buf, sizeof(buf), &bytesRead)); - ASSERT_EQ(bytesRead, 0u); - ASSERT_TRUE(sparseSeek(1000, SEEK_SET)); - ASSERT_TRUE(sparseSeek(1000, SEEK_CUR)); - ASSERT_TRUE(sparseTell(&pos)); - ASSERT_EQ(pos, 2000u); - ASSERT_TRUE(sparseRead(buf, sizeof(buf), &bytesRead)); - ASSERT_EQ(bytesRead, 0u); - ASSERT_TRUE(sparseClose()); + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + + mb::sparse::ChunkHeader chdr = {}; + chdr.chunk_type = mb::sparse::CHUNK_TYPE_RAW; + chdr.chunk_sz = 1; // 4 bytes + chdr.total_sz = shdr.chunk_hdr_sz + chdr.chunk_sz * shdr.blk_sz + 1; + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.write("1234", 4, &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("data blocks"), std::string::npos); } -TEST_F(SparseTest, OpenExtraSizedHeader) +TEST_F(SparseTest, CheckInvalidFillChunkFatal) { + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4; + shdr.total_blks = 1; + shdr.total_chunks = 1; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + char buf[1024]; - uint64_t bytesRead; - uint64_t pos; - buildDataHeaderExtraSized(); - ASSERT_TRUE(sparseOpen()); - ASSERT_TRUE(sparseRead(buf, sizeof(buf), &bytesRead)); - ASSERT_EQ(bytesRead, 0u); - ASSERT_TRUE(sparseSeek(1000, SEEK_SET)); - ASSERT_TRUE(sparseSeek(1000, SEEK_CUR)); - ASSERT_TRUE(sparseTell(&pos)); - ASSERT_EQ(pos, 2000u); - ASSERT_TRUE(sparseRead(buf, sizeof(buf), &bytesRead)); - ASSERT_EQ(bytesRead, 0u); - ASSERT_TRUE(sparseClose()); + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + + mb::sparse::ChunkHeader chdr = {}; + chdr.chunk_type = mb::sparse::CHUNK_TYPE_FILL; + chdr.chunk_sz = 1; // 4 bytes + chdr.total_sz = shdr.chunk_hdr_sz + 0; + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("Data size"), std::string::npos); } -TEST_F(SparseTest, OpenUnderSizedHeader) +TEST_F(SparseTest, CheckInvalidSkipChunkFatal) { + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4; + shdr.total_blks = 1; + shdr.total_chunks = 1; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + char buf[1024]; - uint64_t bytesRead; - uint64_t pos; - buildDataHeaderUnderSized(); - ASSERT_FALSE(sparseOpen()); - ASSERT_FALSE(sparseRead(buf, sizeof(buf), &bytesRead)); - ASSERT_FALSE(sparseSeek(1000, SEEK_SET)); - ASSERT_FALSE(sparseTell(&pos)); - ASSERT_FALSE(sparseClose()); + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + + mb::sparse::ChunkHeader chdr = {}; + chdr.chunk_type = mb::sparse::CHUNK_TYPE_DONT_CARE; + chdr.chunk_sz = 1; // 4 bytes + chdr.total_sz = shdr.chunk_hdr_sz + 1; + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("Data size"), std::string::npos); } -TEST_F(SparseTest, ReadValidSparseFile) +TEST_F(SparseTest, CheckInvalidCrc32ChunkFatal) { - char expected[48] = { - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'a', 'b', 'c', 'd', 'e', 'f', - 0x78, 0x56, 0x34, 0x12, 0x78, 0x56, 0x34, 0x12, 0x78, 0x56, 0x34, 0x12, - 0x78, 0x56, 0x34, 0x12, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4; + shdr.total_blks = 1; + shdr.total_chunks = 1; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + + char buf[1024]; + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + + mb::sparse::ChunkHeader chdr = {}; + chdr.chunk_type = mb::sparse::CHUNK_TYPE_CRC32; + chdr.chunk_sz = 1; // 4 bytes + chdr.total_sz = shdr.chunk_hdr_sz; + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("data size"), std::string::npos); +} + +TEST_F(SparseTest, CheckInvalidChunkTotalSizeFatal) +{ + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4; + shdr.total_blks = 1; + shdr.total_chunks = 1; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); char buf[1024]; - uint64_t bytesRead; + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + + mb::sparse::ChunkHeader chdr = {}; + chdr.chunk_type = mb::sparse::CHUNK_TYPE_FILL; + chdr.chunk_sz = 1; // 4 bytes + chdr.total_sz = shdr.chunk_hdr_sz - 1; + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("chunk size"), std::string::npos); +} + +TEST_F(SparseTest, CheckInvalidChunkTypeFatal) +{ + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4; + shdr.total_blks = 1; + shdr.total_chunks = 1; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + + char buf[1024]; + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + + mb::sparse::ChunkHeader chdr = {}; + chdr.chunk_type = 0xaabb; + chdr.chunk_sz = 1; // 4 bytes + chdr.total_sz = shdr.chunk_hdr_sz; + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("chunk type"), std::string::npos); +} + +TEST_F(SparseTest, CheckReadTruncatedChunkHeaderFatal) +{ + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4; + shdr.total_blks = 1; + shdr.total_chunks = 1; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + + char buf[1024]; + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + + mb::sparse::ChunkHeader chdr = {}; + chdr.chunk_type = mb::sparse::CHUNK_TYPE_FILL; + chdr.chunk_sz = 1; // 4 bytes + chdr.total_sz = shdr.chunk_hdr_sz; + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr) / 2, &n), + mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("chunk header"), std::string::npos); +} + +TEST_F(SparseTest, CheckReadOversizedChunkDataFatal) +{ + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4; + shdr.total_blks = 1; + shdr.total_chunks = 1; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + + char buf[1024]; + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + + mb::sparse::ChunkHeader chdr = {}; + chdr.chunk_type = mb::sparse::CHUNK_TYPE_RAW; + chdr.chunk_sz = 2; // 8 bytes + chdr.total_sz = shdr.chunk_hdr_sz + chdr.chunk_sz * shdr.blk_sz; + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.write("01234567", 8, &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("file size"), std::string::npos); +} + +TEST_F(SparseTest, CheckReadUndersizedChunkDataFatal) +{ + mb::sparse::SparseHeader shdr = {}; + shdr.magic = mb::sparse::SPARSE_HEADER_MAGIC; + shdr.major_version = mb::sparse::SPARSE_HEADER_MAJOR_VER; + shdr.minor_version = 0; + shdr.file_hdr_sz = sizeof(mb::sparse::SparseHeader); + shdr.chunk_hdr_sz = sizeof(mb::sparse::ChunkHeader); + shdr.blk_sz = 4; + shdr.total_blks = 2; + shdr.total_chunks = 1; + shdr.image_checksum = 0; + fix_sparse_header_byte_order(shdr); + + char buf[1024]; + size_t n; + + ASSERT_EQ(_source_file.write(&shdr, sizeof(shdr), &n), mb::FileStatus::OK); + + mb::sparse::ChunkHeader chdr = {}; + chdr.chunk_type = mb::sparse::CHUNK_TYPE_RAW; + chdr.chunk_sz = 1; // 4 bytes + chdr.total_sz = shdr.chunk_hdr_sz + chdr.chunk_sz * shdr.blk_sz; + fix_chunk_header_byte_order(chdr); + + ASSERT_EQ(_source_file.write(&chdr, sizeof(chdr), &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.write("0123", 1, &n), mb::FileStatus::OK); + ASSERT_EQ(_source_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::OK); + + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::FATAL); + ASSERT_NE(_file.error_string().find("chunk does not end"), + std::string::npos); +} + +// All further tests use a valid sparse file + +TEST_F(SparseTest, ReadValidDataWithSeekableFile) +{ + char buf[1024]; + size_t n; uint64_t pos; - buildDataCompleteValid(); + build_valid_data(); // Check that valid sparse header can be opened - ASSERT_TRUE(sparseOpen()); + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); // Check that the entire file could be read and that the contents are // correct - ASSERT_TRUE(sparseRead(buf, sizeof(buf), &bytesRead)); - ASSERT_EQ(bytesRead, 48u); - ASSERT_EQ(memcmp(buf, expected, 48), 0); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::OK); + ASSERT_EQ(n, sizeof(expected_valid_data)); + ASSERT_EQ(memcmp(buf, expected_valid_data, sizeof(expected_valid_data)), 0); // Check that partial read on chunk boundary works - ASSERT_TRUE(sparseSeek(-16, SEEK_END)); - ASSERT_TRUE(sparseRead(buf, sizeof(buf), &bytesRead)); - ASSERT_EQ(bytesRead, 16u); - ASSERT_EQ(memcmp(buf, expected + 32, 16), 0); + ASSERT_EQ(_file.seek(-16, SEEK_END, nullptr), mb::FileStatus::OK); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::OK); + ASSERT_EQ(n, 16u); + ASSERT_EQ(memcmp(buf, expected_valid_data + 32, 16), 0); // Check that partial read not on chunk boundary works - ASSERT_TRUE(sparseSeek(33, SEEK_SET)); - ASSERT_TRUE(sparseRead(buf, sizeof(buf), &bytesRead)); - ASSERT_EQ(bytesRead, 15u); - ASSERT_EQ(memcmp(buf, expected + 33, 15), 0); + ASSERT_EQ(_file.seek(33, SEEK_SET, nullptr), mb::FileStatus::OK); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::OK); + ASSERT_EQ(n, 15u); + ASSERT_EQ(memcmp(buf, expected_valid_data + 33, 15), 0); // Check that seeking past EOF is allowed and doing so returns no data - ASSERT_TRUE(sparseSeek(1000, SEEK_SET)); - ASSERT_TRUE(sparseSeek(1000, SEEK_CUR)); - ASSERT_TRUE(sparseTell(&pos)); + ASSERT_EQ(_file.seek(1000, SEEK_SET, nullptr), mb::FileStatus::OK); + ASSERT_EQ(_file.seek(1000, SEEK_CUR, &pos), mb::FileStatus::OK); ASSERT_EQ(pos, 2000u); - ASSERT_TRUE(sparseRead(buf, sizeof(buf), &bytesRead)); - ASSERT_EQ(bytesRead, 0u); + ASSERT_EQ(_file.read(buf, sizeof(buf), &n), mb::FileStatus::OK); + ASSERT_EQ(n, 0u); // Check that seeking to the end of the sparse file works - ASSERT_TRUE(sparseSeek(0, SEEK_END)); - ASSERT_TRUE(sparseTell(&pos)); + ASSERT_EQ(_file.seek(0, SEEK_END, &pos), mb::FileStatus::OK); ASSERT_EQ(pos, 48u); - ASSERT_TRUE(sparseClose()); + ASSERT_EQ(_file.close(), mb::FileStatus::OK); } -TEST_F(SparseTest, ReadValidSparseFileNoSeek) +TEST_F(SparseTest, ReadValidDataWithSkippableFile) { - char expected[48] = { - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'a', 'b', 'c', 'd', 'e', 'f', - 0x78, 0x56, 0x34, 0x12, 0x78, 0x56, 0x34, 0x12, 0x78, 0x56, 0x34, 0x12, - 0x78, 0x56, 0x34, 0x12, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; + char buf[1024]; + char *ptr = buf; + size_t remaining = sizeof(buf); + size_t n; + uint64_t total = 0; + mb::FileStatus ret; + build_valid_data(); + + // Check that valid sparse header can be opened + _source_file.set_seekability(mb::sparse::Seekability::CAN_SKIP); + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); + // Check that the entire file could be read and that the contents are + // correct. Read one byte at a time to test that a read while the sparse + // file pointer is in the middle of a raw chunk works. + while (remaining > 0 + && (ret = _file.read(ptr, 1, &n)) == mb::FileStatus::OK + && n > 0) { + ptr += n; + total += n; + remaining -= n; + } + ASSERT_EQ(ret, mb::FileStatus::OK); + ASSERT_EQ(total, sizeof(expected_valid_data)); + ASSERT_EQ(remaining, sizeof(buf) - sizeof(expected_valid_data)); + ASSERT_EQ(n, 0u); + ASSERT_EQ(memcmp(buf, expected_valid_data, sizeof(expected_valid_data)), 0); + + // Check that seeking fails + ASSERT_EQ(_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::UNSUPPORTED); + + ASSERT_EQ(_file.close(), mb::FileStatus::OK); +} + +TEST_F(SparseTest, ReadValidDataWithUnseekableFile) +{ char buf[1024]; char *ptr = buf; size_t remaining = sizeof(buf); - uint64_t bytesRead; + size_t n; uint64_t total = 0; - bool ret; - buildDataCompleteValid(); + mb::FileStatus ret; + build_valid_data(); // Check that valid sparse header can be opened - ASSERT_TRUE(sparseOpenNoSeek()); + _source_file.set_seekability(mb::sparse::Seekability::CAN_READ); + ASSERT_EQ(_file.open(&_source_file), mb::FileStatus::OK); // Check that the entire file could be read and that the contents are // correct. Read one byte at a time to test that a read while the sparse - // file pointer is in the middle of a raw chunk works (this normally - // requires seeking when a seek callback is provided). + // file pointer is in the middle of a raw chunk works. while (remaining > 0 - && (ret = sparseRead(ptr, 1, &bytesRead)) - && bytesRead > 0) { - ptr += bytesRead; - total += bytesRead; - remaining -= bytesRead; + && (ret = _file.read(ptr, 1, &n)) == mb::FileStatus::OK + && n > 0) { + ptr += n; + total += n; + remaining -= n; } - ASSERT_TRUE(ret); - ASSERT_EQ(total, 48u); - ASSERT_EQ(remaining, sizeof(buf) - 48); - ASSERT_EQ(bytesRead, 0u); - ASSERT_EQ(memcmp(buf, expected, 48), 0); + ASSERT_EQ(ret, mb::FileStatus::OK); + ASSERT_EQ(total, sizeof(expected_valid_data)); + ASSERT_EQ(remaining, sizeof(buf) - sizeof(expected_valid_data)); + ASSERT_EQ(n, 0u); + ASSERT_EQ(memcmp(buf, expected_valid_data, sizeof(expected_valid_data)), 0); // Check that seeking fails - ASSERT_FALSE(sparseSeek(0, SEEK_SET)); + ASSERT_EQ(_file.seek(0, SEEK_SET, nullptr), mb::FileStatus::UNSUPPORTED); - ASSERT_TRUE(sparseClose()); + ASSERT_EQ(_file.close(), mb::FileStatus::OK); } diff --git a/odinupdater/fuse-sparse.cpp b/odinupdater/fuse-sparse.cpp index 38c32b404..50d2b9149 100644 --- a/odinupdater/fuse-sparse.cpp +++ b/odinupdater/fuse-sparse.cpp @@ -1,5 +1,25 @@ +/* + * Copyright (C) 2016-2017 Andrew Gunnerson + * + * This file is part of DualBootPatcher + * + * DualBootPatcher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * DualBootPatcher is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with DualBootPatcher. If not, see . + */ + #define FUSE_USE_VERSION 26 +#include #include #include @@ -9,7 +29,6 @@ #include #include -#include #include // fuse @@ -33,77 +52,11 @@ static uint64_t sparse_size; struct context { - SparseCtx *sctx; - mb::StandardFile file; - pthread_mutex_t mutex; + mb::StandardFile source_file; + mb::sparse::SparseFile sparse_file; + std::mutex mutex; }; -/*! - * \brief Open callback for sparseOpen() - */ -static bool cb_open(void *userData) -{ - context *ctx = static_cast(userData); - if (ctx->file.open(source_fd_path, mb::FileOpenMode::READ_ONLY) - != mb::FileStatus::OK) { - fprintf(stderr, "%s: Failed to open: %s\n", - source_fd_path, ctx->file.error_string().c_str()); - return false; - } - return true; -} - -/*! - * \brief Close callback for sparseOpen() - */ -static bool cb_close(void *userData) -{ - context *ctx = static_cast(userData); - if (ctx->file.close() != mb::FileStatus::OK) { - fprintf(stderr, "%s: Failed to close: %s\n", - source_fd_path, ctx->file.error_string().c_str()); - return false; - } - return true; -} - -/*! - * \brief Read callback for sparseOpen() - */ -static bool cb_read(void *buf, uint64_t size, uint64_t *bytesRead, - void *userData) -{ - context *ctx = static_cast(userData); - size_t total = 0; - while (size > 0) { - size_t partial; - if (ctx->file.read(buf, size, &partial) != mb::FileStatus::OK) { - fprintf(stderr, "%s: Failed to read: %s\n", - source_fd_path, ctx->file.error_string().c_str()); - return false; - } - size -= partial; - total += partial; - buf = static_cast(buf) + partial; - } - *bytesRead = total; - return true; -} - -/*! - * \brief Seek callback for sparseOpen() - */ -static bool cb_seek(int64_t offset, int whence, void *userData) -{ - context *ctx = static_cast(userData); - if (ctx->file.seek(offset, whence, nullptr) != mb::FileStatus::OK) { - fprintf(stderr, "%s: Failed to seek: %s\n", - source_fd_path, ctx->file.error_string().c_str()); - return false; - } - return true; -} - /*! * \brief Open callback for fuse */ @@ -120,17 +73,21 @@ static int fuse_open(const char *path, fuse_file_info *fi) return -ENOMEM; } - ctx->sctx = sparseCtxNew(); - if (!ctx->sctx) { + if (ctx->source_file.open(source_fd_path, mb::FileOpenMode::READ_ONLY) + != mb::FileStatus::OK) { + fprintf(stderr, "%s: Failed to open file: %s\n", + source_fd_path, ctx->source_file.error_string().c_str()); + auto error = ctx->source_file.error(); delete ctx; - return -ENOMEM; + return error < 0 ? error : -EIO; } - if (!sparseOpen(ctx->sctx, &cb_open, &cb_close, &cb_read, &cb_seek, nullptr, - ctx)) { - sparseCtxFree(ctx->sctx); + if (ctx->sparse_file.open(&ctx->source_file) != mb::FileStatus::OK) { + fprintf(stderr, "%s: Failed to open sparse file: %s\n", + source_fd_path, ctx->sparse_file.error_string().c_str()); + auto error = ctx->sparse_file.error(); delete ctx; - return -EIO; + return error < 0 ? error : -EIO; } fi->fh = reinterpret_cast(ctx); @@ -146,7 +103,6 @@ static int fuse_release(const char *path, fuse_file_info *fi) (void) path; context *ctx = reinterpret_cast(fi->fh); - sparseCtxFree(ctx->sctx); delete ctx; return 0; @@ -162,16 +118,19 @@ static int fuse_read_locked(context *ctx, char *buf, size_t size, OFF_T offset) { // Seek to position - if (!sparseSeek(ctx->sctx, offset, SEEK_SET)) { - return -1; + if (ctx->sparse_file.seek(offset, SEEK_SET, nullptr) + != mb::FileStatus::OK) { + auto error = ctx->sparse_file.error(); + return error < 0 ? error : -EIO; } - uint64_t bytes_read; - if (!sparseRead(ctx->sctx, buf, size, &bytes_read)) { - return -1; + size_t n; + if (ctx->sparse_file.read(buf, size, &n) != mb::FileStatus::OK) { + auto error = ctx->sparse_file.error(); + return error < 0 ? error : -EIO; } - return bytes_read; + return n; } /*! @@ -184,11 +143,8 @@ static int fuse_read(const char *path, char *buf, size_t size, OFF_T offset, context *ctx = reinterpret_cast(fi->fh); - pthread_mutex_lock(&ctx->mutex); - int ret = fuse_read_locked(ctx, buf, size, offset); - pthread_mutex_unlock(&ctx->mutex); - - return ret; + std::lock_guard lock(ctx->mutex); + return fuse_read_locked(ctx, buf, size, offset); } /*! @@ -209,32 +165,26 @@ static int fuse_getattr(const char *path, struct stat *stbuf) */ static int get_sparse_file_size() { - context *ctx = new(std::nothrow) context(); - if (!ctx) { - return -ENOMEM; - } + mb::StandardFile source_file; + mb::sparse::SparseFile sparse_file; - ctx->sctx = sparseCtxNew(); - if (!ctx->sctx) { - delete ctx; - return -ENOMEM; + if (source_file.open(source_fd_path, mb::FileOpenMode::READ_ONLY) + != mb::FileStatus::OK) { + fprintf(stderr, "%s: Failed to open file: %s\n", + source_fd_path, source_file.error_string().c_str()); + auto error = source_file.error(); + return error < 0 ? error : -EIO; } - if (!sparseOpen(ctx->sctx, &cb_open, &cb_close, &cb_read, &cb_seek, nullptr, - ctx)) { - sparseCtxFree(ctx->sctx); - delete ctx; - return -EIO; + if (sparse_file.open(&source_file) != mb::FileStatus::OK) { + fprintf(stderr, "%s: Failed to open sparse file: %s\n", + source_fd_path, sparse_file.error_string().c_str()); + auto error = sparse_file.error(); + return error < 0 ? error : -EIO; } - if (!sparseSize(ctx->sctx, &sparse_size)) { - sparseCtxFree(ctx->sctx); - delete ctx; - return -EIO; - } + sparse_size = sparse_file.size(); - sparseCtxFree(ctx->sctx); - delete ctx; return 0; } diff --git a/odinupdater/odinupdater.cpp b/odinupdater/odinupdater.cpp index 8f16d3e76..17b1c2eef 100644 --- a/odinupdater/odinupdater.cpp +++ b/odinupdater/odinupdater.cpp @@ -31,6 +31,10 @@ #include #include +// libmbcommon +#include "mbcommon/file/callbacks.h" +#include "mbcommon/file/standard.h" + // libmbsparse #include "mbsparse/sparse.h" @@ -71,7 +75,6 @@ #define PROP_BOOT_DEV "boot" typedef std::unique_ptr ScopedArchive; -typedef std::unique_ptr ScopedSparseCtx; enum class ExtractResult { @@ -348,10 +351,12 @@ static bool load_block_devs() return true; } -static bool cb_zip_read(void *buf, uint64_t size, uint64_t *bytes_read, - void *user_data) +static mb::FileStatus cb_zip_read(mb::File &file, void *userdata, + void *buf, size_t size, size_t *bytes_read) { - archive *a = (archive *) user_data; + (void) file; + + archive *a = static_cast(userdata); uint64_t total = 0; while (size > 0) { @@ -359,7 +364,7 @@ static bool cb_zip_read(void *buf, uint64_t size, uint64_t *bytes_read, if (n < 0) { error("libarchive: Failed to read data: %s", archive_error_string(a)); - return false; + return mb::FileStatus::FAILED; } else if (n == 0) { break; } @@ -370,7 +375,7 @@ static bool cb_zip_read(void *buf, uint64_t size, uint64_t *bytes_read, } *bytes_read = total; - return true; + return mb::FileStatus::OK; } #if DEBUG_SKIP_FLASH_SYSTEM @@ -380,18 +385,11 @@ static ExtractResult extract_sparse_file(const char *zip_filename, const char *out_filename) { ScopedArchive a{archive_read_new(), &archive_read_free}; - ScopedSparseCtx ctx{sparseCtxNew(), &sparseCtxFree}; - char buf[10240]; - uint64_t n; - bool sparse_ret; - int fd; - uint64_t cur_bytes = 0; - uint64_t max_bytes = 0; - uint64_t old_bytes = 0; - double old_ratio; - double new_ratio; + mb::CallbackFile file; + mb::sparse::SparseFile sparse_file; + mb::StandardFile out_file; - if (!a || !ctx) { + if (!a) { error("Out of memory"); return ExtractResult::ERROR; } @@ -406,53 +404,69 @@ static ExtractResult extract_sparse_file(const char *zip_filename, return result; } - if (!sparseOpen(ctx.get(), nullptr, nullptr, &cb_zip_read, nullptr, nullptr, - a.get())) { - error("Failed to open sparse file"); + if (file.open(nullptr, nullptr, &cb_zip_read, nullptr, nullptr, nullptr, + a.get()) != mb::FileStatus::OK) { + error("Failed to open sparse file in zip: %s", + file.error_string().c_str()); return ExtractResult::ERROR; } - fd = open64(out_filename, - O_CREAT | O_TRUNC | O_WRONLY | O_CLOEXEC | O_LARGEFILE, 0600); - if (fd < 0) { - error("%s: Failed to open: %s", out_filename, strerror(errno)); + if (sparse_file.open(&file) != mb::FileStatus::OK) { + error("Failed to open sparse file: %s", + sparse_file.error_string().c_str()); return ExtractResult::ERROR; } - auto close_fd = mb::util::finally([fd]{ - close(fd); - }); + if (out_file.open(out_filename, mb::FileOpenMode::WRITE_ONLY) + != mb::FileStatus::OK) { + error("%s: Failed to open for writing: %s", + out_filename, out_file.error_string().c_str()); + return ExtractResult::ERROR; + } - sparseSize(ctx.get(), &max_bytes); + char buf[10240]; + size_t n; + mb::FileStatus ret; + uint64_t cur_bytes = 0; + uint64_t max_bytes = sparse_file.size(); + uint64_t old_bytes = 0; set_progress(0); - while ((sparse_ret = sparseRead(ctx.get(), buf, sizeof(buf), &n)) && n > 0) { + while ((ret = sparse_file.read(buf, sizeof(buf), &n)) == mb::FileStatus::OK + && n > 0) { // Rate limit: update progress only after difference exceeds 0.1% - old_ratio = (double) old_bytes / max_bytes; - new_ratio = (double) cur_bytes / max_bytes; + double old_ratio = static_cast(old_bytes) / max_bytes; + double new_ratio = static_cast(cur_bytes) / max_bytes; if (new_ratio - old_ratio >= 0.001) { set_progress(new_ratio); old_bytes = cur_bytes; } char *out_ptr = buf; - ssize_t nwritten; + size_t nwritten; - do { - if ((nwritten = write(fd, out_ptr, n)) < 0) { - error("%s: Failed to write: %s", - out_filename, strerror(errno)); + while (n > 0) { + if (out_file.write(buf, n, &nwritten) != mb::FileStatus::OK) { + error("%s: Failed to write file: %s", + out_filename, out_file.error_string().c_str()); return ExtractResult::ERROR; } n -= nwritten; out_ptr += nwritten; cur_bytes += nwritten; - } while (n > 0); + } + } + if (ret != mb::FileStatus::OK) { + error("Failed to read sparse file %s: %s", + zip_filename, sparse_file.error_string().c_str()); + return ExtractResult::ERROR; } - if (!sparse_ret) { - error("Failed to read sparse file %s", zip_filename); + + if (out_file.close() != mb::FileStatus::OK) { + error("%s: Failed to close file: %s", + out_filename, out_file.error_string().c_str()); return ExtractResult::ERROR; }