From 671d533ea7e38d6642249c429724f45b4b7005fb Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Sun, 21 Jul 2019 23:22:25 -0400 Subject: [PATCH] Fix seekable decompression in-memory api --- contrib/seekable_format/examples/Makefile | 7 +- .../examples/seekable_decompression_mem.c | 144 ++++++++++++++++++ contrib/seekable_format/zstdseek_decompress.c | 4 +- 3 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 contrib/seekable_format/examples/seekable_decompression_mem.c diff --git a/contrib/seekable_format/examples/Makefile b/contrib/seekable_format/examples/Makefile index 6d9562df8b4..543780f75d3 100644 --- a/contrib/seekable_format/examples/Makefile +++ b/contrib/seekable_format/examples/Makefile @@ -24,7 +24,8 @@ SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB) default: all -all: seekable_compression seekable_decompression parallel_processing +all: seekable_compression seekable_decompression seekable_decompression_mem \ + parallel_processing $(ZSTDLIB): make -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME) @@ -35,6 +36,9 @@ seekable_compression : seekable_compression.c $(SEEKABLE_OBJS) seekable_decompression : seekable_decompression.c $(SEEKABLE_OBJS) $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +seekable_decompression_mem : seekable_decompression_mem.c $(SEEKABLE_OBJS) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + parallel_processing : parallel_processing.c $(SEEKABLE_OBJS) $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ -pthread @@ -44,5 +48,6 @@ parallel_compression : parallel_compression.c $(SEEKABLE_OBJS) clean: @rm -f core *.o tmp* result* *.zst \ seekable_compression seekable_decompression \ + seekable_decompression_mem \ parallel_processing parallel_compression @echo Cleaning completed diff --git a/contrib/seekable_format/examples/seekable_decompression_mem.c b/contrib/seekable_format/examples/seekable_decompression_mem.c new file mode 100644 index 00000000000..c36d2221f97 --- /dev/null +++ b/contrib/seekable_format/examples/seekable_decompression_mem.c @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + + +#include // malloc, exit +#include // fprintf, perror, feof +#include // strerror +#include // errno +#define ZSTD_STATIC_LINKING_ONLY +#include // presumes zstd library is installed +#include + +#include "zstd_seekable.h" + +#define MIN(a, b) ((a) < (b) ? (a) : (b)) + +#define MAX_FILE_SIZE (8 * 1024 * 1024) + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror("malloc"); + exit(1); +} + +static void* realloc_orDie(void* ptr, size_t size) +{ + ptr = realloc(ptr, size); + if (ptr) return ptr; + /* error */ + perror("realloc"); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(3); +} + +static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) +{ + size_t const readSize = fread(buffer, 1, sizeToRead, file); + if (readSize == sizeToRead) return readSize; /* good */ + if (feof(file)) return readSize; /* good, reached end of file */ + /* error */ + perror("fread"); + exit(4); +} + +static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) +{ + size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); + if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ + /* error */ + perror("fwrite"); + exit(5); +} + +static size_t fclose_orDie(FILE* file) +{ + if (!fclose(file)) return 0; + /* error */ + perror("fclose"); + exit(6); +} + +static void fseek_orDie(FILE* file, long int offset, int origin) { + if (!fseek(file, offset, origin)) { + if (!fflush(file)) return; + } + /* error */ + perror("fseek"); + exit(7); +} + + +static void decompressFile_orDie(const char* fname, off_t startOffset, off_t endOffset) +{ + FILE* const fin = fopen_orDie(fname, "rb"); + FILE* const fout = stdout; + // Just for demo purposes, assume file is <= MAX_FILE_SIZE + void* const buffIn = malloc_orDie(MAX_FILE_SIZE); + size_t const inSize = fread_orDie(buffIn, MAX_FILE_SIZE, fin); + size_t const buffOutSize = ZSTD_DStreamOutSize(); /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */ + void* const buffOut = malloc_orDie(buffOutSize); + + ZSTD_seekable* const seekable = ZSTD_seekable_create(); + if (seekable==NULL) { fprintf(stderr, "ZSTD_seekable_create() error \n"); exit(10); } + + size_t const initResult = ZSTD_seekable_initBuff(seekable, buffIn, inSize); + if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } + + while (startOffset < endOffset) { + size_t const result = ZSTD_seekable_decompress(seekable, buffOut, MIN(endOffset - startOffset, buffOutSize), startOffset); + + if (ZSTD_isError(result)) { + fprintf(stderr, "ZSTD_seekable_decompress() error : %s \n", + ZSTD_getErrorName(result)); + exit(12); + } + fwrite_orDie(buffOut, result, fout); + startOffset += result; + } + + ZSTD_seekable_free(seekable); + fclose_orDie(fin); + fclose_orDie(fout); + free(buffIn); + free(buffOut); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc!=4) { + fprintf(stderr, "wrong arguments\n"); + fprintf(stderr, "usage:\n"); + fprintf(stderr, "%s FILE START END\n", exeName); + return 1; + } + + { + const char* const inFilename = argv[1]; + off_t const startOffset = atoll(argv[2]); + off_t const endOffset = atoll(argv[3]); + decompressFile_orDie(inFilename, startOffset, endOffset); + } + + return 0; +} diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c index a0206592299..abfd1e90271 100644 --- a/contrib/seekable_format/zstdseek_decompress.c +++ b/contrib/seekable_format/zstdseek_decompress.c @@ -106,7 +106,7 @@ typedef struct { static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n) { buffWrapper_t* buff = (buffWrapper_t*) opaque; - if (buff->size + n > buff->pos) return -1; + if (buff->pos + n > buff->size) return -1; memcpy(buffer, (const BYTE*)buff->ptr + buff->pos, n); buff->pos += n; return 0; @@ -124,7 +124,7 @@ static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin) newOffset = (unsigned long long)buff->pos + offset; break; case SEEK_END: - newOffset = (unsigned long long)buff->size - offset; + newOffset = (unsigned long long)buff->size + offset; break; default: assert(0); /* not possible */