Merge branch 'rc'

.gitignore

@@ -302,3 +302,4 @@ __pycache__/
 *.xsd.cs
 
 *.map
+*.d

Makefile

@@ -0,0 +1,89 @@
+SRC = $(wildcard *.c)
+OBJ = $(SRC:.c=.o)
+DEP = $(OBJ:.o=.d)
+
+CFLAGS:=-Wall -O2 -pthread -fPIC
+CC:=gcc
+
+ASFLAGS :=
+
+SONAME:=libfast-lzma2.so.1
+REAL_NAME:=libfast-lzma2.so.1.0
+LINKER_NAME=libfast-lzma2.so
+
+x86_64:=0
+
+ifeq ($(OS),Windows_NT)
+	CFLAGS+=-DFL2_DLL_EXPORT=1
+	LINKER_NAME=libfast-lzma2.dll
+	SONAME:=$(LINKER_NAME)
+	REAL_NAME:=$(LINKER_NAME)
+ifeq ($(PROCESSOR_ARCHITECTURE),AMD64)
+	ASFLAGS+=-DMS_x64_CALL=1
+	x86_64:=1
+endif
+else
+	PROC_ARCH:=$(shell uname -p)
+ifneq ($(PROC_ARCH),x86_64)
+	PROC_ARCH:=$(shell uname -m)
+endif
+ifeq ($(PROC_ARCH),x86_64)
+	ASFLAGS+=-DMS_x64_CALL=0
+	x86_64:=1
+endif
+endif
+
+ifeq ($(x86_64),1)
+	CFLAGS+=-DLZMA2_DEC_OPT
+	OBJ+=lzma_dec_x86_64.o
+endif
+
+libfast-lzma2 : $(OBJ)
+	$(CC) -shared -pthread -Wl,-soname,$(SONAME) -o $(REAL_NAME) $(OBJ)
+
+-include $(DEP)
+
+%.d: %.c
+	@$(CC) $(CFLAGS) $< -MM -MT $(@:.d=.o) >$@
+
+DESTDIR:=
+PREFIX:=/usr/local
+LIBDIR:=$(DESTDIR)$(PREFIX)/lib
+
+.PHONY: install
+install:
+ifeq ($(OS),Windows_NT)
+	strip -g $(REAL_NAME)
+else
+	mkdir -p $(LIBDIR)
+	cp $(REAL_NAME) $(LIBDIR)/$(REAL_NAME)
+	strip -g $(LIBDIR)/$(REAL_NAME)
+	chmod 0755 $(LIBDIR)/$(REAL_NAME)
+	cd $(LIBDIR) && ln -sf $(REAL_NAME) $(LINKER_NAME)
+	ldconfig $(LIBDIR)
+	mkdir -p $(DESTDIR)$(PREFIX)/include
+	cp fast-lzma2.h $(DESTDIR)$(PREFIX)/include/
+	cp fl2_errors.h $(DESTDIR)$(PREFIX)/include/
+endif
+
+.PHONY: uninstall
+uninstall:
+ifeq ($(OS),Windows_NT)
+	rm -f libfast-lzma2.dll
+else
+	rm -f $(LIBDIR)/$(LINKER_NAME)
+	rm -f $(LIBDIR)/$(REAL_NAME)
+	ldconfig $(LIBDIR)
+	rm -f $(DESTDIR)$(PREFIX)/include/fast-lzma2.h
+	rm -f $(DESTDIR)$(PREFIX)/include/fl2_errors.h
+endif
+
+.PHONY: test
+test:
+	cd test && make
+	test/file_test radix_engine.h
+	@echo "Test file compressed and decompressed ok."
+
+.PHONY: clean
+clean:
+	rm -f $(REAL_NAME) $(OBJ) $(DEP)

README.md

@@ -1,4 +1,4 @@
-The __Fast LZMA2 Library__ is a lossless high-ratio data compression library based on the LZMA2 codec in 7-zip.
+The __Fast LZMA2 Library__ is a lossless high-ratio data compression library based on Igor Pavlov's LZMA2 codec from 7-zip.
 
 Binaries of 7-Zip forks which use the algorithm are available in the [7-Zip-FL2 project] and the [7-Zip-zstd project]. The library
 is also embedded in a fork of XZ Utils, named [FXZ Utils].
@@ -12,13 +12,13 @@ speed gain at the higher levels over the default LZMA2 algorithm used in 7-zip,
 depend on the nature of the source data. The library also uses some threading, portability, and testing code from Zstandard.
 
 Use of the radix match-finder allows multi-threaded execution employing a simple threading model and with low memory usage. The
-library can compress using many threads without dividing the input into large chunks which require the duplication of the match-
-finder tables and chains. Extra memory used per thread is typically no more than a few megabytes.
+library can compress using many threads without dividing the input into large chunks which require the duplication of the
+match-finder tables and chains. Extra memory used per thread is typically no more than a few megabytes.
 
 The largest caveat is that the match-finder is a block algorithm, and to achieve about the same ratio as 7-Zip requires double the
 dictionary size, which raises the decompression memory usage. By default it uses the same dictionary size as 7-Zip, resulting in
-output that is larger by about 1% of the compressed size. A high-compression option is provided to select parameters which achieve
-higher compression on smaller dictionaries. The speed/ratio tradeoff is less optimal with this enabled.
+output that is larger by about 1%-5% of the compressed size. A high-compression option is provided to select parameters which
+achieve higher compression on smaller dictionaries. The speed/ratio tradeoff is less optimal with this enabled.
 
 Here are the results of an in-memory benchmark using two threads on the [Silesia compression corpus] vs the 7-zip 19.00 LZMA2
 encoder. The design goal for the encoder and compression level parameters was to move the line as far as possible toward the top
@@ -31,21 +31,41 @@ Compression data rate vs ratio
 ------------------------------
 ![Compression data rate vs ratio](doc/images/bench_mt2.png "Compression data rate vs ratio")
 
-### Build
+## Build
 
-Build methods are not yet comprehensive for all systems. There are VS 2015 projects for building a benchmark program, fuzz tester,
-and a DLL. Makefiles for gcc are included for the benchmark, fuzzer and DLL, and Evan Nemerson has contributed a CMake file, which
-currently does not enable the assembler-optimized decoder. If anyone would like to help improve the build methods, please do so.
-On POSIX systems, FXZ Utils provides a derivative of liblzma from XZ Utils as a wrapper for Fast LZMA2. It is built using GNU
-autotools.
+### Windows
 
-### Status
+The build\VS folder contains a solution for VS2015. It includes projects for a benchmark program, fuzz tester, file compression
+tester, and DLL.
 
-The library has passed a large amount of fuzz testing, and testing on file sets selected at random in the Radyx file archiver. An
+### POSIX
+
+Run `make` in the root directory to build the shared library, then `make install` to allow other programs to use the headers and
+libfast-lzma2. Use `make test` to build the file compression tester and run it on a test file.
+
+The bench, fuzzer and test directories have makefiles for these programs. The CMake file present in earlier releases does not
+have an installation script so is not currently included.
+
+If a build fails on any system please open an issue on github.
+
+[FXZ Utils] provides a derivative of liblzma from XZ Utils as a wrapper for Fast LZMA2. It is built using GNU autotools.
+
+## Status
+
+The library has passed long periods of fuzz testing, and testing on file sets selected at random in the Radyx file archiver. An
 earlier version was released in the 7-Zip forks linked above. The library is considered suitable for production environments.
 However, no warranty or fitness for a particular purpose is expressed or implied.
 
 
+Changes in v1.0.1:
+
+- The root makefile for GNU make now builds and installs a shared library and headers.
+- Fixed a potential crash on memory allocation failure during structure allocations for multi-threaded decoding.
+- Added a file compression test program.
+- Renamed the VS DLL project and some structures / types.
+- Removed some duplicated typedefs.
+
+
 Changes in v1.0.0:
 
 - Breaking changes have been made to the API functions.
@@ -58,7 +78,7 @@ Changes in v1.0.0:
 - Replaced the callbacks for writing and progress with timeouts and new functions to gain direct access to the dictionary buffer and
   the compressed data buffers.
 - Added Igor Pavlov's assembler-optimized decoder.
-- Multithreaded decompression.
+- Multi-threaded decompression.
 
 
 Changes in v0.9.2:
@@ -72,6 +92,6 @@ Changes in v0.9.1:
 - Added an incompressibility checker which processes high-entropy (e.g. encrypted or already compressed) data about twice as fast
   as before.
 
-### License
+## License
 
 Fast LZMA2 is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).

bench/Makefile

@@ -1,67 +1,45 @@
-objects = \
-../dict_buffer.o \
-../fl2_common.o \
-../fl2_compress.o \
-../fl2_decompress.o \
-../fl2_pool.o \
-../fl2_threading.o \
-../lzma2_dec.o \
-../lzma2_enc.o \
-../radix_bitpack.o \
-../radix_mf.o \
-../radix_struct.o \
-../range_enc.o \
-../util.o \
-../xxhash.o \
-../tests/datagen.o \
-bench.o
+SRC = $(wildcard ../*.c) ../fuzzer/datagen.c bench.c
+OBJ = $(SRC:.c=.o)
+DEP = $(OBJ:.o=.d)
 
-CFLAGS := -Wall -O3 -pthread
-CC := gcc
+CFLAGS:=-Wall -O2 -pthread
+CC:=gcc
 
-ASFLAGS :=
+ASFLAGS:=
 
-x86_64 := 0
-
-exe_ext :=
+x86_64:=0
+EXT:=
 
 ifeq ($(OS),Windows_NT)
-	exe_ext :=.exe
+	CFLAGS+=-DFL2_DLL_EXPORT=1
+	EXT:=.exe
 ifeq ($(PROCESSOR_ARCHITECTURE),AMD64)
-    ASFLAGS += -D MS_x64_CALL=1
-	x86_64 := 1
+	ASFLAGS+=-DMS_x64_CALL=1
+	x86_64:=1
 endif
 else
-	UNAME_P := $(shell uname -p)
-ifeq ($(UNAME_P),x86_64)
-    ASFLAGS += -D MS_x64_CALL=0
-	x86_64 := 1
+	PROC_ARCH:=$(shell uname -p)
+ifneq ($(PROC_ARCH),x86_64)
+	PROC_ARCH:=$(shell uname -m)
+endif
+ifeq ($(PROC_ARCH),x86_64)
+	ASFLAGS+=-DMS_x64_CALL=0
+	x86_64:=1
 endif
 endif
 
 ifeq ($(x86_64),1)
-	CFLAGS += -DLZMA2_DEC_OPT
-	objects += ../lzma_dec_x86_64.o
+	CFLAGS+=-DLZMA2_DEC_OPT
+	OBJ+=../lzma_dec_x86_64.o
 endif
 
-bench : $(objects)
-	$(CC) -pthread -o bench$(exe_ext) $(objects)
+bench : $(OBJ)
+	$(CC) -pthread -o bench$(EXT) $(OBJ)
+
+-include $(DEP)
 
-dict_buffer.o : ../dict_buffer.h ../fl2_internal.h ../mem.h ../data_block.h
-fl2_common.o : ../fast-lzma2.h ../fl2_errors.h ../fl2_internal.h
-fl2_compress.o : ../dict_buffer.h ../fast-lzma2.h ../fl2_internal.h ../mem.h ../util.h ../fl2_compress_internal.h ../fl2_threading.h ../fl2_pool.h ../radix_mf.h ../lzma2_enc.h
-fl2_decompress.o : ../fast-lzma2.h ../fl2_internal.h ../mem.h ../util.h ../lzma2_dec.h ../xxhash.h
-fl2_pool.o : ../fl2_pool.h ../fl2_internal.h
-fl2_threading.o : ../fl2_threading.h
-lzma2_dec.o : ../lzma2_dec.h ../fl2_internal.h
-lzma2_enc.o : ../fl2_internal.h ../mem.h ../lzma2_enc.h ../fl2_compress_internal.h ../radix_get.h ../radix_mf.h ../range_enc.h ../count.h
-radix_bitpack.o : ../fast-lzma2.h ../mem.h ../fl2_threading.h ../fl2_internal.h ../radix_internal.h ../radix_engine.h
-radix_mf.o : ../fast-lzma2.h ../mem.h ../fl2_internal.h ../radix_internal.h
-radix_struct.o : ../fast-lzma2.h ../mem.h ../fl2_threading.h ../fl2_internal.h ../radix_internal.h ../radix_engine.h
-range_enc.o : ../fl2_internal.h ../mem.h ../range_enc.h
-util.o : ../util.h
-xxhash.o : ../xxhash.h
-bench.o : ../fast-lzma2.h ../mem.h ../util.h ../tests/datagen.h
+%.d: %.c
+	@$(CC) $(CFLAGS) $< -MM -MT $(@:.d=.o) >$@
 
 clean:
-	rm bench$(exe_ext) $(objects)
+	rm -f bench$(EXT) $(OBJ) $(DEP)

bench/bench.c

@@ -4,7 +4,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include "../fast-lzma2.h"
-#include "../tests/datagen.h"
+#include "../fuzzer/datagen.h"
 #include "../mem.h"
 #include "../util.h"
 
@@ -125,24 +125,6 @@ static void benchmark(FL2_CCtx* fcs, FL2_DCtx* dctx, char* srcBuffer, size_t src
     }
 }
 
-#define kHash3Bits 14
-#define GET_HASH_3(data) (((data << 8) * 506832829U) >> (32 - kHash3Bits))
-
-int hash_test(void)
-{
-    for (U32 i = 0; i < 0xFFFF; ++i) {
-        BYTE flags[1 << kHash3Bits];
-        memset(flags, 0, sizeof(flags));
-        for (U32 j = 0; j <= 0xFF; ++j) {
-            U32 hash = GET_HASH_3(j | (i << 8));
-            if (flags[hash])
-                return 0;
-            flags[hash] = 1;
-        }
-    }
-    return 1;
-}
-
 static int parse_params(FL2_CCtx* fcs, int argc, char** argv)
 {
     for (int i = 2; i < argc; ++i) {

bench/bench.vcxproj

@@ -19,14 +19,14 @@
     </ProjectConfiguration>
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="..\tests\datagen.c" />
+    <ClCompile Include="..\fuzzer\datagen.c" />
     <ClCompile Include="..\util.c" />
     <ClCompile Include="bench.c" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\fast-lzma2.h" />
     <ClInclude Include="..\fl2_errors.h" />
-    <ClInclude Include="..\tests\datagen.h" />
+    <ClInclude Include="..\fuzzer\datagen.h" />
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <ProjectGuid>{E49A84FD-DE73-4A88-8968-F3B1C328B238}</ProjectGuid>
@@ -102,7 +102,7 @@
     <Link>
       <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>$(SolutionDir)$(Configuration)\libflzma2-x86.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>$(SolutionDir)$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
@@ -118,7 +118,7 @@
     <Link>
       <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>$(SolutionDir)$(Platform)\$(Configuration)\libflzma2-x64.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>$(SolutionDir)$(Platform)\$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@@ -143,7 +143,7 @@
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
-      <AdditionalDependencies>$(SolutionDir)$(Configuration)\libflzma2-x86.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>$(SolutionDir)$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
@@ -169,7 +169,7 @@
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
-      <AdditionalDependencies>$(SolutionDir)$(Platform)\$(Configuration)\libflzma2-x64.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>$(SolutionDir)$(Platform)\$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />