Exercise fuzzers during CI builds

.circleci/config.yml

@@ -143,6 +143,85 @@ jobs:
             - ~/.cargo
             - ~/.rustup
 
+  fuzz:
+    docker:
+      - image: cossacklabs/build:ubuntu-bionic
+    environment:
+      FUZZ_TIMEOUT: 30s
+      WITH_FATAL_WARNINGS: yes
+      WITH_FATAL_SANITIZERS: yes
+    steps:
+      - run:
+          name: Install AFL and 32-bit toolchain for ASAN builds
+          command: |
+            sudo dpkg --add-architecture i386
+            sudo apt update
+            sudo DEBIAN_FRONTEND=noninteractive apt install --yes \
+               afl zip gcc-multilib libc6-dev:i386 \
+               libssl-dev:amd64 libssl-dev:i386
+      - checkout
+      - run:
+          name: Pull BoringSSL submodule
+          command: |
+            git reset HEAD
+            git submodule sync
+            git submodule update --init
+      - run:
+          name: Check "make fuzz" builds
+          command: |
+            make fuzz AFL_CC=afl-clang
+            make clean
+            make fuzz AFL_CC=afl-gcc
+            make clean
+      # Don't run them for too long, we aim for low-hanging fruit here.
+      # Ideally we'd like to wait for AFL to make one cycle and stop.
+      - run:
+          name: Fuzzing with Address Sanitizer
+          when: always
+          command: |
+            make clean
+            make fuzz AFL_CC=afl-gcc WITH_ASAN=1
+            for tool in tools/afl/input/*
+            do
+                timeout -s INT "$FUZZ_TIMEOUT" \
+                  make fuzz AFL_CC=afl-gcc WITH_ASAN=1 \
+                    AFL_FUZZ="afl-fuzz -m 1024" \
+                    FUZZ_BIN=$(basename $tool) \
+                | cat -u || true
+            done
+            cd build/afl
+            zip -r results-asan.zip output
+            cd -
+            echo
+            echo Analyzing results...
+            echo
+            ./tools/afl/analyze_crashes.sh --no-debugger
+      - store_artifacts:
+          path: build/afl/results-asan.zip
+      - run:
+          name: Fuzzing with Undefined Behavior Sanitizer
+          when: always
+          command: |
+            make clean
+            make fuzz AFL_CC=afl-clang WITH_UBSAN=1
+            for tool in tools/afl/input/*
+            do
+                timeout -s INT "$FUZZ_TIMEOUT" \
+                  make fuzz AFL_CC=afl-clang WITH_UBSAN=1 \
+                    FUZZ_BIN=$(basename $tool) \
+                | cat -u || true
+            done
+            cd build/afl
+            zip -r results-ubsan.zip output
+            cd -
+            echo
+            echo Analyzing results...
+            echo
+            ./tools/afl/analyze_crashes.sh --no-debugger
+      - store_artifacts:
+          path: build/afl/results-ubsan.zip
+      # TODO: 32-bit builds WITH_UBSAN=1
+
   x86_64:
     docker:
       - image: cossacklabs/android-build:2019.01
@@ -231,7 +310,6 @@ jobs:
       - run: make test_python
       - run: make test_ruby
       - run: make test_rust
-      - run: make fuzz
       - run: $HOME/valgrind/bin/valgrind build/tests/soter_test 2>&1 | grep "ERROR SUMMARY\|definitely lost\|indirectly lost\|possibly lost" | awk '{sum += $4} END {print $0; if ( sum > 0 ) { exit 1 } }'
       - run: $HOME/valgrind/bin/valgrind build/tests/themis_test 2>&1 | grep "ERROR SUMMARY\|definitely lost\|indirectly lost\|possibly lost" | awk '{sum += $4} END {print $0; if ( sum > 0 ) { exit 1 } }'
       - run: cover_build/tests/soter_test
@@ -624,6 +702,7 @@ workflows:
   tests:
     jobs:
       - analyze
+      - fuzz
       - benchmark
       - android
       - x86_64
@@ -646,6 +725,7 @@ workflows:
                 - stable
     jobs:
       - analyze
+      - fuzz
       - benchmark
       - android
       - x86_64

tools/afl/README.md

@@ -69,10 +69,6 @@ by tweaking the following environment variables:
     path to the fuzzer instrumentation compiler
     (`afl-clang` from $PATH by default)
 
-  - `AFL_CFLAGS` and `AFL_LDFLAGS` —
-    additional flags for compiler and linker
-    if you need them
-
 ### Analyzing results
 
 Fuzzing results are put into the build directory.
@@ -112,6 +108,42 @@ and prints a report with results and backtraces,
 formatted as Markdown.
 Run the tool with `--help` to learn more.
 
+## Caveats
+
+### Fuzzing with sanitizers
+
+Make sure to run `WITH_FATAL_SANITIZERS=yes`
+so that sanitizers call abort() on errors instead of just printing messages.
+AFL reacts to abnormal process terminations, not stderr traffic.
+
+When building `WITH_ASAN`, we produce and test 32-bit binaries.
+You will need to have 32-bit support installed
+(e.g., `gcc-multilib`, `libc6-dev:i386`, `libssl-dev:i386` on Debian).
+AFL cannot reliably work with address-sanitized 64-bit binaries
+because ASAN allocates (but does not use) terabytes of virtual memory.
+Unfortunately, AFL cannot distinguish that behavior from an allocation bug.
+If you have to test 64-bit binaries, there are workarounds (like using cgroups on Linux).
+Read AFL documentation to learn more
+(`notes_for_asan.txt`, [mirror](https://github.com/mirrorer/afl/blob/master/docs/notes_for_asan.txt)).
+
+Please note that glibc 2.25⁓2.27 has a bug that prevents ASAN from working with 32-bit binaries
+[[1](https://github.com/google/sanitizers/issues/914),
+ [2](https://github.com/google/sanitizers/issues/954),
+ [3](https://sourceware.org/ml/libc-alpha/2018-02/msg00567.html),
+ [4](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84761)].
+Currently, Clang does not have fixes applied so make sure to set `AFL_CC=afl-gcc`.
+
+You may also need to bump the memory limit up a bit.
+If AFL screams at you
+```
+[-] Whoops, the target binary crashed suddenly, before receiving any input
+    from the fuzzer! Since it seems to be built with ASAN and you have a
+    restrictive memory limit configured, this is expected; please read
+    /usr/share/doc/afl-doc/docs/notes_for_asan.txt for help.
+```
+then try setting `AFL_FUZZ="afl-fuzz -m 1024"` to increase the limit to 1 GB,
+that should be enough in most cases.
+
 ## Developing fuzzing tests
 
 ### Directory layout

tools/afl/analyze_crashes.sh

@@ -32,6 +32,7 @@ options:
                         instead of quietly producing an automated report
     -d, --debugger <path>
                         set the debugger to use, we support GDB and LLDB
+        --no-debugger   disable backtrace printing, just check for failures
 EOF
 }
 
@@ -61,6 +62,11 @@ do
         shift 2
         ;;
 
+      --no-debugger)
+        DEBUGGER=/dev/null
+        shift
+        ;;
+
       *)
         exec >&2
         echo "invalid argument: $1"
@@ -99,7 +105,10 @@ fi
 DEBUGGER_TYPE=
 
 check_debugger() {
-    if "$DEBUGGER" --version 2>/dev/null | grep --quiet lldb
+    if [ "$DEBUGGER" = "/dev/null" ]
+    then
+        DEBUGGER_TYPE=none
+    elif "$DEBUGGER" --version 2>/dev/null | grep --quiet lldb
     then
         DEBUGGER_TYPE=lldb
     elif "$DEBUGGER" --version 2>/dev/null | grep --quiet gdb
@@ -145,6 +154,18 @@ Run:
 
     $tool $file
 
+Input (base64):
+
+EOF
+    # BSD and GNU have an ongoing feud over flag names...
+    if base64 --wrap 64 </dev/null >/dev/null 2>/dev/null
+    then
+        cat "$file" | base64 --wrap 64
+    else
+        cat "$file" | base64 --break 64
+    fi
+    cat <<EOF
+
 Debugger output:
 
 EOF
@@ -188,7 +209,7 @@ EOF
         ;;
 
       *)
-        "$tool" "$file"
+        "$tool" "$file" || true
         ;;
     esac
 
@@ -200,6 +221,7 @@ EOF
 #
 
 print_banner=no
+have_failures=no
 
 for tool in $(ls "$FUZZ_BIN_PATH" 2>/dev/null)
 do
@@ -240,12 +262,14 @@ do
             echo
 
             analyze_crash "$FUZZ_BIN_PATH/$tool" "$FUZZ_OUTPUT_PATH/$tool/$run/crashes/$crash"
+
+            have_failures=yes
         done
     done
 done
 
 # Exit with non-zero status if we have printed a crash report
-if [ "$print_banner" = "yes" ] && [ "$INTERACTIVE" = "no" ]
+if [ "$have_failures" = "yes" ] && [ "$INTERACTIVE" = "no" ]
 then
     exit 1
 fi

tools/afl/fuzzy.mk

@@ -21,6 +21,7 @@ FUZZ_PATH = tools/afl
 FUZZ_BIN_PATH = $(BIN_PATH)/afl
 FUZZ_SRC_PATH = $(FUZZ_PATH)/src
 FUZZ_THEMIS_PATH = $(BIN_PATH)/afl-themis
+FUZZ_SOTER_LIB  = $(FUZZ_THEMIS_PATH)/$(LIBSOTER_A)
 FUZZ_THEMIS_LIB = $(FUZZ_THEMIS_PATH)/$(LIBTHEMIS_A)
 
 FUZZ_SOURCES = $(wildcard $(FUZZ_SRC_PATH)/*.c)
@@ -30,13 +31,28 @@ FUZZ_TOOLS = $(addprefix $(FUZZ_BIN_PATH)/,$(notdir $(wildcard $(FUZZ_PATH)/inpu
 FUZZ_OBJS  = $(patsubst $(FUZZ_SRC_PATH)/%.c,$(FUZZ_BIN_PATH)/%.o,$(FUZZ_SOURCES))
 FUZZ_UTILS = $(filter-out $(addsuffix .o,$(FUZZ_TOOLS)),$(FUZZ_OBJS))
 
-AFL_CFLAGS  += -I$(FUZZ_SRC_PATH)
-AFL_LDFLAGS += -L$(FUZZ_THEMIS_PATH) -lthemis -lsoter
-
-# We would like to use all other compilation flags as well, but some of them
-# (like warnings) might be supported by CC but not AFL_CC. Filter them out.
-AFL_CFLAGS  := $(AFL_CFLAGS)  $(foreach flag,$(CFLAGS),$(if $(call supported,$(flag),$(AFL_CC)),$(flag),))
-AFL_LDFLAGS := $(AFL_LDFLAGS) $(LDFLAGS) $(CRYPTO_ENGINE_LDFLAGS)
+# Build sources with access to fuzzing headers and link tools to $(FUZZ_THEMIS_LIB).
+$(FUZZ_OBJS): CFLAGS += -I$(FUZZ_SRC_PATH)
+$(FUZZ_TOOLS): LDFLAGS += $(FUZZ_THEMIS_LIB) $(FUZZ_SOTER_LIB) $(CRYPTO_ENGINE_LDFLAGS)
+
+# afl-clang is partially configured via environment variables. For one, it likes to
+# talk on stdout so tell it to pipe down a bit. Additionally, address sanitizer builds
+# are usually 32-bit (to keep virtual memory at bay, read AFL docs to learn more).
+AFL_CC_ENV += AFL_QUIET=1
+ifdef WITH_ASAN
+AFL_CC_ENV += AFL_USE_ASAN=1
+$(FUZZ_OBJS): CFLAGS += -m32
+$(FUZZ_TOOLS): LDFLAGS += -m32
+endif
+# We do not pass CFLAGS or LDFLAGS to child processes, add the flags again for them
+# to be used during recursive make invocation for building Themis.
+ifeq ($(AFL_USE_ASAN),1)
+CFLAGS  += -m32
+LDFLAGS += -m32
+endif
+# afl-gcc uses AFL_CC environment variable itself. Do not export it to avoid silly
+# infinite loops of afl-gcc calling afl-gcc. (afl-clang is fine though.)
+unexport AFL_CC
 
 # We don't really track dependencies of $(FUZZ_THEMIS_LIB) here,
 # so ask our make to rebuild it every time. The recursively called
@@ -66,17 +82,17 @@ endif
 $(FUZZ_BIN_PATH)/%.o: $(FUZZ_SRC_PATH)/%.c
 	@mkdir -p $(@D)
 	@echo -n "compile "
-	@AFL_QUIET=1 $(AFL_CC) $(AFL_CFLAGS) -c -o $@ $<
+	@$(AFL_CC_ENV) $(AFL_CC) $(CFLAGS) -c -o $@ $<
 	@$(PRINT_OK)
 
 $(FUZZ_BIN_PATH)/%: $(FUZZ_BIN_PATH)/%.o $(FUZZ_UTILS) $(FUZZ_THEMIS_LIB)
 	@mkdir -p $(@D)
 	@echo -n "link "
-	@AFL_QUIET=1 $(AFL_CC) -o $@ $< $(FUZZ_UTILS) $(AFL_LDFLAGS)
+	@$(AFL_CC_ENV) $(AFL_CC) -o $@ $< $(FUZZ_UTILS) $(LDFLAGS)
 	@$(PRINT_OK)
 
-$(FUZZ_THEMIS_LIB):
-	@AFL_QUIET=1 $(MAKE) themis_static soter_static CC=$(AFL_CC) BUILD_PATH=$(FUZZ_THEMIS_PATH)
+$(FUZZ_THEMIS_LIB): $(SOTER_ENGINE_DEPS)
+	@$(AFL_CC_ENV) $(MAKE) themis_static soter_static CC=$(AFL_CC) BUILD_PATH=$(FUZZ_THEMIS_PATH)
 
 FMT_FIXUP += $(patsubst $(FUZZ_SRC_PATH)/%,$(FUZZ_BIN_PATH)/%.fmt_fixup,$(FUZZ_SOURCES) $(FUZZ_HEADERS))
 FMT_CHECK += $(patsubst $(FUZZ_SRC_PATH)/%,$(FUZZ_BIN_PATH)/%.fmt_check,$(FUZZ_SOURCES) $(FUZZ_HEADERS))

tools/afl/src/readline.c

@@ -16,21 +16,28 @@
 
 #include "readline.h"
 
+#include <errno.h>
 #include <stdlib.h>
 
+#define MAX_SANE_LENGTH (50 * 1024 * 1024)
+
 int read_line_binary(FILE* input, uint8_t** out_bytes, size_t* out_size)
 {
     uint8_t length_bytes[4] = {0};
     uint32_t length = 0;
 
     if (!input || !out_bytes || !out_size) {
+        errno = EINVAL;
         return -1;
     }
 
     *out_bytes = NULL;
     *out_size = 0;
 
     if (fread(length_bytes, 1, 4, input) != 4) {
+        if (!errno) {
+            errno = EFAULT;
+        }
         return -1;
     }
 
@@ -39,14 +46,28 @@ int read_line_binary(FILE* input, uint8_t** out_bytes, size_t* out_size)
     length = (length << 8) | length_bytes[2];
     length = (length << 8) | length_bytes[3];
 
+    /*
+     * We use correct buffer lengths in our input data, but don't let AFL
+     * mess with that by tricking us into allocating gigabytes before we
+     * even started using Themis. This fails with ASAN on 32-bit archs.
+     */
+    if (length > MAX_SANE_LENGTH) {
+        errno = ERANGE;
+        return -1;
+    }
+
     *out_bytes = malloc(length);
     if (!*out_bytes) {
+        errno = ENOMEM;
         return -1;
     }
 
     if (fread(*out_bytes, 1, length, input) != length) {
         free(*out_bytes);
         *out_bytes = NULL;
+        if (!errno) {
+            errno = EIO;
+        }
         return -1;
     }