Automatic Generation of Test Cases for Incremental Static Analysis

incremental-test-generation/.gitignore

@@ -0,0 +1,7 @@
+temp/**
+config.yaml
+api-key.yaml
+out/**
+witness.*
+finished*
+transformed.c

incremental-test-generation/README.md

@@ -0,0 +1,20 @@
+# Create the clang tidy checks
+Read [this](clang-mutations/README.md) for creating the clang tidy checks needed to generate mutations.
+
+# The mutations
+Read [this](clang-mutations/MUTATIONS.md) to find out more about the different types of mutations.
+
+# Sample files
+In `samle-files/` you find some examples you can use as input to get started.
+
+# Mutation Generator
+Run `./RUN.sh` to start the Command Line Interface. With `./RUN.sh -h` you can see the command line options for skipping the interactive user input.
+
+# Run on regression tests
+Run `./RUN_BATCH <directory> [additional arguments...]` for running all c files in the directory. The additional arguments are passed to goblint to skip the interactive cli (Recommended: `--enable-mutations --disable-precision --enable-running --disable-create-tests --enable-cfg --goblint-config {}`). When there are arguments defined in the file by the `PARAM` keyword they are recognized and passed to the incremental tester. When running on the regression test `28-race_reach` you should add this include `--include YOUR_PATH_TO_GOBLINT/analyzer/tests/regression/28-race_reach/racemacros.h`. You can also ignore files using a `.ignore` file with all the paths seperated by a new line. It is recommended to ignore the files `--ignore sample-files/batch-ignore-files/regression-parsing-error.ignore` and `--ignore sample-files/batch-ignore-files/regression-exceptions.ignore`. Use `--no-print` to print only the status updates on input level. This results in the command:
+```
+./RUN_BATCH.sh YOUR_PATH_TO_GOBLINT/analyzer/tests/regression/ --no-print --ignore sample-files/batch-ignore-files/regression-parsing-error.ignore --ignore sample-files/batch-ignore-files/regression-exceptions.ignore -m -dp -er -dt -ec -c {} --include YOUR_PATH_TO_GOBLINT/analyzer/tests/regression/28-race_reach/racemacros.h
+```
+
+# Implementation
+To find out more about the implementation check out the `./scripts` directory and its [readme](./scripts/README.md) file.

incremental-test-generation/RUN.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Check if the script is run from the correct location
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+CURRENT_DIR=$(pwd)
+if [ "$SCRIPT_DIR" != "$CURRENT_DIR" ]; then
+  echo "Script is NOT run from its own directory! Please change the directory to the location of the script."
+  exit 1
+fi
+
+# Check for python imports
+modules=("argparse" "difflib" "glob" "threading" "re" "json" "os" "subprocess" "sys" "yaml" "pydriller" "ast" "random" "time" "concurrent.futures" "multiprocessing" "openai" "questionary" "pathlib" "enum" "shutil")
+for module in "${modules[@]}"
+do
+    python3 -c "import $module" &> /dev/null
+    if [ $? -ne 0 ]; then
+        echo "Python module '$module' is not installed."
+        exit 1
+    fi
+done
+
+# Check if 'git' is installed
+command -v git > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+    echo "'git' is not installed."
+    exit 1
+fi
+
+# Run cli
+python3 ./scripts/run_cli.py "$@"

incremental-test-generation/RUN_BATCH.sh

@@ -0,0 +1,201 @@
+#!/bin/bash
+
+# Check if the script is run from the correct location
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+CURRENT_DIR=$(pwd)
+if [ "$SCRIPT_DIR" != "$CURRENT_DIR" ]; then
+  echo "Script is NOT run from its own directory! Please change the directory to the location of the script."
+  exit 1
+fi
+
+# Check if realpath is available
+if ! command -v realpath &> /dev/null; then
+    echo "realpath command is not available. Please install it."
+    exit 1
+fi
+
+color_grey="\033[90m"
+color_red="\033[31m"
+color_blue='\033[34m'
+color_green='\033[32m'
+color_yellow="\033[33m"
+color_orange="\033[38;5;208m"
+color_reset="\033[0m"
+
+no_print=false
+ignore_files=()
+
+# Check for at least one argument
+if [ $# -lt 1 ]; then
+    printf "Usage: $0 <directory> [--no-print] [--ignore <file_path>]* [additional arguments...]\n"
+    printf "Please order the arguments and options as stated above"
+    printf "<directory>: path to a directory with the .c files for the batch\n"
+    printf "[--no-print]: Do not print the mutation generator output\n"
+    printf "[--ignore <file_path>]* : paths to files containing paths to be ignored, separated by newlines\n"
+    printf "[additional arguments...]: Arguments passed to Goblint to skip interactive cli.\n"
+    printf "    -> Recommended: --enable-mutations --disable-precision --enable-running --disable-create-tests --enable-cfg --Goblint-config {}\n"
+    printf "       or in short: -m -dp -er -dt -ec -c {}\n"
+    exit 1
+fi
+
+# Store directory and shift to get other arguments
+dir=$1
+shift
+
+# Check if directory exists
+if [ ! -d "$dir" ]; then
+    printf "Directory $dir does not exist.\n"
+    exit 1
+fi
+
+# Check for --no-print option
+if [[ $1 == "--no-print" ]]; then
+    no_print=true
+    shift
+fi
+
+# Check for --ignore options and store ignore files in an array
+while [[ $1 == "--ignore" ]]; do
+    ignore_files+=($2)
+    shift 2
+done
+
+# Normalize paths in ignore files
+ignore_patterns=()
+for ignore_file in "${ignore_files[@]}"; do
+    while IFS= read -r line; do
+        ignore_patterns+=("$(realpath "$line" 2>/dev/null)")
+    done < "$ignore_file"
+done
+
+# Find .c files in subdirectories of the specified directory
+files=$(find "$dir" -type f -name "*.c" | sort)
+files_length=$(find "$dir" -type f -name "*.c" | wc -l)
+
+# Iterate over the found files
+index=0
+for file in $files
+do
+    ((index=index+1))
+
+    # Check if the file should be ignored
+    file_realpath=$(realpath "$file")
+    for pattern in "${ignore_patterns[@]}"; do
+        if [[ "$file_realpath" == $pattern ]]; then
+            printf "${color_grey}[BATCH][${index}/${files_length}] Ignoring file $file${color_reset}\n"
+            ignored_files+=("$file")
+            continue 2
+        fi
+    done
+
+    # Run the command with remaining arguments
+    printf "${color_blue}[BATCH][${index}/${files_length}] Running file $file${color_reset}"
+    if [ "$no_print" = true ]; then
+        ./RUN.sh -i "$file" "$@" > /dev/null
+    else
+        printf "\n"
+        ./RUN.sh -i "$file" "$@"
+    fi
+
+    # Check for different return values
+    case $? in
+        0)
+            printf "$\r${color_green}[BATCH][${index}/${files_length}] Test succeeded for file $file"
+            success_files+=("$file")
+            ;;
+        100)
+            printf "$\r${color_orange}[BATCH][${index}/${files_length}] Test failed for file $file"
+            failed_files+=("$file")
+            ;;
+        *)
+            printf "$\r${color_red}[BATCH][${index}/${files_length}] Exception during execution for file $file"
+            exception_files+=("$file")
+            ;;
+    esac
+    printf "${color_reset}\n"
+done
+
+ignored_length=${#ignored_files[@]}
+success_length=${#success_files[@]}
+failed_length=${#failed_files[@]}
+exception_length=${#exception_files[@]}
+printf "\n\n${color_blue}[BATCH] Batch finished running $files_length input files${color_reset}\n\n"
+
+# Print all ignored files
+if [ ${#ignored_files[@]} -ne 0 ]; then
+    printf "${color_yellow}The following $ignored_length files were ignored:\n"
+    for file in "${ignored_files[@]}"; do
+        printf "$file"
+        if [[ $first_line == *"SKIP"* ]]; then
+            printf " (Contained SKIP keyword in first line)\n"
+        else
+            printf "\n"
+        fi
+    done
+    printf "${color_reset}\n"
+fi
+
+# Print all success files
+if [ ${#success_files[@]} -ne 0 ]; then
+    printf "${color_green}The following $success_length files were run successfully with all tests passing:\n"
+    for file in "${success_files[@]}"; do
+        printf "$file"
+        # Check if the first line of the file contains "SKIP"
+        first_line=$(head -n 1 "$file")
+        if [[ $first_line == *"SKIP"* ]]; then
+            printf "${color_yellow} (Contained SKIP keyword in first line)${color_green}\n"
+        else
+            printf "\n"
+        fi
+    done
+    printf "${color_reset}\n"
+fi
+
+# Print all files for which the test failed
+if [ ${#failed_files[@]} -ne 0 ]; then
+    printf "${color_orange}The following $failed_length files failed the tests:\n"
+    for file in "${failed_files[@]}"; do
+        printf "$file"
+        # Check if the first line of the file contains "SKIP"
+        first_line=$(head -n 1 "$file")
+        if [[ $first_line == *"SKIP"* ]]; then
+            printf "${color_yellow} (Contained SKIP keyword in first line)${color_orange}\n"
+        else
+            printf "\n"
+        fi
+    done
+    printf "${color_reset}\n"
+fi
+
+
+# Print all files for which an exception occurred during execution
+if [ ${#exception_files[@]} -ne 0 ]; then
+    printf "${color_red}The following $exception_length files experienced an exception during execution:\n"
+    for file in "${exception_files[@]}"; do
+        printf "$file"
+        # Check if the first line of the file contains "SKIP"
+        first_line=$(head -n 1 "$file")
+        if [[ $first_line == *"SKIP"* ]]; then
+            printf "${color_yellow} (Contained SKIP keyword in first line)${color_red}\n"
+        else
+            printf "\n"
+        fi
+    done
+    printf "${color_reset}\n"
+fi
+
+# Print summary
+printf "\n[BATCH] Summary:\n"
+printf "Total number of executed files: $files_length\n"
+
+if [ "$ignored_length" -eq 0 ]; then color=${color_grey}; else color=${color_yellow}; fi
+printf "${color}Number of ignored files: $ignored_length\n"
+
+if [ "$success_length" -eq 0 ]; then color=${color_grey}; else color=${color_green}; fi
+printf "${color}Number of successfully executed files: $success_length\n"
+
+if [ "$failed_length" -eq 0 ]; then color=${color_grey}; else color=${color_orange}; fi
+printf "${color}Number of files that failed the tests: $failed_length\n"
+
+if [ "$exception_length" -eq 0 ]; then color=${color_grey}; else color=${color_red}; fi
+printf "${color}Number of files that experienced an exception during execution: $exception_length\n"

incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp

@@ -0,0 +1,40 @@
+#include "ConstantReplacementCheck.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang::tidy::readability {
+
+void ConstantReplacementCheck::registerMatchers(MatchFinder *Finder) {
+  Finder->addMatcher(integerLiteral(isExpansionInMainFile(), unless(anyOf(equals(0), equals(1)))).bind("constant-replacement"), this);
+}
+
+// Replaces all Integer Literals != 0 with 1
+void ConstantReplacementCheck::check(const MatchFinder::MatchResult &Result) {
+  const auto *MatchedDecl = Result.Nodes.getNodeAs<IntegerLiteral>("constant-replacement");  
+
+  // Get old int value
+  llvm::APInt value = MatchedDecl->getValue();
+  std::string valueAsString;
+  llvm::SmallString<8> buffer;
+  value.toString(buffer, 10, true);
+  valueAsString = buffer.str();
+  // Get locations
+  SourceLocation Start = MatchedDecl->getBeginLoc();
+  auto Range = MatchedDecl->getSourceRange();
+  // Check if it is a Macro
+  std::string MacroInfo = " ";
+  if (MatchedDecl->getLocation().isMacroID()) {
+    std::string MacroId = Lexer::getImmediateMacroName(MatchedDecl->getLocation(), *Result.SourceManager, Result.Context->getLangOpts()).str();
+    MacroInfo = "[MACRO][" + MacroId + "] ";
+  }
+  // Replace with 1
+  std::string Replacement = "1 /* [MUTATION][CR]" + MacroInfo + "Replaced Constant " + valueAsString + " */";
+  diag(Start, "[MUTATION][CR]" + MacroInfo + "Replaced Constant %0 with 1")
+    << valueAsString
+    << FixItHint::CreateReplacement(Range, Replacement);
+}
+
+} // namespace clang::tidy::readability

incremental-test-generation/clang-mutations/ConstantReplacementCheck.h

@@ -0,0 +1,19 @@
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_CONSTANTREPLACEMENTCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_CONSTANTREPLACEMENTCHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang::tidy::readability {
+
+/// Constant Replacement
+class ConstantReplacementCheck : public ClangTidyCheck {
+public:
+  ConstantReplacementCheck(StringRef Name, ClangTidyContext *Context)
+      : ClangTidyCheck(Name, Context) {}
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+};
+
+} // namespace clang::tidy::readability
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_CONSTANTREPLACEMENTCHECK_H

incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp

@@ -0,0 +1,40 @@
+#include "LogicalConnectorReplacementCheck.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang::tidy::readability {
+
+void LogicalConnectorReplacementCheck::registerMatchers(MatchFinder *Finder) {
+  Finder->addMatcher(stmt(expr(binaryOperator())).bind("logical-connector-replacement"), this);
+}
+
+// "||" <-> "&&"
+void LogicalConnectorReplacementCheck::check(const MatchFinder::MatchResult &Result) {
+  const auto *MatchedDecl = Result.Nodes.getNodeAs<BinaryOperator>("logical-connector-replacement");
+
+  std::string Replacement;
+
+  if (MatchedDecl->getOpcode() == BO_LOr) {
+    // || -> &&
+    Replacement = "&&";
+  } else if (MatchedDecl->getOpcode() == BO_LAnd) {
+    // && -> ||
+    Replacement = "||";
+  } else {
+    return;
+  }
+  Replacement += " /* [MUTATION][LCR] Replaced Logical Connector */";
+
+  // Get locations
+  SourceLocation Start = MatchedDecl->getOperatorLoc();
+  SourceLocation End = MatchedDecl->getOperatorLoc().getLocWithOffset(2);
+  auto Range = CharSourceRange::getCharRange(Start, End);
+
+  diag(Start, "[MUTATION][LCR] Replaced Logical Connector %0")
+      << MatchedDecl->getOpcodeStr()
+      << FixItHint::CreateReplacement(Range, Replacement);
+}
+
+} // namespace clang::tidy::readability

incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.h

@@ -0,0 +1,19 @@
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_LOGICALCONNECTORREPLACEMENTCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_LOGICALCONNECTORREPLACEMENTCHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang::tidy::readability {
+
+/// Logical Connector Replacement
+class LogicalConnectorReplacementCheck : public ClangTidyCheck {
+public:
+  LogicalConnectorReplacementCheck(StringRef Name, ClangTidyContext *Context)
+      : ClangTidyCheck(Name, Context) {}
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+};
+
+} // namespace clang::tidy::readability
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_LOGICALCONNECTORREPLACEMENTCHECK_H